Big backend refactor for upcoming features
This commit is contained in:
@@ -60,17 +60,20 @@ python auto-claude/run.py --spec 001 --qa-status
|
||||
|
||||
### Testing
|
||||
```bash
|
||||
# Run all tests
|
||||
pytest tests/ -v
|
||||
# Install test dependencies (required first time)
|
||||
cd auto-claude && uv pip install -r ../tests/requirements-test.txt
|
||||
|
||||
# Run all tests (use virtual environment pytest)
|
||||
auto-claude/.venv/bin/pytest tests/ -v
|
||||
|
||||
# Run single test file
|
||||
pytest tests/test_security.py -v
|
||||
auto-claude/.venv/bin/pytest tests/test_security.py -v
|
||||
|
||||
# Run specific test
|
||||
pytest tests/test_security.py::test_bash_command_validation -v
|
||||
auto-claude/.venv/bin/pytest tests/test_security.py::test_bash_command_validation -v
|
||||
|
||||
# Skip slow tests
|
||||
pytest tests/ -m "not slow"
|
||||
auto-claude/.venv/bin/pytest tests/ -m "not slow"
|
||||
```
|
||||
|
||||
### Spec Validation
|
||||
|
||||
@@ -75,7 +75,7 @@ export function GitHubIssues({ onOpenSettings }: GitHubIssuesProps) {
|
||||
<div className="flex-1 flex flex-col h-full">
|
||||
{/* Header */}
|
||||
<IssueListHeader
|
||||
repoFullName={syncStatus.repoFullName}
|
||||
repoFullName={syncStatus.repoFullName ?? ''}
|
||||
openIssuesCount={getOpenIssuesCount()}
|
||||
isLoading={isLoading}
|
||||
searchQuery={searchQuery}
|
||||
|
||||
@@ -249,7 +249,7 @@ export function ProjectSettings({ project, open, onOpenChange }: ProjectSettings
|
||||
envConfig={envConfig}
|
||||
settings={settings}
|
||||
onUpdateConfig={updateEnvConfig}
|
||||
onUpdateSettings={setSettings}
|
||||
onUpdateSettings={(updates) => setSettings({ ...settings, ...updates })}
|
||||
infrastructureStatus={infrastructureStatus}
|
||||
isCheckingInfrastructure={isCheckingInfrastructure}
|
||||
isStartingFalkorDB={isStartingFalkorDB}
|
||||
|
||||
@@ -5,5 +5,5 @@ export { useClaudeAuth } from './useClaudeAuth';
|
||||
export { useLinearConnection } from './useLinearConnection';
|
||||
export { useGitHubConnection } from './useGitHubConnection';
|
||||
export { useInfrastructureStatus } from './useInfrastructureStatus';
|
||||
export { useIpc } from './useIpc';
|
||||
export { useIpcListeners } from './useIpc';
|
||||
export { useVirtualizedTree } from './useVirtualizedTree';
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
import type { ProjectEnvConfig } from '../../../shared/types';
|
||||
import type { ProjectEnvConfig } from '../../shared/types';
|
||||
|
||||
type AuthStatus = 'checking' | 'authenticated' | 'not_authenticated' | 'error';
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
import type { ProjectEnvConfig } from '../../../shared/types';
|
||||
import type { ProjectEnvConfig } from '../../shared/types';
|
||||
|
||||
export function useEnvironmentConfig(projectId: string, autoBuildPath: string | null, open: boolean) {
|
||||
const [envConfig, setEnvConfig] = useState<ProjectEnvConfig | null>(null);
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
import type { GitHubSyncStatus } from '../../../shared/types';
|
||||
import type { GitHubSyncStatus } from '../../shared/types';
|
||||
|
||||
export function useGitHubConnection(
|
||||
projectId: string,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
import type { InfrastructureStatus } from '../../../shared/types';
|
||||
import type { InfrastructureStatus } from '../../shared/types';
|
||||
|
||||
export function useInfrastructureStatus(
|
||||
graphitiEnabled: boolean | undefined,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
import type { LinearSyncStatus } from '../../../shared/types';
|
||||
import type { LinearSyncStatus } from '../../shared/types';
|
||||
|
||||
export function useLinearConnection(
|
||||
projectId: string,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
import type { Project, ProjectSettings, AutoBuildVersionInfo, ProjectEnvConfig } from '../../../shared/types';
|
||||
import type { Project, ProjectSettings, AutoBuildVersionInfo, ProjectEnvConfig } from '../../shared/types';
|
||||
import { checkProjectVersion } from '../stores/project-store';
|
||||
|
||||
export function useProjectSettings(project: Project, open: boolean) {
|
||||
|
||||
@@ -11,9 +11,16 @@ export const claudeProfileMock = {
|
||||
}
|
||||
}),
|
||||
|
||||
saveClaudeProfile: async (profile: unknown) => ({
|
||||
saveClaudeProfile: async (profile: { id: string; name: string; oauthToken?: string; email?: string; isDefault?: boolean; createdAt?: Date }) => ({
|
||||
success: true,
|
||||
data: profile
|
||||
data: {
|
||||
id: profile.id,
|
||||
name: profile.name,
|
||||
oauthToken: profile.oauthToken,
|
||||
email: profile.email,
|
||||
isDefault: profile.isDefault ?? false,
|
||||
createdAt: profile.createdAt ?? new Date(),
|
||||
}
|
||||
}),
|
||||
|
||||
deleteClaudeProfile: async () => ({ success: true }),
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
"""
|
||||
Auto Claude - Autonomous Coding Framework
|
||||
==========================================
|
||||
|
||||
Multi-agent autonomous coding framework that builds software through
|
||||
coordinated AI agent sessions.
|
||||
"""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
+2
-63
@@ -1,63 +1,2 @@
|
||||
"""
|
||||
Agent Session Logic
|
||||
===================
|
||||
|
||||
Core agent interaction functions for running autonomous coding sessions.
|
||||
Uses subtask-based implementation plans with minimal, focused prompts.
|
||||
|
||||
Architecture:
|
||||
- Orchestrator (Python) handles all bookkeeping: memory, commits, progress
|
||||
- Agent focuses ONLY on implementing code
|
||||
- Post-session processing updates memory automatically (100% reliable)
|
||||
|
||||
Enhanced with status file updates for ccstatusline integration.
|
||||
Enhanced with Graphiti memory for cross-session context retrieval.
|
||||
|
||||
NOTE: This module is now a facade that imports from agents/ submodules.
|
||||
All logic has been refactored into focused modules for better maintainability.
|
||||
"""
|
||||
|
||||
# Re-export everything from the agents module to maintain backwards compatibility
|
||||
from agents import (
|
||||
# Constants
|
||||
AUTO_CONTINUE_DELAY_SECONDS,
|
||||
HUMAN_INTERVENTION_FILE,
|
||||
# Memory functions
|
||||
debug_memory_system_status,
|
||||
find_phase_for_subtask,
|
||||
find_subtask_in_plan,
|
||||
get_commit_count,
|
||||
get_graphiti_context,
|
||||
# Utility functions
|
||||
get_latest_commit,
|
||||
load_implementation_plan,
|
||||
post_session_processing,
|
||||
# Session management
|
||||
run_agent_session,
|
||||
# Main API
|
||||
run_autonomous_agent,
|
||||
run_followup_planner,
|
||||
save_session_memory,
|
||||
save_session_to_graphiti,
|
||||
sync_plan_to_source,
|
||||
)
|
||||
|
||||
# Ensure all exports are available at module level
|
||||
__all__ = [
|
||||
"run_autonomous_agent",
|
||||
"run_followup_planner",
|
||||
"debug_memory_system_status",
|
||||
"get_graphiti_context",
|
||||
"save_session_memory",
|
||||
"save_session_to_graphiti",
|
||||
"run_agent_session",
|
||||
"post_session_processing",
|
||||
"get_latest_commit",
|
||||
"get_commit_count",
|
||||
"load_implementation_plan",
|
||||
"find_subtask_in_plan",
|
||||
"find_phase_for_subtask",
|
||||
"sync_plan_to_source",
|
||||
"AUTO_CONTINUE_DELAY_SECONDS",
|
||||
"HUMAN_INTERVENTION_FILE",
|
||||
]
|
||||
"""Backward compatibility shim - import from core.agent instead."""
|
||||
from core.agent import *
|
||||
|
||||
@@ -21,7 +21,7 @@ from .base import (
|
||||
from .coder import run_autonomous_agent
|
||||
|
||||
# Memory functions
|
||||
from .memory import (
|
||||
from .memory_manager import (
|
||||
debug_memory_system_status,
|
||||
get_graphiti_context,
|
||||
save_session_memory,
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
"""
|
||||
Custom MCP Tools for Auto-Claude Agents
|
||||
========================================
|
||||
|
||||
DEPRECATED: This module is now a compatibility shim.
|
||||
Please import from the tools_pkg package instead:
|
||||
|
||||
from agents.tools_pkg import create_auto_claude_mcp_server, get_allowed_tools
|
||||
|
||||
This file remains for backward compatibility with existing imports.
|
||||
All functionality has been moved to the tools_pkg package for better
|
||||
organization and maintainability.
|
||||
"""
|
||||
|
||||
# Import everything from the package to maintain backward compatibility
|
||||
# Use try/except to handle both relative and absolute imports
|
||||
try:
|
||||
from .tools_pkg import (
|
||||
ELECTRON_TOOLS,
|
||||
TOOL_GET_BUILD_PROGRESS,
|
||||
TOOL_GET_SESSION_CONTEXT,
|
||||
TOOL_RECORD_DISCOVERY,
|
||||
TOOL_RECORD_GOTCHA,
|
||||
TOOL_UPDATE_QA_STATUS,
|
||||
TOOL_UPDATE_SUBTASK_STATUS,
|
||||
create_auto_claude_mcp_server,
|
||||
get_allowed_tools,
|
||||
is_electron_mcp_enabled,
|
||||
is_tools_available,
|
||||
)
|
||||
except ImportError:
|
||||
# Fallback for direct execution - import from tools_pkg directly
|
||||
from tools_pkg import (
|
||||
ELECTRON_TOOLS,
|
||||
TOOL_GET_BUILD_PROGRESS,
|
||||
TOOL_GET_SESSION_CONTEXT,
|
||||
TOOL_RECORD_DISCOVERY,
|
||||
TOOL_RECORD_GOTCHA,
|
||||
TOOL_UPDATE_QA_STATUS,
|
||||
TOOL_UPDATE_SUBTASK_STATUS,
|
||||
create_auto_claude_mcp_server,
|
||||
get_allowed_tools,
|
||||
is_electron_mcp_enabled,
|
||||
is_tools_available,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# Main API
|
||||
"create_auto_claude_mcp_server",
|
||||
"get_allowed_tools",
|
||||
"is_tools_available",
|
||||
# Tool name constants
|
||||
"TOOL_UPDATE_SUBTASK_STATUS",
|
||||
"TOOL_GET_BUILD_PROGRESS",
|
||||
"TOOL_RECORD_DISCOVERY",
|
||||
"TOOL_RECORD_GOTCHA",
|
||||
"TOOL_GET_SESSION_CONTEXT",
|
||||
"TOOL_UPDATE_QA_STATUS",
|
||||
# Electron MCP
|
||||
"ELECTRON_TOOLS",
|
||||
"is_electron_mcp_enabled",
|
||||
]
|
||||
@@ -53,7 +53,7 @@ from ui import (
|
||||
)
|
||||
|
||||
from .base import AUTO_CONTINUE_DELAY_SECONDS, HUMAN_INTERVENTION_FILE
|
||||
from .memory import debug_memory_system_status, get_graphiti_context
|
||||
from .memory_manager import debug_memory_system_status, get_graphiti_context
|
||||
from .session import post_session_processing, run_agent_session
|
||||
from .utils import (
|
||||
find_phase_for_subtask,
|
||||
|
||||
@@ -8,6 +8,7 @@ Handles session memory storage using dual-layer approach:
|
||||
"""
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from debug import (
|
||||
@@ -20,6 +21,9 @@ from debug import (
|
||||
is_debug_enabled,
|
||||
)
|
||||
from graphiti_config import get_graphiti_status, is_graphiti_enabled
|
||||
|
||||
# Import from parent memory package
|
||||
# Now safe since this module is named memory_manager (not memory)
|
||||
from memory import save_session_insights as save_file_based_memory
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -32,7 +32,7 @@ from ui import (
|
||||
print_status,
|
||||
)
|
||||
|
||||
from .memory import save_session_memory
|
||||
from .memory_manager import save_session_memory
|
||||
from .utils import (
|
||||
find_subtask_in_plan,
|
||||
get_commit_count,
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
"""
|
||||
Analysis Module
|
||||
===============
|
||||
|
||||
Code analysis and project scanning tools.
|
||||
"""
|
||||
|
||||
# Import from analyzers subpackage (these are the modular analyzers)
|
||||
from .analyzers import (
|
||||
ProjectAnalyzer as ModularProjectAnalyzer,
|
||||
ServiceAnalyzer,
|
||||
analyze_project,
|
||||
analyze_service,
|
||||
)
|
||||
|
||||
# Import from analysis module root (these are other analysis tools)
|
||||
from .project_analyzer import ProjectAnalyzer
|
||||
from .risk_classifier import RiskClassifier
|
||||
from .security_scanner import SecurityScanner
|
||||
from .ci_discovery import CIDiscovery
|
||||
from .test_discovery import TestDiscovery
|
||||
|
||||
# insight_extractor is a module with functions, not a class, so don't import it here
|
||||
# Import it directly when needed: from analysis import insight_extractor
|
||||
|
||||
__all__ = [
|
||||
"ProjectAnalyzer",
|
||||
"ModularProjectAnalyzer",
|
||||
"ServiceAnalyzer",
|
||||
"analyze_project",
|
||||
"analyze_service",
|
||||
"RiskClassifier",
|
||||
"SecurityScanner",
|
||||
"CIDiscovery",
|
||||
"TestDiscovery",
|
||||
]
|
||||
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Codebase Analyzer
|
||||
=================
|
||||
|
||||
Automatically detects project structure, frameworks, and services.
|
||||
Supports monorepos with multiple services.
|
||||
|
||||
Usage:
|
||||
# Index entire project (creates project_index.json)
|
||||
python auto-claude/analyzer.py --index
|
||||
|
||||
# Analyze specific service
|
||||
python auto-claude/analyzer.py --service backend
|
||||
|
||||
# Output to specific file
|
||||
python auto-claude/analyzer.py --index --output path/to/output.json
|
||||
|
||||
The analyzer will:
|
||||
1. Detect if this is a monorepo or single project
|
||||
2. Find all services/packages and analyze each separately
|
||||
3. Map interdependencies between services
|
||||
4. Identify infrastructure (Docker, CI/CD)
|
||||
5. Document conventions (linting, testing)
|
||||
|
||||
This module now serves as a facade to the modular analyzer system in the analyzers/ package.
|
||||
All actual implementation is in focused submodules for better maintainability.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
# Import from the new modular structure
|
||||
from .analyzers import (
|
||||
ProjectAnalyzer,
|
||||
ServiceAnalyzer,
|
||||
analyze_project,
|
||||
analyze_service,
|
||||
)
|
||||
|
||||
# Re-export for backward compatibility
|
||||
__all__ = [
|
||||
"ServiceAnalyzer",
|
||||
"ProjectAnalyzer",
|
||||
"analyze_project",
|
||||
"analyze_service",
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
"""CLI entry point."""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Analyze project structure, frameworks, and services"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--project-dir",
|
||||
type=Path,
|
||||
default=Path.cwd(),
|
||||
help="Project directory to analyze (default: current directory)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--index",
|
||||
action="store_true",
|
||||
help="Create full project index (default behavior)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--service",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Analyze a specific service only",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Output file for JSON results",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--quiet",
|
||||
action="store_true",
|
||||
help="Only output JSON, no status messages",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Determine what to analyze
|
||||
if args.service:
|
||||
results = analyze_service(args.project_dir, args.service, args.output)
|
||||
else:
|
||||
results = analyze_project(args.project_dir, args.output)
|
||||
|
||||
# Print results
|
||||
if not args.quiet or not args.output:
|
||||
print(json.dumps(results, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,92 @@
|
||||
"""
|
||||
Analyzers Package
|
||||
=================
|
||||
|
||||
Modular analyzer system for detecting project structure, frameworks, and services.
|
||||
|
||||
Main exports:
|
||||
- ServiceAnalyzer: Analyzes a single service/package
|
||||
- ProjectAnalyzer: Analyzes entire projects (single or monorepo)
|
||||
- analyze_project: Convenience function for project analysis
|
||||
- analyze_service: Convenience function for service analysis
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .project_analyzer_module import ProjectAnalyzer
|
||||
from .service_analyzer import ServiceAnalyzer
|
||||
|
||||
# Re-export main classes
|
||||
__all__ = [
|
||||
"ServiceAnalyzer",
|
||||
"ProjectAnalyzer",
|
||||
"analyze_project",
|
||||
"analyze_service",
|
||||
]
|
||||
|
||||
|
||||
def analyze_project(project_dir: Path, output_file: Path | None = None) -> dict:
|
||||
"""
|
||||
Analyze a project and optionally save results.
|
||||
|
||||
Args:
|
||||
project_dir: Path to the project root
|
||||
output_file: Optional path to save JSON output
|
||||
|
||||
Returns:
|
||||
Project index as a dictionary
|
||||
"""
|
||||
import json
|
||||
|
||||
analyzer = ProjectAnalyzer(project_dir)
|
||||
results = analyzer.analyze()
|
||||
|
||||
if output_file:
|
||||
output_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_file, "w") as f:
|
||||
json.dump(results, f, indent=2)
|
||||
print(f"Project index saved to: {output_file}")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def analyze_service(
|
||||
project_dir: Path, service_name: str, output_file: Path | None = None
|
||||
) -> dict:
|
||||
"""
|
||||
Analyze a specific service within a project.
|
||||
|
||||
Args:
|
||||
project_dir: Path to the project root
|
||||
service_name: Name of the service to analyze
|
||||
output_file: Optional path to save JSON output
|
||||
|
||||
Returns:
|
||||
Service analysis as a dictionary
|
||||
"""
|
||||
import json
|
||||
|
||||
# Find the service
|
||||
service_path = project_dir / service_name
|
||||
if not service_path.exists():
|
||||
# Check common locations
|
||||
for parent in ["packages", "apps", "services"]:
|
||||
candidate = project_dir / parent / service_name
|
||||
if candidate.exists():
|
||||
service_path = candidate
|
||||
break
|
||||
|
||||
if not service_path.exists():
|
||||
raise ValueError(f"Service '{service_name}' not found in {project_dir}")
|
||||
|
||||
analyzer = ServiceAnalyzer(service_path, service_name)
|
||||
results = analyzer.analyze()
|
||||
|
||||
if output_file:
|
||||
output_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_file, "w") as f:
|
||||
json.dump(results, f, indent=2)
|
||||
print(f"Service analysis saved to: {output_file}")
|
||||
|
||||
return results
|
||||
@@ -0,0 +1,587 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
CI Discovery Module
|
||||
===================
|
||||
|
||||
Parses CI/CD configuration files to extract test commands and workflows.
|
||||
Supports GitHub Actions, GitLab CI, CircleCI, and Jenkins.
|
||||
|
||||
The CI discovery results are used by:
|
||||
- QA Agent: To understand existing CI test patterns
|
||||
- Validation Strategy: To match CI commands
|
||||
- Planner: To align verification with CI
|
||||
|
||||
Usage:
|
||||
from ci_discovery import CIDiscovery
|
||||
|
||||
discovery = CIDiscovery()
|
||||
result = discovery.discover(project_dir)
|
||||
|
||||
if result:
|
||||
print(f"CI System: {result.ci_system}")
|
||||
print(f"Test Commands: {result.test_commands}")
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# Try to import yaml, fall back gracefully
|
||||
try:
|
||||
import yaml
|
||||
|
||||
HAS_YAML = True
|
||||
except ImportError:
|
||||
HAS_YAML = False
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# DATA CLASSES
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@dataclass
|
||||
class CIWorkflow:
|
||||
"""
|
||||
Represents a CI workflow or job.
|
||||
|
||||
Attributes:
|
||||
name: Name of the workflow/job
|
||||
trigger: What triggers this workflow (push, pull_request, etc.)
|
||||
steps: List of step names or commands
|
||||
test_related: Whether this appears to be test-related
|
||||
"""
|
||||
|
||||
name: str
|
||||
trigger: list[str] = field(default_factory=list)
|
||||
steps: list[str] = field(default_factory=list)
|
||||
test_related: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class CIConfig:
|
||||
"""
|
||||
Result of CI configuration discovery.
|
||||
|
||||
Attributes:
|
||||
ci_system: Name of CI system (github_actions, gitlab, circleci, jenkins)
|
||||
config_files: List of CI config files found
|
||||
test_commands: Extracted test commands by type
|
||||
coverage_command: Coverage command if found
|
||||
workflows: List of discovered workflows
|
||||
environment_variables: Environment variables used
|
||||
"""
|
||||
|
||||
ci_system: str
|
||||
config_files: list[str] = field(default_factory=list)
|
||||
test_commands: dict[str, str] = field(default_factory=dict)
|
||||
coverage_command: str | None = None
|
||||
workflows: list[CIWorkflow] = field(default_factory=list)
|
||||
environment_variables: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CI PARSERS
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class CIDiscovery:
|
||||
"""
|
||||
Discovers CI/CD configurations in a project.
|
||||
|
||||
Analyzes:
|
||||
- GitHub Actions (.github/workflows/*.yml)
|
||||
- GitLab CI (.gitlab-ci.yml)
|
||||
- CircleCI (.circleci/config.yml)
|
||||
- Jenkins (Jenkinsfile)
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialize CI discovery."""
|
||||
self._cache: dict[str, CIConfig | None] = {}
|
||||
|
||||
def discover(self, project_dir: Path) -> CIConfig | None:
|
||||
"""
|
||||
Discover CI configuration in the project.
|
||||
|
||||
Args:
|
||||
project_dir: Path to the project root
|
||||
|
||||
Returns:
|
||||
CIConfig if CI found, None otherwise
|
||||
"""
|
||||
project_dir = Path(project_dir)
|
||||
cache_key = str(project_dir.resolve())
|
||||
|
||||
if cache_key in self._cache:
|
||||
return self._cache[cache_key]
|
||||
|
||||
# Try each CI system
|
||||
result = None
|
||||
|
||||
# GitHub Actions
|
||||
github_workflows = project_dir / ".github" / "workflows"
|
||||
if github_workflows.exists():
|
||||
result = self._parse_github_actions(github_workflows)
|
||||
|
||||
# GitLab CI
|
||||
if not result:
|
||||
gitlab_ci = project_dir / ".gitlab-ci.yml"
|
||||
if gitlab_ci.exists():
|
||||
result = self._parse_gitlab_ci(gitlab_ci)
|
||||
|
||||
# CircleCI
|
||||
if not result:
|
||||
circleci = project_dir / ".circleci" / "config.yml"
|
||||
if circleci.exists():
|
||||
result = self._parse_circleci(circleci)
|
||||
|
||||
# Jenkins
|
||||
if not result:
|
||||
jenkinsfile = project_dir / "Jenkinsfile"
|
||||
if jenkinsfile.exists():
|
||||
result = self._parse_jenkinsfile(jenkinsfile)
|
||||
|
||||
self._cache[cache_key] = result
|
||||
return result
|
||||
|
||||
def _parse_github_actions(self, workflows_dir: Path) -> CIConfig:
|
||||
"""Parse GitHub Actions workflow files."""
|
||||
result = CIConfig(ci_system="github_actions")
|
||||
|
||||
workflow_files = list(workflows_dir.glob("*.yml")) + list(
|
||||
workflows_dir.glob("*.yaml")
|
||||
)
|
||||
|
||||
for wf_file in workflow_files:
|
||||
result.config_files.append(
|
||||
str(wf_file.relative_to(workflows_dir.parent.parent))
|
||||
)
|
||||
|
||||
try:
|
||||
content = wf_file.read_text()
|
||||
workflow_data = self._parse_yaml(content)
|
||||
|
||||
if not workflow_data:
|
||||
continue
|
||||
|
||||
# Get workflow name
|
||||
wf_name = workflow_data.get("name", wf_file.stem)
|
||||
|
||||
# Get triggers
|
||||
triggers = []
|
||||
on_trigger = workflow_data.get("on", {})
|
||||
if isinstance(on_trigger, str):
|
||||
triggers = [on_trigger]
|
||||
elif isinstance(on_trigger, list):
|
||||
triggers = on_trigger
|
||||
elif isinstance(on_trigger, dict):
|
||||
triggers = list(on_trigger.keys())
|
||||
|
||||
# Parse jobs
|
||||
jobs = workflow_data.get("jobs", {})
|
||||
for job_name, job_config in jobs.items():
|
||||
if not isinstance(job_config, dict):
|
||||
continue
|
||||
|
||||
steps = job_config.get("steps", [])
|
||||
step_commands = []
|
||||
test_related = False
|
||||
|
||||
for step in steps:
|
||||
if not isinstance(step, dict):
|
||||
continue
|
||||
|
||||
# Get step name or command
|
||||
step_name = step.get("name", "")
|
||||
run_cmd = step.get("run", "")
|
||||
uses = step.get("uses", "")
|
||||
|
||||
if step_name:
|
||||
step_commands.append(step_name)
|
||||
if run_cmd:
|
||||
step_commands.append(run_cmd)
|
||||
# Extract test commands
|
||||
self._extract_test_commands(run_cmd, result)
|
||||
if uses:
|
||||
step_commands.append(f"uses: {uses}")
|
||||
|
||||
# Check if test-related
|
||||
test_keywords = ["test", "pytest", "jest", "vitest", "coverage"]
|
||||
if any(kw in str(step).lower() for kw in test_keywords):
|
||||
test_related = True
|
||||
|
||||
result.workflows.append(
|
||||
CIWorkflow(
|
||||
name=f"{wf_name}/{job_name}",
|
||||
trigger=triggers,
|
||||
steps=step_commands,
|
||||
test_related=test_related,
|
||||
)
|
||||
)
|
||||
|
||||
# Extract environment variables
|
||||
env = workflow_data.get("env", {})
|
||||
if isinstance(env, dict):
|
||||
result.environment_variables.extend(env.keys())
|
||||
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return result
|
||||
|
||||
def _parse_gitlab_ci(self, config_file: Path) -> CIConfig:
|
||||
"""Parse GitLab CI configuration."""
|
||||
result = CIConfig(
|
||||
ci_system="gitlab",
|
||||
config_files=[".gitlab-ci.yml"],
|
||||
)
|
||||
|
||||
try:
|
||||
content = config_file.read_text()
|
||||
data = self._parse_yaml(content)
|
||||
|
||||
if not data:
|
||||
return result
|
||||
|
||||
# Parse jobs (top-level keys that aren't special keywords)
|
||||
special_keys = {
|
||||
"stages",
|
||||
"variables",
|
||||
"image",
|
||||
"services",
|
||||
"before_script",
|
||||
"after_script",
|
||||
"cache",
|
||||
"include",
|
||||
"default",
|
||||
"workflow",
|
||||
}
|
||||
|
||||
for key, value in data.items():
|
||||
if key.startswith(".") or key in special_keys:
|
||||
continue
|
||||
|
||||
if not isinstance(value, dict):
|
||||
continue
|
||||
|
||||
job_config = value
|
||||
script = job_config.get("script", [])
|
||||
if isinstance(script, str):
|
||||
script = [script]
|
||||
|
||||
test_related = any(
|
||||
kw in str(script).lower()
|
||||
for kw in ["test", "pytest", "jest", "vitest", "coverage"]
|
||||
)
|
||||
|
||||
result.workflows.append(
|
||||
CIWorkflow(
|
||||
name=key,
|
||||
trigger=job_config.get("only", [])
|
||||
or job_config.get("rules", []),
|
||||
steps=script,
|
||||
test_related=test_related,
|
||||
)
|
||||
)
|
||||
|
||||
# Extract test commands
|
||||
for cmd in script:
|
||||
if isinstance(cmd, str):
|
||||
self._extract_test_commands(cmd, result)
|
||||
|
||||
# Extract variables
|
||||
variables = data.get("variables", {})
|
||||
if isinstance(variables, dict):
|
||||
result.environment_variables.extend(variables.keys())
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
def _parse_circleci(self, config_file: Path) -> CIConfig:
|
||||
"""Parse CircleCI configuration."""
|
||||
result = CIConfig(
|
||||
ci_system="circleci",
|
||||
config_files=[".circleci/config.yml"],
|
||||
)
|
||||
|
||||
try:
|
||||
content = config_file.read_text()
|
||||
data = self._parse_yaml(content)
|
||||
|
||||
if not data:
|
||||
return result
|
||||
|
||||
# Parse jobs
|
||||
jobs = data.get("jobs", {})
|
||||
for job_name, job_config in jobs.items():
|
||||
if not isinstance(job_config, dict):
|
||||
continue
|
||||
|
||||
steps = job_config.get("steps", [])
|
||||
step_commands = []
|
||||
test_related = False
|
||||
|
||||
for step in steps:
|
||||
if isinstance(step, str):
|
||||
step_commands.append(step)
|
||||
elif isinstance(step, dict):
|
||||
if "run" in step:
|
||||
run = step["run"]
|
||||
if isinstance(run, str):
|
||||
step_commands.append(run)
|
||||
self._extract_test_commands(run, result)
|
||||
elif isinstance(run, dict):
|
||||
cmd = run.get("command", "")
|
||||
step_commands.append(cmd)
|
||||
self._extract_test_commands(cmd, result)
|
||||
|
||||
if any(
|
||||
kw in str(step).lower()
|
||||
for kw in ["test", "pytest", "jest", "coverage"]
|
||||
):
|
||||
test_related = True
|
||||
|
||||
result.workflows.append(
|
||||
CIWorkflow(
|
||||
name=job_name,
|
||||
trigger=[],
|
||||
steps=step_commands,
|
||||
test_related=test_related,
|
||||
)
|
||||
)
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
def _parse_jenkinsfile(self, jenkinsfile: Path) -> CIConfig:
|
||||
"""Parse Jenkinsfile (basic extraction)."""
|
||||
result = CIConfig(
|
||||
ci_system="jenkins",
|
||||
config_files=["Jenkinsfile"],
|
||||
)
|
||||
|
||||
try:
|
||||
content = jenkinsfile.read_text()
|
||||
|
||||
# Extract sh commands using regex
|
||||
sh_pattern = re.compile(r'sh\s+[\'"]([^\'"]+)[\'"]')
|
||||
matches = sh_pattern.findall(content)
|
||||
|
||||
steps = []
|
||||
test_related = False
|
||||
|
||||
for cmd in matches:
|
||||
steps.append(cmd)
|
||||
self._extract_test_commands(cmd, result)
|
||||
|
||||
if any(
|
||||
kw in cmd.lower() for kw in ["test", "pytest", "jest", "coverage"]
|
||||
):
|
||||
test_related = True
|
||||
|
||||
# Extract stage names
|
||||
stage_pattern = re.compile(r'stage\s*\([\'"]([^\'"]+)[\'"]\)')
|
||||
stages = stage_pattern.findall(content)
|
||||
|
||||
for stage in stages:
|
||||
result.workflows.append(
|
||||
CIWorkflow(
|
||||
name=stage,
|
||||
trigger=[],
|
||||
steps=steps if "test" in stage.lower() else [],
|
||||
test_related="test" in stage.lower(),
|
||||
)
|
||||
)
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
def _parse_yaml(self, content: str) -> dict | None:
|
||||
"""Parse YAML content, with fallback to basic parsing if yaml not available."""
|
||||
if HAS_YAML:
|
||||
try:
|
||||
return yaml.safe_load(content)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
# Basic fallback for simple YAML (very limited)
|
||||
# This won't work for complex structures
|
||||
return None
|
||||
|
||||
def _extract_test_commands(self, cmd: str, result: CIConfig) -> None:
|
||||
"""Extract test commands from a command string."""
|
||||
cmd_lower = cmd.lower()
|
||||
|
||||
# Python pytest
|
||||
if "pytest" in cmd_lower:
|
||||
if "pytest" not in result.test_commands:
|
||||
result.test_commands["unit"] = cmd.strip()
|
||||
if "--cov" in cmd_lower:
|
||||
result.coverage_command = cmd.strip()
|
||||
|
||||
# Node.js test commands
|
||||
if (
|
||||
"npm test" in cmd_lower
|
||||
or "yarn test" in cmd_lower
|
||||
or "pnpm test" in cmd_lower
|
||||
):
|
||||
if "unit" not in result.test_commands:
|
||||
result.test_commands["unit"] = cmd.strip()
|
||||
|
||||
# Jest/Vitest
|
||||
if "jest" in cmd_lower or "vitest" in cmd_lower:
|
||||
if "unit" not in result.test_commands:
|
||||
result.test_commands["unit"] = cmd.strip()
|
||||
if "--coverage" in cmd_lower:
|
||||
result.coverage_command = cmd.strip()
|
||||
|
||||
# E2E testing
|
||||
if "playwright" in cmd_lower:
|
||||
result.test_commands["e2e"] = cmd.strip()
|
||||
if "cypress" in cmd_lower:
|
||||
result.test_commands["e2e"] = cmd.strip()
|
||||
|
||||
# Integration tests
|
||||
if "integration" in cmd_lower:
|
||||
result.test_commands["integration"] = cmd.strip()
|
||||
|
||||
# Go tests
|
||||
if "go test" in cmd_lower:
|
||||
if "unit" not in result.test_commands:
|
||||
result.test_commands["unit"] = cmd.strip()
|
||||
|
||||
# Rust tests
|
||||
if "cargo test" in cmd_lower:
|
||||
if "unit" not in result.test_commands:
|
||||
result.test_commands["unit"] = cmd.strip()
|
||||
|
||||
def to_dict(self, result: CIConfig) -> dict[str, Any]:
|
||||
"""Convert result to dictionary for JSON serialization."""
|
||||
return {
|
||||
"ci_system": result.ci_system,
|
||||
"config_files": result.config_files,
|
||||
"test_commands": result.test_commands,
|
||||
"coverage_command": result.coverage_command,
|
||||
"workflows": [
|
||||
{
|
||||
"name": w.name,
|
||||
"trigger": w.trigger,
|
||||
"steps": w.steps,
|
||||
"test_related": w.test_related,
|
||||
}
|
||||
for w in result.workflows
|
||||
],
|
||||
"environment_variables": result.environment_variables,
|
||||
}
|
||||
|
||||
def clear_cache(self) -> None:
|
||||
"""Clear the internal cache."""
|
||||
self._cache.clear()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CONVENIENCE FUNCTIONS
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def discover_ci(project_dir: Path) -> CIConfig | None:
|
||||
"""
|
||||
Convenience function to discover CI configuration.
|
||||
|
||||
Args:
|
||||
project_dir: Path to project root
|
||||
|
||||
Returns:
|
||||
CIConfig if found, None otherwise
|
||||
"""
|
||||
discovery = CIDiscovery()
|
||||
return discovery.discover(project_dir)
|
||||
|
||||
|
||||
def get_ci_test_commands(project_dir: Path) -> dict[str, str]:
|
||||
"""
|
||||
Get test commands from CI configuration.
|
||||
|
||||
Args:
|
||||
project_dir: Path to project root
|
||||
|
||||
Returns:
|
||||
Dictionary of test type to command
|
||||
"""
|
||||
discovery = CIDiscovery()
|
||||
result = discovery.discover(project_dir)
|
||||
if result:
|
||||
return result.test_commands
|
||||
return {}
|
||||
|
||||
|
||||
def get_ci_system(project_dir: Path) -> str | None:
|
||||
"""
|
||||
Get the CI system name if configured.
|
||||
|
||||
Args:
|
||||
project_dir: Path to project root
|
||||
|
||||
Returns:
|
||||
CI system name or None
|
||||
"""
|
||||
discovery = CIDiscovery()
|
||||
result = discovery.discover(project_dir)
|
||||
if result:
|
||||
return result.ci_system
|
||||
return None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLI
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""CLI entry point for testing."""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Discover CI configuration")
|
||||
parser.add_argument("project_dir", type=Path, help="Path to project root")
|
||||
parser.add_argument("--json", action="store_true", help="Output as JSON")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
discovery = CIDiscovery()
|
||||
result = discovery.discover(args.project_dir)
|
||||
|
||||
if not result:
|
||||
print("No CI configuration found")
|
||||
return
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(discovery.to_dict(result), indent=2))
|
||||
else:
|
||||
print(f"CI System: {result.ci_system}")
|
||||
print(f"Config Files: {', '.join(result.config_files)}")
|
||||
print("\nTest Commands:")
|
||||
for test_type, cmd in result.test_commands.items():
|
||||
print(f" {test_type}: {cmd}")
|
||||
if result.coverage_command:
|
||||
print(f"\nCoverage Command: {result.coverage_command}")
|
||||
print(f"\nWorkflows ({len(result.workflows)}):")
|
||||
for w in result.workflows:
|
||||
marker = "[TEST]" if w.test_related else ""
|
||||
print(f" - {w.name} {marker}")
|
||||
if w.trigger:
|
||||
print(f" Triggers: {', '.join(str(t) for t in w.trigger)}")
|
||||
if result.environment_variables:
|
||||
print(f"\nEnvironment Variables: {', '.join(result.environment_variables)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,584 @@
|
||||
"""
|
||||
Insight Extractor
|
||||
=================
|
||||
|
||||
Automatically extracts structured insights from completed coding sessions.
|
||||
Runs after each session to capture rich, actionable knowledge for Graphiti memory.
|
||||
|
||||
Uses the Claude Agent SDK (same as the rest of the system) for extraction.
|
||||
Falls back to generic insights if extraction fails (never blocks the build).
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Check for Claude SDK availability
|
||||
try:
|
||||
from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient
|
||||
SDK_AVAILABLE = True
|
||||
except ImportError:
|
||||
SDK_AVAILABLE = False
|
||||
ClaudeAgentOptions = None
|
||||
ClaudeSDKClient = None
|
||||
|
||||
# Default model for insight extraction (fast and cheap)
|
||||
DEFAULT_EXTRACTION_MODEL = "claude-3-5-haiku-latest"
|
||||
|
||||
# Maximum diff size to send to the LLM (avoid context limits)
|
||||
MAX_DIFF_CHARS = 15000
|
||||
|
||||
# Maximum attempt history entries to include
|
||||
MAX_ATTEMPTS_TO_INCLUDE = 3
|
||||
|
||||
|
||||
def is_extraction_enabled() -> bool:
|
||||
"""Check if insight extraction is enabled."""
|
||||
# Extraction requires Claude SDK and OAuth token
|
||||
if not SDK_AVAILABLE:
|
||||
return False
|
||||
if not os.environ.get("CLAUDE_CODE_OAUTH_TOKEN"):
|
||||
return False
|
||||
enabled_str = os.environ.get("INSIGHT_EXTRACTION_ENABLED", "true").lower()
|
||||
return enabled_str in ("true", "1", "yes")
|
||||
|
||||
|
||||
def get_extraction_model() -> str:
|
||||
"""Get the model to use for insight extraction."""
|
||||
return os.environ.get("INSIGHT_EXTRACTOR_MODEL", DEFAULT_EXTRACTION_MODEL)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Git Helpers
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def get_session_diff(
|
||||
project_dir: Path,
|
||||
commit_before: str | None,
|
||||
commit_after: str | None,
|
||||
) -> str:
|
||||
"""
|
||||
Get the git diff between two commits.
|
||||
|
||||
Args:
|
||||
project_dir: Project root directory
|
||||
commit_before: Commit hash before session (or None)
|
||||
commit_after: Commit hash after session (or None)
|
||||
|
||||
Returns:
|
||||
Diff text (truncated if too large)
|
||||
"""
|
||||
if not commit_before or not commit_after:
|
||||
return "(No commits to diff)"
|
||||
|
||||
if commit_before == commit_after:
|
||||
return "(No changes - same commit)"
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "diff", commit_before, commit_after],
|
||||
cwd=project_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
diff = result.stdout
|
||||
|
||||
if len(diff) > MAX_DIFF_CHARS:
|
||||
# Truncate and add note
|
||||
diff = (
|
||||
diff[:MAX_DIFF_CHARS] + f"\n\n... (truncated, {len(diff)} chars total)"
|
||||
)
|
||||
|
||||
return diff if diff else "(Empty diff)"
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("Git diff timed out")
|
||||
return "(Git diff timed out)"
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to get git diff: {e}")
|
||||
return f"(Failed to get diff: {e})"
|
||||
|
||||
|
||||
def get_changed_files(
|
||||
project_dir: Path,
|
||||
commit_before: str | None,
|
||||
commit_after: str | None,
|
||||
) -> list[str]:
|
||||
"""
|
||||
Get list of files changed between two commits.
|
||||
|
||||
Args:
|
||||
project_dir: Project root directory
|
||||
commit_before: Commit hash before session
|
||||
commit_after: Commit hash after session
|
||||
|
||||
Returns:
|
||||
List of changed file paths
|
||||
"""
|
||||
if not commit_before or not commit_after or commit_before == commit_after:
|
||||
return []
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "diff", "--name-only", commit_before, commit_after],
|
||||
cwd=project_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
files = [f.strip() for f in result.stdout.strip().split("\n") if f.strip()]
|
||||
return files
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to get changed files: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def get_commit_messages(
|
||||
project_dir: Path,
|
||||
commit_before: str | None,
|
||||
commit_after: str | None,
|
||||
) -> str:
|
||||
"""Get commit messages between two commits."""
|
||||
if not commit_before or not commit_after or commit_before == commit_after:
|
||||
return "(No commits)"
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "log", "--oneline", f"{commit_before}..{commit_after}"],
|
||||
cwd=project_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
return result.stdout.strip() if result.stdout.strip() else "(No commits)"
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to get commit messages: {e}")
|
||||
return f"(Failed: {e})"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Input Gathering
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def gather_extraction_inputs(
|
||||
spec_dir: Path,
|
||||
project_dir: Path,
|
||||
subtask_id: str,
|
||||
session_num: int,
|
||||
commit_before: str | None,
|
||||
commit_after: str | None,
|
||||
success: bool,
|
||||
recovery_manager: Any,
|
||||
) -> dict:
|
||||
"""
|
||||
Gather all inputs needed for insight extraction.
|
||||
|
||||
Args:
|
||||
spec_dir: Spec directory
|
||||
project_dir: Project root
|
||||
subtask_id: The subtask that was worked on
|
||||
session_num: Session number
|
||||
commit_before: Commit before session
|
||||
commit_after: Commit after session
|
||||
success: Whether session succeeded
|
||||
recovery_manager: Recovery manager with attempt history
|
||||
|
||||
Returns:
|
||||
Dict with all inputs for the extractor
|
||||
"""
|
||||
# Get subtask description from implementation plan
|
||||
subtask_description = _get_subtask_description(spec_dir, subtask_id)
|
||||
|
||||
# Get git diff
|
||||
diff = get_session_diff(project_dir, commit_before, commit_after)
|
||||
|
||||
# Get changed files
|
||||
changed_files = get_changed_files(project_dir, commit_before, commit_after)
|
||||
|
||||
# Get commit messages
|
||||
commit_messages = get_commit_messages(project_dir, commit_before, commit_after)
|
||||
|
||||
# Get attempt history
|
||||
attempt_history = _get_attempt_history(recovery_manager, subtask_id)
|
||||
|
||||
return {
|
||||
"subtask_id": subtask_id,
|
||||
"subtask_description": subtask_description,
|
||||
"session_num": session_num,
|
||||
"success": success,
|
||||
"diff": diff,
|
||||
"changed_files": changed_files,
|
||||
"commit_messages": commit_messages,
|
||||
"attempt_history": attempt_history,
|
||||
}
|
||||
|
||||
|
||||
def _get_subtask_description(spec_dir: Path, subtask_id: str) -> str:
|
||||
"""Get subtask description from implementation plan."""
|
||||
plan_file = spec_dir / "implementation_plan.json"
|
||||
if not plan_file.exists():
|
||||
return f"Subtask: {subtask_id}"
|
||||
|
||||
try:
|
||||
with open(plan_file) as f:
|
||||
plan = json.load(f)
|
||||
|
||||
# Search through phases for the subtask
|
||||
for phase in plan.get("phases", []):
|
||||
for subtask in phase.get("subtasks", []):
|
||||
if subtask.get("id") == subtask_id:
|
||||
return subtask.get("description", f"Subtask: {subtask_id}")
|
||||
|
||||
return f"Subtask: {subtask_id}"
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load subtask description: {e}")
|
||||
return f"Subtask: {subtask_id}"
|
||||
|
||||
|
||||
def _get_attempt_history(recovery_manager: Any, subtask_id: str) -> list[dict]:
|
||||
"""Get previous attempt history for this subtask."""
|
||||
if not recovery_manager:
|
||||
return []
|
||||
|
||||
try:
|
||||
history = recovery_manager.get_subtask_history(subtask_id)
|
||||
attempts = history.get("attempts", [])
|
||||
|
||||
# Limit to recent attempts
|
||||
return attempts[-MAX_ATTEMPTS_TO_INCLUDE:]
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to get attempt history: {e}")
|
||||
return []
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# LLM Extraction
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def _build_extraction_prompt(inputs: dict) -> str:
|
||||
"""Build the prompt for insight extraction."""
|
||||
prompt_file = Path(__file__).parent / "prompts" / "insight_extractor.md"
|
||||
|
||||
if prompt_file.exists():
|
||||
base_prompt = prompt_file.read_text()
|
||||
else:
|
||||
# Fallback if prompt file missing
|
||||
base_prompt = """Extract structured insights from this coding session.
|
||||
Output ONLY valid JSON with: file_insights, patterns_discovered, gotchas_discovered, approach_outcome, recommendations"""
|
||||
|
||||
# Build session context
|
||||
session_context = f"""
|
||||
---
|
||||
|
||||
## SESSION DATA
|
||||
|
||||
### Subtask
|
||||
- **ID**: {inputs["subtask_id"]}
|
||||
- **Description**: {inputs["subtask_description"]}
|
||||
- **Session Number**: {inputs["session_num"]}
|
||||
- **Outcome**: {"SUCCESS" if inputs["success"] else "FAILED"}
|
||||
|
||||
### Files Changed
|
||||
{chr(10).join(f"- {f}" for f in inputs["changed_files"]) if inputs["changed_files"] else "(No files changed)"}
|
||||
|
||||
### Commit Messages
|
||||
{inputs["commit_messages"]}
|
||||
|
||||
### Git Diff
|
||||
```diff
|
||||
{inputs["diff"]}
|
||||
```
|
||||
|
||||
### Previous Attempts
|
||||
{_format_attempt_history(inputs["attempt_history"])}
|
||||
|
||||
---
|
||||
|
||||
Now analyze this session and output ONLY the JSON object.
|
||||
"""
|
||||
|
||||
return base_prompt + session_context
|
||||
|
||||
|
||||
def _format_attempt_history(attempts: list[dict]) -> str:
|
||||
"""Format attempt history for the prompt."""
|
||||
if not attempts:
|
||||
return "(First attempt - no previous history)"
|
||||
|
||||
lines = []
|
||||
for i, attempt in enumerate(attempts, 1):
|
||||
success = "SUCCESS" if attempt.get("success") else "FAILED"
|
||||
approach = attempt.get("approach", "Unknown approach")
|
||||
error = attempt.get("error", "")
|
||||
lines.append(f"**Attempt {i}** ({success}): {approach}")
|
||||
if error:
|
||||
lines.append(f" Error: {error}")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
async def run_insight_extraction(inputs: dict, project_dir: Path | None = None) -> dict | None:
|
||||
"""
|
||||
Run the insight extraction using Claude Agent SDK.
|
||||
|
||||
Args:
|
||||
inputs: Gathered session inputs
|
||||
project_dir: Project directory for SDK context (optional)
|
||||
|
||||
Returns:
|
||||
Extracted insights dict or None if failed
|
||||
"""
|
||||
if not SDK_AVAILABLE:
|
||||
logger.warning("Claude SDK not available, skipping insight extraction")
|
||||
return None
|
||||
|
||||
oauth_token = os.environ.get("CLAUDE_CODE_OAUTH_TOKEN")
|
||||
if not oauth_token:
|
||||
logger.warning("CLAUDE_CODE_OAUTH_TOKEN not set, skipping insight extraction")
|
||||
return None
|
||||
|
||||
model = get_extraction_model()
|
||||
prompt = _build_extraction_prompt(inputs)
|
||||
|
||||
# Use current directory if project_dir not specified
|
||||
cwd = str(project_dir.resolve()) if project_dir else os.getcwd()
|
||||
|
||||
try:
|
||||
# Create a minimal SDK client for insight extraction
|
||||
# No tools needed - just text generation
|
||||
client = ClaudeSDKClient(
|
||||
options=ClaudeAgentOptions(
|
||||
model=model,
|
||||
system_prompt=(
|
||||
"You are an expert code analyst. You extract structured insights from coding sessions. "
|
||||
"Always respond with valid JSON only, no markdown formatting or explanations."
|
||||
),
|
||||
allowed_tools=[], # No tools needed for extraction
|
||||
max_turns=1, # Single turn extraction
|
||||
cwd=cwd,
|
||||
)
|
||||
)
|
||||
|
||||
# Use async context manager
|
||||
async with client:
|
||||
await client.query(prompt)
|
||||
|
||||
# Collect the response
|
||||
response_text = ""
|
||||
async for msg in client.receive_response():
|
||||
msg_type = type(msg).__name__
|
||||
if msg_type == "AssistantMessage" and hasattr(msg, "content"):
|
||||
for block in msg.content:
|
||||
if hasattr(block, "text"):
|
||||
response_text += block.text
|
||||
|
||||
# Parse JSON from response
|
||||
return parse_insights(response_text)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Insight extraction failed: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def parse_insights(response_text: str) -> dict | None:
|
||||
"""
|
||||
Parse the LLM response into structured insights.
|
||||
|
||||
Args:
|
||||
response_text: Raw LLM response
|
||||
|
||||
Returns:
|
||||
Parsed insights dict or None if parsing failed
|
||||
"""
|
||||
# Try to extract JSON from the response
|
||||
text = response_text.strip()
|
||||
|
||||
# Handle markdown code blocks
|
||||
if text.startswith("```"):
|
||||
# Remove code block markers
|
||||
lines = text.split("\n")
|
||||
# Remove first line (```json or ```)
|
||||
if lines[0].startswith("```"):
|
||||
lines = lines[1:]
|
||||
# Remove last line if it's ``
|
||||
if lines and lines[-1].strip() == "```":
|
||||
lines = lines[:-1]
|
||||
text = "\n".join(lines)
|
||||
|
||||
try:
|
||||
insights = json.loads(text)
|
||||
|
||||
# Validate structure
|
||||
if not isinstance(insights, dict):
|
||||
logger.warning("Insights is not a dict")
|
||||
return None
|
||||
|
||||
# Ensure required keys exist with defaults
|
||||
insights.setdefault("file_insights", [])
|
||||
insights.setdefault("patterns_discovered", [])
|
||||
insights.setdefault("gotchas_discovered", [])
|
||||
insights.setdefault("approach_outcome", {})
|
||||
insights.setdefault("recommendations", [])
|
||||
|
||||
return insights
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning(f"Failed to parse insights JSON: {e}")
|
||||
logger.debug(f"Response text was: {text[:500]}")
|
||||
return None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Main Entry Point
|
||||
# =============================================================================
|
||||
|
||||
|
||||
async def extract_session_insights(
|
||||
spec_dir: Path,
|
||||
project_dir: Path,
|
||||
subtask_id: str,
|
||||
session_num: int,
|
||||
commit_before: str | None,
|
||||
commit_after: str | None,
|
||||
success: bool,
|
||||
recovery_manager: Any,
|
||||
) -> dict:
|
||||
"""
|
||||
Extract insights from a completed coding session.
|
||||
|
||||
This is the main entry point called from post_session_processing().
|
||||
Falls back to generic insights if extraction fails.
|
||||
|
||||
Args:
|
||||
spec_dir: Spec directory
|
||||
project_dir: Project root
|
||||
subtask_id: Subtask that was worked on
|
||||
session_num: Session number
|
||||
commit_before: Commit before session
|
||||
commit_after: Commit after session
|
||||
success: Whether session succeeded
|
||||
recovery_manager: Recovery manager with attempt history
|
||||
|
||||
Returns:
|
||||
Insights dict (rich if extraction succeeded, generic if failed)
|
||||
"""
|
||||
# Check if extraction is enabled
|
||||
if not is_extraction_enabled():
|
||||
logger.info("Insight extraction disabled")
|
||||
return _get_generic_insights(subtask_id, success)
|
||||
|
||||
# Check for no changes
|
||||
if commit_before == commit_after:
|
||||
logger.info("No changes to extract insights from")
|
||||
return _get_generic_insights(subtask_id, success)
|
||||
|
||||
try:
|
||||
# Gather inputs
|
||||
inputs = gather_extraction_inputs(
|
||||
spec_dir=spec_dir,
|
||||
project_dir=project_dir,
|
||||
subtask_id=subtask_id,
|
||||
session_num=session_num,
|
||||
commit_before=commit_before,
|
||||
commit_after=commit_after,
|
||||
success=success,
|
||||
recovery_manager=recovery_manager,
|
||||
)
|
||||
|
||||
# Run extraction
|
||||
extracted = await run_insight_extraction(inputs, project_dir=project_dir)
|
||||
|
||||
if extracted:
|
||||
# Add metadata
|
||||
extracted["subtask_id"] = subtask_id
|
||||
extracted["session_num"] = session_num
|
||||
extracted["success"] = success
|
||||
extracted["changed_files"] = inputs["changed_files"]
|
||||
|
||||
logger.info(
|
||||
f"Extracted insights: {len(extracted.get('file_insights', []))} file insights, "
|
||||
f"{len(extracted.get('patterns_discovered', []))} patterns, "
|
||||
f"{len(extracted.get('gotchas_discovered', []))} gotchas"
|
||||
)
|
||||
return extracted
|
||||
else:
|
||||
logger.warning("Extraction returned no results, using generic insights")
|
||||
return _get_generic_insights(subtask_id, success)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Insight extraction failed: {e}, using generic insights")
|
||||
return _get_generic_insights(subtask_id, success)
|
||||
|
||||
|
||||
def _get_generic_insights(subtask_id: str, success: bool) -> dict:
|
||||
"""Return generic insights when extraction fails or is disabled."""
|
||||
return {
|
||||
"file_insights": [],
|
||||
"patterns_discovered": [],
|
||||
"gotchas_discovered": [],
|
||||
"approach_outcome": {
|
||||
"success": success,
|
||||
"approach_used": f"Implemented subtask: {subtask_id}",
|
||||
"why_it_worked": None,
|
||||
"why_it_failed": None,
|
||||
"alternatives_tried": [],
|
||||
},
|
||||
"recommendations": [],
|
||||
"subtask_id": subtask_id,
|
||||
"success": success,
|
||||
"changed_files": [],
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLI for Testing
|
||||
# =============================================================================
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
import asyncio
|
||||
|
||||
parser = argparse.ArgumentParser(description="Test insight extraction")
|
||||
parser.add_argument("--spec-dir", type=Path, required=True, help="Spec directory")
|
||||
parser.add_argument(
|
||||
"--project-dir", type=Path, required=True, help="Project directory"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--commit-before", type=str, required=True, help="Commit before session"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--commit-after", type=str, required=True, help="Commit after session"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--subtask-id", type=str, default="test-subtask", help="Subtask ID"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
async def main():
|
||||
insights = await extract_session_insights(
|
||||
spec_dir=args.spec_dir,
|
||||
project_dir=args.project_dir,
|
||||
subtask_id=args.subtask_id,
|
||||
session_num=1,
|
||||
commit_before=args.commit_before,
|
||||
commit_after=args.commit_after,
|
||||
success=True,
|
||||
recovery_manager=None,
|
||||
)
|
||||
print(json.dumps(insights, indent=2))
|
||||
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,106 @@
|
||||
"""
|
||||
Smart Project Analyzer for Dynamic Security Profiles
|
||||
=====================================================
|
||||
|
||||
FACADE MODULE: This module re-exports all functionality from the
|
||||
auto-claude/project/ package for backward compatibility.
|
||||
|
||||
The implementation has been refactored into focused modules:
|
||||
- project/command_registry.py - Command registries
|
||||
- project/models.py - Data structures
|
||||
- project/config_parser.py - Config file parsing
|
||||
- project/stack_detector.py - Stack detection
|
||||
- project/framework_detector.py - Framework detection
|
||||
- project/structure_analyzer.py - Project structure analysis
|
||||
- project/analyzer.py - Main orchestration
|
||||
|
||||
This file maintains the original API so existing imports continue to work.
|
||||
|
||||
This system:
|
||||
1. Detects languages, frameworks, databases, and infrastructure
|
||||
2. Parses package.json scripts, Makefile targets, pyproject.toml scripts
|
||||
3. Builds a tailored security profile for the specific project
|
||||
4. Caches the profile for subsequent runs
|
||||
5. Can re-analyze when project structure changes
|
||||
|
||||
The goal: Allow an AI developer to run any command that's legitimately
|
||||
needed for the detected tech stack, while blocking dangerous operations.
|
||||
"""
|
||||
|
||||
# Re-export all public API from the project module
|
||||
from project import (
|
||||
# Command registries
|
||||
BASE_COMMANDS,
|
||||
VALIDATED_COMMANDS,
|
||||
CustomScripts,
|
||||
# Main classes
|
||||
ProjectAnalyzer,
|
||||
SecurityProfile,
|
||||
TechnologyStack,
|
||||
# Utility functions
|
||||
get_or_create_profile,
|
||||
is_command_allowed,
|
||||
needs_validation,
|
||||
)
|
||||
|
||||
# Also re-export command registries for backward compatibility
|
||||
from project.command_registry import (
|
||||
CLOUD_COMMANDS,
|
||||
CODE_QUALITY_COMMANDS,
|
||||
DATABASE_COMMANDS,
|
||||
FRAMEWORK_COMMANDS,
|
||||
INFRASTRUCTURE_COMMANDS,
|
||||
LANGUAGE_COMMANDS,
|
||||
PACKAGE_MANAGER_COMMANDS,
|
||||
VERSION_MANAGER_COMMANDS,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# Main classes
|
||||
"ProjectAnalyzer",
|
||||
"SecurityProfile",
|
||||
"TechnologyStack",
|
||||
"CustomScripts",
|
||||
# Utility functions
|
||||
"get_or_create_profile",
|
||||
"is_command_allowed",
|
||||
"needs_validation",
|
||||
# Base command sets
|
||||
"BASE_COMMANDS",
|
||||
"VALIDATED_COMMANDS",
|
||||
# Technology-specific command sets
|
||||
"LANGUAGE_COMMANDS",
|
||||
"PACKAGE_MANAGER_COMMANDS",
|
||||
"FRAMEWORK_COMMANDS",
|
||||
"DATABASE_COMMANDS",
|
||||
"INFRASTRUCTURE_COMMANDS",
|
||||
"CLOUD_COMMANDS",
|
||||
"CODE_QUALITY_COMMANDS",
|
||||
"VERSION_MANAGER_COMMANDS",
|
||||
]
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLI for testing
|
||||
# =============================================================================
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python project_analyzer.py <project_dir> [--force]")
|
||||
sys.exit(1)
|
||||
|
||||
project_dir = Path(sys.argv[1])
|
||||
force = "--force" in sys.argv
|
||||
|
||||
if not project_dir.exists():
|
||||
print(f"Error: {project_dir} does not exist")
|
||||
sys.exit(1)
|
||||
|
||||
profile = get_or_create_profile(project_dir, force_reanalyze=force)
|
||||
|
||||
print("\nAllowed commands:")
|
||||
for cmd in sorted(profile.get_all_allowed_commands()):
|
||||
print(f" {cmd}")
|
||||
@@ -0,0 +1,589 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Risk Classifier Module
|
||||
======================
|
||||
|
||||
Reads the AI-generated complexity_assessment.json and provides programmatic
|
||||
access to risk classification and validation recommendations.
|
||||
|
||||
This module serves as the bridge between the AI complexity assessor prompt
|
||||
and the rest of the validation system.
|
||||
|
||||
Usage:
|
||||
from risk_classifier import RiskClassifier
|
||||
|
||||
classifier = RiskClassifier()
|
||||
assessment = classifier.load_assessment(spec_dir)
|
||||
|
||||
if classifier.should_skip_validation(spec_dir):
|
||||
print("Validation can be skipped for this task")
|
||||
|
||||
test_types = classifier.get_required_test_types(spec_dir)
|
||||
"""
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# =============================================================================
|
||||
# DATA CLASSES
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScopeAnalysis:
|
||||
"""Analysis of task scope."""
|
||||
|
||||
estimated_files: int = 0
|
||||
estimated_services: int = 0
|
||||
is_cross_cutting: bool = False
|
||||
notes: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class IntegrationAnalysis:
|
||||
"""Analysis of external integrations."""
|
||||
|
||||
external_services: list[str] = field(default_factory=list)
|
||||
new_dependencies: list[str] = field(default_factory=list)
|
||||
research_needed: bool = False
|
||||
notes: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class InfrastructureAnalysis:
|
||||
"""Analysis of infrastructure requirements."""
|
||||
|
||||
docker_changes: bool = False
|
||||
database_changes: bool = False
|
||||
config_changes: bool = False
|
||||
notes: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class KnowledgeAnalysis:
|
||||
"""Analysis of knowledge requirements."""
|
||||
|
||||
patterns_exist: bool = True
|
||||
research_required: bool = False
|
||||
unfamiliar_tech: list[str] = field(default_factory=list)
|
||||
notes: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class RiskAnalysis:
|
||||
"""Analysis of task risk."""
|
||||
|
||||
level: str = "low" # low, medium, high
|
||||
concerns: list[str] = field(default_factory=list)
|
||||
notes: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class ComplexityAnalysis:
|
||||
"""Full complexity analysis from the AI assessor."""
|
||||
|
||||
scope: ScopeAnalysis = field(default_factory=ScopeAnalysis)
|
||||
integrations: IntegrationAnalysis = field(default_factory=IntegrationAnalysis)
|
||||
infrastructure: InfrastructureAnalysis = field(
|
||||
default_factory=InfrastructureAnalysis
|
||||
)
|
||||
knowledge: KnowledgeAnalysis = field(default_factory=KnowledgeAnalysis)
|
||||
risk: RiskAnalysis = field(default_factory=RiskAnalysis)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ValidationRecommendations:
|
||||
"""Validation recommendations from the AI assessor."""
|
||||
|
||||
risk_level: str = "medium" # trivial, low, medium, high, critical
|
||||
skip_validation: bool = False
|
||||
minimal_mode: bool = False
|
||||
test_types_required: list[str] = field(default_factory=lambda: ["unit"])
|
||||
security_scan_required: bool = False
|
||||
staging_deployment_required: bool = False
|
||||
reasoning: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class AssessmentFlags:
|
||||
"""Flags indicating special requirements."""
|
||||
|
||||
needs_research: bool = False
|
||||
needs_self_critique: bool = False
|
||||
needs_infrastructure_setup: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class RiskAssessment:
|
||||
"""Complete risk assessment from complexity_assessment.json."""
|
||||
|
||||
complexity: str # simple, standard, complex
|
||||
workflow_type: str # feature, refactor, investigation, migration, simple
|
||||
confidence: float
|
||||
reasoning: str
|
||||
analysis: ComplexityAnalysis
|
||||
recommended_phases: list[str]
|
||||
flags: AssessmentFlags
|
||||
validation: ValidationRecommendations
|
||||
created_at: str | None = None
|
||||
|
||||
@property
|
||||
def risk_level(self) -> str:
|
||||
"""Get the risk level from validation recommendations."""
|
||||
return self.validation.risk_level
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# RISK CLASSIFIER
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class RiskClassifier:
|
||||
"""
|
||||
Reads AI-generated complexity_assessment.json and provides risk classification.
|
||||
|
||||
The complexity_assessment.json is generated by the AI complexity assessor
|
||||
agent using the complexity_assessor.md prompt. This module parses that output
|
||||
and provides programmatic access to the risk classification.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialize the risk classifier."""
|
||||
self._cache: dict[str, RiskAssessment] = {}
|
||||
|
||||
def load_assessment(self, spec_dir: Path) -> RiskAssessment | None:
|
||||
"""
|
||||
Load complexity_assessment.json from spec directory.
|
||||
|
||||
Args:
|
||||
spec_dir: Path to the spec directory containing complexity_assessment.json
|
||||
|
||||
Returns:
|
||||
RiskAssessment object if file exists and is valid, None otherwise
|
||||
"""
|
||||
spec_dir = Path(spec_dir)
|
||||
cache_key = str(spec_dir.resolve())
|
||||
|
||||
# Return cached result if available
|
||||
if cache_key in self._cache:
|
||||
return self._cache[cache_key]
|
||||
|
||||
assessment_file = spec_dir / "complexity_assessment.json"
|
||||
if not assessment_file.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(assessment_file, encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
assessment = self._parse_assessment(data)
|
||||
self._cache[cache_key] = assessment
|
||||
return assessment
|
||||
|
||||
except (json.JSONDecodeError, KeyError, TypeError) as e:
|
||||
# Log error but don't crash - return None to allow fallback behavior
|
||||
print(f"Warning: Failed to parse complexity_assessment.json: {e}")
|
||||
return None
|
||||
|
||||
def _parse_assessment(self, data: dict[str, Any]) -> RiskAssessment:
|
||||
"""Parse raw JSON data into a RiskAssessment object."""
|
||||
# Parse analysis sections
|
||||
analysis_data = data.get("analysis", {})
|
||||
analysis = ComplexityAnalysis(
|
||||
scope=self._parse_scope(analysis_data.get("scope", {})),
|
||||
integrations=self._parse_integrations(
|
||||
analysis_data.get("integrations", {})
|
||||
),
|
||||
infrastructure=self._parse_infrastructure(
|
||||
analysis_data.get("infrastructure", {})
|
||||
),
|
||||
knowledge=self._parse_knowledge(analysis_data.get("knowledge", {})),
|
||||
risk=self._parse_risk(analysis_data.get("risk", {})),
|
||||
)
|
||||
|
||||
# Parse flags
|
||||
flags_data = data.get("flags", {})
|
||||
flags = AssessmentFlags(
|
||||
needs_research=flags_data.get("needs_research", False),
|
||||
needs_self_critique=flags_data.get("needs_self_critique", False),
|
||||
needs_infrastructure_setup=flags_data.get(
|
||||
"needs_infrastructure_setup", False
|
||||
),
|
||||
)
|
||||
|
||||
# Parse validation recommendations
|
||||
validation_data = data.get("validation_recommendations", {})
|
||||
validation = self._parse_validation_recommendations(validation_data, analysis)
|
||||
|
||||
return RiskAssessment(
|
||||
complexity=data.get("complexity", "standard"),
|
||||
workflow_type=data.get("workflow_type", "feature"),
|
||||
confidence=float(data.get("confidence", 0.5)),
|
||||
reasoning=data.get("reasoning", ""),
|
||||
analysis=analysis,
|
||||
recommended_phases=data.get("recommended_phases", []),
|
||||
flags=flags,
|
||||
validation=validation,
|
||||
created_at=data.get("created_at"),
|
||||
)
|
||||
|
||||
def _parse_scope(self, data: dict[str, Any]) -> ScopeAnalysis:
|
||||
"""Parse scope analysis section."""
|
||||
return ScopeAnalysis(
|
||||
estimated_files=int(data.get("estimated_files", 0)),
|
||||
estimated_services=int(data.get("estimated_services", 0)),
|
||||
is_cross_cutting=bool(data.get("is_cross_cutting", False)),
|
||||
notes=str(data.get("notes", "")),
|
||||
)
|
||||
|
||||
def _parse_integrations(self, data: dict[str, Any]) -> IntegrationAnalysis:
|
||||
"""Parse integrations analysis section."""
|
||||
return IntegrationAnalysis(
|
||||
external_services=list(data.get("external_services", [])),
|
||||
new_dependencies=list(data.get("new_dependencies", [])),
|
||||
research_needed=bool(data.get("research_needed", False)),
|
||||
notes=str(data.get("notes", "")),
|
||||
)
|
||||
|
||||
def _parse_infrastructure(self, data: dict[str, Any]) -> InfrastructureAnalysis:
|
||||
"""Parse infrastructure analysis section."""
|
||||
return InfrastructureAnalysis(
|
||||
docker_changes=bool(data.get("docker_changes", False)),
|
||||
database_changes=bool(data.get("database_changes", False)),
|
||||
config_changes=bool(data.get("config_changes", False)),
|
||||
notes=str(data.get("notes", "")),
|
||||
)
|
||||
|
||||
def _parse_knowledge(self, data: dict[str, Any]) -> KnowledgeAnalysis:
|
||||
"""Parse knowledge analysis section."""
|
||||
return KnowledgeAnalysis(
|
||||
patterns_exist=bool(data.get("patterns_exist", True)),
|
||||
research_required=bool(data.get("research_required", False)),
|
||||
unfamiliar_tech=list(data.get("unfamiliar_tech", [])),
|
||||
notes=str(data.get("notes", "")),
|
||||
)
|
||||
|
||||
def _parse_risk(self, data: dict[str, Any]) -> RiskAnalysis:
|
||||
"""Parse risk analysis section."""
|
||||
return RiskAnalysis(
|
||||
level=str(data.get("level", "low")),
|
||||
concerns=list(data.get("concerns", [])),
|
||||
notes=str(data.get("notes", "")),
|
||||
)
|
||||
|
||||
def _parse_validation_recommendations(
|
||||
self, data: dict[str, Any], analysis: ComplexityAnalysis
|
||||
) -> ValidationRecommendations:
|
||||
"""
|
||||
Parse validation recommendations section.
|
||||
|
||||
If validation_recommendations is not present in the JSON (older assessments),
|
||||
infer appropriate values from the analysis.
|
||||
"""
|
||||
if data:
|
||||
# New format with explicit validation recommendations
|
||||
return ValidationRecommendations(
|
||||
risk_level=str(data.get("risk_level", "medium")),
|
||||
skip_validation=bool(data.get("skip_validation", False)),
|
||||
minimal_mode=bool(data.get("minimal_mode", False)),
|
||||
test_types_required=list(data.get("test_types_required", ["unit"])),
|
||||
security_scan_required=bool(data.get("security_scan_required", False)),
|
||||
staging_deployment_required=bool(
|
||||
data.get("staging_deployment_required", False)
|
||||
),
|
||||
reasoning=str(data.get("reasoning", "")),
|
||||
)
|
||||
else:
|
||||
# Infer from analysis (backward compatibility)
|
||||
return self._infer_validation_recommendations(analysis)
|
||||
|
||||
def _infer_validation_recommendations(
|
||||
self, analysis: ComplexityAnalysis
|
||||
) -> ValidationRecommendations:
|
||||
"""
|
||||
Infer validation recommendations from analysis when not explicitly provided.
|
||||
|
||||
This provides backward compatibility with older complexity assessments
|
||||
that don't have the validation_recommendations section.
|
||||
"""
|
||||
risk_level = analysis.risk.level
|
||||
|
||||
# Map old risk levels to new ones
|
||||
risk_mapping = {
|
||||
"low": "low",
|
||||
"medium": "medium",
|
||||
"high": "high",
|
||||
}
|
||||
normalized_risk = risk_mapping.get(risk_level, "medium")
|
||||
|
||||
# Infer test types based on risk
|
||||
test_types_map = {
|
||||
"low": ["unit"],
|
||||
"medium": ["unit", "integration"],
|
||||
"high": ["unit", "integration", "e2e"],
|
||||
}
|
||||
test_types = test_types_map.get(normalized_risk, ["unit", "integration"])
|
||||
|
||||
# Security scan for high risk or security-related concerns
|
||||
security_keywords = [
|
||||
"security",
|
||||
"auth",
|
||||
"password",
|
||||
"credential",
|
||||
"token",
|
||||
"api key",
|
||||
]
|
||||
has_security_concerns = any(
|
||||
kw in str(analysis.risk.concerns).lower() for kw in security_keywords
|
||||
)
|
||||
security_scan_required = normalized_risk == "high" or has_security_concerns
|
||||
|
||||
# Staging for database or infrastructure changes
|
||||
staging_required = (
|
||||
analysis.infrastructure.database_changes
|
||||
and normalized_risk in ["medium", "high"]
|
||||
)
|
||||
|
||||
# Minimal mode for simple changes
|
||||
minimal_mode = (
|
||||
analysis.scope.estimated_files <= 2
|
||||
and analysis.scope.estimated_services <= 1
|
||||
and not analysis.integrations.external_services
|
||||
)
|
||||
|
||||
return ValidationRecommendations(
|
||||
risk_level=normalized_risk,
|
||||
skip_validation=False, # Never skip by inference
|
||||
minimal_mode=minimal_mode,
|
||||
test_types_required=test_types,
|
||||
security_scan_required=security_scan_required,
|
||||
staging_deployment_required=staging_required,
|
||||
reasoning="Inferred from complexity analysis (no explicit recommendations found)",
|
||||
)
|
||||
|
||||
def should_skip_validation(self, spec_dir: Path) -> bool:
|
||||
"""
|
||||
Quick check if validation can be skipped entirely.
|
||||
|
||||
Args:
|
||||
spec_dir: Path to the spec directory
|
||||
|
||||
Returns:
|
||||
True if validation can be skipped (trivial changes), False otherwise
|
||||
"""
|
||||
assessment = self.load_assessment(spec_dir)
|
||||
if not assessment:
|
||||
return False # When in doubt, don't skip
|
||||
|
||||
return assessment.validation.skip_validation
|
||||
|
||||
def should_use_minimal_mode(self, spec_dir: Path) -> bool:
|
||||
"""
|
||||
Check if minimal validation mode should be used.
|
||||
|
||||
Args:
|
||||
spec_dir: Path to the spec directory
|
||||
|
||||
Returns:
|
||||
True if minimal mode is recommended, False otherwise
|
||||
"""
|
||||
assessment = self.load_assessment(spec_dir)
|
||||
if not assessment:
|
||||
return False
|
||||
|
||||
return assessment.validation.minimal_mode
|
||||
|
||||
def get_required_test_types(self, spec_dir: Path) -> list[str]:
|
||||
"""
|
||||
Get list of required test types based on risk.
|
||||
|
||||
Args:
|
||||
spec_dir: Path to the spec directory
|
||||
|
||||
Returns:
|
||||
List of test types (e.g., ["unit", "integration", "e2e"])
|
||||
"""
|
||||
assessment = self.load_assessment(spec_dir)
|
||||
if not assessment:
|
||||
return ["unit"] # Default to unit tests
|
||||
|
||||
return assessment.validation.test_types_required
|
||||
|
||||
def requires_security_scan(self, spec_dir: Path) -> bool:
|
||||
"""
|
||||
Check if security scanning is required.
|
||||
|
||||
Args:
|
||||
spec_dir: Path to the spec directory
|
||||
|
||||
Returns:
|
||||
True if security scan is required, False otherwise
|
||||
"""
|
||||
assessment = self.load_assessment(spec_dir)
|
||||
if not assessment:
|
||||
return False
|
||||
|
||||
return assessment.validation.security_scan_required
|
||||
|
||||
def requires_staging_deployment(self, spec_dir: Path) -> bool:
|
||||
"""
|
||||
Check if staging deployment is required.
|
||||
|
||||
Args:
|
||||
spec_dir: Path to the spec directory
|
||||
|
||||
Returns:
|
||||
True if staging deployment is required, False otherwise
|
||||
"""
|
||||
assessment = self.load_assessment(spec_dir)
|
||||
if not assessment:
|
||||
return False
|
||||
|
||||
return assessment.validation.staging_deployment_required
|
||||
|
||||
def get_risk_level(self, spec_dir: Path) -> str:
|
||||
"""
|
||||
Get the risk level for the task.
|
||||
|
||||
Args:
|
||||
spec_dir: Path to the spec directory
|
||||
|
||||
Returns:
|
||||
Risk level string (trivial, low, medium, high, critical)
|
||||
"""
|
||||
assessment = self.load_assessment(spec_dir)
|
||||
if not assessment:
|
||||
return "medium" # Default to medium when unknown
|
||||
|
||||
return assessment.validation.risk_level
|
||||
|
||||
def get_complexity(self, spec_dir: Path) -> str:
|
||||
"""
|
||||
Get the complexity level for the task.
|
||||
|
||||
Args:
|
||||
spec_dir: Path to the spec directory
|
||||
|
||||
Returns:
|
||||
Complexity level string (simple, standard, complex)
|
||||
"""
|
||||
assessment = self.load_assessment(spec_dir)
|
||||
if not assessment:
|
||||
return "standard" # Default to standard when unknown
|
||||
|
||||
return assessment.complexity
|
||||
|
||||
def get_validation_summary(self, spec_dir: Path) -> dict[str, Any]:
|
||||
"""
|
||||
Get a summary of validation requirements.
|
||||
|
||||
Args:
|
||||
spec_dir: Path to the spec directory
|
||||
|
||||
Returns:
|
||||
Dictionary with validation summary
|
||||
"""
|
||||
assessment = self.load_assessment(spec_dir)
|
||||
if not assessment:
|
||||
return {
|
||||
"risk_level": "unknown",
|
||||
"complexity": "unknown",
|
||||
"skip_validation": False,
|
||||
"minimal_mode": False,
|
||||
"test_types": ["unit"],
|
||||
"security_scan": False,
|
||||
"staging_deployment": False,
|
||||
"confidence": 0.0,
|
||||
}
|
||||
|
||||
return {
|
||||
"risk_level": assessment.validation.risk_level,
|
||||
"complexity": assessment.complexity,
|
||||
"skip_validation": assessment.validation.skip_validation,
|
||||
"minimal_mode": assessment.validation.minimal_mode,
|
||||
"test_types": assessment.validation.test_types_required,
|
||||
"security_scan": assessment.validation.security_scan_required,
|
||||
"staging_deployment": assessment.validation.staging_deployment_required,
|
||||
"confidence": assessment.confidence,
|
||||
"reasoning": assessment.validation.reasoning,
|
||||
}
|
||||
|
||||
def clear_cache(self) -> None:
|
||||
"""Clear the internal cache of loaded assessments."""
|
||||
self._cache.clear()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CONVENIENCE FUNCTIONS
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def load_risk_assessment(spec_dir: Path) -> RiskAssessment | None:
|
||||
"""
|
||||
Convenience function to load a risk assessment.
|
||||
|
||||
Args:
|
||||
spec_dir: Path to the spec directory
|
||||
|
||||
Returns:
|
||||
RiskAssessment object or None
|
||||
"""
|
||||
classifier = RiskClassifier()
|
||||
return classifier.load_assessment(spec_dir)
|
||||
|
||||
|
||||
def get_validation_requirements(spec_dir: Path) -> dict[str, Any]:
|
||||
"""
|
||||
Convenience function to get validation requirements.
|
||||
|
||||
Args:
|
||||
spec_dir: Path to the spec directory
|
||||
|
||||
Returns:
|
||||
Dictionary with validation requirements
|
||||
"""
|
||||
classifier = RiskClassifier()
|
||||
return classifier.get_validation_summary(spec_dir)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLI
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""CLI entry point for testing."""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Load and display risk assessment")
|
||||
parser.add_argument(
|
||||
"spec_dir",
|
||||
type=Path,
|
||||
help="Path to spec directory with complexity_assessment.json",
|
||||
)
|
||||
parser.add_argument("--json", action="store_true", help="Output as JSON")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
classifier = RiskClassifier()
|
||||
summary = classifier.get_validation_summary(args.spec_dir)
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(summary, indent=2))
|
||||
else:
|
||||
print(f"Risk Level: {summary['risk_level']}")
|
||||
print(f"Complexity: {summary['complexity']}")
|
||||
print(f"Skip Validation: {summary['skip_validation']}")
|
||||
print(f"Minimal Mode: {summary['minimal_mode']}")
|
||||
print(f"Test Types: {', '.join(summary['test_types'])}")
|
||||
print(f"Security Scan: {summary['security_scan']}")
|
||||
print(f"Staging Deployment: {summary['staging_deployment']}")
|
||||
print(f"Confidence: {summary['confidence']:.2f}")
|
||||
if summary.get("reasoning"):
|
||||
print(f"Reasoning: {summary['reasoning']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,597 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Security Scanner Module
|
||||
=======================
|
||||
|
||||
Consolidates security scanning including secrets detection and SAST tools.
|
||||
This module integrates the existing scan_secrets.py and provides a unified
|
||||
interface for all security scanning.
|
||||
|
||||
The security scanner is used by:
|
||||
- QA Agent: To verify no secrets are committed
|
||||
- Validation Strategy: To run security scans for high-risk changes
|
||||
|
||||
Usage:
|
||||
from analysis.security_scanner import SecurityScanner
|
||||
|
||||
scanner = SecurityScanner()
|
||||
results = scanner.scan(project_dir, spec_dir)
|
||||
|
||||
if results.has_critical_issues:
|
||||
print("Security issues found - blocking QA approval")
|
||||
"""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# Import the existing secrets scanner
|
||||
try:
|
||||
from security.scan_secrets import SecretMatch, get_all_tracked_files, scan_files
|
||||
|
||||
HAS_SECRETS_SCANNER = True
|
||||
except ImportError:
|
||||
HAS_SECRETS_SCANNER = False
|
||||
SecretMatch = None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# DATA CLASSES
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@dataclass
|
||||
class SecurityVulnerability:
|
||||
"""
|
||||
Represents a security vulnerability found during scanning.
|
||||
|
||||
Attributes:
|
||||
severity: Severity level (critical, high, medium, low, info)
|
||||
source: Which scanner found this (secrets, bandit, npm_audit, etc.)
|
||||
title: Short title of the vulnerability
|
||||
description: Detailed description
|
||||
file: File where vulnerability was found (if applicable)
|
||||
line: Line number (if applicable)
|
||||
cwe: CWE identifier if available
|
||||
"""
|
||||
|
||||
severity: str # critical, high, medium, low, info
|
||||
source: str # secrets, bandit, npm_audit, semgrep, etc.
|
||||
title: str
|
||||
description: str
|
||||
file: str | None = None
|
||||
line: int | None = None
|
||||
cwe: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SecurityScanResult:
|
||||
"""
|
||||
Result of a security scan.
|
||||
|
||||
Attributes:
|
||||
secrets: List of detected secrets
|
||||
vulnerabilities: List of security vulnerabilities
|
||||
scan_errors: List of errors during scanning
|
||||
has_critical_issues: Whether any critical issues were found
|
||||
should_block_qa: Whether these results should block QA approval
|
||||
"""
|
||||
|
||||
secrets: list[dict[str, Any]] = field(default_factory=list)
|
||||
vulnerabilities: list[SecurityVulnerability] = field(default_factory=list)
|
||||
scan_errors: list[str] = field(default_factory=list)
|
||||
has_critical_issues: bool = False
|
||||
should_block_qa: bool = False
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# SECURITY SCANNER
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class SecurityScanner:
|
||||
"""
|
||||
Consolidates all security scanning operations.
|
||||
|
||||
Integrates:
|
||||
- scan_secrets.py for secrets detection
|
||||
- Bandit for Python SAST (if available)
|
||||
- npm audit for JavaScript vulnerabilities (if applicable)
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialize the security scanner."""
|
||||
self._bandit_available: bool | None = None
|
||||
self._npm_available: bool | None = None
|
||||
|
||||
def scan(
|
||||
self,
|
||||
project_dir: Path,
|
||||
spec_dir: Path | None = None,
|
||||
changed_files: list[str] | None = None,
|
||||
run_secrets: bool = True,
|
||||
run_sast: bool = True,
|
||||
run_dependency_audit: bool = True,
|
||||
) -> SecurityScanResult:
|
||||
"""
|
||||
Run all applicable security scans.
|
||||
|
||||
Args:
|
||||
project_dir: Path to the project root
|
||||
spec_dir: Path to the spec directory (for storing results)
|
||||
changed_files: Optional list of files to scan (if None, scans all)
|
||||
run_secrets: Whether to run secrets scanning
|
||||
run_sast: Whether to run SAST tools
|
||||
run_dependency_audit: Whether to run dependency audits
|
||||
|
||||
Returns:
|
||||
SecurityScanResult with all findings
|
||||
"""
|
||||
project_dir = Path(project_dir)
|
||||
result = SecurityScanResult()
|
||||
|
||||
# Run secrets scan
|
||||
if run_secrets:
|
||||
self._run_secrets_scan(project_dir, changed_files, result)
|
||||
|
||||
# Run SAST based on project type
|
||||
if run_sast:
|
||||
self._run_sast_scans(project_dir, result)
|
||||
|
||||
# Run dependency audits
|
||||
if run_dependency_audit:
|
||||
self._run_dependency_audits(project_dir, result)
|
||||
|
||||
# Determine if should block QA
|
||||
result.has_critical_issues = (
|
||||
any(v.severity in ["critical", "high"] for v in result.vulnerabilities)
|
||||
or len(result.secrets) > 0
|
||||
)
|
||||
|
||||
# Any secrets always block, critical vulnerabilities block
|
||||
result.should_block_qa = len(result.secrets) > 0 or any(
|
||||
v.severity == "critical" for v in result.vulnerabilities
|
||||
)
|
||||
|
||||
# Save results if spec_dir provided
|
||||
if spec_dir:
|
||||
self._save_results(spec_dir, result)
|
||||
|
||||
return result
|
||||
|
||||
def _run_secrets_scan(
|
||||
self,
|
||||
project_dir: Path,
|
||||
changed_files: list[str] | None,
|
||||
result: SecurityScanResult,
|
||||
) -> None:
|
||||
"""Run secrets scanning using scan_secrets.py."""
|
||||
if not HAS_SECRETS_SCANNER:
|
||||
result.scan_errors.append("scan_secrets module not available")
|
||||
return
|
||||
|
||||
try:
|
||||
# Get files to scan
|
||||
if changed_files:
|
||||
files_to_scan = changed_files
|
||||
else:
|
||||
files_to_scan = get_all_tracked_files()
|
||||
|
||||
# Run scan
|
||||
matches = scan_files(files_to_scan, project_dir)
|
||||
|
||||
# Convert matches to result format
|
||||
for match in matches:
|
||||
result.secrets.append(
|
||||
{
|
||||
"file": match.file_path,
|
||||
"line": match.line_number,
|
||||
"pattern": match.pattern_name,
|
||||
"matched_text": self._redact_secret(match.matched_text),
|
||||
}
|
||||
)
|
||||
|
||||
# Also add as vulnerability
|
||||
result.vulnerabilities.append(
|
||||
SecurityVulnerability(
|
||||
severity="critical",
|
||||
source="secrets",
|
||||
title=f"Potential secret: {match.pattern_name}",
|
||||
description=f"Found potential {match.pattern_name} in file",
|
||||
file=match.file_path,
|
||||
line=match.line_number,
|
||||
)
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
result.scan_errors.append(f"Secrets scan error: {str(e)}")
|
||||
|
||||
def _run_sast_scans(self, project_dir: Path, result: SecurityScanResult) -> None:
|
||||
"""Run SAST tools based on project type."""
|
||||
# Python SAST with Bandit
|
||||
if self._is_python_project(project_dir):
|
||||
self._run_bandit(project_dir, result)
|
||||
|
||||
# JavaScript/Node.js - npm audit
|
||||
# (handled in dependency audits for Node projects)
|
||||
|
||||
def _run_bandit(self, project_dir: Path, result: SecurityScanResult) -> None:
|
||||
"""Run Bandit security scanner for Python projects."""
|
||||
if not self._check_bandit_available():
|
||||
return
|
||||
|
||||
try:
|
||||
# Find Python source directories
|
||||
src_dirs = []
|
||||
for candidate in ["src", "app", project_dir.name, "."]:
|
||||
candidate_path = project_dir / candidate
|
||||
if (
|
||||
candidate_path.exists()
|
||||
and (candidate_path / "__init__.py").exists()
|
||||
):
|
||||
src_dirs.append(str(candidate_path))
|
||||
|
||||
if not src_dirs:
|
||||
# Try to find any Python files
|
||||
py_files = list(project_dir.glob("**/*.py"))
|
||||
if not py_files:
|
||||
return
|
||||
src_dirs = ["."]
|
||||
|
||||
# Run bandit
|
||||
cmd = [
|
||||
"bandit",
|
||||
"-r",
|
||||
*src_dirs,
|
||||
"-f",
|
||||
"json",
|
||||
"--exit-zero", # Don't fail on findings
|
||||
]
|
||||
|
||||
proc = subprocess.run(
|
||||
cmd,
|
||||
cwd=project_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
|
||||
if proc.stdout:
|
||||
try:
|
||||
bandit_output = json.loads(proc.stdout)
|
||||
for finding in bandit_output.get("results", []):
|
||||
severity = finding.get("issue_severity", "MEDIUM").lower()
|
||||
if severity == "high":
|
||||
severity = "high"
|
||||
elif severity == "medium":
|
||||
severity = "medium"
|
||||
else:
|
||||
severity = "low"
|
||||
|
||||
result.vulnerabilities.append(
|
||||
SecurityVulnerability(
|
||||
severity=severity,
|
||||
source="bandit",
|
||||
title=finding.get("issue_text", "Unknown issue"),
|
||||
description=finding.get("issue_text", ""),
|
||||
file=finding.get("filename"),
|
||||
line=finding.get("line_number"),
|
||||
cwe=finding.get("issue_cwe", {}).get("id"),
|
||||
)
|
||||
)
|
||||
except json.JSONDecodeError:
|
||||
result.scan_errors.append("Failed to parse Bandit output")
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
result.scan_errors.append("Bandit scan timed out")
|
||||
except FileNotFoundError:
|
||||
result.scan_errors.append("Bandit not found")
|
||||
except Exception as e:
|
||||
result.scan_errors.append(f"Bandit error: {str(e)}")
|
||||
|
||||
def _run_dependency_audits(
|
||||
self, project_dir: Path, result: SecurityScanResult
|
||||
) -> None:
|
||||
"""Run dependency vulnerability audits."""
|
||||
# npm audit for JavaScript projects
|
||||
if (project_dir / "package.json").exists():
|
||||
self._run_npm_audit(project_dir, result)
|
||||
|
||||
# pip-audit for Python projects (if available)
|
||||
if self._is_python_project(project_dir):
|
||||
self._run_pip_audit(project_dir, result)
|
||||
|
||||
def _run_npm_audit(self, project_dir: Path, result: SecurityScanResult) -> None:
|
||||
"""Run npm audit for JavaScript projects."""
|
||||
try:
|
||||
cmd = ["npm", "audit", "--json"]
|
||||
|
||||
proc = subprocess.run(
|
||||
cmd,
|
||||
cwd=project_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
|
||||
if proc.stdout:
|
||||
try:
|
||||
audit_output = json.loads(proc.stdout)
|
||||
|
||||
# npm audit v2+ format
|
||||
vulnerabilities = audit_output.get("vulnerabilities", {})
|
||||
for pkg_name, vuln_info in vulnerabilities.items():
|
||||
severity = vuln_info.get("severity", "moderate")
|
||||
if severity == "critical":
|
||||
severity = "critical"
|
||||
elif severity == "high":
|
||||
severity = "high"
|
||||
elif severity == "moderate":
|
||||
severity = "medium"
|
||||
else:
|
||||
severity = "low"
|
||||
|
||||
result.vulnerabilities.append(
|
||||
SecurityVulnerability(
|
||||
severity=severity,
|
||||
source="npm_audit",
|
||||
title=f"Vulnerable dependency: {pkg_name}",
|
||||
description=vuln_info.get("via", [{}])[0].get(
|
||||
"title", ""
|
||||
)
|
||||
if isinstance(vuln_info.get("via"), list)
|
||||
and vuln_info.get("via")
|
||||
else str(vuln_info.get("via", "")),
|
||||
file="package.json",
|
||||
)
|
||||
)
|
||||
except json.JSONDecodeError:
|
||||
pass # npm audit may return invalid JSON on no findings
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
result.scan_errors.append("npm audit timed out")
|
||||
except FileNotFoundError:
|
||||
pass # npm not available
|
||||
except Exception as e:
|
||||
result.scan_errors.append(f"npm audit error: {str(e)}")
|
||||
|
||||
def _run_pip_audit(self, project_dir: Path, result: SecurityScanResult) -> None:
|
||||
"""Run pip-audit for Python projects (if available)."""
|
||||
try:
|
||||
cmd = ["pip-audit", "--format", "json"]
|
||||
|
||||
proc = subprocess.run(
|
||||
cmd,
|
||||
cwd=project_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
|
||||
if proc.stdout:
|
||||
try:
|
||||
audit_output = json.loads(proc.stdout)
|
||||
for vuln in audit_output:
|
||||
severity = "high" if vuln.get("fix_versions") else "medium"
|
||||
|
||||
result.vulnerabilities.append(
|
||||
SecurityVulnerability(
|
||||
severity=severity,
|
||||
source="pip_audit",
|
||||
title=f"Vulnerable package: {vuln.get('name')}",
|
||||
description=vuln.get("description", ""),
|
||||
cwe=vuln.get("aliases", [""])[0]
|
||||
if vuln.get("aliases")
|
||||
else None,
|
||||
)
|
||||
)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
except FileNotFoundError:
|
||||
pass # pip-audit not available
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _is_python_project(self, project_dir: Path) -> bool:
|
||||
"""Check if this is a Python project."""
|
||||
indicators = [
|
||||
project_dir / "pyproject.toml",
|
||||
project_dir / "requirements.txt",
|
||||
project_dir / "setup.py",
|
||||
project_dir / "setup.cfg",
|
||||
]
|
||||
return any(p.exists() for p in indicators)
|
||||
|
||||
def _check_bandit_available(self) -> bool:
|
||||
"""Check if Bandit is available."""
|
||||
if self._bandit_available is None:
|
||||
try:
|
||||
subprocess.run(
|
||||
["bandit", "--version"],
|
||||
capture_output=True,
|
||||
timeout=5,
|
||||
)
|
||||
self._bandit_available = True
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||||
self._bandit_available = False
|
||||
return self._bandit_available
|
||||
|
||||
def _redact_secret(self, text: str) -> str:
|
||||
"""Redact a secret for safe logging."""
|
||||
if len(text) <= 8:
|
||||
return "*" * len(text)
|
||||
return text[:4] + "*" * (len(text) - 8) + text[-4:]
|
||||
|
||||
def _save_results(self, spec_dir: Path, result: SecurityScanResult) -> None:
|
||||
"""Save scan results to spec directory."""
|
||||
spec_dir = Path(spec_dir)
|
||||
spec_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
output_file = spec_dir / "security_scan_results.json"
|
||||
output_data = self.to_dict(result)
|
||||
|
||||
with open(output_file, "w", encoding="utf-8") as f:
|
||||
json.dump(output_data, f, indent=2)
|
||||
|
||||
def to_dict(self, result: SecurityScanResult) -> dict[str, Any]:
|
||||
"""Convert result to dictionary for JSON serialization."""
|
||||
return {
|
||||
"secrets": result.secrets,
|
||||
"vulnerabilities": [
|
||||
{
|
||||
"severity": v.severity,
|
||||
"source": v.source,
|
||||
"title": v.title,
|
||||
"description": v.description,
|
||||
"file": v.file,
|
||||
"line": v.line,
|
||||
"cwe": v.cwe,
|
||||
}
|
||||
for v in result.vulnerabilities
|
||||
],
|
||||
"scan_errors": result.scan_errors,
|
||||
"has_critical_issues": result.has_critical_issues,
|
||||
"should_block_qa": result.should_block_qa,
|
||||
"summary": {
|
||||
"total_secrets": len(result.secrets),
|
||||
"total_vulnerabilities": len(result.vulnerabilities),
|
||||
"critical_count": sum(
|
||||
1 for v in result.vulnerabilities if v.severity == "critical"
|
||||
),
|
||||
"high_count": sum(
|
||||
1 for v in result.vulnerabilities if v.severity == "high"
|
||||
),
|
||||
"medium_count": sum(
|
||||
1 for v in result.vulnerabilities if v.severity == "medium"
|
||||
),
|
||||
"low_count": sum(
|
||||
1 for v in result.vulnerabilities if v.severity == "low"
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CONVENIENCE FUNCTIONS
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def scan_for_security_issues(
|
||||
project_dir: Path,
|
||||
spec_dir: Path | None = None,
|
||||
changed_files: list[str] | None = None,
|
||||
) -> SecurityScanResult:
|
||||
"""
|
||||
Convenience function to run security scan.
|
||||
|
||||
Args:
|
||||
project_dir: Path to project root
|
||||
spec_dir: Optional spec directory to save results
|
||||
changed_files: Optional list of files to scan
|
||||
|
||||
Returns:
|
||||
SecurityScanResult with all findings
|
||||
"""
|
||||
scanner = SecurityScanner()
|
||||
return scanner.scan(project_dir, spec_dir, changed_files)
|
||||
|
||||
|
||||
def has_security_issues(project_dir: Path) -> bool:
|
||||
"""
|
||||
Quick check if project has security issues.
|
||||
|
||||
Args:
|
||||
project_dir: Path to project root
|
||||
|
||||
Returns:
|
||||
True if any critical/high issues found
|
||||
"""
|
||||
scanner = SecurityScanner()
|
||||
result = scanner.scan(project_dir, run_sast=False, run_dependency_audit=False)
|
||||
return result.has_critical_issues
|
||||
|
||||
|
||||
def scan_secrets_only(
|
||||
project_dir: Path,
|
||||
changed_files: list[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Scan only for secrets (quick scan).
|
||||
|
||||
Args:
|
||||
project_dir: Path to project root
|
||||
changed_files: Optional list of files to scan
|
||||
|
||||
Returns:
|
||||
List of detected secrets
|
||||
"""
|
||||
scanner = SecurityScanner()
|
||||
result = scanner.scan(
|
||||
project_dir,
|
||||
changed_files=changed_files,
|
||||
run_sast=False,
|
||||
run_dependency_audit=False,
|
||||
)
|
||||
return result.secrets
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLI
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""CLI entry point for testing."""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Run security scans")
|
||||
parser.add_argument("project_dir", type=Path, help="Path to project root")
|
||||
parser.add_argument("--spec-dir", type=Path, help="Path to spec directory")
|
||||
parser.add_argument(
|
||||
"--secrets-only", action="store_true", help="Only scan for secrets"
|
||||
)
|
||||
parser.add_argument("--json", action="store_true", help="Output as JSON")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
scanner = SecurityScanner()
|
||||
result = scanner.scan(
|
||||
args.project_dir,
|
||||
spec_dir=args.spec_dir,
|
||||
run_sast=not args.secrets_only,
|
||||
run_dependency_audit=not args.secrets_only,
|
||||
)
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(scanner.to_dict(result), indent=2))
|
||||
else:
|
||||
print(f"Secrets Found: {len(result.secrets)}")
|
||||
print(f"Vulnerabilities: {len(result.vulnerabilities)}")
|
||||
print(f"Has Critical Issues: {result.has_critical_issues}")
|
||||
print(f"Should Block QA: {result.should_block_qa}")
|
||||
|
||||
if result.secrets:
|
||||
print("\nSecrets Detected:")
|
||||
for secret in result.secrets:
|
||||
print(f" - {secret['pattern']} in {secret['file']}:{secret['line']}")
|
||||
|
||||
if result.vulnerabilities:
|
||||
print(f"\nVulnerabilities ({len(result.vulnerabilities)}):")
|
||||
for v in result.vulnerabilities:
|
||||
print(f" [{v.severity.upper()}] {v.title}")
|
||||
if v.file:
|
||||
print(f" File: {v.file}:{v.line or ''}")
|
||||
|
||||
if result.scan_errors:
|
||||
print(f"\nScan Errors ({len(result.scan_errors)}):")
|
||||
for error in result.scan_errors:
|
||||
print(f" - {error}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,688 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test Discovery Module
|
||||
=====================
|
||||
|
||||
Detects test frameworks, test commands, and test directories in a project.
|
||||
This module analyzes project configuration files to discover how tests
|
||||
should be run.
|
||||
|
||||
The test discovery results are used by:
|
||||
- QA Agent: To determine what test commands to run
|
||||
- Test Creator: To know what framework to use when creating tests
|
||||
- Planner: To include correct test commands in verification strategy
|
||||
|
||||
Usage:
|
||||
from test_discovery import TestDiscovery
|
||||
|
||||
discovery = TestDiscovery()
|
||||
result = discovery.discover(project_dir)
|
||||
|
||||
print(f"Test frameworks: {result['frameworks']}")
|
||||
print(f"Test command: {result['test_command']}")
|
||||
"""
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# =============================================================================
|
||||
# DATA CLASSES
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@dataclass
|
||||
class TestFramework:
|
||||
"""
|
||||
Represents a detected test framework.
|
||||
|
||||
Attributes:
|
||||
name: Name of the framework (e.g., "pytest", "jest", "vitest")
|
||||
type: Type of testing (unit, integration, e2e, all)
|
||||
command: Command to run tests
|
||||
config_file: Configuration file if found
|
||||
version: Version if detected
|
||||
coverage_command: Command for coverage if available
|
||||
"""
|
||||
|
||||
__test__ = False # Prevent pytest from collecting this as a test class
|
||||
|
||||
name: str
|
||||
type: str # unit, integration, e2e, all
|
||||
command: str
|
||||
config_file: str | None = None
|
||||
version: str | None = None
|
||||
coverage_command: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class TestDiscoveryResult:
|
||||
"""
|
||||
Result of test framework discovery.
|
||||
|
||||
Attributes:
|
||||
frameworks: List of detected test frameworks
|
||||
test_command: Primary test command to run
|
||||
test_directories: Discovered test directories
|
||||
package_manager: Detected package manager
|
||||
has_tests: Whether any test files were found
|
||||
coverage_command: Command for coverage if available
|
||||
"""
|
||||
|
||||
__test__ = False # Prevent pytest from collecting this as a test class
|
||||
|
||||
frameworks: list[TestFramework] = field(default_factory=list)
|
||||
test_command: str = ""
|
||||
test_directories: list[str] = field(default_factory=list)
|
||||
package_manager: str = ""
|
||||
has_tests: bool = False
|
||||
coverage_command: str | None = None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# FRAMEWORK DETECTORS
|
||||
# =============================================================================
|
||||
|
||||
|
||||
# Pattern-based framework detection
|
||||
FRAMEWORK_PATTERNS = {
|
||||
# JavaScript/TypeScript
|
||||
"jest": {
|
||||
"config_files": [
|
||||
"jest.config.js",
|
||||
"jest.config.ts",
|
||||
"jest.config.mjs",
|
||||
"jest.config.cjs",
|
||||
],
|
||||
"package_key": "jest",
|
||||
"type": "unit",
|
||||
"command": "npx jest",
|
||||
"coverage_command": "npx jest --coverage",
|
||||
},
|
||||
"vitest": {
|
||||
"config_files": ["vitest.config.js", "vitest.config.ts", "vitest.config.mjs"],
|
||||
"package_key": "vitest",
|
||||
"type": "unit",
|
||||
"command": "npx vitest run",
|
||||
"coverage_command": "npx vitest run --coverage",
|
||||
},
|
||||
"mocha": {
|
||||
"config_files": [
|
||||
".mocharc.js",
|
||||
".mocharc.json",
|
||||
".mocharc.yaml",
|
||||
".mocharc.yml",
|
||||
],
|
||||
"package_key": "mocha",
|
||||
"type": "unit",
|
||||
"command": "npx mocha",
|
||||
"coverage_command": "npx nyc mocha",
|
||||
},
|
||||
"playwright": {
|
||||
"config_files": ["playwright.config.js", "playwright.config.ts"],
|
||||
"package_key": "@playwright/test",
|
||||
"type": "e2e",
|
||||
"command": "npx playwright test",
|
||||
"coverage_command": None,
|
||||
},
|
||||
"cypress": {
|
||||
"config_files": ["cypress.config.js", "cypress.config.ts", "cypress.json"],
|
||||
"package_key": "cypress",
|
||||
"type": "e2e",
|
||||
"command": "npx cypress run",
|
||||
"coverage_command": None,
|
||||
},
|
||||
# Python
|
||||
"pytest": {
|
||||
"config_files": ["pytest.ini", "pyproject.toml", "setup.cfg", "conftest.py"],
|
||||
"pyproject_key": "pytest",
|
||||
"requirements_key": "pytest",
|
||||
"type": "all",
|
||||
"command": "pytest",
|
||||
"coverage_command": "pytest --cov",
|
||||
},
|
||||
"unittest": {
|
||||
"config_files": [],
|
||||
"type": "unit",
|
||||
"command": "python -m unittest discover",
|
||||
"coverage_command": "coverage run -m unittest discover",
|
||||
},
|
||||
# Rust
|
||||
"cargo_test": {
|
||||
"config_files": ["Cargo.toml"],
|
||||
"type": "all",
|
||||
"command": "cargo test",
|
||||
"coverage_command": "cargo tarpaulin",
|
||||
},
|
||||
# Go
|
||||
"go_test": {
|
||||
"config_files": ["go.mod"],
|
||||
"type": "all",
|
||||
"command": "go test ./...",
|
||||
"coverage_command": "go test -cover ./...",
|
||||
},
|
||||
# Ruby
|
||||
"rspec": {
|
||||
"config_files": [".rspec", "spec/spec_helper.rb"],
|
||||
"gemfile_key": "rspec",
|
||||
"type": "all",
|
||||
"command": "bundle exec rspec",
|
||||
"coverage_command": "bundle exec rspec --format documentation",
|
||||
},
|
||||
"minitest": {
|
||||
"config_files": [],
|
||||
"gemfile_key": "minitest",
|
||||
"type": "unit",
|
||||
"command": "bundle exec rake test",
|
||||
"coverage_command": None,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST DISCOVERY
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class TestDiscovery:
|
||||
"""
|
||||
Discovers test frameworks and configurations in a project.
|
||||
|
||||
Analyzes:
|
||||
- Package files (package.json, pyproject.toml, Cargo.toml, etc.)
|
||||
- Configuration files (jest.config.js, pytest.ini, etc.)
|
||||
- Directory structure (tests/, spec/, __tests__/)
|
||||
"""
|
||||
|
||||
__test__ = False # Prevent pytest from collecting this as a test class
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialize the test discovery."""
|
||||
self._cache: dict[str, TestDiscoveryResult] = {}
|
||||
|
||||
def discover(self, project_dir: Path) -> TestDiscoveryResult:
|
||||
"""
|
||||
Discover test frameworks and configuration in the project.
|
||||
|
||||
Args:
|
||||
project_dir: Path to the project root
|
||||
|
||||
Returns:
|
||||
TestDiscoveryResult with detected frameworks and commands
|
||||
"""
|
||||
project_dir = Path(project_dir)
|
||||
cache_key = str(project_dir.resolve())
|
||||
|
||||
if cache_key in self._cache:
|
||||
return self._cache[cache_key]
|
||||
|
||||
result = TestDiscoveryResult()
|
||||
|
||||
# Detect package manager
|
||||
result.package_manager = self._detect_package_manager(project_dir)
|
||||
|
||||
# Discover frameworks based on project type
|
||||
if (project_dir / "package.json").exists():
|
||||
self._discover_js_frameworks(project_dir, result)
|
||||
|
||||
# Check for Python project indicators
|
||||
python_indicators = [
|
||||
project_dir / "pyproject.toml",
|
||||
project_dir / "requirements.txt",
|
||||
project_dir / "setup.py",
|
||||
project_dir / "pytest.ini",
|
||||
project_dir / "conftest.py",
|
||||
project_dir / "tests" / "conftest.py",
|
||||
]
|
||||
if any(p.exists() for p in python_indicators):
|
||||
self._discover_python_frameworks(project_dir, result)
|
||||
|
||||
if (project_dir / "Cargo.toml").exists():
|
||||
self._discover_rust_frameworks(project_dir, result)
|
||||
if (project_dir / "go.mod").exists():
|
||||
self._discover_go_frameworks(project_dir, result)
|
||||
if (project_dir / "Gemfile").exists():
|
||||
self._discover_ruby_frameworks(project_dir, result)
|
||||
|
||||
# Find test directories
|
||||
result.test_directories = self._find_test_directories(project_dir)
|
||||
|
||||
# Check if tests exist
|
||||
result.has_tests = self._has_test_files(project_dir, result.test_directories)
|
||||
|
||||
# Set primary test command
|
||||
if result.frameworks:
|
||||
result.test_command = result.frameworks[0].command
|
||||
|
||||
# Set coverage command from first framework that has one
|
||||
if not result.coverage_command:
|
||||
for framework in result.frameworks:
|
||||
if framework.coverage_command:
|
||||
result.coverage_command = framework.coverage_command
|
||||
break
|
||||
|
||||
self._cache[cache_key] = result
|
||||
return result
|
||||
|
||||
def _detect_package_manager(self, project_dir: Path) -> str:
|
||||
"""Detect the package manager used by the project."""
|
||||
if (project_dir / "pnpm-lock.yaml").exists():
|
||||
return "pnpm"
|
||||
if (project_dir / "yarn.lock").exists():
|
||||
return "yarn"
|
||||
if (project_dir / "package-lock.json").exists():
|
||||
return "npm"
|
||||
if (project_dir / "bun.lockb").exists():
|
||||
return "bun"
|
||||
if (project_dir / "uv.lock").exists():
|
||||
return "uv"
|
||||
if (project_dir / "poetry.lock").exists():
|
||||
return "poetry"
|
||||
if (project_dir / "Pipfile.lock").exists():
|
||||
return "pipenv"
|
||||
if (project_dir / "Cargo.lock").exists():
|
||||
return "cargo"
|
||||
if (project_dir / "go.sum").exists():
|
||||
return "go"
|
||||
if (project_dir / "Gemfile.lock").exists():
|
||||
return "bundler"
|
||||
return ""
|
||||
|
||||
def _discover_js_frameworks(
|
||||
self, project_dir: Path, result: TestDiscoveryResult
|
||||
) -> None:
|
||||
"""Discover JavaScript/TypeScript test frameworks."""
|
||||
package_json = project_dir / "package.json"
|
||||
if not package_json.exists():
|
||||
return
|
||||
|
||||
try:
|
||||
with open(package_json, encoding="utf-8") as f:
|
||||
pkg = json.load(f)
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return
|
||||
|
||||
deps = pkg.get("dependencies", {})
|
||||
dev_deps = pkg.get("devDependencies", {})
|
||||
all_deps = {**deps, **dev_deps}
|
||||
scripts = pkg.get("scripts", {})
|
||||
|
||||
# Check for test frameworks in dependencies
|
||||
for name, pattern in FRAMEWORK_PATTERNS.items():
|
||||
if "package_key" not in pattern:
|
||||
continue
|
||||
|
||||
if pattern["package_key"] in all_deps:
|
||||
# Check for config file
|
||||
config_file = None
|
||||
for cf in pattern.get("config_files", []):
|
||||
if (project_dir / cf).exists():
|
||||
config_file = cf
|
||||
break
|
||||
|
||||
# Get version
|
||||
version = all_deps.get(pattern["package_key"], "")
|
||||
if version.startswith("^") or version.startswith("~"):
|
||||
version = version[1:]
|
||||
|
||||
# Determine command - prefer npm scripts if available
|
||||
command = pattern["command"]
|
||||
if "test" in scripts and pattern["package_key"] in scripts.get(
|
||||
"test", ""
|
||||
):
|
||||
command = f"{result.package_manager or 'npm'} test"
|
||||
|
||||
result.frameworks.append(
|
||||
TestFramework(
|
||||
name=name,
|
||||
type=pattern["type"],
|
||||
command=command,
|
||||
config_file=config_file,
|
||||
version=version,
|
||||
coverage_command=pattern.get("coverage_command"),
|
||||
)
|
||||
)
|
||||
|
||||
# Check npm scripts for test commands
|
||||
if not result.frameworks and "test" in scripts:
|
||||
test_script = scripts["test"]
|
||||
if (
|
||||
test_script
|
||||
and test_script != 'echo "Error: no test specified" && exit 1'
|
||||
):
|
||||
# Try to infer framework from script
|
||||
framework_name = "npm_test"
|
||||
framework_type = "unit"
|
||||
|
||||
if "jest" in test_script:
|
||||
framework_name = "jest"
|
||||
elif "vitest" in test_script:
|
||||
framework_name = "vitest"
|
||||
elif "mocha" in test_script:
|
||||
framework_name = "mocha"
|
||||
elif "playwright" in test_script:
|
||||
framework_name = "playwright"
|
||||
framework_type = "e2e"
|
||||
elif "cypress" in test_script:
|
||||
framework_name = "cypress"
|
||||
framework_type = "e2e"
|
||||
|
||||
result.frameworks.append(
|
||||
TestFramework(
|
||||
name=framework_name,
|
||||
type=framework_type,
|
||||
command=f"{result.package_manager or 'npm'} test",
|
||||
config_file=None,
|
||||
)
|
||||
)
|
||||
|
||||
def _discover_python_frameworks(
|
||||
self, project_dir: Path, result: TestDiscoveryResult
|
||||
) -> None:
|
||||
"""Discover Python test frameworks."""
|
||||
# Check for pytest.ini first (explicit pytest config)
|
||||
if (project_dir / "pytest.ini").exists():
|
||||
if not any(f.name == "pytest" for f in result.frameworks):
|
||||
result.frameworks.append(
|
||||
TestFramework(
|
||||
name="pytest",
|
||||
type="all",
|
||||
command="pytest",
|
||||
config_file="pytest.ini",
|
||||
)
|
||||
)
|
||||
|
||||
# Check pyproject.toml
|
||||
pyproject = project_dir / "pyproject.toml"
|
||||
if pyproject.exists():
|
||||
content = pyproject.read_text()
|
||||
|
||||
# Check for pytest
|
||||
if "pytest" in content:
|
||||
if not any(f.name == "pytest" for f in result.frameworks):
|
||||
config_file = (
|
||||
"pyproject.toml" if "[tool.pytest" in content else None
|
||||
)
|
||||
result.frameworks.append(
|
||||
TestFramework(
|
||||
name="pytest",
|
||||
type="all",
|
||||
command="pytest",
|
||||
config_file=config_file,
|
||||
)
|
||||
)
|
||||
|
||||
# Check requirements.txt
|
||||
requirements = project_dir / "requirements.txt"
|
||||
if requirements.exists():
|
||||
content = requirements.read_text().lower()
|
||||
if "pytest" in content and not any(
|
||||
f.name == "pytest" for f in result.frameworks
|
||||
):
|
||||
result.frameworks.append(
|
||||
TestFramework(
|
||||
name="pytest",
|
||||
type="all",
|
||||
command="pytest",
|
||||
config_file=None,
|
||||
)
|
||||
)
|
||||
|
||||
# Check for conftest.py (pytest marker)
|
||||
conftest_root = project_dir / "conftest.py"
|
||||
conftest_tests = project_dir / "tests" / "conftest.py"
|
||||
if conftest_root.exists() or conftest_tests.exists():
|
||||
if not any(f.name == "pytest" for f in result.frameworks):
|
||||
result.frameworks.append(
|
||||
TestFramework(
|
||||
name="pytest",
|
||||
type="all",
|
||||
command="pytest",
|
||||
config_file="conftest.py",
|
||||
)
|
||||
)
|
||||
|
||||
# Fall back to unittest if test files exist but no framework detected
|
||||
if not result.frameworks:
|
||||
test_dirs = self._find_test_directories(project_dir)
|
||||
if test_dirs:
|
||||
result.frameworks.append(
|
||||
TestFramework(
|
||||
name="unittest",
|
||||
type="unit",
|
||||
command="python -m unittest discover",
|
||||
config_file=None,
|
||||
)
|
||||
)
|
||||
|
||||
def _discover_rust_frameworks(
|
||||
self, project_dir: Path, result: TestDiscoveryResult
|
||||
) -> None:
|
||||
"""Discover Rust test frameworks."""
|
||||
cargo_toml = project_dir / "Cargo.toml"
|
||||
if cargo_toml.exists():
|
||||
result.frameworks.append(
|
||||
TestFramework(
|
||||
name="cargo_test",
|
||||
type="all",
|
||||
command="cargo test",
|
||||
config_file="Cargo.toml",
|
||||
)
|
||||
)
|
||||
|
||||
def _discover_go_frameworks(
|
||||
self, project_dir: Path, result: TestDiscoveryResult
|
||||
) -> None:
|
||||
"""Discover Go test frameworks."""
|
||||
go_mod = project_dir / "go.mod"
|
||||
if go_mod.exists():
|
||||
result.frameworks.append(
|
||||
TestFramework(
|
||||
name="go_test",
|
||||
type="all",
|
||||
command="go test ./...",
|
||||
config_file="go.mod",
|
||||
)
|
||||
)
|
||||
|
||||
def _discover_ruby_frameworks(
|
||||
self, project_dir: Path, result: TestDiscoveryResult
|
||||
) -> None:
|
||||
"""Discover Ruby test frameworks."""
|
||||
gemfile = project_dir / "Gemfile"
|
||||
if not gemfile.exists():
|
||||
return
|
||||
|
||||
content = gemfile.read_text().lower()
|
||||
|
||||
if "rspec" in content or (project_dir / ".rspec").exists():
|
||||
result.frameworks.append(
|
||||
TestFramework(
|
||||
name="rspec",
|
||||
type="all",
|
||||
command="bundle exec rspec",
|
||||
config_file=".rspec" if (project_dir / ".rspec").exists() else None,
|
||||
)
|
||||
)
|
||||
elif "minitest" in content:
|
||||
result.frameworks.append(
|
||||
TestFramework(
|
||||
name="minitest",
|
||||
type="unit",
|
||||
command="bundle exec rake test",
|
||||
config_file=None,
|
||||
)
|
||||
)
|
||||
|
||||
def _find_test_directories(self, project_dir: Path) -> list[str]:
|
||||
"""Find test directories in the project."""
|
||||
test_dir_patterns = [
|
||||
"tests",
|
||||
"test",
|
||||
"spec",
|
||||
"__tests__",
|
||||
"specs",
|
||||
"test_*",
|
||||
]
|
||||
|
||||
found_dirs = []
|
||||
for pattern in test_dir_patterns:
|
||||
if pattern.endswith("*"):
|
||||
# Glob pattern
|
||||
for d in project_dir.glob(pattern):
|
||||
if d.is_dir():
|
||||
found_dirs.append(str(d.relative_to(project_dir)))
|
||||
else:
|
||||
# Exact name
|
||||
test_dir = project_dir / pattern
|
||||
if test_dir.is_dir():
|
||||
found_dirs.append(pattern)
|
||||
|
||||
return found_dirs
|
||||
|
||||
def _has_test_files(self, project_dir: Path, test_directories: list[str]) -> bool:
|
||||
"""Check if any test files exist."""
|
||||
test_file_patterns = [
|
||||
"**/test_*.py",
|
||||
"**/*_test.py",
|
||||
"**/*.test.js",
|
||||
"**/*.test.ts",
|
||||
"**/*.test.tsx",
|
||||
"**/*.spec.js",
|
||||
"**/*.spec.ts",
|
||||
"**/*.spec.tsx",
|
||||
"**/test_*.go",
|
||||
"**/*_test.go",
|
||||
"**/*_test.rs",
|
||||
"**/spec/**/*_spec.rb",
|
||||
]
|
||||
|
||||
# Check in test directories
|
||||
for test_dir in test_directories:
|
||||
test_path = project_dir / test_dir
|
||||
if test_path.exists():
|
||||
for pattern in test_file_patterns:
|
||||
if list(test_path.glob(pattern.replace("**/", ""))):
|
||||
return True
|
||||
|
||||
# Check project-wide
|
||||
for pattern in test_file_patterns:
|
||||
if list(project_dir.glob(pattern)):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def to_dict(self, result: TestDiscoveryResult) -> dict[str, Any]:
|
||||
"""Convert result to dictionary for JSON serialization."""
|
||||
return {
|
||||
"frameworks": [
|
||||
{
|
||||
"name": f.name,
|
||||
"type": f.type,
|
||||
"command": f.command,
|
||||
"config_file": f.config_file,
|
||||
"version": f.version,
|
||||
"coverage_command": f.coverage_command,
|
||||
}
|
||||
for f in result.frameworks
|
||||
],
|
||||
"test_command": result.test_command,
|
||||
"test_directories": result.test_directories,
|
||||
"package_manager": result.package_manager,
|
||||
"has_tests": result.has_tests,
|
||||
"coverage_command": result.coverage_command,
|
||||
}
|
||||
|
||||
def clear_cache(self) -> None:
|
||||
"""Clear the internal cache."""
|
||||
self._cache.clear()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CONVENIENCE FUNCTIONS
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def discover_tests(project_dir: Path) -> TestDiscoveryResult:
|
||||
"""
|
||||
Convenience function to discover tests in a project.
|
||||
|
||||
Args:
|
||||
project_dir: Path to project root
|
||||
|
||||
Returns:
|
||||
TestDiscoveryResult with detected frameworks
|
||||
"""
|
||||
discovery = TestDiscovery()
|
||||
return discovery.discover(project_dir)
|
||||
|
||||
|
||||
def get_test_command(project_dir: Path) -> str:
|
||||
"""
|
||||
Get the primary test command for a project.
|
||||
|
||||
Args:
|
||||
project_dir: Path to project root
|
||||
|
||||
Returns:
|
||||
Test command string, or empty string if not found
|
||||
"""
|
||||
discovery = TestDiscovery()
|
||||
result = discovery.discover(project_dir)
|
||||
return result.test_command
|
||||
|
||||
|
||||
def get_test_frameworks(project_dir: Path) -> list[str]:
|
||||
"""
|
||||
Get list of test framework names in a project.
|
||||
|
||||
Args:
|
||||
project_dir: Path to project root
|
||||
|
||||
Returns:
|
||||
List of framework names
|
||||
"""
|
||||
discovery = TestDiscovery()
|
||||
result = discovery.discover(project_dir)
|
||||
return [f.name for f in result.frameworks]
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLI
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""CLI entry point for testing."""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Discover test frameworks")
|
||||
parser.add_argument("project_dir", type=Path, help="Path to project root")
|
||||
parser.add_argument("--json", action="store_true", help="Output as JSON")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
discovery = TestDiscovery()
|
||||
result = discovery.discover(args.project_dir)
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(discovery.to_dict(result), indent=2))
|
||||
else:
|
||||
print(f"Package Manager: {result.package_manager or 'unknown'}")
|
||||
print(f"Has Tests: {result.has_tests}")
|
||||
print(f"Test Command: {result.test_command or 'none'}")
|
||||
print(f"Test Directories: {', '.join(result.test_directories) or 'none'}")
|
||||
print(f"Coverage Command: {result.coverage_command or 'none'}")
|
||||
print(f"\nFrameworks ({len(result.frameworks)}):")
|
||||
for f in result.frameworks:
|
||||
print(f" - {f.name} ({f.type})")
|
||||
print(f" Command: {f.command}")
|
||||
if f.config_file:
|
||||
print(f" Config: {f.config_file}")
|
||||
if f.version:
|
||||
print(f" Version: {f.version}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+2
-100
@@ -1,100 +1,2 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Codebase Analyzer
|
||||
=================
|
||||
|
||||
Automatically detects project structure, frameworks, and services.
|
||||
Supports monorepos with multiple services.
|
||||
|
||||
Usage:
|
||||
# Index entire project (creates project_index.json)
|
||||
python auto-claude/analyzer.py --index
|
||||
|
||||
# Analyze specific service
|
||||
python auto-claude/analyzer.py --service backend
|
||||
|
||||
# Output to specific file
|
||||
python auto-claude/analyzer.py --index --output path/to/output.json
|
||||
|
||||
The analyzer will:
|
||||
1. Detect if this is a monorepo or single project
|
||||
2. Find all services/packages and analyze each separately
|
||||
3. Map interdependencies between services
|
||||
4. Identify infrastructure (Docker, CI/CD)
|
||||
5. Document conventions (linting, testing)
|
||||
|
||||
This module now serves as a facade to the modular analyzer system in the analyzers/ package.
|
||||
All actual implementation is in focused submodules for better maintainability.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
# Import from the new modular structure
|
||||
from analyzers import (
|
||||
ProjectAnalyzer,
|
||||
ServiceAnalyzer,
|
||||
analyze_project,
|
||||
analyze_service,
|
||||
)
|
||||
|
||||
# Re-export for backward compatibility
|
||||
__all__ = [
|
||||
"ServiceAnalyzer",
|
||||
"ProjectAnalyzer",
|
||||
"analyze_project",
|
||||
"analyze_service",
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
"""CLI entry point."""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Analyze project structure, frameworks, and services"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--project-dir",
|
||||
type=Path,
|
||||
default=Path.cwd(),
|
||||
help="Project directory to analyze (default: current directory)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--index",
|
||||
action="store_true",
|
||||
help="Create full project index (default behavior)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--service",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Analyze a specific service only",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Output file for JSON results",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--quiet",
|
||||
action="store_true",
|
||||
help="Only output JSON, no status messages",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Determine what to analyze
|
||||
if args.service:
|
||||
results = analyze_service(args.project_dir, args.service, args.output)
|
||||
else:
|
||||
results = analyze_project(args.project_dir, args.output)
|
||||
|
||||
# Print results
|
||||
if not args.quiet or not args.output:
|
||||
print(json.dumps(results, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
"""Backward compatibility shim - import from analysis.analyzer instead."""
|
||||
from analysis.analyzer import *
|
||||
|
||||
@@ -1,92 +1,2 @@
|
||||
"""
|
||||
Analyzers Package
|
||||
=================
|
||||
|
||||
Modular analyzer system for detecting project structure, frameworks, and services.
|
||||
|
||||
Main exports:
|
||||
- ServiceAnalyzer: Analyzes a single service/package
|
||||
- ProjectAnalyzer: Analyzes entire projects (single or monorepo)
|
||||
- analyze_project: Convenience function for project analysis
|
||||
- analyze_service: Convenience function for service analysis
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .project_analyzer_module import ProjectAnalyzer
|
||||
from .service_analyzer import ServiceAnalyzer
|
||||
|
||||
# Re-export main classes
|
||||
__all__ = [
|
||||
"ServiceAnalyzer",
|
||||
"ProjectAnalyzer",
|
||||
"analyze_project",
|
||||
"analyze_service",
|
||||
]
|
||||
|
||||
|
||||
def analyze_project(project_dir: Path, output_file: Path | None = None) -> dict:
|
||||
"""
|
||||
Analyze a project and optionally save results.
|
||||
|
||||
Args:
|
||||
project_dir: Path to the project root
|
||||
output_file: Optional path to save JSON output
|
||||
|
||||
Returns:
|
||||
Project index as a dictionary
|
||||
"""
|
||||
import json
|
||||
|
||||
analyzer = ProjectAnalyzer(project_dir)
|
||||
results = analyzer.analyze()
|
||||
|
||||
if output_file:
|
||||
output_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_file, "w") as f:
|
||||
json.dump(results, f, indent=2)
|
||||
print(f"Project index saved to: {output_file}")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def analyze_service(
|
||||
project_dir: Path, service_name: str, output_file: Path | None = None
|
||||
) -> dict:
|
||||
"""
|
||||
Analyze a specific service within a project.
|
||||
|
||||
Args:
|
||||
project_dir: Path to the project root
|
||||
service_name: Name of the service to analyze
|
||||
output_file: Optional path to save JSON output
|
||||
|
||||
Returns:
|
||||
Service analysis as a dictionary
|
||||
"""
|
||||
import json
|
||||
|
||||
# Find the service
|
||||
service_path = project_dir / service_name
|
||||
if not service_path.exists():
|
||||
# Check common locations
|
||||
for parent in ["packages", "apps", "services"]:
|
||||
candidate = project_dir / parent / service_name
|
||||
if candidate.exists():
|
||||
service_path = candidate
|
||||
break
|
||||
|
||||
if not service_path.exists():
|
||||
raise ValueError(f"Service '{service_name}' not found in {project_dir}")
|
||||
|
||||
analyzer = ServiceAnalyzer(service_path, service_name)
|
||||
results = analyzer.analyze()
|
||||
|
||||
if output_file:
|
||||
output_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_file, "w") as f:
|
||||
json.dump(results, f, indent=2)
|
||||
print(f"Service analysis saved to: {output_file}")
|
||||
|
||||
return results
|
||||
"""Backward compatibility shim - import from analysis.analyzers instead."""
|
||||
from analysis.analyzers import *
|
||||
|
||||
@@ -1,45 +1,13 @@
|
||||
"""
|
||||
Custom MCP Tools for Auto-Claude Agents
|
||||
========================================
|
||||
"""Backward compatibility shim - import from agents.tools_pkg instead."""
|
||||
|
||||
DEPRECATED: This module is now a compatibility shim.
|
||||
Please import from the auto_claude_tools package instead:
|
||||
# Direct import to avoid triggering agents.__init__ circular dependencies
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from auto_claude_tools import create_auto_claude_mcp_server, get_allowed_tools
|
||||
# Add agents directory to path if needed
|
||||
agents_dir = Path(__file__).parent / "agents"
|
||||
if str(agents_dir) not in sys.path:
|
||||
sys.path.insert(0, str(agents_dir))
|
||||
|
||||
This file remains for backward compatibility with existing imports.
|
||||
All functionality has been moved to the auto_claude_tools package for better
|
||||
organization and maintainability.
|
||||
"""
|
||||
|
||||
# Import everything from the package to maintain backward compatibility
|
||||
from auto_claude_tools import (
|
||||
ELECTRON_TOOLS,
|
||||
TOOL_GET_BUILD_PROGRESS,
|
||||
TOOL_GET_SESSION_CONTEXT,
|
||||
TOOL_RECORD_DISCOVERY,
|
||||
TOOL_RECORD_GOTCHA,
|
||||
TOOL_UPDATE_QA_STATUS,
|
||||
TOOL_UPDATE_SUBTASK_STATUS,
|
||||
create_auto_claude_mcp_server,
|
||||
get_allowed_tools,
|
||||
is_electron_mcp_enabled,
|
||||
is_tools_available,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# Main API
|
||||
"create_auto_claude_mcp_server",
|
||||
"get_allowed_tools",
|
||||
"is_tools_available",
|
||||
# Tool name constants
|
||||
"TOOL_UPDATE_SUBTASK_STATUS",
|
||||
"TOOL_GET_BUILD_PROGRESS",
|
||||
"TOOL_RECORD_DISCOVERY",
|
||||
"TOOL_RECORD_GOTCHA",
|
||||
"TOOL_GET_SESSION_CONTEXT",
|
||||
"TOOL_UPDATE_QA_STATUS",
|
||||
# Electron MCP
|
||||
"ELECTRON_TOOLS",
|
||||
"is_electron_mcp_enabled",
|
||||
]
|
||||
# Import directly from tools_pkg to avoid agents.__init__ circular imports
|
||||
from tools_pkg import * # noqa: F403, E402
|
||||
|
||||
+20
-587
@@ -1,587 +1,20 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
CI Discovery Module
|
||||
===================
|
||||
|
||||
Parses CI/CD configuration files to extract test commands and workflows.
|
||||
Supports GitHub Actions, GitLab CI, CircleCI, and Jenkins.
|
||||
|
||||
The CI discovery results are used by:
|
||||
- QA Agent: To understand existing CI test patterns
|
||||
- Validation Strategy: To match CI commands
|
||||
- Planner: To align verification with CI
|
||||
|
||||
Usage:
|
||||
from ci_discovery import CIDiscovery
|
||||
|
||||
discovery = CIDiscovery()
|
||||
result = discovery.discover(project_dir)
|
||||
|
||||
if result:
|
||||
print(f"CI System: {result.ci_system}")
|
||||
print(f"Test Commands: {result.test_commands}")
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# Try to import yaml, fall back gracefully
|
||||
try:
|
||||
import yaml
|
||||
|
||||
HAS_YAML = True
|
||||
except ImportError:
|
||||
HAS_YAML = False
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# DATA CLASSES
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@dataclass
|
||||
class CIWorkflow:
|
||||
"""
|
||||
Represents a CI workflow or job.
|
||||
|
||||
Attributes:
|
||||
name: Name of the workflow/job
|
||||
trigger: What triggers this workflow (push, pull_request, etc.)
|
||||
steps: List of step names or commands
|
||||
test_related: Whether this appears to be test-related
|
||||
"""
|
||||
|
||||
name: str
|
||||
trigger: list[str] = field(default_factory=list)
|
||||
steps: list[str] = field(default_factory=list)
|
||||
test_related: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class CIConfig:
|
||||
"""
|
||||
Result of CI configuration discovery.
|
||||
|
||||
Attributes:
|
||||
ci_system: Name of CI system (github_actions, gitlab, circleci, jenkins)
|
||||
config_files: List of CI config files found
|
||||
test_commands: Extracted test commands by type
|
||||
coverage_command: Coverage command if found
|
||||
workflows: List of discovered workflows
|
||||
environment_variables: Environment variables used
|
||||
"""
|
||||
|
||||
ci_system: str
|
||||
config_files: list[str] = field(default_factory=list)
|
||||
test_commands: dict[str, str] = field(default_factory=dict)
|
||||
coverage_command: str | None = None
|
||||
workflows: list[CIWorkflow] = field(default_factory=list)
|
||||
environment_variables: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CI PARSERS
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class CIDiscovery:
|
||||
"""
|
||||
Discovers CI/CD configurations in a project.
|
||||
|
||||
Analyzes:
|
||||
- GitHub Actions (.github/workflows/*.yml)
|
||||
- GitLab CI (.gitlab-ci.yml)
|
||||
- CircleCI (.circleci/config.yml)
|
||||
- Jenkins (Jenkinsfile)
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialize CI discovery."""
|
||||
self._cache: dict[str, CIConfig | None] = {}
|
||||
|
||||
def discover(self, project_dir: Path) -> CIConfig | None:
|
||||
"""
|
||||
Discover CI configuration in the project.
|
||||
|
||||
Args:
|
||||
project_dir: Path to the project root
|
||||
|
||||
Returns:
|
||||
CIConfig if CI found, None otherwise
|
||||
"""
|
||||
project_dir = Path(project_dir)
|
||||
cache_key = str(project_dir.resolve())
|
||||
|
||||
if cache_key in self._cache:
|
||||
return self._cache[cache_key]
|
||||
|
||||
# Try each CI system
|
||||
result = None
|
||||
|
||||
# GitHub Actions
|
||||
github_workflows = project_dir / ".github" / "workflows"
|
||||
if github_workflows.exists():
|
||||
result = self._parse_github_actions(github_workflows)
|
||||
|
||||
# GitLab CI
|
||||
if not result:
|
||||
gitlab_ci = project_dir / ".gitlab-ci.yml"
|
||||
if gitlab_ci.exists():
|
||||
result = self._parse_gitlab_ci(gitlab_ci)
|
||||
|
||||
# CircleCI
|
||||
if not result:
|
||||
circleci = project_dir / ".circleci" / "config.yml"
|
||||
if circleci.exists():
|
||||
result = self._parse_circleci(circleci)
|
||||
|
||||
# Jenkins
|
||||
if not result:
|
||||
jenkinsfile = project_dir / "Jenkinsfile"
|
||||
if jenkinsfile.exists():
|
||||
result = self._parse_jenkinsfile(jenkinsfile)
|
||||
|
||||
self._cache[cache_key] = result
|
||||
return result
|
||||
|
||||
def _parse_github_actions(self, workflows_dir: Path) -> CIConfig:
|
||||
"""Parse GitHub Actions workflow files."""
|
||||
result = CIConfig(ci_system="github_actions")
|
||||
|
||||
workflow_files = list(workflows_dir.glob("*.yml")) + list(
|
||||
workflows_dir.glob("*.yaml")
|
||||
)
|
||||
|
||||
for wf_file in workflow_files:
|
||||
result.config_files.append(
|
||||
str(wf_file.relative_to(workflows_dir.parent.parent))
|
||||
)
|
||||
|
||||
try:
|
||||
content = wf_file.read_text()
|
||||
workflow_data = self._parse_yaml(content)
|
||||
|
||||
if not workflow_data:
|
||||
continue
|
||||
|
||||
# Get workflow name
|
||||
wf_name = workflow_data.get("name", wf_file.stem)
|
||||
|
||||
# Get triggers
|
||||
triggers = []
|
||||
on_trigger = workflow_data.get("on", {})
|
||||
if isinstance(on_trigger, str):
|
||||
triggers = [on_trigger]
|
||||
elif isinstance(on_trigger, list):
|
||||
triggers = on_trigger
|
||||
elif isinstance(on_trigger, dict):
|
||||
triggers = list(on_trigger.keys())
|
||||
|
||||
# Parse jobs
|
||||
jobs = workflow_data.get("jobs", {})
|
||||
for job_name, job_config in jobs.items():
|
||||
if not isinstance(job_config, dict):
|
||||
continue
|
||||
|
||||
steps = job_config.get("steps", [])
|
||||
step_commands = []
|
||||
test_related = False
|
||||
|
||||
for step in steps:
|
||||
if not isinstance(step, dict):
|
||||
continue
|
||||
|
||||
# Get step name or command
|
||||
step_name = step.get("name", "")
|
||||
run_cmd = step.get("run", "")
|
||||
uses = step.get("uses", "")
|
||||
|
||||
if step_name:
|
||||
step_commands.append(step_name)
|
||||
if run_cmd:
|
||||
step_commands.append(run_cmd)
|
||||
# Extract test commands
|
||||
self._extract_test_commands(run_cmd, result)
|
||||
if uses:
|
||||
step_commands.append(f"uses: {uses}")
|
||||
|
||||
# Check if test-related
|
||||
test_keywords = ["test", "pytest", "jest", "vitest", "coverage"]
|
||||
if any(kw in str(step).lower() for kw in test_keywords):
|
||||
test_related = True
|
||||
|
||||
result.workflows.append(
|
||||
CIWorkflow(
|
||||
name=f"{wf_name}/{job_name}",
|
||||
trigger=triggers,
|
||||
steps=step_commands,
|
||||
test_related=test_related,
|
||||
)
|
||||
)
|
||||
|
||||
# Extract environment variables
|
||||
env = workflow_data.get("env", {})
|
||||
if isinstance(env, dict):
|
||||
result.environment_variables.extend(env.keys())
|
||||
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return result
|
||||
|
||||
def _parse_gitlab_ci(self, config_file: Path) -> CIConfig:
|
||||
"""Parse GitLab CI configuration."""
|
||||
result = CIConfig(
|
||||
ci_system="gitlab",
|
||||
config_files=[".gitlab-ci.yml"],
|
||||
)
|
||||
|
||||
try:
|
||||
content = config_file.read_text()
|
||||
data = self._parse_yaml(content)
|
||||
|
||||
if not data:
|
||||
return result
|
||||
|
||||
# Parse jobs (top-level keys that aren't special keywords)
|
||||
special_keys = {
|
||||
"stages",
|
||||
"variables",
|
||||
"image",
|
||||
"services",
|
||||
"before_script",
|
||||
"after_script",
|
||||
"cache",
|
||||
"include",
|
||||
"default",
|
||||
"workflow",
|
||||
}
|
||||
|
||||
for key, value in data.items():
|
||||
if key.startswith(".") or key in special_keys:
|
||||
continue
|
||||
|
||||
if not isinstance(value, dict):
|
||||
continue
|
||||
|
||||
job_config = value
|
||||
script = job_config.get("script", [])
|
||||
if isinstance(script, str):
|
||||
script = [script]
|
||||
|
||||
test_related = any(
|
||||
kw in str(script).lower()
|
||||
for kw in ["test", "pytest", "jest", "vitest", "coverage"]
|
||||
)
|
||||
|
||||
result.workflows.append(
|
||||
CIWorkflow(
|
||||
name=key,
|
||||
trigger=job_config.get("only", [])
|
||||
or job_config.get("rules", []),
|
||||
steps=script,
|
||||
test_related=test_related,
|
||||
)
|
||||
)
|
||||
|
||||
# Extract test commands
|
||||
for cmd in script:
|
||||
if isinstance(cmd, str):
|
||||
self._extract_test_commands(cmd, result)
|
||||
|
||||
# Extract variables
|
||||
variables = data.get("variables", {})
|
||||
if isinstance(variables, dict):
|
||||
result.environment_variables.extend(variables.keys())
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
def _parse_circleci(self, config_file: Path) -> CIConfig:
|
||||
"""Parse CircleCI configuration."""
|
||||
result = CIConfig(
|
||||
ci_system="circleci",
|
||||
config_files=[".circleci/config.yml"],
|
||||
)
|
||||
|
||||
try:
|
||||
content = config_file.read_text()
|
||||
data = self._parse_yaml(content)
|
||||
|
||||
if not data:
|
||||
return result
|
||||
|
||||
# Parse jobs
|
||||
jobs = data.get("jobs", {})
|
||||
for job_name, job_config in jobs.items():
|
||||
if not isinstance(job_config, dict):
|
||||
continue
|
||||
|
||||
steps = job_config.get("steps", [])
|
||||
step_commands = []
|
||||
test_related = False
|
||||
|
||||
for step in steps:
|
||||
if isinstance(step, str):
|
||||
step_commands.append(step)
|
||||
elif isinstance(step, dict):
|
||||
if "run" in step:
|
||||
run = step["run"]
|
||||
if isinstance(run, str):
|
||||
step_commands.append(run)
|
||||
self._extract_test_commands(run, result)
|
||||
elif isinstance(run, dict):
|
||||
cmd = run.get("command", "")
|
||||
step_commands.append(cmd)
|
||||
self._extract_test_commands(cmd, result)
|
||||
|
||||
if any(
|
||||
kw in str(step).lower()
|
||||
for kw in ["test", "pytest", "jest", "coverage"]
|
||||
):
|
||||
test_related = True
|
||||
|
||||
result.workflows.append(
|
||||
CIWorkflow(
|
||||
name=job_name,
|
||||
trigger=[],
|
||||
steps=step_commands,
|
||||
test_related=test_related,
|
||||
)
|
||||
)
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
def _parse_jenkinsfile(self, jenkinsfile: Path) -> CIConfig:
|
||||
"""Parse Jenkinsfile (basic extraction)."""
|
||||
result = CIConfig(
|
||||
ci_system="jenkins",
|
||||
config_files=["Jenkinsfile"],
|
||||
)
|
||||
|
||||
try:
|
||||
content = jenkinsfile.read_text()
|
||||
|
||||
# Extract sh commands using regex
|
||||
sh_pattern = re.compile(r'sh\s+[\'"]([^\'"]+)[\'"]')
|
||||
matches = sh_pattern.findall(content)
|
||||
|
||||
steps = []
|
||||
test_related = False
|
||||
|
||||
for cmd in matches:
|
||||
steps.append(cmd)
|
||||
self._extract_test_commands(cmd, result)
|
||||
|
||||
if any(
|
||||
kw in cmd.lower() for kw in ["test", "pytest", "jest", "coverage"]
|
||||
):
|
||||
test_related = True
|
||||
|
||||
# Extract stage names
|
||||
stage_pattern = re.compile(r'stage\s*\([\'"]([^\'"]+)[\'"]\)')
|
||||
stages = stage_pattern.findall(content)
|
||||
|
||||
for stage in stages:
|
||||
result.workflows.append(
|
||||
CIWorkflow(
|
||||
name=stage,
|
||||
trigger=[],
|
||||
steps=steps if "test" in stage.lower() else [],
|
||||
test_related="test" in stage.lower(),
|
||||
)
|
||||
)
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
def _parse_yaml(self, content: str) -> dict | None:
|
||||
"""Parse YAML content, with fallback to basic parsing if yaml not available."""
|
||||
if HAS_YAML:
|
||||
try:
|
||||
return yaml.safe_load(content)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
# Basic fallback for simple YAML (very limited)
|
||||
# This won't work for complex structures
|
||||
return None
|
||||
|
||||
def _extract_test_commands(self, cmd: str, result: CIConfig) -> None:
|
||||
"""Extract test commands from a command string."""
|
||||
cmd_lower = cmd.lower()
|
||||
|
||||
# Python pytest
|
||||
if "pytest" in cmd_lower:
|
||||
if "pytest" not in result.test_commands:
|
||||
result.test_commands["unit"] = cmd.strip()
|
||||
if "--cov" in cmd_lower:
|
||||
result.coverage_command = cmd.strip()
|
||||
|
||||
# Node.js test commands
|
||||
if (
|
||||
"npm test" in cmd_lower
|
||||
or "yarn test" in cmd_lower
|
||||
or "pnpm test" in cmd_lower
|
||||
):
|
||||
if "unit" not in result.test_commands:
|
||||
result.test_commands["unit"] = cmd.strip()
|
||||
|
||||
# Jest/Vitest
|
||||
if "jest" in cmd_lower or "vitest" in cmd_lower:
|
||||
if "unit" not in result.test_commands:
|
||||
result.test_commands["unit"] = cmd.strip()
|
||||
if "--coverage" in cmd_lower:
|
||||
result.coverage_command = cmd.strip()
|
||||
|
||||
# E2E testing
|
||||
if "playwright" in cmd_lower:
|
||||
result.test_commands["e2e"] = cmd.strip()
|
||||
if "cypress" in cmd_lower:
|
||||
result.test_commands["e2e"] = cmd.strip()
|
||||
|
||||
# Integration tests
|
||||
if "integration" in cmd_lower:
|
||||
result.test_commands["integration"] = cmd.strip()
|
||||
|
||||
# Go tests
|
||||
if "go test" in cmd_lower:
|
||||
if "unit" not in result.test_commands:
|
||||
result.test_commands["unit"] = cmd.strip()
|
||||
|
||||
# Rust tests
|
||||
if "cargo test" in cmd_lower:
|
||||
if "unit" not in result.test_commands:
|
||||
result.test_commands["unit"] = cmd.strip()
|
||||
|
||||
def to_dict(self, result: CIConfig) -> dict[str, Any]:
|
||||
"""Convert result to dictionary for JSON serialization."""
|
||||
return {
|
||||
"ci_system": result.ci_system,
|
||||
"config_files": result.config_files,
|
||||
"test_commands": result.test_commands,
|
||||
"coverage_command": result.coverage_command,
|
||||
"workflows": [
|
||||
{
|
||||
"name": w.name,
|
||||
"trigger": w.trigger,
|
||||
"steps": w.steps,
|
||||
"test_related": w.test_related,
|
||||
}
|
||||
for w in result.workflows
|
||||
],
|
||||
"environment_variables": result.environment_variables,
|
||||
}
|
||||
|
||||
def clear_cache(self) -> None:
|
||||
"""Clear the internal cache."""
|
||||
self._cache.clear()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CONVENIENCE FUNCTIONS
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def discover_ci(project_dir: Path) -> CIConfig | None:
|
||||
"""
|
||||
Convenience function to discover CI configuration.
|
||||
|
||||
Args:
|
||||
project_dir: Path to project root
|
||||
|
||||
Returns:
|
||||
CIConfig if found, None otherwise
|
||||
"""
|
||||
discovery = CIDiscovery()
|
||||
return discovery.discover(project_dir)
|
||||
|
||||
|
||||
def get_ci_test_commands(project_dir: Path) -> dict[str, str]:
|
||||
"""
|
||||
Get test commands from CI configuration.
|
||||
|
||||
Args:
|
||||
project_dir: Path to project root
|
||||
|
||||
Returns:
|
||||
Dictionary of test type to command
|
||||
"""
|
||||
discovery = CIDiscovery()
|
||||
result = discovery.discover(project_dir)
|
||||
if result:
|
||||
return result.test_commands
|
||||
return {}
|
||||
|
||||
|
||||
def get_ci_system(project_dir: Path) -> str | None:
|
||||
"""
|
||||
Get the CI system name if configured.
|
||||
|
||||
Args:
|
||||
project_dir: Path to project root
|
||||
|
||||
Returns:
|
||||
CI system name or None
|
||||
"""
|
||||
discovery = CIDiscovery()
|
||||
result = discovery.discover(project_dir)
|
||||
if result:
|
||||
return result.ci_system
|
||||
return None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLI
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""CLI entry point for testing."""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Discover CI configuration")
|
||||
parser.add_argument("project_dir", type=Path, help="Path to project root")
|
||||
parser.add_argument("--json", action="store_true", help="Output as JSON")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
discovery = CIDiscovery()
|
||||
result = discovery.discover(args.project_dir)
|
||||
|
||||
if not result:
|
||||
print("No CI configuration found")
|
||||
return
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(discovery.to_dict(result), indent=2))
|
||||
else:
|
||||
print(f"CI System: {result.ci_system}")
|
||||
print(f"Config Files: {', '.join(result.config_files)}")
|
||||
print("\nTest Commands:")
|
||||
for test_type, cmd in result.test_commands.items():
|
||||
print(f" {test_type}: {cmd}")
|
||||
if result.coverage_command:
|
||||
print(f"\nCoverage Command: {result.coverage_command}")
|
||||
print(f"\nWorkflows ({len(result.workflows)}):")
|
||||
for w in result.workflows:
|
||||
marker = "[TEST]" if w.test_related else ""
|
||||
print(f" - {w.name} {marker}")
|
||||
if w.trigger:
|
||||
print(f" Triggers: {', '.join(str(t) for t in w.trigger)}")
|
||||
if result.environment_variables:
|
||||
print(f"\nEnvironment Variables: {', '.join(result.environment_variables)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
"""Backward compatibility shim - import from analysis.ci_discovery instead."""
|
||||
from analysis.ci_discovery import (
|
||||
CIConfig,
|
||||
CIWorkflow,
|
||||
CIDiscovery,
|
||||
discover_ci,
|
||||
get_ci_test_commands,
|
||||
get_ci_system,
|
||||
HAS_YAML,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"CIConfig",
|
||||
"CIWorkflow",
|
||||
"CIDiscovery",
|
||||
"discover_ci",
|
||||
"get_ci_test_commands",
|
||||
"get_ci_system",
|
||||
"HAS_YAML",
|
||||
]
|
||||
|
||||
+19
-310
@@ -1,315 +1,24 @@
|
||||
"""
|
||||
Claude SDK Client Configuration
|
||||
===============================
|
||||
|
||||
Functions for creating and configuring the Claude Agent SDK client.
|
||||
"""
|
||||
|
||||
import json
|
||||
"""Backward compatibility shim - import from core.client instead."""
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from auto_claude_tools import (
|
||||
create_auto_claude_mcp_server,
|
||||
is_tools_available,
|
||||
)
|
||||
from auto_claude_tools import (
|
||||
get_allowed_tools as get_agent_allowed_tools,
|
||||
)
|
||||
from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient
|
||||
from claude_agent_sdk.types import HookMatcher
|
||||
from linear_updater import is_linear_enabled
|
||||
from security import bash_security_hook
|
||||
# Add auto-claude to path if not present
|
||||
_auto_claude_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
if _auto_claude_dir not in sys.path:
|
||||
sys.path.insert(0, _auto_claude_dir)
|
||||
|
||||
# Use lazy imports to avoid circular dependency
|
||||
def __getattr__(name):
|
||||
"""Lazy import to avoid circular imports with auto_claude_tools."""
|
||||
from core import client as _client
|
||||
return getattr(_client, name)
|
||||
|
||||
def is_graphiti_mcp_enabled() -> bool:
|
||||
"""
|
||||
Check if Graphiti MCP server integration is enabled.
|
||||
|
||||
Requires GRAPHITI_MCP_URL to be set (e.g., http://localhost:8000/mcp/)
|
||||
This is separate from GRAPHITI_ENABLED which controls the Python library integration.
|
||||
"""
|
||||
return bool(os.environ.get("GRAPHITI_MCP_URL"))
|
||||
|
||||
|
||||
def get_graphiti_mcp_url() -> str:
|
||||
"""Get the Graphiti MCP server URL."""
|
||||
return os.environ.get("GRAPHITI_MCP_URL", "http://localhost:8000/mcp/")
|
||||
|
||||
|
||||
def is_electron_mcp_enabled() -> bool:
|
||||
"""
|
||||
Check if Electron MCP server integration is enabled.
|
||||
|
||||
Requires ELECTRON_MCP_ENABLED to be set to 'true'.
|
||||
When enabled, QA agents can use Puppeteer MCP tools to connect to Electron apps
|
||||
via Chrome DevTools Protocol on the configured debug port.
|
||||
"""
|
||||
return os.environ.get("ELECTRON_MCP_ENABLED", "").lower() == "true"
|
||||
|
||||
|
||||
def get_electron_debug_port() -> int:
|
||||
"""Get the Electron remote debugging port (default: 9222)."""
|
||||
return int(os.environ.get("ELECTRON_DEBUG_PORT", "9222"))
|
||||
|
||||
|
||||
# Puppeteer MCP tools for browser automation
|
||||
PUPPETEER_TOOLS = [
|
||||
"mcp__puppeteer__puppeteer_connect_active_tab",
|
||||
"mcp__puppeteer__puppeteer_navigate",
|
||||
"mcp__puppeteer__puppeteer_screenshot",
|
||||
"mcp__puppeteer__puppeteer_click",
|
||||
"mcp__puppeteer__puppeteer_fill",
|
||||
"mcp__puppeteer__puppeteer_select",
|
||||
"mcp__puppeteer__puppeteer_hover",
|
||||
"mcp__puppeteer__puppeteer_evaluate",
|
||||
# For wildcard imports, define __all__
|
||||
__all__ = [
|
||||
"create_claude_client",
|
||||
"ClaudeClient",
|
||||
"is_graphiti_mcp_enabled",
|
||||
"get_graphiti_mcp_url",
|
||||
"is_electron_mcp_enabled",
|
||||
"get_electron_debug_port",
|
||||
]
|
||||
|
||||
# Linear MCP tools for project management (when LINEAR_API_KEY is set)
|
||||
LINEAR_TOOLS = [
|
||||
"mcp__linear-server__list_teams",
|
||||
"mcp__linear-server__get_team",
|
||||
"mcp__linear-server__list_projects",
|
||||
"mcp__linear-server__get_project",
|
||||
"mcp__linear-server__create_project",
|
||||
"mcp__linear-server__update_project",
|
||||
"mcp__linear-server__list_issues",
|
||||
"mcp__linear-server__get_issue",
|
||||
"mcp__linear-server__create_issue",
|
||||
"mcp__linear-server__update_issue",
|
||||
"mcp__linear-server__list_comments",
|
||||
"mcp__linear-server__create_comment",
|
||||
"mcp__linear-server__list_issue_statuses",
|
||||
"mcp__linear-server__list_issue_labels",
|
||||
"mcp__linear-server__list_users",
|
||||
"mcp__linear-server__get_user",
|
||||
]
|
||||
|
||||
# Context7 MCP tools for documentation lookup (always enabled)
|
||||
CONTEXT7_TOOLS = [
|
||||
"mcp__context7__resolve-library-id",
|
||||
"mcp__context7__get-library-docs",
|
||||
]
|
||||
|
||||
# Graphiti MCP tools for knowledge graph memory (when GRAPHITI_MCP_ENABLED is set)
|
||||
# See: https://docs.falkordb.com/agentic-memory/graphiti-mcp-server.html
|
||||
GRAPHITI_MCP_TOOLS = [
|
||||
"mcp__graphiti-memory__search_nodes", # Search entity summaries
|
||||
"mcp__graphiti-memory__search_facts", # Search relationships between entities
|
||||
"mcp__graphiti-memory__add_episode", # Add data to knowledge graph
|
||||
"mcp__graphiti-memory__get_episodes", # Retrieve recent episodes
|
||||
"mcp__graphiti-memory__get_entity_edge", # Get specific entity/relationship
|
||||
]
|
||||
|
||||
# Electron MCP tools for desktop app automation (when ELECTRON_MCP_ENABLED is set)
|
||||
# Uses puppeteer-mcp-server to connect to Electron apps via Chrome DevTools Protocol.
|
||||
# Electron app must be started with --remote-debugging-port=9222 (or ELECTRON_DEBUG_PORT).
|
||||
# These tools are only available to QA agents (qa_reviewer, qa_fixer), not Coder/Planner.
|
||||
ELECTRON_TOOLS = [
|
||||
"mcp__electron__electron_connect", # Connect to Electron app via DevTools
|
||||
"mcp__electron__electron_screenshot", # Take screenshot of Electron window
|
||||
"mcp__electron__electron_click", # Click element in Electron app
|
||||
"mcp__electron__electron_fill", # Fill input field in Electron app
|
||||
"mcp__electron__electron_evaluate", # Execute JS in Electron renderer
|
||||
"mcp__electron__electron_get_window_info", # Get window state/bounds
|
||||
"mcp__electron__electron_get_console", # Get console logs from renderer
|
||||
]
|
||||
|
||||
# Built-in tools
|
||||
BUILTIN_TOOLS = [
|
||||
"Read",
|
||||
"Write",
|
||||
"Edit",
|
||||
"Glob",
|
||||
"Grep",
|
||||
"Bash",
|
||||
]
|
||||
|
||||
|
||||
def create_client(
|
||||
project_dir: Path,
|
||||
spec_dir: Path,
|
||||
model: str,
|
||||
agent_type: str = "coder",
|
||||
) -> ClaudeSDKClient:
|
||||
"""
|
||||
Create a Claude Agent SDK client with multi-layered security.
|
||||
|
||||
Args:
|
||||
project_dir: Root directory for the project (working directory)
|
||||
spec_dir: Directory containing the spec (for settings file)
|
||||
model: Claude model to use
|
||||
agent_type: Type of agent - 'planner', 'coder', 'qa_reviewer', or 'qa_fixer'
|
||||
This determines which custom auto-claude tools are available.
|
||||
|
||||
Returns:
|
||||
Configured ClaudeSDKClient
|
||||
|
||||
Security layers (defense in depth):
|
||||
1. Sandbox - OS-level bash command isolation prevents filesystem escape
|
||||
2. Permissions - File operations restricted to project_dir only
|
||||
3. Security hooks - Bash commands validated against an allowlist
|
||||
(see security.py for ALLOWED_COMMANDS)
|
||||
4. Tool filtering - Each agent type only sees relevant tools (prevents misuse)
|
||||
"""
|
||||
oauth_token = os.environ.get("CLAUDE_CODE_OAUTH_TOKEN")
|
||||
if not oauth_token:
|
||||
raise ValueError(
|
||||
"CLAUDE_CODE_OAUTH_TOKEN environment variable not set.\n"
|
||||
"Get your token by running: claude setup-token"
|
||||
)
|
||||
|
||||
# Check if Linear integration is enabled
|
||||
linear_enabled = is_linear_enabled()
|
||||
linear_api_key = os.environ.get("LINEAR_API_KEY", "")
|
||||
|
||||
# Check if custom auto-claude tools are available
|
||||
auto_claude_tools_enabled = is_tools_available()
|
||||
|
||||
# Build the list of allowed tools
|
||||
# Start with agent-specific tools (includes base tools + auto-claude tools)
|
||||
if auto_claude_tools_enabled:
|
||||
allowed_tools_list = get_agent_allowed_tools(agent_type)
|
||||
else:
|
||||
allowed_tools_list = [*BUILTIN_TOOLS]
|
||||
|
||||
# Check if Graphiti MCP is enabled
|
||||
graphiti_mcp_enabled = is_graphiti_mcp_enabled()
|
||||
|
||||
# Check if Electron MCP is enabled (for QA agents testing Electron apps)
|
||||
electron_mcp_enabled = is_electron_mcp_enabled()
|
||||
|
||||
# Add external MCP tools
|
||||
allowed_tools_list.extend(PUPPETEER_TOOLS)
|
||||
allowed_tools_list.extend(CONTEXT7_TOOLS)
|
||||
if linear_enabled:
|
||||
allowed_tools_list.extend(LINEAR_TOOLS)
|
||||
if graphiti_mcp_enabled:
|
||||
allowed_tools_list.extend(GRAPHITI_MCP_TOOLS)
|
||||
# Add Electron MCP tools only for QA agents (qa_reviewer, qa_fixer) when enabled
|
||||
# This prevents context bloat for coder/planner agents who don't need desktop automation
|
||||
if electron_mcp_enabled and agent_type in ("qa_reviewer", "qa_fixer"):
|
||||
allowed_tools_list.extend(ELECTRON_TOOLS)
|
||||
|
||||
# Create comprehensive security settings
|
||||
# Note: Using relative paths ("./**") restricts access to project directory
|
||||
# since cwd is set to project_dir
|
||||
security_settings = {
|
||||
"sandbox": {"enabled": True, "autoAllowBashIfSandboxed": True},
|
||||
"permissions": {
|
||||
"defaultMode": "acceptEdits", # Auto-approve edits within allowed directories
|
||||
"allow": [
|
||||
# Allow all file operations within the project directory
|
||||
"Read(./**)",
|
||||
"Write(./**)",
|
||||
"Edit(./**)",
|
||||
"Glob(./**)",
|
||||
"Grep(./**)",
|
||||
# Bash permission granted here, but actual commands are validated
|
||||
# by the bash_security_hook (see security.py for allowed commands)
|
||||
"Bash(*)",
|
||||
# Allow Puppeteer MCP tools for browser automation
|
||||
*PUPPETEER_TOOLS,
|
||||
# Allow Context7 MCP tools for documentation lookup
|
||||
*CONTEXT7_TOOLS,
|
||||
# Allow Linear MCP tools for project management (if enabled)
|
||||
*(LINEAR_TOOLS if linear_enabled else []),
|
||||
# Allow Graphiti MCP tools for knowledge graph memory (if enabled)
|
||||
*(GRAPHITI_MCP_TOOLS if graphiti_mcp_enabled else []),
|
||||
# Allow Electron MCP tools for QA agents only (if enabled)
|
||||
*(ELECTRON_TOOLS if electron_mcp_enabled and agent_type in ("qa_reviewer", "qa_fixer") else []),
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
# Write settings to a file in the project directory
|
||||
settings_file = project_dir / ".claude_settings.json"
|
||||
with open(settings_file, "w") as f:
|
||||
json.dump(security_settings, f, indent=2)
|
||||
|
||||
print(f"Security settings: {settings_file}")
|
||||
print(" - Sandbox enabled (OS-level bash isolation)")
|
||||
print(f" - Filesystem restricted to: {project_dir.resolve()}")
|
||||
print(" - Bash commands restricted to allowlist")
|
||||
|
||||
mcp_servers_list = ["puppeteer (browser automation)", "context7 (documentation)"]
|
||||
if linear_enabled:
|
||||
mcp_servers_list.append("linear (project management)")
|
||||
if graphiti_mcp_enabled:
|
||||
mcp_servers_list.append("graphiti-memory (knowledge graph)")
|
||||
if electron_mcp_enabled:
|
||||
mcp_servers_list.append(f"electron (desktop automation, port {get_electron_debug_port()})")
|
||||
if auto_claude_tools_enabled:
|
||||
mcp_servers_list.append(f"auto-claude ({agent_type} tools)")
|
||||
print(f" - MCP servers: {', '.join(mcp_servers_list)}")
|
||||
print()
|
||||
|
||||
# Configure MCP servers
|
||||
mcp_servers = {
|
||||
"puppeteer": {"command": "npx", "args": ["puppeteer-mcp-server"]},
|
||||
"context7": {"command": "npx", "args": ["-y", "@upstash/context7-mcp"]},
|
||||
}
|
||||
|
||||
# Add Linear MCP server if enabled
|
||||
if linear_enabled:
|
||||
mcp_servers["linear"] = {
|
||||
"type": "http",
|
||||
"url": "https://mcp.linear.app/mcp",
|
||||
"headers": {"Authorization": f"Bearer {linear_api_key}"},
|
||||
}
|
||||
|
||||
# Add Graphiti MCP server if enabled
|
||||
# Requires running: docker run -d -p 8000:8000 falkordb/graphiti-knowledge-graph-mcp
|
||||
if graphiti_mcp_enabled:
|
||||
mcp_servers["graphiti-memory"] = {
|
||||
"type": "http",
|
||||
"url": get_graphiti_mcp_url(),
|
||||
}
|
||||
|
||||
# Add Electron MCP server if enabled
|
||||
# Electron app must be started with --remote-debugging-port=<port> for connection
|
||||
# Uses puppeteer-mcp-server to connect to Electron via Chrome DevTools Protocol
|
||||
if electron_mcp_enabled:
|
||||
electron_port = get_electron_debug_port()
|
||||
mcp_servers["electron"] = {
|
||||
"command": "npx",
|
||||
"args": [
|
||||
"-y",
|
||||
"@anthropic/puppeteer-mcp-server",
|
||||
"--debug-port",
|
||||
str(electron_port),
|
||||
],
|
||||
}
|
||||
|
||||
# Add custom auto-claude MCP server if available
|
||||
auto_claude_mcp_server = None
|
||||
if auto_claude_tools_enabled:
|
||||
auto_claude_mcp_server = create_auto_claude_mcp_server(spec_dir, project_dir)
|
||||
if auto_claude_mcp_server:
|
||||
mcp_servers["auto-claude"] = auto_claude_mcp_server
|
||||
|
||||
return ClaudeSDKClient(
|
||||
options=ClaudeAgentOptions(
|
||||
model=model,
|
||||
system_prompt=(
|
||||
f"You are an expert full-stack developer building production-quality software. "
|
||||
f"Your working directory is: {project_dir.resolve()}\n"
|
||||
f"Your filesystem access is RESTRICTED to this directory only. "
|
||||
f"Use relative paths (starting with ./) for all file operations. "
|
||||
f"Never use absolute paths or try to access files outside your working directory.\n\n"
|
||||
f"You follow existing code patterns, write clean maintainable code, and verify "
|
||||
f"your work through thorough testing. You communicate progress through Git commits "
|
||||
f"and build-progress.txt updates."
|
||||
),
|
||||
allowed_tools=allowed_tools_list,
|
||||
mcp_servers=mcp_servers,
|
||||
hooks={
|
||||
"PreToolUse": [
|
||||
HookMatcher(matcher="Bash", hooks=[bash_security_hook]),
|
||||
],
|
||||
},
|
||||
max_turns=1000,
|
||||
cwd=str(project_dir.resolve()),
|
||||
settings=str(settings_file.resolve()),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
"""
|
||||
Core Framework Module
|
||||
=====================
|
||||
|
||||
Core components for the Auto Claude autonomous coding framework.
|
||||
"""
|
||||
|
||||
# Note: We use lazy imports here because the full agent module has many dependencies
|
||||
# that may not be needed for basic operations like workspace management.
|
||||
|
||||
__all__ = [
|
||||
"run_autonomous_agent",
|
||||
"run_followup_planner",
|
||||
"WorkspaceManager",
|
||||
"WorktreeManager",
|
||||
"ProgressTracker",
|
||||
]
|
||||
|
||||
def __getattr__(name):
|
||||
"""Lazy imports to avoid circular dependencies and heavy imports."""
|
||||
if name in ("run_autonomous_agent", "run_followup_planner"):
|
||||
from .agent import run_autonomous_agent, run_followup_planner
|
||||
return locals()[name]
|
||||
elif name == "WorkspaceManager":
|
||||
from .workspace import WorkspaceManager
|
||||
return WorkspaceManager
|
||||
elif name == "WorktreeManager":
|
||||
from .worktree import WorktreeManager
|
||||
return WorktreeManager
|
||||
elif name == "ProgressTracker":
|
||||
from .progress import ProgressTracker
|
||||
return ProgressTracker
|
||||
elif name in ("create_claude_client", "ClaudeClient"):
|
||||
from . import client as _client
|
||||
return getattr(_client, name)
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
@@ -0,0 +1,63 @@
|
||||
"""
|
||||
Agent Session Logic
|
||||
===================
|
||||
|
||||
Core agent interaction functions for running autonomous coding sessions.
|
||||
Uses subtask-based implementation plans with minimal, focused prompts.
|
||||
|
||||
Architecture:
|
||||
- Orchestrator (Python) handles all bookkeeping: memory, commits, progress
|
||||
- Agent focuses ONLY on implementing code
|
||||
- Post-session processing updates memory automatically (100% reliable)
|
||||
|
||||
Enhanced with status file updates for ccstatusline integration.
|
||||
Enhanced with Graphiti memory for cross-session context retrieval.
|
||||
|
||||
NOTE: This module is now a facade that imports from agents/ submodules.
|
||||
All logic has been refactored into focused modules for better maintainability.
|
||||
"""
|
||||
|
||||
# Re-export everything from the agents module to maintain backwards compatibility
|
||||
from agents import (
|
||||
# Constants
|
||||
AUTO_CONTINUE_DELAY_SECONDS,
|
||||
HUMAN_INTERVENTION_FILE,
|
||||
# Memory functions
|
||||
debug_memory_system_status,
|
||||
find_phase_for_subtask,
|
||||
find_subtask_in_plan,
|
||||
get_commit_count,
|
||||
get_graphiti_context,
|
||||
# Utility functions
|
||||
get_latest_commit,
|
||||
load_implementation_plan,
|
||||
post_session_processing,
|
||||
# Session management
|
||||
run_agent_session,
|
||||
# Main API
|
||||
run_autonomous_agent,
|
||||
run_followup_planner,
|
||||
save_session_memory,
|
||||
save_session_to_graphiti,
|
||||
sync_plan_to_source,
|
||||
)
|
||||
|
||||
# Ensure all exports are available at module level
|
||||
__all__ = [
|
||||
"run_autonomous_agent",
|
||||
"run_followup_planner",
|
||||
"debug_memory_system_status",
|
||||
"get_graphiti_context",
|
||||
"save_session_memory",
|
||||
"save_session_to_graphiti",
|
||||
"run_agent_session",
|
||||
"post_session_processing",
|
||||
"get_latest_commit",
|
||||
"get_commit_count",
|
||||
"load_implementation_plan",
|
||||
"find_subtask_in_plan",
|
||||
"find_phase_for_subtask",
|
||||
"sync_plan_to_source",
|
||||
"AUTO_CONTINUE_DELAY_SECONDS",
|
||||
"HUMAN_INTERVENTION_FILE",
|
||||
]
|
||||
@@ -0,0 +1,315 @@
|
||||
"""
|
||||
Claude SDK Client Configuration
|
||||
===============================
|
||||
|
||||
Functions for creating and configuring the Claude Agent SDK client.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from auto_claude_tools import (
|
||||
create_auto_claude_mcp_server,
|
||||
is_tools_available,
|
||||
)
|
||||
from auto_claude_tools import (
|
||||
get_allowed_tools as get_agent_allowed_tools,
|
||||
)
|
||||
from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient
|
||||
from claude_agent_sdk.types import HookMatcher
|
||||
from linear_updater import is_linear_enabled
|
||||
from security import bash_security_hook
|
||||
|
||||
|
||||
def is_graphiti_mcp_enabled() -> bool:
|
||||
"""
|
||||
Check if Graphiti MCP server integration is enabled.
|
||||
|
||||
Requires GRAPHITI_MCP_URL to be set (e.g., http://localhost:8000/mcp/)
|
||||
This is separate from GRAPHITI_ENABLED which controls the Python library integration.
|
||||
"""
|
||||
return bool(os.environ.get("GRAPHITI_MCP_URL"))
|
||||
|
||||
|
||||
def get_graphiti_mcp_url() -> str:
|
||||
"""Get the Graphiti MCP server URL."""
|
||||
return os.environ.get("GRAPHITI_MCP_URL", "http://localhost:8000/mcp/")
|
||||
|
||||
|
||||
def is_electron_mcp_enabled() -> bool:
|
||||
"""
|
||||
Check if Electron MCP server integration is enabled.
|
||||
|
||||
Requires ELECTRON_MCP_ENABLED to be set to 'true'.
|
||||
When enabled, QA agents can use Puppeteer MCP tools to connect to Electron apps
|
||||
via Chrome DevTools Protocol on the configured debug port.
|
||||
"""
|
||||
return os.environ.get("ELECTRON_MCP_ENABLED", "").lower() == "true"
|
||||
|
||||
|
||||
def get_electron_debug_port() -> int:
|
||||
"""Get the Electron remote debugging port (default: 9222)."""
|
||||
return int(os.environ.get("ELECTRON_DEBUG_PORT", "9222"))
|
||||
|
||||
|
||||
# Puppeteer MCP tools for browser automation
|
||||
PUPPETEER_TOOLS = [
|
||||
"mcp__puppeteer__puppeteer_connect_active_tab",
|
||||
"mcp__puppeteer__puppeteer_navigate",
|
||||
"mcp__puppeteer__puppeteer_screenshot",
|
||||
"mcp__puppeteer__puppeteer_click",
|
||||
"mcp__puppeteer__puppeteer_fill",
|
||||
"mcp__puppeteer__puppeteer_select",
|
||||
"mcp__puppeteer__puppeteer_hover",
|
||||
"mcp__puppeteer__puppeteer_evaluate",
|
||||
]
|
||||
|
||||
# Linear MCP tools for project management (when LINEAR_API_KEY is set)
|
||||
LINEAR_TOOLS = [
|
||||
"mcp__linear-server__list_teams",
|
||||
"mcp__linear-server__get_team",
|
||||
"mcp__linear-server__list_projects",
|
||||
"mcp__linear-server__get_project",
|
||||
"mcp__linear-server__create_project",
|
||||
"mcp__linear-server__update_project",
|
||||
"mcp__linear-server__list_issues",
|
||||
"mcp__linear-server__get_issue",
|
||||
"mcp__linear-server__create_issue",
|
||||
"mcp__linear-server__update_issue",
|
||||
"mcp__linear-server__list_comments",
|
||||
"mcp__linear-server__create_comment",
|
||||
"mcp__linear-server__list_issue_statuses",
|
||||
"mcp__linear-server__list_issue_labels",
|
||||
"mcp__linear-server__list_users",
|
||||
"mcp__linear-server__get_user",
|
||||
]
|
||||
|
||||
# Context7 MCP tools for documentation lookup (always enabled)
|
||||
CONTEXT7_TOOLS = [
|
||||
"mcp__context7__resolve-library-id",
|
||||
"mcp__context7__get-library-docs",
|
||||
]
|
||||
|
||||
# Graphiti MCP tools for knowledge graph memory (when GRAPHITI_MCP_ENABLED is set)
|
||||
# See: https://docs.falkordb.com/agentic-memory/graphiti-mcp-server.html
|
||||
GRAPHITI_MCP_TOOLS = [
|
||||
"mcp__graphiti-memory__search_nodes", # Search entity summaries
|
||||
"mcp__graphiti-memory__search_facts", # Search relationships between entities
|
||||
"mcp__graphiti-memory__add_episode", # Add data to knowledge graph
|
||||
"mcp__graphiti-memory__get_episodes", # Retrieve recent episodes
|
||||
"mcp__graphiti-memory__get_entity_edge", # Get specific entity/relationship
|
||||
]
|
||||
|
||||
# Electron MCP tools for desktop app automation (when ELECTRON_MCP_ENABLED is set)
|
||||
# Uses puppeteer-mcp-server to connect to Electron apps via Chrome DevTools Protocol.
|
||||
# Electron app must be started with --remote-debugging-port=9222 (or ELECTRON_DEBUG_PORT).
|
||||
# These tools are only available to QA agents (qa_reviewer, qa_fixer), not Coder/Planner.
|
||||
ELECTRON_TOOLS = [
|
||||
"mcp__electron__electron_connect", # Connect to Electron app via DevTools
|
||||
"mcp__electron__electron_screenshot", # Take screenshot of Electron window
|
||||
"mcp__electron__electron_click", # Click element in Electron app
|
||||
"mcp__electron__electron_fill", # Fill input field in Electron app
|
||||
"mcp__electron__electron_evaluate", # Execute JS in Electron renderer
|
||||
"mcp__electron__electron_get_window_info", # Get window state/bounds
|
||||
"mcp__electron__electron_get_console", # Get console logs from renderer
|
||||
]
|
||||
|
||||
# Built-in tools
|
||||
BUILTIN_TOOLS = [
|
||||
"Read",
|
||||
"Write",
|
||||
"Edit",
|
||||
"Glob",
|
||||
"Grep",
|
||||
"Bash",
|
||||
]
|
||||
|
||||
|
||||
def create_client(
|
||||
project_dir: Path,
|
||||
spec_dir: Path,
|
||||
model: str,
|
||||
agent_type: str = "coder",
|
||||
) -> ClaudeSDKClient:
|
||||
"""
|
||||
Create a Claude Agent SDK client with multi-layered security.
|
||||
|
||||
Args:
|
||||
project_dir: Root directory for the project (working directory)
|
||||
spec_dir: Directory containing the spec (for settings file)
|
||||
model: Claude model to use
|
||||
agent_type: Type of agent - 'planner', 'coder', 'qa_reviewer', or 'qa_fixer'
|
||||
This determines which custom auto-claude tools are available.
|
||||
|
||||
Returns:
|
||||
Configured ClaudeSDKClient
|
||||
|
||||
Security layers (defense in depth):
|
||||
1. Sandbox - OS-level bash command isolation prevents filesystem escape
|
||||
2. Permissions - File operations restricted to project_dir only
|
||||
3. Security hooks - Bash commands validated against an allowlist
|
||||
(see security.py for ALLOWED_COMMANDS)
|
||||
4. Tool filtering - Each agent type only sees relevant tools (prevents misuse)
|
||||
"""
|
||||
oauth_token = os.environ.get("CLAUDE_CODE_OAUTH_TOKEN")
|
||||
if not oauth_token:
|
||||
raise ValueError(
|
||||
"CLAUDE_CODE_OAUTH_TOKEN environment variable not set.\n"
|
||||
"Get your token by running: claude setup-token"
|
||||
)
|
||||
|
||||
# Check if Linear integration is enabled
|
||||
linear_enabled = is_linear_enabled()
|
||||
linear_api_key = os.environ.get("LINEAR_API_KEY", "")
|
||||
|
||||
# Check if custom auto-claude tools are available
|
||||
auto_claude_tools_enabled = is_tools_available()
|
||||
|
||||
# Build the list of allowed tools
|
||||
# Start with agent-specific tools (includes base tools + auto-claude tools)
|
||||
if auto_claude_tools_enabled:
|
||||
allowed_tools_list = get_agent_allowed_tools(agent_type)
|
||||
else:
|
||||
allowed_tools_list = [*BUILTIN_TOOLS]
|
||||
|
||||
# Check if Graphiti MCP is enabled
|
||||
graphiti_mcp_enabled = is_graphiti_mcp_enabled()
|
||||
|
||||
# Check if Electron MCP is enabled (for QA agents testing Electron apps)
|
||||
electron_mcp_enabled = is_electron_mcp_enabled()
|
||||
|
||||
# Add external MCP tools
|
||||
allowed_tools_list.extend(PUPPETEER_TOOLS)
|
||||
allowed_tools_list.extend(CONTEXT7_TOOLS)
|
||||
if linear_enabled:
|
||||
allowed_tools_list.extend(LINEAR_TOOLS)
|
||||
if graphiti_mcp_enabled:
|
||||
allowed_tools_list.extend(GRAPHITI_MCP_TOOLS)
|
||||
# Add Electron MCP tools only for QA agents (qa_reviewer, qa_fixer) when enabled
|
||||
# This prevents context bloat for coder/planner agents who don't need desktop automation
|
||||
if electron_mcp_enabled and agent_type in ("qa_reviewer", "qa_fixer"):
|
||||
allowed_tools_list.extend(ELECTRON_TOOLS)
|
||||
|
||||
# Create comprehensive security settings
|
||||
# Note: Using relative paths ("./**") restricts access to project directory
|
||||
# since cwd is set to project_dir
|
||||
security_settings = {
|
||||
"sandbox": {"enabled": True, "autoAllowBashIfSandboxed": True},
|
||||
"permissions": {
|
||||
"defaultMode": "acceptEdits", # Auto-approve edits within allowed directories
|
||||
"allow": [
|
||||
# Allow all file operations within the project directory
|
||||
"Read(./**)",
|
||||
"Write(./**)",
|
||||
"Edit(./**)",
|
||||
"Glob(./**)",
|
||||
"Grep(./**)",
|
||||
# Bash permission granted here, but actual commands are validated
|
||||
# by the bash_security_hook (see security.py for allowed commands)
|
||||
"Bash(*)",
|
||||
# Allow Puppeteer MCP tools for browser automation
|
||||
*PUPPETEER_TOOLS,
|
||||
# Allow Context7 MCP tools for documentation lookup
|
||||
*CONTEXT7_TOOLS,
|
||||
# Allow Linear MCP tools for project management (if enabled)
|
||||
*(LINEAR_TOOLS if linear_enabled else []),
|
||||
# Allow Graphiti MCP tools for knowledge graph memory (if enabled)
|
||||
*(GRAPHITI_MCP_TOOLS if graphiti_mcp_enabled else []),
|
||||
# Allow Electron MCP tools for QA agents only (if enabled)
|
||||
*(ELECTRON_TOOLS if electron_mcp_enabled and agent_type in ("qa_reviewer", "qa_fixer") else []),
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
# Write settings to a file in the project directory
|
||||
settings_file = project_dir / ".claude_settings.json"
|
||||
with open(settings_file, "w") as f:
|
||||
json.dump(security_settings, f, indent=2)
|
||||
|
||||
print(f"Security settings: {settings_file}")
|
||||
print(" - Sandbox enabled (OS-level bash isolation)")
|
||||
print(f" - Filesystem restricted to: {project_dir.resolve()}")
|
||||
print(" - Bash commands restricted to allowlist")
|
||||
|
||||
mcp_servers_list = ["puppeteer (browser automation)", "context7 (documentation)"]
|
||||
if linear_enabled:
|
||||
mcp_servers_list.append("linear (project management)")
|
||||
if graphiti_mcp_enabled:
|
||||
mcp_servers_list.append("graphiti-memory (knowledge graph)")
|
||||
if electron_mcp_enabled:
|
||||
mcp_servers_list.append(f"electron (desktop automation, port {get_electron_debug_port()})")
|
||||
if auto_claude_tools_enabled:
|
||||
mcp_servers_list.append(f"auto-claude ({agent_type} tools)")
|
||||
print(f" - MCP servers: {', '.join(mcp_servers_list)}")
|
||||
print()
|
||||
|
||||
# Configure MCP servers
|
||||
mcp_servers = {
|
||||
"puppeteer": {"command": "npx", "args": ["puppeteer-mcp-server"]},
|
||||
"context7": {"command": "npx", "args": ["-y", "@upstash/context7-mcp"]},
|
||||
}
|
||||
|
||||
# Add Linear MCP server if enabled
|
||||
if linear_enabled:
|
||||
mcp_servers["linear"] = {
|
||||
"type": "http",
|
||||
"url": "https://mcp.linear.app/mcp",
|
||||
"headers": {"Authorization": f"Bearer {linear_api_key}"},
|
||||
}
|
||||
|
||||
# Add Graphiti MCP server if enabled
|
||||
# Requires running: docker run -d -p 8000:8000 falkordb/graphiti-knowledge-graph-mcp
|
||||
if graphiti_mcp_enabled:
|
||||
mcp_servers["graphiti-memory"] = {
|
||||
"type": "http",
|
||||
"url": get_graphiti_mcp_url(),
|
||||
}
|
||||
|
||||
# Add Electron MCP server if enabled
|
||||
# Electron app must be started with --remote-debugging-port=<port> for connection
|
||||
# Uses puppeteer-mcp-server to connect to Electron via Chrome DevTools Protocol
|
||||
if electron_mcp_enabled:
|
||||
electron_port = get_electron_debug_port()
|
||||
mcp_servers["electron"] = {
|
||||
"command": "npx",
|
||||
"args": [
|
||||
"-y",
|
||||
"@anthropic/puppeteer-mcp-server",
|
||||
"--debug-port",
|
||||
str(electron_port),
|
||||
],
|
||||
}
|
||||
|
||||
# Add custom auto-claude MCP server if available
|
||||
auto_claude_mcp_server = None
|
||||
if auto_claude_tools_enabled:
|
||||
auto_claude_mcp_server = create_auto_claude_mcp_server(spec_dir, project_dir)
|
||||
if auto_claude_mcp_server:
|
||||
mcp_servers["auto-claude"] = auto_claude_mcp_server
|
||||
|
||||
return ClaudeSDKClient(
|
||||
options=ClaudeAgentOptions(
|
||||
model=model,
|
||||
system_prompt=(
|
||||
f"You are an expert full-stack developer building production-quality software. "
|
||||
f"Your working directory is: {project_dir.resolve()}\n"
|
||||
f"Your filesystem access is RESTRICTED to this directory only. "
|
||||
f"Use relative paths (starting with ./) for all file operations. "
|
||||
f"Never use absolute paths or try to access files outside your working directory.\n\n"
|
||||
f"You follow existing code patterns, write clean maintainable code, and verify "
|
||||
f"your work through thorough testing. You communicate progress through Git commits "
|
||||
f"and build-progress.txt updates."
|
||||
),
|
||||
allowed_tools=allowed_tools_list,
|
||||
mcp_servers=mcp_servers,
|
||||
hooks={
|
||||
"PreToolUse": [
|
||||
HookMatcher(matcher="Bash", hooks=[bash_security_hook]),
|
||||
],
|
||||
},
|
||||
max_turns=1000,
|
||||
cwd=str(project_dir.resolve()),
|
||||
settings=str(settings_file.resolve()),
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,349 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Debug Logging Utility
|
||||
=====================
|
||||
|
||||
Centralized debug logging for the Auto-Claude framework.
|
||||
Controlled via environment variables:
|
||||
- DEBUG=true Enable debug mode
|
||||
- DEBUG_LEVEL=1|2|3 Log verbosity (1=basic, 2=detailed, 3=verbose)
|
||||
- DEBUG_LOG_FILE=path Optional file output
|
||||
|
||||
Usage:
|
||||
from debug import debug, debug_detailed, debug_verbose, is_debug_enabled
|
||||
|
||||
debug("run.py", "Starting task execution", task_id="001")
|
||||
debug_detailed("agent", "Agent response received", response_length=1234)
|
||||
debug_verbose("client", "Full request payload", payload=data)
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from functools import wraps
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
# ANSI color codes for terminal output
|
||||
class Colors:
|
||||
RESET = "\033[0m"
|
||||
BOLD = "\033[1m"
|
||||
DIM = "\033[2m"
|
||||
|
||||
# Debug colors
|
||||
DEBUG = "\033[36m" # Cyan
|
||||
DEBUG_DIM = "\033[96m" # Light cyan
|
||||
TIMESTAMP = "\033[90m" # Gray
|
||||
MODULE = "\033[33m" # Yellow
|
||||
KEY = "\033[35m" # Magenta
|
||||
VALUE = "\033[37m" # White
|
||||
SUCCESS = "\033[32m" # Green
|
||||
WARNING = "\033[33m" # Yellow
|
||||
ERROR = "\033[31m" # Red
|
||||
|
||||
|
||||
def _get_debug_enabled() -> bool:
|
||||
"""Check if debug mode is enabled via environment variable."""
|
||||
return os.environ.get("DEBUG", "").lower() in ("true", "1", "yes", "on")
|
||||
|
||||
|
||||
def _get_debug_level() -> int:
|
||||
"""Get debug verbosity level (1-3)."""
|
||||
try:
|
||||
level = int(os.environ.get("DEBUG_LEVEL", "1"))
|
||||
return max(1, min(3, level)) # Clamp to 1-3
|
||||
except ValueError:
|
||||
return 1
|
||||
|
||||
|
||||
def _get_log_file() -> Path | None:
|
||||
"""Get optional log file path."""
|
||||
log_file = os.environ.get("DEBUG_LOG_FILE")
|
||||
if log_file:
|
||||
return Path(log_file)
|
||||
return None
|
||||
|
||||
|
||||
def is_debug_enabled() -> bool:
|
||||
"""Check if debug mode is enabled."""
|
||||
return _get_debug_enabled()
|
||||
|
||||
|
||||
def get_debug_level() -> int:
|
||||
"""Get current debug level."""
|
||||
return _get_debug_level()
|
||||
|
||||
|
||||
def _format_value(value: Any, max_length: int = 200) -> str:
|
||||
"""Format a value for debug output, truncating if necessary."""
|
||||
if value is None:
|
||||
return "None"
|
||||
|
||||
if isinstance(value, (dict, list)):
|
||||
try:
|
||||
formatted = json.dumps(value, indent=2, default=str)
|
||||
if len(formatted) > max_length:
|
||||
formatted = formatted[:max_length] + "..."
|
||||
return formatted
|
||||
except (TypeError, ValueError):
|
||||
return str(value)[:max_length]
|
||||
|
||||
str_value = str(value)
|
||||
if len(str_value) > max_length:
|
||||
return str_value[:max_length] + "..."
|
||||
return str_value
|
||||
|
||||
|
||||
def _write_log(message: str, to_file: bool = True) -> None:
|
||||
"""Write log message to stdout and optionally to file."""
|
||||
print(message, file=sys.stderr)
|
||||
|
||||
if to_file:
|
||||
log_file = _get_log_file()
|
||||
if log_file:
|
||||
try:
|
||||
log_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Strip ANSI codes for file output
|
||||
import re
|
||||
|
||||
clean_message = re.sub(r"\033\[[0-9;]*m", "", message)
|
||||
with open(log_file, "a") as f:
|
||||
f.write(clean_message + "\n")
|
||||
except Exception:
|
||||
pass # Silently fail file logging
|
||||
|
||||
|
||||
def debug(module: str, message: str, level: int = 1, **kwargs) -> None:
|
||||
"""
|
||||
Log a debug message.
|
||||
|
||||
Args:
|
||||
module: Source module name (e.g., "run.py", "ideation_runner")
|
||||
message: Debug message
|
||||
level: Required debug level (1=basic, 2=detailed, 3=verbose)
|
||||
**kwargs: Additional key-value pairs to log
|
||||
"""
|
||||
if not _get_debug_enabled():
|
||||
return
|
||||
|
||||
if _get_debug_level() < level:
|
||||
return
|
||||
|
||||
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
|
||||
|
||||
# Build the log line
|
||||
parts = [
|
||||
f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET}",
|
||||
f"{Colors.DEBUG}[DEBUG]{Colors.RESET}",
|
||||
f"{Colors.MODULE}[{module}]{Colors.RESET}",
|
||||
f"{Colors.DEBUG_DIM}{message}{Colors.RESET}",
|
||||
]
|
||||
|
||||
log_line = " ".join(parts)
|
||||
|
||||
# Add kwargs on separate lines if present
|
||||
if kwargs:
|
||||
for key, value in kwargs.items():
|
||||
formatted_value = _format_value(value)
|
||||
if "\n" in formatted_value:
|
||||
# Multi-line value
|
||||
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}:"
|
||||
for line in formatted_value.split("\n"):
|
||||
log_line += f"\n {Colors.VALUE}{line}{Colors.RESET}"
|
||||
else:
|
||||
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{formatted_value}{Colors.RESET}"
|
||||
|
||||
_write_log(log_line)
|
||||
|
||||
|
||||
def debug_detailed(module: str, message: str, **kwargs) -> None:
|
||||
"""Log a detailed debug message (level 2)."""
|
||||
debug(module, message, level=2, **kwargs)
|
||||
|
||||
|
||||
def debug_verbose(module: str, message: str, **kwargs) -> None:
|
||||
"""Log a verbose debug message (level 3)."""
|
||||
debug(module, message, level=3, **kwargs)
|
||||
|
||||
|
||||
def debug_success(module: str, message: str, **kwargs) -> None:
|
||||
"""Log a success debug message."""
|
||||
if not _get_debug_enabled():
|
||||
return
|
||||
|
||||
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
|
||||
log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.SUCCESS}[OK]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {message}"
|
||||
|
||||
if kwargs:
|
||||
for key, value in kwargs.items():
|
||||
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}"
|
||||
|
||||
_write_log(log_line)
|
||||
|
||||
|
||||
def debug_info(module: str, message: str, **kwargs) -> None:
|
||||
"""Log an info debug message."""
|
||||
if not _get_debug_enabled():
|
||||
return
|
||||
|
||||
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
|
||||
log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.DEBUG}[INFO]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {message}"
|
||||
|
||||
if kwargs:
|
||||
for key, value in kwargs.items():
|
||||
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}"
|
||||
|
||||
_write_log(log_line)
|
||||
|
||||
|
||||
def debug_error(module: str, message: str, **kwargs) -> None:
|
||||
"""Log an error debug message (always shown if debug enabled)."""
|
||||
if not _get_debug_enabled():
|
||||
return
|
||||
|
||||
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
|
||||
log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.ERROR}[ERROR]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {Colors.ERROR}{message}{Colors.RESET}"
|
||||
|
||||
if kwargs:
|
||||
for key, value in kwargs.items():
|
||||
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}"
|
||||
|
||||
_write_log(log_line)
|
||||
|
||||
|
||||
def debug_warning(module: str, message: str, **kwargs) -> None:
|
||||
"""Log a warning debug message."""
|
||||
if not _get_debug_enabled():
|
||||
return
|
||||
|
||||
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
|
||||
log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.WARNING}[WARN]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {Colors.WARNING}{message}{Colors.RESET}"
|
||||
|
||||
if kwargs:
|
||||
for key, value in kwargs.items():
|
||||
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}"
|
||||
|
||||
_write_log(log_line)
|
||||
|
||||
|
||||
def debug_section(module: str, title: str) -> None:
|
||||
"""Log a section header for organizing debug output."""
|
||||
if not _get_debug_enabled():
|
||||
return
|
||||
|
||||
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
|
||||
separator = "─" * 60
|
||||
log_line = f"\n{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.DEBUG}{Colors.BOLD}┌{separator}┐{Colors.RESET}"
|
||||
log_line += f"\n{Colors.TIMESTAMP} {Colors.RESET} {Colors.DEBUG}{Colors.BOLD}│ {module}: {title}{' ' * (58 - len(module) - len(title) - 2)}│{Colors.RESET}"
|
||||
log_line += f"\n{Colors.TIMESTAMP} {Colors.RESET} {Colors.DEBUG}{Colors.BOLD}└{separator}┘{Colors.RESET}"
|
||||
|
||||
_write_log(log_line)
|
||||
|
||||
|
||||
def debug_timer(module: str):
|
||||
"""
|
||||
Decorator to time function execution.
|
||||
|
||||
Usage:
|
||||
@debug_timer("run.py")
|
||||
def my_function():
|
||||
...
|
||||
"""
|
||||
|
||||
def decorator(func):
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
if not _get_debug_enabled():
|
||||
return func(*args, **kwargs)
|
||||
|
||||
start = time.time()
|
||||
debug_detailed(module, f"Starting {func.__name__}()")
|
||||
|
||||
try:
|
||||
result = func(*args, **kwargs)
|
||||
elapsed = time.time() - start
|
||||
debug_success(
|
||||
module,
|
||||
f"Completed {func.__name__}()",
|
||||
elapsed_ms=f"{elapsed * 1000:.1f}ms",
|
||||
)
|
||||
return result
|
||||
except Exception as e:
|
||||
elapsed = time.time() - start
|
||||
debug_error(
|
||||
module,
|
||||
f"Failed {func.__name__}()",
|
||||
error=str(e),
|
||||
elapsed_ms=f"{elapsed * 1000:.1f}ms",
|
||||
)
|
||||
raise
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def debug_async_timer(module: str):
|
||||
"""
|
||||
Decorator to time async function execution.
|
||||
|
||||
Usage:
|
||||
@debug_async_timer("ideation_runner")
|
||||
async def my_async_function():
|
||||
...
|
||||
"""
|
||||
|
||||
def decorator(func):
|
||||
@wraps(func)
|
||||
async def wrapper(*args, **kwargs):
|
||||
if not _get_debug_enabled():
|
||||
return await func(*args, **kwargs)
|
||||
|
||||
start = time.time()
|
||||
debug_detailed(module, f"Starting {func.__name__}()")
|
||||
|
||||
try:
|
||||
result = await func(*args, **kwargs)
|
||||
elapsed = time.time() - start
|
||||
debug_success(
|
||||
module,
|
||||
f"Completed {func.__name__}()",
|
||||
elapsed_ms=f"{elapsed * 1000:.1f}ms",
|
||||
)
|
||||
return result
|
||||
except Exception as e:
|
||||
elapsed = time.time() - start
|
||||
debug_error(
|
||||
module,
|
||||
f"Failed {func.__name__}()",
|
||||
error=str(e),
|
||||
elapsed_ms=f"{elapsed * 1000:.1f}ms",
|
||||
)
|
||||
raise
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def debug_env_status() -> None:
|
||||
"""Print debug environment status on startup."""
|
||||
if not _get_debug_enabled():
|
||||
return
|
||||
|
||||
debug_section("debug", "Debug Mode Enabled")
|
||||
debug(
|
||||
"debug",
|
||||
"Environment configuration",
|
||||
DEBUG=os.environ.get("DEBUG", "not set"),
|
||||
DEBUG_LEVEL=_get_debug_level(),
|
||||
DEBUG_LOG_FILE=os.environ.get("DEBUG_LOG_FILE", "not set"),
|
||||
)
|
||||
|
||||
|
||||
# Print status on import if debug is enabled
|
||||
if _get_debug_enabled():
|
||||
debug_env_status()
|
||||
@@ -0,0 +1,467 @@
|
||||
"""
|
||||
Progress Tracking Utilities
|
||||
===========================
|
||||
|
||||
Functions for tracking and displaying progress of the autonomous coding agent.
|
||||
Uses subtask-based implementation plans (implementation_plan.json).
|
||||
|
||||
Enhanced with colored output, icons, and better visual formatting.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from ui import (
|
||||
Icons,
|
||||
bold,
|
||||
box,
|
||||
highlight,
|
||||
icon,
|
||||
muted,
|
||||
print_phase_status,
|
||||
print_status,
|
||||
progress_bar,
|
||||
success,
|
||||
warning,
|
||||
)
|
||||
|
||||
|
||||
def count_subtasks(spec_dir: Path) -> tuple[int, int]:
|
||||
"""
|
||||
Count completed and total subtasks in implementation_plan.json.
|
||||
|
||||
Args:
|
||||
spec_dir: Directory containing implementation_plan.json
|
||||
|
||||
Returns:
|
||||
(completed_count, total_count)
|
||||
"""
|
||||
plan_file = spec_dir / "implementation_plan.json"
|
||||
|
||||
if not plan_file.exists():
|
||||
return 0, 0
|
||||
|
||||
try:
|
||||
with open(plan_file) as f:
|
||||
plan = json.load(f)
|
||||
|
||||
total = 0
|
||||
completed = 0
|
||||
|
||||
for phase in plan.get("phases", []):
|
||||
for subtask in phase.get("subtasks", []):
|
||||
total += 1
|
||||
if subtask.get("status") == "completed":
|
||||
completed += 1
|
||||
|
||||
return completed, total
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return 0, 0
|
||||
|
||||
|
||||
def count_subtasks_detailed(spec_dir: Path) -> dict:
|
||||
"""
|
||||
Count subtasks by status.
|
||||
|
||||
Returns:
|
||||
Dict with completed, in_progress, pending, failed counts
|
||||
"""
|
||||
plan_file = spec_dir / "implementation_plan.json"
|
||||
|
||||
result = {
|
||||
"completed": 0,
|
||||
"in_progress": 0,
|
||||
"pending": 0,
|
||||
"failed": 0,
|
||||
"total": 0,
|
||||
}
|
||||
|
||||
if not plan_file.exists():
|
||||
return result
|
||||
|
||||
try:
|
||||
with open(plan_file) as f:
|
||||
plan = json.load(f)
|
||||
|
||||
for phase in plan.get("phases", []):
|
||||
for subtask in phase.get("subtasks", []):
|
||||
result["total"] += 1
|
||||
status = subtask.get("status", "pending")
|
||||
if status in result:
|
||||
result[status] += 1
|
||||
else:
|
||||
result["pending"] += 1
|
||||
|
||||
return result
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return result
|
||||
|
||||
|
||||
def is_build_complete(spec_dir: Path) -> bool:
|
||||
"""
|
||||
Check if all subtasks are completed.
|
||||
|
||||
Args:
|
||||
spec_dir: Directory containing implementation_plan.json
|
||||
|
||||
Returns:
|
||||
True if all subtasks complete, False otherwise
|
||||
"""
|
||||
completed, total = count_subtasks(spec_dir)
|
||||
return total > 0 and completed == total
|
||||
|
||||
|
||||
def get_progress_percentage(spec_dir: Path) -> float:
|
||||
"""
|
||||
Get the progress as a percentage.
|
||||
|
||||
Args:
|
||||
spec_dir: Directory containing implementation_plan.json
|
||||
|
||||
Returns:
|
||||
Percentage of subtasks completed (0-100)
|
||||
"""
|
||||
completed, total = count_subtasks(spec_dir)
|
||||
if total == 0:
|
||||
return 0.0
|
||||
return (completed / total) * 100
|
||||
|
||||
|
||||
def print_session_header(
|
||||
session_num: int,
|
||||
is_planner: bool,
|
||||
subtask_id: str = None,
|
||||
subtask_desc: str = None,
|
||||
phase_name: str = None,
|
||||
attempt: int = 1,
|
||||
) -> None:
|
||||
"""Print a formatted header for the session."""
|
||||
session_type = "PLANNER AGENT" if is_planner else "CODING AGENT"
|
||||
session_icon = Icons.GEAR if is_planner else Icons.LIGHTNING
|
||||
|
||||
content = [
|
||||
bold(f"{icon(session_icon)} SESSION {session_num}: {session_type}"),
|
||||
]
|
||||
|
||||
if subtask_id:
|
||||
content.append("")
|
||||
subtask_line = f"{icon(Icons.SUBTASK)} Subtask: {highlight(subtask_id)}"
|
||||
if subtask_desc:
|
||||
# Truncate long descriptions
|
||||
desc = subtask_desc[:50] + "..." if len(subtask_desc) > 50 else subtask_desc
|
||||
subtask_line += f" - {desc}"
|
||||
content.append(subtask_line)
|
||||
|
||||
if phase_name:
|
||||
content.append(f"{icon(Icons.PHASE)} Phase: {phase_name}")
|
||||
|
||||
if attempt > 1:
|
||||
content.append(warning(f"{icon(Icons.WARNING)} Attempt: {attempt}"))
|
||||
|
||||
print()
|
||||
print(box(content, width=70, style="heavy"))
|
||||
print()
|
||||
|
||||
|
||||
def print_progress_summary(spec_dir: Path, show_next: bool = True) -> None:
|
||||
"""Print a summary of current progress with enhanced formatting."""
|
||||
completed, total = count_subtasks(spec_dir)
|
||||
|
||||
if total > 0:
|
||||
print()
|
||||
# Progress bar
|
||||
print(f"Progress: {progress_bar(completed, total, width=40)}")
|
||||
|
||||
# Status message
|
||||
if completed == total:
|
||||
print_status("BUILD COMPLETE - All subtasks completed!", "success")
|
||||
else:
|
||||
remaining = total - completed
|
||||
print_status(f"{remaining} subtasks remaining", "info")
|
||||
|
||||
# Phase summary
|
||||
try:
|
||||
with open(spec_dir / "implementation_plan.json") as f:
|
||||
plan = json.load(f)
|
||||
|
||||
print("\nPhases:")
|
||||
for phase in plan.get("phases", []):
|
||||
phase_subtasks = phase.get("subtasks", [])
|
||||
phase_completed = sum(
|
||||
1 for s in phase_subtasks if s.get("status") == "completed"
|
||||
)
|
||||
phase_total = len(phase_subtasks)
|
||||
phase_name = phase.get("name", phase.get("id", "Unknown"))
|
||||
|
||||
if phase_completed == phase_total:
|
||||
status = "complete"
|
||||
elif phase_completed > 0 or any(
|
||||
s.get("status") == "in_progress" for s in phase_subtasks
|
||||
):
|
||||
status = "in_progress"
|
||||
else:
|
||||
# Check if blocked by dependencies
|
||||
deps = phase.get("depends_on", [])
|
||||
all_deps_complete = True
|
||||
for dep_id in deps:
|
||||
for p in plan.get("phases", []):
|
||||
if p.get("id") == dep_id or p.get("phase") == dep_id:
|
||||
p_subtasks = p.get("subtasks", [])
|
||||
if not all(
|
||||
s.get("status") == "completed" for s in p_subtasks
|
||||
):
|
||||
all_deps_complete = False
|
||||
break
|
||||
status = "pending" if all_deps_complete else "blocked"
|
||||
|
||||
print_phase_status(phase_name, phase_completed, phase_total, status)
|
||||
|
||||
# Show next subtask if requested
|
||||
if show_next and completed < total:
|
||||
next_subtask = get_next_subtask(spec_dir)
|
||||
if next_subtask:
|
||||
print()
|
||||
next_id = next_subtask.get("id", "unknown")
|
||||
next_desc = next_subtask.get("description", "")
|
||||
if len(next_desc) > 60:
|
||||
next_desc = next_desc[:57] + "..."
|
||||
print(
|
||||
f" {icon(Icons.ARROW_RIGHT)} Next: {highlight(next_id)} - {next_desc}"
|
||||
)
|
||||
|
||||
except (OSError, json.JSONDecodeError):
|
||||
pass
|
||||
else:
|
||||
print()
|
||||
print_status("No implementation subtasks yet - planner needs to run", "pending")
|
||||
|
||||
|
||||
def print_build_complete_banner(spec_dir: Path) -> None:
|
||||
"""Print a completion banner."""
|
||||
content = [
|
||||
success(f"{icon(Icons.SUCCESS)} BUILD COMPLETE!"),
|
||||
"",
|
||||
"All subtasks have been implemented successfully.",
|
||||
"",
|
||||
muted("Next steps:"),
|
||||
f" 1. Review the {highlight('auto-claude/*')} branch",
|
||||
" 2. Run manual tests",
|
||||
" 3. Create a PR and merge to main",
|
||||
]
|
||||
|
||||
print()
|
||||
print(box(content, width=70, style="heavy"))
|
||||
print()
|
||||
|
||||
|
||||
def print_paused_banner(
|
||||
spec_dir: Path,
|
||||
spec_name: str,
|
||||
has_worktree: bool = False,
|
||||
) -> None:
|
||||
"""Print a paused banner with resume instructions."""
|
||||
completed, total = count_subtasks(spec_dir)
|
||||
|
||||
content = [
|
||||
warning(f"{icon(Icons.PAUSE)} BUILD PAUSED"),
|
||||
"",
|
||||
f"Progress saved: {completed}/{total} subtasks complete",
|
||||
]
|
||||
|
||||
if has_worktree:
|
||||
content.append("")
|
||||
content.append(muted("Your build is in a separate workspace and is safe."))
|
||||
|
||||
print()
|
||||
print(box(content, width=70, style="heavy"))
|
||||
|
||||
|
||||
def get_plan_summary(spec_dir: Path) -> dict:
|
||||
"""
|
||||
Get a detailed summary of implementation plan status.
|
||||
|
||||
Args:
|
||||
spec_dir: Directory containing implementation_plan.json
|
||||
|
||||
Returns:
|
||||
Dictionary with plan statistics
|
||||
"""
|
||||
plan_file = spec_dir / "implementation_plan.json"
|
||||
|
||||
if not plan_file.exists():
|
||||
return {
|
||||
"workflow_type": None,
|
||||
"total_phases": 0,
|
||||
"total_subtasks": 0,
|
||||
"completed_subtasks": 0,
|
||||
"pending_subtasks": 0,
|
||||
"in_progress_subtasks": 0,
|
||||
"failed_subtasks": 0,
|
||||
"phases": [],
|
||||
}
|
||||
|
||||
try:
|
||||
with open(plan_file) as f:
|
||||
plan = json.load(f)
|
||||
|
||||
summary = {
|
||||
"workflow_type": plan.get("workflow_type"),
|
||||
"total_phases": len(plan.get("phases", [])),
|
||||
"total_subtasks": 0,
|
||||
"completed_subtasks": 0,
|
||||
"pending_subtasks": 0,
|
||||
"in_progress_subtasks": 0,
|
||||
"failed_subtasks": 0,
|
||||
"phases": [],
|
||||
}
|
||||
|
||||
for phase in plan.get("phases", []):
|
||||
phase_info = {
|
||||
"id": phase.get("id"),
|
||||
"phase": phase.get("phase"),
|
||||
"name": phase.get("name"),
|
||||
"depends_on": phase.get("depends_on", []),
|
||||
"subtasks": [],
|
||||
"completed": 0,
|
||||
"total": 0,
|
||||
}
|
||||
|
||||
for subtask in phase.get("subtasks", []):
|
||||
status = subtask.get("status", "pending")
|
||||
summary["total_subtasks"] += 1
|
||||
phase_info["total"] += 1
|
||||
|
||||
if status == "completed":
|
||||
summary["completed_subtasks"] += 1
|
||||
phase_info["completed"] += 1
|
||||
elif status == "in_progress":
|
||||
summary["in_progress_subtasks"] += 1
|
||||
elif status == "failed":
|
||||
summary["failed_subtasks"] += 1
|
||||
else:
|
||||
summary["pending_subtasks"] += 1
|
||||
|
||||
phase_info["subtasks"].append(
|
||||
{
|
||||
"id": subtask.get("id"),
|
||||
"description": subtask.get("description"),
|
||||
"status": status,
|
||||
"service": subtask.get("service"),
|
||||
}
|
||||
)
|
||||
|
||||
summary["phases"].append(phase_info)
|
||||
|
||||
return summary
|
||||
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return {
|
||||
"workflow_type": None,
|
||||
"total_phases": 0,
|
||||
"total_subtasks": 0,
|
||||
"completed_subtasks": 0,
|
||||
"pending_subtasks": 0,
|
||||
"in_progress_subtasks": 0,
|
||||
"failed_subtasks": 0,
|
||||
"phases": [],
|
||||
}
|
||||
|
||||
|
||||
def get_current_phase(spec_dir: Path) -> dict | None:
|
||||
"""Get the current phase being worked on."""
|
||||
plan_file = spec_dir / "implementation_plan.json"
|
||||
|
||||
if not plan_file.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(plan_file) as f:
|
||||
plan = json.load(f)
|
||||
|
||||
for phase in plan.get("phases", []):
|
||||
subtasks = phase.get("subtasks", [])
|
||||
# Phase is current if it has incomplete subtasks and dependencies are met
|
||||
has_incomplete = any(s.get("status") != "completed" for s in subtasks)
|
||||
if has_incomplete:
|
||||
return {
|
||||
"id": phase.get("id"),
|
||||
"phase": phase.get("phase"),
|
||||
"name": phase.get("name"),
|
||||
"completed": sum(
|
||||
1 for s in subtasks if s.get("status") == "completed"
|
||||
),
|
||||
"total": len(subtasks),
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return None
|
||||
|
||||
|
||||
def get_next_subtask(spec_dir: Path) -> dict | None:
|
||||
"""
|
||||
Find the next subtask to work on, respecting phase dependencies.
|
||||
|
||||
Args:
|
||||
spec_dir: Directory containing implementation_plan.json
|
||||
|
||||
Returns:
|
||||
The next subtask dict to work on, or None if all complete
|
||||
"""
|
||||
plan_file = spec_dir / "implementation_plan.json"
|
||||
|
||||
if not plan_file.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(plan_file) as f:
|
||||
plan = json.load(f)
|
||||
|
||||
phases = plan.get("phases", [])
|
||||
|
||||
# Build a map of phase completion
|
||||
phase_complete = {}
|
||||
for phase in phases:
|
||||
phase_id = phase.get("id") or phase.get("phase")
|
||||
subtasks = phase.get("subtasks", [])
|
||||
phase_complete[phase_id] = all(
|
||||
s.get("status") == "completed" for s in subtasks
|
||||
)
|
||||
|
||||
# Find next available subtask
|
||||
for phase in phases:
|
||||
phase_id = phase.get("id") or phase.get("phase")
|
||||
depends_on = phase.get("depends_on", [])
|
||||
|
||||
# Check if dependencies are satisfied
|
||||
deps_satisfied = all(phase_complete.get(dep, False) for dep in depends_on)
|
||||
if not deps_satisfied:
|
||||
continue
|
||||
|
||||
# Find first pending subtask in this phase
|
||||
for subtask in phase.get("subtasks", []):
|
||||
if subtask.get("status") == "pending":
|
||||
return {
|
||||
"phase_id": phase_id,
|
||||
"phase_name": phase.get("name"),
|
||||
"phase_num": phase.get("phase"),
|
||||
**subtask,
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return None
|
||||
|
||||
|
||||
def format_duration(seconds: float) -> str:
|
||||
"""Format a duration in human-readable form."""
|
||||
if seconds < 60:
|
||||
return f"{seconds:.0f}s"
|
||||
elif seconds < 3600:
|
||||
minutes = seconds / 60
|
||||
return f"{minutes:.1f}m"
|
||||
else:
|
||||
hours = seconds / 3600
|
||||
return f"{hours:.1f}h"
|
||||
@@ -6,19 +6,15 @@ Workspace Management - Per-Spec Architecture
|
||||
Handles workspace isolation through Git worktrees, where each spec
|
||||
gets its own isolated worktree in .worktrees/{spec-name}/.
|
||||
|
||||
Key changes from old design:
|
||||
- Each spec has its own worktree (not shared)
|
||||
- Worktree path: .worktrees/{spec-name}/
|
||||
- Branch name: auto-claude/{spec-name}
|
||||
- Fixed: get_existing_build_worktree() now properly checks spec_name
|
||||
- Fixed: finalize_workspace() skips prompts in auto_continue mode
|
||||
This module has been refactored for better maintainability:
|
||||
- Models and enums: workspace/models.py
|
||||
- Git utilities: workspace/git_utils.py
|
||||
- Setup functions: workspace/setup.py
|
||||
- Display functions: workspace/display.py
|
||||
- Finalization: workspace/finalization.py
|
||||
- Complex merge operations: remain here (workspace.py)
|
||||
|
||||
Terminology mapping (technical -> user-friendly):
|
||||
- worktree -> "separate workspace"
|
||||
- branch -> "version of your project"
|
||||
- uncommitted changes -> "unsaved work"
|
||||
- merge -> "add to your project"
|
||||
- working directory -> "your project"
|
||||
Public API is exported via workspace/__init__.py for backward compatibility.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
@@ -27,7 +23,6 @@ import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
@@ -69,645 +64,81 @@ from merge import (
|
||||
FileTimelineTracker,
|
||||
)
|
||||
|
||||
# Track if we've already tried to install the git hook this session
|
||||
_git_hook_check_done = False
|
||||
# Import from refactored modules in core/workspace/
|
||||
from core.workspace.models import (
|
||||
WorkspaceMode,
|
||||
WorkspaceChoice,
|
||||
ParallelMergeTask,
|
||||
ParallelMergeResult,
|
||||
MergeLock,
|
||||
MergeLockError,
|
||||
)
|
||||
|
||||
from core.workspace.git_utils import (
|
||||
has_uncommitted_changes,
|
||||
get_current_branch,
|
||||
get_existing_build_worktree,
|
||||
get_file_content_from_ref as _get_file_content_from_ref,
|
||||
get_changed_files_from_branch as _get_changed_files_from_branch,
|
||||
is_process_running as _is_process_running,
|
||||
is_binary_file as _is_binary_file,
|
||||
is_lock_file as _is_lock_file,
|
||||
validate_merged_syntax as _validate_merged_syntax,
|
||||
create_conflict_file_with_git as _create_conflict_file_with_git,
|
||||
MAX_FILE_LINES_FOR_AI,
|
||||
MAX_PARALLEL_AI_MERGES,
|
||||
MAX_SYNTAX_FIX_RETRIES,
|
||||
BINARY_EXTENSIONS,
|
||||
LOCK_FILES,
|
||||
MERGE_LOCK_TIMEOUT,
|
||||
)
|
||||
|
||||
from core.workspace.setup import (
|
||||
choose_workspace,
|
||||
copy_spec_to_worktree,
|
||||
setup_workspace,
|
||||
ensure_timeline_hook_installed as _ensure_timeline_hook_installed,
|
||||
initialize_timeline_tracking as _initialize_timeline_tracking,
|
||||
)
|
||||
|
||||
from core.workspace.display import (
|
||||
show_build_summary,
|
||||
show_changed_files,
|
||||
print_merge_success as _print_merge_success,
|
||||
print_conflict_info as _print_conflict_info,
|
||||
)
|
||||
|
||||
from core.workspace.finalization import (
|
||||
finalize_workspace,
|
||||
handle_workspace_choice,
|
||||
review_existing_build,
|
||||
discard_existing_build,
|
||||
check_existing_build,
|
||||
list_all_worktrees,
|
||||
cleanup_all_worktrees,
|
||||
)
|
||||
|
||||
MODULE = "workspace"
|
||||
|
||||
|
||||
class WorkspaceMode(Enum):
|
||||
"""How auto-claude should work."""
|
||||
|
||||
ISOLATED = "isolated" # Work in a separate worktree (safe)
|
||||
DIRECT = "direct" # Work directly in user's project
|
||||
|
||||
|
||||
class WorkspaceChoice(Enum):
|
||||
"""User's choice after build completes."""
|
||||
|
||||
MERGE = "merge" # Add changes to project
|
||||
REVIEW = "review" # Show what changed
|
||||
TEST = "test" # Test the feature in the staging worktree
|
||||
LATER = "later" # Decide later
|
||||
|
||||
|
||||
def has_uncommitted_changes(project_dir: Path) -> bool:
|
||||
"""Check if user has unsaved work."""
|
||||
result = subprocess.run(
|
||||
["git", "status", "--porcelain"],
|
||||
cwd=project_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return bool(result.stdout.strip())
|
||||
|
||||
|
||||
def get_current_branch(project_dir: Path) -> str:
|
||||
"""Get the current branch name."""
|
||||
result = subprocess.run(
|
||||
["git", "rev-parse", "--abbrev-ref", "HEAD"],
|
||||
cwd=project_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def get_existing_build_worktree(project_dir: Path, spec_name: str) -> Path | None:
|
||||
"""
|
||||
Check if there's an existing worktree for this specific spec.
|
||||
|
||||
Args:
|
||||
project_dir: The main project directory
|
||||
spec_name: The spec folder name (e.g., "001-feature-name")
|
||||
|
||||
Returns:
|
||||
Path to the worktree if it exists for this spec, None otherwise
|
||||
"""
|
||||
# Per-spec worktree path: .worktrees/{spec-name}/
|
||||
worktree_path = project_dir / ".worktrees" / spec_name
|
||||
if worktree_path.exists():
|
||||
return worktree_path
|
||||
return None
|
||||
|
||||
|
||||
def choose_workspace(
|
||||
project_dir: Path,
|
||||
spec_name: str,
|
||||
force_isolated: bool = False,
|
||||
force_direct: bool = False,
|
||||
auto_continue: bool = False,
|
||||
) -> WorkspaceMode:
|
||||
"""
|
||||
Let user choose where auto-claude should work.
|
||||
|
||||
Uses simple, non-technical language. Safe defaults.
|
||||
|
||||
Args:
|
||||
project_dir: The project directory
|
||||
spec_name: Name of the spec being built
|
||||
force_isolated: Skip prompts and use isolated mode
|
||||
force_direct: Skip prompts and use direct mode
|
||||
auto_continue: Non-interactive mode (for UI integration) - skip all prompts
|
||||
|
||||
Returns:
|
||||
WorkspaceMode indicating where to work
|
||||
"""
|
||||
# Handle forced modes
|
||||
if force_isolated:
|
||||
return WorkspaceMode.ISOLATED
|
||||
if force_direct:
|
||||
return WorkspaceMode.DIRECT
|
||||
|
||||
# Non-interactive mode: default to isolated for safety
|
||||
if auto_continue:
|
||||
print("Auto-continue: Using isolated workspace for safety.")
|
||||
return WorkspaceMode.ISOLATED
|
||||
|
||||
# Check for unsaved work
|
||||
has_unsaved = has_uncommitted_changes(project_dir)
|
||||
|
||||
if has_unsaved:
|
||||
# Unsaved work detected - use isolated mode for safety
|
||||
content = [
|
||||
success(f"{icon(Icons.SHIELD)} YOUR WORK IS PROTECTED"),
|
||||
"",
|
||||
"You have unsaved work in your project.",
|
||||
"",
|
||||
"To keep your work safe, the AI will build in a",
|
||||
"separate workspace. Your current files won't be",
|
||||
"touched until you're ready.",
|
||||
]
|
||||
print()
|
||||
print(box(content, width=60, style="heavy"))
|
||||
print()
|
||||
|
||||
try:
|
||||
input("Press Enter to continue...")
|
||||
except KeyboardInterrupt:
|
||||
print()
|
||||
print_status("Cancelled.", "info")
|
||||
sys.exit(0)
|
||||
|
||||
return WorkspaceMode.ISOLATED
|
||||
|
||||
# Clean working directory - give choice with enhanced menu
|
||||
options = [
|
||||
MenuOption(
|
||||
key="isolated",
|
||||
label="Separate workspace (Recommended)",
|
||||
icon=Icons.SHIELD,
|
||||
description="Your current files stay untouched. Easy to review and undo.",
|
||||
),
|
||||
MenuOption(
|
||||
key="direct",
|
||||
label="Right here in your project",
|
||||
icon=Icons.LIGHTNING,
|
||||
description="Changes happen directly. Best if you're not working on anything else.",
|
||||
),
|
||||
]
|
||||
|
||||
choice = select_menu(
|
||||
title="Where should the AI build your feature?",
|
||||
options=options,
|
||||
allow_quit=True,
|
||||
)
|
||||
|
||||
if choice is None:
|
||||
print()
|
||||
print_status("Cancelled.", "info")
|
||||
sys.exit(0)
|
||||
|
||||
if choice == "direct":
|
||||
print()
|
||||
print_status("Working directly in your project.", "info")
|
||||
return WorkspaceMode.DIRECT
|
||||
else:
|
||||
print()
|
||||
print_status("Using a separate workspace for safety.", "success")
|
||||
return WorkspaceMode.ISOLATED
|
||||
|
||||
|
||||
def copy_spec_to_worktree(
|
||||
source_spec_dir: Path,
|
||||
worktree_path: Path,
|
||||
spec_name: str,
|
||||
) -> Path:
|
||||
"""
|
||||
Copy spec files into the worktree so the AI can access them.
|
||||
|
||||
The AI's filesystem is restricted to the worktree, so spec files
|
||||
must be copied inside for access.
|
||||
|
||||
Args:
|
||||
source_spec_dir: Original spec directory (may be outside worktree)
|
||||
worktree_path: Path to the worktree
|
||||
spec_name: Name of the spec folder
|
||||
|
||||
Returns:
|
||||
Path to the spec directory inside the worktree
|
||||
"""
|
||||
# Determine target location inside worktree
|
||||
# Use .auto-claude/specs/{spec_name}/ as the standard location
|
||||
# Note: auto-claude/ is source code, .auto-claude/ is the installed instance
|
||||
target_spec_dir = worktree_path / ".auto-claude" / "specs" / spec_name
|
||||
|
||||
# Create parent directories if needed
|
||||
target_spec_dir.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Copy spec files (overwrite if exists to get latest)
|
||||
if target_spec_dir.exists():
|
||||
shutil.rmtree(target_spec_dir)
|
||||
|
||||
shutil.copytree(source_spec_dir, target_spec_dir)
|
||||
|
||||
return target_spec_dir
|
||||
|
||||
|
||||
def setup_workspace(
|
||||
project_dir: Path,
|
||||
spec_name: str,
|
||||
mode: WorkspaceMode,
|
||||
source_spec_dir: Path | None = None,
|
||||
) -> tuple[Path, WorktreeManager | None, Path | None]:
|
||||
"""
|
||||
Set up the workspace based on user's choice.
|
||||
|
||||
Uses per-spec worktrees - each spec gets its own isolated worktree.
|
||||
|
||||
Args:
|
||||
project_dir: The project directory
|
||||
spec_name: Name of the spec being built (e.g., "001-feature-name")
|
||||
mode: The workspace mode to use
|
||||
source_spec_dir: Optional source spec directory to copy to worktree
|
||||
|
||||
Returns:
|
||||
Tuple of (working_directory, worktree_manager or None, localized_spec_dir or None)
|
||||
|
||||
When using isolated mode with source_spec_dir:
|
||||
- working_directory: Path to the worktree
|
||||
- worktree_manager: Manager for the worktree
|
||||
- localized_spec_dir: Path to spec files INSIDE the worktree (accessible to AI)
|
||||
"""
|
||||
if mode == WorkspaceMode.DIRECT:
|
||||
# Work directly in project - spec_dir stays as-is
|
||||
return project_dir, None, source_spec_dir
|
||||
|
||||
# Create isolated workspace using per-spec worktree
|
||||
print()
|
||||
print_status("Setting up separate workspace...", "progress")
|
||||
|
||||
# Ensure timeline tracking hook is installed (once per session)
|
||||
_ensure_timeline_hook_installed(project_dir)
|
||||
|
||||
manager = WorktreeManager(project_dir)
|
||||
manager.setup()
|
||||
|
||||
# Get or create worktree for THIS SPECIFIC SPEC
|
||||
worktree_info = manager.get_or_create_worktree(spec_name)
|
||||
|
||||
# Copy spec files to worktree if provided
|
||||
localized_spec_dir = None
|
||||
if source_spec_dir and source_spec_dir.exists():
|
||||
localized_spec_dir = copy_spec_to_worktree(
|
||||
source_spec_dir, worktree_info.path, spec_name
|
||||
)
|
||||
print_status("Spec files copied to workspace", "success")
|
||||
|
||||
print_status(f"Workspace ready: {worktree_info.path.name}", "success")
|
||||
print()
|
||||
|
||||
# Initialize FileTimelineTracker for this task
|
||||
_initialize_timeline_tracking(
|
||||
project_dir=project_dir,
|
||||
spec_name=spec_name,
|
||||
worktree_path=worktree_info.path,
|
||||
source_spec_dir=localized_spec_dir or source_spec_dir,
|
||||
)
|
||||
|
||||
return worktree_info.path, manager, localized_spec_dir
|
||||
|
||||
|
||||
def _ensure_timeline_hook_installed(project_dir: Path) -> None:
|
||||
"""
|
||||
Ensure the FileTimelineTracker git post-commit hook is installed.
|
||||
|
||||
This enables tracking human commits to main branch for drift detection.
|
||||
Called once per session during first workspace setup.
|
||||
"""
|
||||
global _git_hook_check_done
|
||||
if _git_hook_check_done:
|
||||
return
|
||||
|
||||
_git_hook_check_done = True
|
||||
|
||||
try:
|
||||
git_dir = project_dir / ".git"
|
||||
if not git_dir.exists():
|
||||
return # Not a git repo
|
||||
|
||||
# Handle worktrees (where .git is a file, not directory)
|
||||
if git_dir.is_file():
|
||||
content = git_dir.read_text().strip()
|
||||
if content.startswith("gitdir:"):
|
||||
git_dir = Path(content.split(":", 1)[1].strip())
|
||||
else:
|
||||
return
|
||||
|
||||
hook_path = git_dir / "hooks" / "post-commit"
|
||||
|
||||
# Check if hook already installed
|
||||
if hook_path.exists():
|
||||
if "FileTimelineTracker" in hook_path.read_text():
|
||||
debug(MODULE, "FileTimelineTracker hook already installed")
|
||||
return
|
||||
|
||||
# Auto-install the hook (silent, non-intrusive)
|
||||
from merge.install_hook import install_hook
|
||||
install_hook(project_dir)
|
||||
debug(MODULE, "Auto-installed FileTimelineTracker git hook")
|
||||
|
||||
except Exception as e:
|
||||
# Non-fatal - hook installation is optional
|
||||
debug_warning(MODULE, f"Could not auto-install timeline hook: {e}")
|
||||
|
||||
|
||||
def _initialize_timeline_tracking(
|
||||
project_dir: Path,
|
||||
spec_name: str,
|
||||
worktree_path: Path,
|
||||
source_spec_dir: Path | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize FileTimelineTracker for a new task.
|
||||
|
||||
This registers the task's branch point and the files it intends to modify,
|
||||
enabling intent-aware merge conflict resolution later.
|
||||
"""
|
||||
try:
|
||||
tracker = FileTimelineTracker(project_dir)
|
||||
|
||||
# Get task intent from implementation plan
|
||||
task_intent = ""
|
||||
task_title = spec_name
|
||||
files_to_modify = []
|
||||
|
||||
if source_spec_dir:
|
||||
plan_path = source_spec_dir / "implementation_plan.json"
|
||||
if plan_path.exists():
|
||||
import json
|
||||
with open(plan_path) as f:
|
||||
plan = json.load(f)
|
||||
task_title = plan.get("title", spec_name)
|
||||
task_intent = plan.get("description", "")
|
||||
|
||||
# Extract files from phases/subtasks
|
||||
for phase in plan.get("phases", []):
|
||||
for subtask in phase.get("subtasks", []):
|
||||
files_to_modify.extend(subtask.get("files", []))
|
||||
|
||||
# Get the current branch point commit
|
||||
import subprocess
|
||||
result = subprocess.run(
|
||||
["git", "rev-parse", "HEAD"],
|
||||
cwd=project_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
branch_point = result.stdout.strip() if result.returncode == 0 else None
|
||||
|
||||
if files_to_modify and branch_point:
|
||||
# Register the task with known files
|
||||
tracker.on_task_start(
|
||||
task_id=spec_name,
|
||||
files_to_modify=list(set(files_to_modify)), # Dedupe
|
||||
branch_point_commit=branch_point,
|
||||
task_intent=task_intent,
|
||||
task_title=task_title,
|
||||
)
|
||||
debug(MODULE, f"Timeline tracking initialized for {spec_name}",
|
||||
files_tracked=len(files_to_modify),
|
||||
branch_point=branch_point[:8] if branch_point else None)
|
||||
else:
|
||||
# Initialize retroactively from worktree if no plan
|
||||
tracker.initialize_from_worktree(
|
||||
task_id=spec_name,
|
||||
worktree_path=worktree_path,
|
||||
task_intent=task_intent,
|
||||
task_title=task_title,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
# Non-fatal - timeline tracking is supplementary
|
||||
debug_warning(MODULE, f"Could not initialize timeline tracking: {e}")
|
||||
print(muted(f" Note: Timeline tracking could not be initialized: {e}"))
|
||||
|
||||
|
||||
def show_build_summary(manager: WorktreeManager, spec_name: str) -> None:
|
||||
"""Show a summary of what was built."""
|
||||
summary = manager.get_change_summary(spec_name)
|
||||
files = manager.get_changed_files(spec_name)
|
||||
|
||||
total = summary["new_files"] + summary["modified_files"] + summary["deleted_files"]
|
||||
|
||||
if total == 0:
|
||||
print_status("No changes were made.", "info")
|
||||
return
|
||||
|
||||
print()
|
||||
print(bold("What was built:"))
|
||||
if summary["new_files"] > 0:
|
||||
print(
|
||||
success(
|
||||
f" + {summary['new_files']} new file{'s' if summary['new_files'] != 1 else ''}"
|
||||
)
|
||||
)
|
||||
if summary["modified_files"] > 0:
|
||||
print(
|
||||
info(
|
||||
f" ~ {summary['modified_files']} modified file{'s' if summary['modified_files'] != 1 else ''}"
|
||||
)
|
||||
)
|
||||
if summary["deleted_files"] > 0:
|
||||
print(
|
||||
error(
|
||||
f" - {summary['deleted_files']} deleted file{'s' if summary['deleted_files'] != 1 else ''}"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def show_changed_files(manager: WorktreeManager, spec_name: str) -> None:
|
||||
"""Show detailed list of changed files."""
|
||||
files = manager.get_changed_files(spec_name)
|
||||
|
||||
if not files:
|
||||
print_status("No changes.", "info")
|
||||
return
|
||||
|
||||
print()
|
||||
print(bold("Changed files:"))
|
||||
for status, filepath in files:
|
||||
if status == "A":
|
||||
print(success(f" + {filepath}"))
|
||||
elif status == "M":
|
||||
print(info(f" ~ {filepath}"))
|
||||
elif status == "D":
|
||||
print(error(f" - {filepath}"))
|
||||
else:
|
||||
print(f" {status} {filepath}")
|
||||
|
||||
|
||||
def finalize_workspace(
|
||||
project_dir: Path,
|
||||
spec_name: str,
|
||||
manager: WorktreeManager | None,
|
||||
auto_continue: bool = False,
|
||||
) -> WorkspaceChoice:
|
||||
"""
|
||||
Handle post-build workflow - let user decide what to do with changes.
|
||||
|
||||
Safe design:
|
||||
- No "discard" option (requires separate --discard command)
|
||||
- Default is "test" - encourages testing before merging
|
||||
- Everything is preserved until user explicitly merges or discards
|
||||
|
||||
Args:
|
||||
project_dir: The project directory
|
||||
spec_name: Name of the spec that was built
|
||||
manager: The worktree manager (None if direct mode was used)
|
||||
auto_continue: If True, skip interactive prompts (UI mode)
|
||||
|
||||
Returns:
|
||||
WorkspaceChoice indicating what user wants to do
|
||||
"""
|
||||
if manager is None:
|
||||
# Direct mode - nothing to finalize
|
||||
content = [
|
||||
success(f"{icon(Icons.SUCCESS)} BUILD COMPLETE!"),
|
||||
"",
|
||||
"Changes were made directly to your project.",
|
||||
muted("Use 'git status' to see what changed."),
|
||||
]
|
||||
print()
|
||||
print(box(content, width=60, style="heavy"))
|
||||
return WorkspaceChoice.MERGE # Already merged
|
||||
|
||||
# In auto_continue mode (UI), skip interactive prompts
|
||||
# The worktree stays for the UI to manage
|
||||
if auto_continue:
|
||||
worktree_info = manager.get_worktree_info(spec_name)
|
||||
if worktree_info:
|
||||
print()
|
||||
print(success(f"Build complete in worktree: {worktree_info.path}"))
|
||||
print(muted("Worktree preserved for UI review."))
|
||||
return WorkspaceChoice.LATER
|
||||
|
||||
# Isolated mode - show options with testing as the recommended path
|
||||
content = [
|
||||
success(f"{icon(Icons.SUCCESS)} BUILD COMPLETE!"),
|
||||
"",
|
||||
"The AI built your feature in a separate workspace.",
|
||||
]
|
||||
print()
|
||||
print(box(content, width=60, style="heavy"))
|
||||
|
||||
show_build_summary(manager, spec_name)
|
||||
|
||||
# Get the worktree path for test instructions
|
||||
worktree_info = manager.get_worktree_info(spec_name)
|
||||
staging_path = worktree_info.path if worktree_info else None
|
||||
|
||||
# Enhanced menu for post-build options
|
||||
options = [
|
||||
MenuOption(
|
||||
key="test",
|
||||
label="Test the feature (Recommended)",
|
||||
icon=Icons.PLAY,
|
||||
description="Run the app and try it out before adding to your project",
|
||||
),
|
||||
MenuOption(
|
||||
key="merge",
|
||||
label="Add to my project now",
|
||||
icon=Icons.SUCCESS,
|
||||
description="Merge the changes into your files immediately",
|
||||
),
|
||||
MenuOption(
|
||||
key="review",
|
||||
label="Review what changed",
|
||||
icon=Icons.FILE,
|
||||
description="See exactly what files were modified",
|
||||
),
|
||||
MenuOption(
|
||||
key="later",
|
||||
label="Decide later",
|
||||
icon=Icons.PAUSE,
|
||||
description="Your build is saved - you can come back anytime",
|
||||
),
|
||||
]
|
||||
|
||||
print()
|
||||
choice = select_menu(
|
||||
title="What would you like to do?",
|
||||
options=options,
|
||||
allow_quit=False,
|
||||
)
|
||||
|
||||
if choice == "test":
|
||||
return WorkspaceChoice.TEST
|
||||
elif choice == "merge":
|
||||
return WorkspaceChoice.MERGE
|
||||
elif choice == "review":
|
||||
return WorkspaceChoice.REVIEW
|
||||
else:
|
||||
return WorkspaceChoice.LATER
|
||||
|
||||
|
||||
def handle_workspace_choice(
|
||||
choice: WorkspaceChoice,
|
||||
project_dir: Path,
|
||||
spec_name: str,
|
||||
manager: WorktreeManager,
|
||||
) -> None:
|
||||
"""
|
||||
Execute the user's choice.
|
||||
|
||||
Args:
|
||||
choice: What the user wants to do
|
||||
project_dir: The project directory
|
||||
spec_name: Name of the spec
|
||||
manager: The worktree manager
|
||||
"""
|
||||
worktree_info = manager.get_worktree_info(spec_name)
|
||||
staging_path = worktree_info.path if worktree_info else None
|
||||
|
||||
if choice == WorkspaceChoice.TEST:
|
||||
# Show testing instructions
|
||||
content = [
|
||||
bold(f"{icon(Icons.PLAY)} TEST YOUR FEATURE"),
|
||||
"",
|
||||
"Your feature is ready to test in a separate workspace.",
|
||||
]
|
||||
print()
|
||||
print(box(content, width=60, style="heavy"))
|
||||
|
||||
print()
|
||||
print("To test it, open a NEW terminal and run:")
|
||||
print()
|
||||
if staging_path:
|
||||
print(highlight(f" cd {staging_path}"))
|
||||
else:
|
||||
print(highlight(f" cd {project_dir}/.worktrees/{spec_name}"))
|
||||
|
||||
# Show likely test/run commands
|
||||
if staging_path:
|
||||
commands = manager.get_test_commands(spec_name)
|
||||
print()
|
||||
print("Then run your project:")
|
||||
for cmd in commands[:2]: # Show top 2 commands
|
||||
print(f" {cmd}")
|
||||
|
||||
print()
|
||||
print(muted("-" * 60))
|
||||
print()
|
||||
print("When you're done testing:")
|
||||
print(highlight(f" python auto-claude/run.py --spec {spec_name} --merge"))
|
||||
print()
|
||||
print("To discard (if you don't like it):")
|
||||
print(muted(f" python auto-claude/run.py --spec {spec_name} --discard"))
|
||||
print()
|
||||
|
||||
elif choice == WorkspaceChoice.MERGE:
|
||||
print()
|
||||
print_status("Adding changes to your project...", "progress")
|
||||
success_result = manager.merge_worktree(spec_name, delete_after=True)
|
||||
|
||||
if success_result:
|
||||
print()
|
||||
print_status("Your feature has been added to your project.", "success")
|
||||
else:
|
||||
print()
|
||||
print_status("There was a conflict merging the changes.", "error")
|
||||
print(muted("Your build is still saved in the separate workspace."))
|
||||
print(muted("You may need to merge manually or ask for help."))
|
||||
|
||||
elif choice == WorkspaceChoice.REVIEW:
|
||||
show_changed_files(manager, spec_name)
|
||||
print()
|
||||
print(muted("-" * 60))
|
||||
print()
|
||||
print("To see full details of changes:")
|
||||
if worktree_info:
|
||||
print(
|
||||
muted(
|
||||
f" git diff {worktree_info.base_branch}...{worktree_info.branch}"
|
||||
)
|
||||
)
|
||||
print()
|
||||
print("To test the feature:")
|
||||
if staging_path:
|
||||
print(highlight(f" cd {staging_path}"))
|
||||
print()
|
||||
print("To add these changes to your project:")
|
||||
print(highlight(f" python auto-claude/run.py --spec {spec_name} --merge"))
|
||||
print()
|
||||
|
||||
else: # LATER
|
||||
print()
|
||||
print_status("No problem! Your build is saved.", "success")
|
||||
print()
|
||||
print("To test the feature:")
|
||||
if staging_path:
|
||||
print(highlight(f" cd {staging_path}"))
|
||||
else:
|
||||
print(highlight(f" cd {project_dir}/.worktrees/{spec_name}"))
|
||||
print()
|
||||
print("When you're ready to add it:")
|
||||
print(highlight(f" python auto-claude/run.py --spec {spec_name} --merge"))
|
||||
print()
|
||||
print("To see what was built:")
|
||||
print(muted(f" python auto-claude/run.py --spec {spec_name} --review"))
|
||||
print()
|
||||
|
||||
# The following functions are now imported from refactored modules above.
|
||||
# They are kept here only to avoid breaking the existing code that still needs
|
||||
# the complex merge operations below.
|
||||
|
||||
# Remaining complex merge operations that reference each other:
|
||||
# - merge_existing_build
|
||||
# - _try_smart_merge
|
||||
# - _try_smart_merge_inner
|
||||
# - _check_git_conflicts
|
||||
# - _resolve_git_conflicts_with_ai
|
||||
# - _create_async_claude_client
|
||||
# - _async_ai_call
|
||||
# - _merge_file_with_ai_async
|
||||
# - _run_parallel_merges
|
||||
# - _record_merge_completion
|
||||
# - _get_task_intent
|
||||
# - _get_recent_merges_context
|
||||
# - _merge_file_with_ai
|
||||
# - _heuristic_merge
|
||||
|
||||
def merge_existing_build(
|
||||
project_dir: Path,
|
||||
@@ -1236,15 +667,22 @@ def _resolve_git_conflicts_with_ai(
|
||||
simple_merges.append((file_path, None)) # None = delete
|
||||
debug(MODULE, f" {file_path}: deleted (no AI needed)")
|
||||
else:
|
||||
# File exists in both - needs AI merge
|
||||
files_needing_ai_merge.append(ParallelMergeTask(
|
||||
file_path=file_path,
|
||||
main_content=main_content,
|
||||
worktree_content=worktree_content,
|
||||
base_content=base_content,
|
||||
spec_name=spec_name,
|
||||
))
|
||||
debug(MODULE, f" {file_path}: needs AI merge")
|
||||
# File exists in both - check if it's a lock file
|
||||
if _is_lock_file(file_path):
|
||||
# Lock files should never go through AI - just take worktree version
|
||||
# User can run package manager install to regenerate if needed
|
||||
simple_merges.append((file_path, worktree_content))
|
||||
debug(MODULE, f" {file_path}: lock file (taking worktree version)")
|
||||
else:
|
||||
# Regular file - needs AI merge
|
||||
files_needing_ai_merge.append(ParallelMergeTask(
|
||||
file_path=file_path,
|
||||
main_content=main_content,
|
||||
worktree_content=worktree_content,
|
||||
base_content=base_content,
|
||||
spec_name=spec_name,
|
||||
))
|
||||
debug(MODULE, f" {file_path}: needs AI merge")
|
||||
|
||||
except Exception as e:
|
||||
print(error(f" ✗ Failed to categorize {file_path}: {e}"))
|
||||
@@ -1265,7 +703,11 @@ def _resolve_git_conflicts_with_ai(
|
||||
target_path.write_text(merged_content, encoding="utf-8")
|
||||
subprocess.run(["git", "add", file_path], cwd=project_dir, capture_output=True)
|
||||
resolved_files.append(file_path)
|
||||
print(success(f" ✓ {file_path} (new file)"))
|
||||
# Determine the type for display
|
||||
if _is_lock_file(file_path):
|
||||
print(success(f" ✓ {file_path} (lock file - took worktree version)"))
|
||||
else:
|
||||
print(success(f" ✓ {file_path} (new file)"))
|
||||
else:
|
||||
# Delete the file
|
||||
target_path = project_dir / file_path
|
||||
@@ -1330,14 +772,8 @@ def _resolve_git_conflicts_with_ai(
|
||||
if remaining_conflicts:
|
||||
print(muted(f" Failed: {len(remaining_conflicts)}"))
|
||||
|
||||
if remaining_conflicts:
|
||||
return {
|
||||
"success": False,
|
||||
"resolved_files": resolved_files,
|
||||
"remaining_conflicts": remaining_conflicts,
|
||||
}
|
||||
|
||||
# All conflicts resolved - now merge remaining non-conflicting files
|
||||
# ALWAYS process non-conflicting files, even if some conflicts failed
|
||||
# This ensures we get as much of the build as possible
|
||||
# (New files were already copied at the start)
|
||||
print(muted(" Merging remaining files..."))
|
||||
|
||||
@@ -1371,18 +807,31 @@ def _resolve_git_conflicts_with_ai(
|
||||
if resolved_files:
|
||||
_record_merge_completion(project_dir, spec_name, resolved_files)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
# Build result - partial success if some files failed but we got others
|
||||
result = {
|
||||
"success": len(remaining_conflicts) == 0,
|
||||
"resolved_files": resolved_files,
|
||||
"stats": {
|
||||
"files_merged": len(resolved_files),
|
||||
"conflicts_resolved": len(conflicting_files),
|
||||
"conflicts_resolved": len(conflicting_files) - len(remaining_conflicts),
|
||||
"ai_assisted": ai_merged_count,
|
||||
"auto_merged": auto_merged_count,
|
||||
"parallel_ai_merges": len(files_needing_ai_merge),
|
||||
},
|
||||
}
|
||||
|
||||
# Add remaining conflicts if any (for UI to show what needs manual attention)
|
||||
if remaining_conflicts:
|
||||
result["remaining_conflicts"] = remaining_conflicts
|
||||
result["partial_success"] = len(resolved_files) > 0
|
||||
print()
|
||||
print(warning(f" ⚠ {len(remaining_conflicts)} file(s) could not be auto-merged:"))
|
||||
for conflict in remaining_conflicts:
|
||||
print(muted(f" - {conflict['file']}: {conflict['reason']}"))
|
||||
print(muted(" These files may need manual review."))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _get_file_content_from_ref(project_dir: Path, ref: str, file_path: str) -> Optional[str]:
|
||||
"""Get file content from a git ref (branch, commit, etc.)."""
|
||||
@@ -1702,14 +1151,55 @@ async def _merge_file_with_ai_async(
|
||||
resolutions = extract_conflict_resolutions(response, conflicts, language)
|
||||
if resolutions:
|
||||
merged = reassemble_with_resolutions(merged_content, conflicts, resolutions)
|
||||
is_valid, _ = _validate_merged_syntax(file_path, merged, project_dir)
|
||||
is_valid, error_msg = _validate_merged_syntax(file_path, merged, project_dir)
|
||||
|
||||
# Retry loop: if syntax is invalid, give AI feedback to fix it
|
||||
retry_count = 0
|
||||
while not is_valid and retry_count < MAX_SYNTAX_FIX_RETRIES:
|
||||
retry_count += 1
|
||||
debug(MODULE, f"[PARALLEL] Syntax error in merge, retry {retry_count}/{MAX_SYNTAX_FIX_RETRIES}: {file_path}")
|
||||
|
||||
# Build fix prompt with error feedback
|
||||
fix_prompt = f"""The merged code you produced has a syntax error:
|
||||
|
||||
{error_msg}
|
||||
|
||||
Here is your merged code that has the error:
|
||||
```{language}
|
||||
{merged}
|
||||
```
|
||||
|
||||
Please fix the syntax error and output the corrected code. Make sure:
|
||||
1. All lines are properly separated (no concatenated lines)
|
||||
2. All brackets/braces are properly matched
|
||||
3. All statements end correctly
|
||||
|
||||
Output ONLY the fixed code, no explanations."""
|
||||
|
||||
fix_response = await _async_ai_call(
|
||||
client,
|
||||
"You are an expert code fixer. Fix the syntax error in the code.",
|
||||
fix_prompt,
|
||||
)
|
||||
|
||||
if fix_response:
|
||||
fixed = resolver._extract_code_block(fix_response, language)
|
||||
if not fixed and resolver._looks_like_code(fix_response, language):
|
||||
fixed = fix_response.strip()
|
||||
if fixed:
|
||||
merged = fixed
|
||||
is_valid, error_msg = _validate_merged_syntax(file_path, merged, project_dir)
|
||||
|
||||
if is_valid:
|
||||
debug_success(MODULE, f"[PARALLEL] Conflict-only merge succeeded: {file_path}")
|
||||
debug_success(MODULE, f"[PARALLEL] Conflict-only merge succeeded: {file_path}" +
|
||||
(f" (after {retry_count} fix retries)" if retry_count > 0 else ""))
|
||||
return ParallelMergeResult(
|
||||
file_path=file_path,
|
||||
merged_content=merged,
|
||||
success=True
|
||||
)
|
||||
else:
|
||||
debug(MODULE, f"[PARALLEL] Conflict-only merge failed validation after {retry_count} retries, falling back to full-file: {file_path}")
|
||||
|
||||
# Case 3: Full-file AI merge (fallback)
|
||||
debug(MODULE, f"[PARALLEL] Full-file merge for: {file_path}")
|
||||
@@ -1736,21 +1226,62 @@ async def _merge_file_with_ai_async(
|
||||
merged = response.strip()
|
||||
|
||||
if merged:
|
||||
is_valid, _ = _validate_merged_syntax(file_path, merged, project_dir)
|
||||
is_valid, error_msg = _validate_merged_syntax(file_path, merged, project_dir)
|
||||
|
||||
# Retry loop: if syntax is invalid, give AI feedback to fix it
|
||||
retry_count = 0
|
||||
while not is_valid and retry_count < MAX_SYNTAX_FIX_RETRIES:
|
||||
retry_count += 1
|
||||
debug(MODULE, f"[PARALLEL] Syntax error in full-file merge, retry {retry_count}/{MAX_SYNTAX_FIX_RETRIES}: {file_path}")
|
||||
|
||||
# Build fix prompt with error feedback
|
||||
fix_prompt = f"""The merged code you produced has a syntax error:
|
||||
|
||||
{error_msg}
|
||||
|
||||
Here is your merged code that has the error:
|
||||
```{language}
|
||||
{merged}
|
||||
```
|
||||
|
||||
Please fix the syntax error and output the corrected code. Make sure:
|
||||
1. All lines are properly separated (no concatenated lines)
|
||||
2. All brackets/braces are properly matched
|
||||
3. All statements end correctly
|
||||
|
||||
Output ONLY the fixed code, no explanations."""
|
||||
|
||||
fix_response = await _async_ai_call(
|
||||
client,
|
||||
"You are an expert code fixer. Fix the syntax error in the code.",
|
||||
fix_prompt,
|
||||
)
|
||||
|
||||
if fix_response:
|
||||
fixed = resolver._extract_code_block(fix_response, language)
|
||||
if not fixed and resolver._looks_like_code(fix_response, language):
|
||||
fixed = fix_response.strip()
|
||||
if fixed:
|
||||
merged = fixed
|
||||
is_valid, error_msg = _validate_merged_syntax(file_path, merged, project_dir)
|
||||
|
||||
if is_valid:
|
||||
debug_success(MODULE, f"[PARALLEL] Full-file merge succeeded: {file_path}")
|
||||
debug_success(MODULE, f"[PARALLEL] Full-file merge succeeded: {file_path}" +
|
||||
(f" (after {retry_count} fix retries)" if retry_count > 0 else ""))
|
||||
return ParallelMergeResult(
|
||||
file_path=file_path,
|
||||
merged_content=merged,
|
||||
success=True
|
||||
)
|
||||
else:
|
||||
debug_error(MODULE, f"[PARALLEL] Full-file merge failed validation after {retry_count} retries: {file_path}")
|
||||
|
||||
# AI couldn't merge
|
||||
return ParallelMergeResult(
|
||||
file_path=file_path,
|
||||
merged_content=None,
|
||||
success=False,
|
||||
error="AI could not merge file"
|
||||
error="AI could not merge file - syntax validation failed after retries"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
@@ -104,6 +104,7 @@ from .finalization import (
|
||||
__all__ = [
|
||||
# Merge Operations (from workspace.py)
|
||||
'merge_existing_build',
|
||||
'_run_parallel_merges', # Private but used by tests
|
||||
# Models
|
||||
'WorkspaceMode',
|
||||
'WorkspaceChoice',
|
||||
@@ -0,0 +1,561 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Git Worktree Manager - Per-Spec Architecture
|
||||
=============================================
|
||||
|
||||
Each spec gets its own worktree:
|
||||
- Worktree path: .worktrees/{spec-name}/
|
||||
- Branch name: auto-claude/{spec-name}
|
||||
|
||||
This allows:
|
||||
1. Multiple specs to be worked on simultaneously
|
||||
2. Each spec's changes are isolated
|
||||
3. Branches persist until explicitly merged
|
||||
4. Clear 1:1:1 mapping: spec → worktree → branch
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class WorktreeError(Exception):
|
||||
"""Error during worktree operations."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorktreeInfo:
|
||||
"""Information about a spec's worktree."""
|
||||
|
||||
path: Path
|
||||
branch: str
|
||||
spec_name: str
|
||||
base_branch: str
|
||||
is_active: bool = True
|
||||
commit_count: int = 0
|
||||
files_changed: int = 0
|
||||
additions: int = 0
|
||||
deletions: int = 0
|
||||
|
||||
|
||||
class WorktreeManager:
|
||||
"""
|
||||
Manages per-spec Git worktrees.
|
||||
|
||||
Each spec gets its own worktree in .worktrees/{spec-name}/ with
|
||||
a corresponding branch auto-claude/{spec-name}.
|
||||
"""
|
||||
|
||||
def __init__(self, project_dir: Path, base_branch: str | None = None):
|
||||
self.project_dir = project_dir
|
||||
self.base_branch = base_branch or self._get_current_branch()
|
||||
self.worktrees_dir = project_dir / ".worktrees"
|
||||
self._merge_lock = asyncio.Lock()
|
||||
|
||||
def _get_current_branch(self) -> str:
|
||||
"""Get the current git branch."""
|
||||
result = subprocess.run(
|
||||
["git", "rev-parse", "--abbrev-ref", "HEAD"],
|
||||
cwd=self.project_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise WorktreeError(f"Failed to get current branch: {result.stderr}")
|
||||
return result.stdout.strip()
|
||||
|
||||
def _run_git(
|
||||
self, args: list[str], cwd: Path | None = None
|
||||
) -> subprocess.CompletedProcess:
|
||||
"""Run a git command and return the result."""
|
||||
return subprocess.run(
|
||||
["git"] + args,
|
||||
cwd=cwd or self.project_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
def _unstage_gitignored_files(self) -> None:
|
||||
"""
|
||||
Unstage any staged files that are gitignored in the current branch.
|
||||
|
||||
This is needed after a --no-commit merge because files that exist in the
|
||||
source branch (like spec files in auto-claude/specs/) get staged even if
|
||||
they're gitignored in the target branch.
|
||||
"""
|
||||
# Get list of staged files
|
||||
result = self._run_git(["diff", "--cached", "--name-only"])
|
||||
if result.returncode != 0 or not result.stdout.strip():
|
||||
return
|
||||
|
||||
staged_files = result.stdout.strip().split("\n")
|
||||
|
||||
# Check which staged files are gitignored
|
||||
# git check-ignore returns the files that ARE ignored
|
||||
result = self._run_git(["check-ignore", "--stdin"], cwd=self.project_dir)
|
||||
# We need to pass the files via stdin
|
||||
result = subprocess.run(
|
||||
["git", "check-ignore", "--stdin"],
|
||||
cwd=self.project_dir,
|
||||
input="\n".join(staged_files),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
if not result.stdout.strip():
|
||||
return
|
||||
|
||||
ignored_files = result.stdout.strip().split("\n")
|
||||
|
||||
if ignored_files:
|
||||
print(f"Unstaging {len(ignored_files)} gitignored file(s)...")
|
||||
# Unstage each ignored file
|
||||
for file in ignored_files:
|
||||
if file.strip():
|
||||
self._run_git(["reset", "HEAD", "--", file.strip()])
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Create worktrees directory if needed."""
|
||||
self.worktrees_dir.mkdir(exist_ok=True)
|
||||
|
||||
# ==================== Per-Spec Worktree Methods ====================
|
||||
|
||||
def get_worktree_path(self, spec_name: str) -> Path:
|
||||
"""Get the worktree path for a spec."""
|
||||
return self.worktrees_dir / spec_name
|
||||
|
||||
def get_branch_name(self, spec_name: str) -> str:
|
||||
"""Get the branch name for a spec."""
|
||||
return f"auto-claude/{spec_name}"
|
||||
|
||||
def worktree_exists(self, spec_name: str) -> bool:
|
||||
"""Check if a worktree exists for a spec."""
|
||||
return self.get_worktree_path(spec_name).exists()
|
||||
|
||||
def get_worktree_info(self, spec_name: str) -> WorktreeInfo | None:
|
||||
"""Get info about a spec's worktree."""
|
||||
worktree_path = self.get_worktree_path(spec_name)
|
||||
if not worktree_path.exists():
|
||||
return None
|
||||
|
||||
# Verify the branch exists in the worktree
|
||||
result = self._run_git(["rev-parse", "--abbrev-ref", "HEAD"], cwd=worktree_path)
|
||||
if result.returncode != 0:
|
||||
return None
|
||||
|
||||
actual_branch = result.stdout.strip()
|
||||
|
||||
# Get statistics
|
||||
stats = self._get_worktree_stats(spec_name)
|
||||
|
||||
return WorktreeInfo(
|
||||
path=worktree_path,
|
||||
branch=actual_branch,
|
||||
spec_name=spec_name,
|
||||
base_branch=self.base_branch,
|
||||
is_active=True,
|
||||
**stats,
|
||||
)
|
||||
|
||||
def _get_worktree_stats(self, spec_name: str) -> dict:
|
||||
"""Get diff statistics for a worktree."""
|
||||
worktree_path = self.get_worktree_path(spec_name)
|
||||
|
||||
stats = {
|
||||
"commit_count": 0,
|
||||
"files_changed": 0,
|
||||
"additions": 0,
|
||||
"deletions": 0,
|
||||
}
|
||||
|
||||
if not worktree_path.exists():
|
||||
return stats
|
||||
|
||||
# Commit count
|
||||
result = self._run_git(
|
||||
["rev-list", "--count", f"{self.base_branch}..HEAD"], cwd=worktree_path
|
||||
)
|
||||
if result.returncode == 0:
|
||||
stats["commit_count"] = int(result.stdout.strip() or "0")
|
||||
|
||||
# Diff stats
|
||||
result = self._run_git(
|
||||
["diff", "--shortstat", f"{self.base_branch}...HEAD"], cwd=worktree_path
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
# Parse: "3 files changed, 50 insertions(+), 10 deletions(-)"
|
||||
match = re.search(r"(\d+) files? changed", result.stdout)
|
||||
if match:
|
||||
stats["files_changed"] = int(match.group(1))
|
||||
match = re.search(r"(\d+) insertions?", result.stdout)
|
||||
if match:
|
||||
stats["additions"] = int(match.group(1))
|
||||
match = re.search(r"(\d+) deletions?", result.stdout)
|
||||
if match:
|
||||
stats["deletions"] = int(match.group(1))
|
||||
|
||||
return stats
|
||||
|
||||
def create_worktree(self, spec_name: str) -> WorktreeInfo:
|
||||
"""
|
||||
Create a worktree for a spec.
|
||||
|
||||
Args:
|
||||
spec_name: The spec folder name (e.g., "002-implement-memory")
|
||||
|
||||
Returns:
|
||||
WorktreeInfo for the created worktree
|
||||
"""
|
||||
worktree_path = self.get_worktree_path(spec_name)
|
||||
branch_name = self.get_branch_name(spec_name)
|
||||
|
||||
# Remove existing if present (from crashed previous run)
|
||||
if worktree_path.exists():
|
||||
self._run_git(["worktree", "remove", "--force", str(worktree_path)])
|
||||
|
||||
# Delete branch if it exists (from previous attempt)
|
||||
self._run_git(["branch", "-D", branch_name])
|
||||
|
||||
# Create worktree with new branch from base
|
||||
result = self._run_git(
|
||||
["worktree", "add", "-b", branch_name, str(worktree_path), self.base_branch]
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
raise WorktreeError(
|
||||
f"Failed to create worktree for {spec_name}: {result.stderr}"
|
||||
)
|
||||
|
||||
print(f"Created worktree: {worktree_path.name} on branch {branch_name}")
|
||||
|
||||
return WorktreeInfo(
|
||||
path=worktree_path,
|
||||
branch=branch_name,
|
||||
spec_name=spec_name,
|
||||
base_branch=self.base_branch,
|
||||
is_active=True,
|
||||
)
|
||||
|
||||
def get_or_create_worktree(self, spec_name: str) -> WorktreeInfo:
|
||||
"""
|
||||
Get existing worktree or create a new one for a spec.
|
||||
|
||||
Args:
|
||||
spec_name: The spec folder name
|
||||
|
||||
Returns:
|
||||
WorktreeInfo for the worktree
|
||||
"""
|
||||
existing = self.get_worktree_info(spec_name)
|
||||
if existing:
|
||||
print(f"Using existing worktree: {existing.path}")
|
||||
return existing
|
||||
|
||||
return self.create_worktree(spec_name)
|
||||
|
||||
def remove_worktree(self, spec_name: str, delete_branch: bool = False) -> None:
|
||||
"""
|
||||
Remove a spec's worktree.
|
||||
|
||||
Args:
|
||||
spec_name: The spec folder name
|
||||
delete_branch: Whether to also delete the branch
|
||||
"""
|
||||
worktree_path = self.get_worktree_path(spec_name)
|
||||
branch_name = self.get_branch_name(spec_name)
|
||||
|
||||
if worktree_path.exists():
|
||||
result = self._run_git(
|
||||
["worktree", "remove", "--force", str(worktree_path)]
|
||||
)
|
||||
if result.returncode == 0:
|
||||
print(f"Removed worktree: {worktree_path.name}")
|
||||
else:
|
||||
print(f"Warning: Could not remove worktree: {result.stderr}")
|
||||
shutil.rmtree(worktree_path, ignore_errors=True)
|
||||
|
||||
if delete_branch:
|
||||
self._run_git(["branch", "-D", branch_name])
|
||||
print(f"Deleted branch: {branch_name}")
|
||||
|
||||
self._run_git(["worktree", "prune"])
|
||||
|
||||
def merge_worktree(
|
||||
self, spec_name: str, delete_after: bool = False, no_commit: bool = False
|
||||
) -> bool:
|
||||
"""
|
||||
Merge a spec's worktree branch back to base branch.
|
||||
|
||||
Args:
|
||||
spec_name: The spec folder name
|
||||
delete_after: Whether to remove worktree and branch after merge
|
||||
no_commit: If True, merge changes but don't commit (stage only for review)
|
||||
|
||||
Returns:
|
||||
True if merge succeeded
|
||||
"""
|
||||
info = self.get_worktree_info(spec_name)
|
||||
if not info:
|
||||
print(f"No worktree found for spec: {spec_name}")
|
||||
return False
|
||||
|
||||
if no_commit:
|
||||
print(
|
||||
f"Merging {info.branch} into {self.base_branch} (staged, not committed)..."
|
||||
)
|
||||
else:
|
||||
print(f"Merging {info.branch} into {self.base_branch}...")
|
||||
|
||||
# Switch to base branch in main project
|
||||
result = self._run_git(["checkout", self.base_branch])
|
||||
if result.returncode != 0:
|
||||
print(f"Error: Could not checkout base branch: {result.stderr}")
|
||||
return False
|
||||
|
||||
# Merge the spec branch
|
||||
merge_args = ["merge", "--no-ff", info.branch]
|
||||
if no_commit:
|
||||
# --no-commit stages the merge but doesn't create the commit
|
||||
merge_args.append("--no-commit")
|
||||
else:
|
||||
merge_args.extend(["-m", f"auto-claude: Merge {info.branch}"])
|
||||
|
||||
result = self._run_git(merge_args)
|
||||
|
||||
if result.returncode != 0:
|
||||
print("Merge conflict! Aborting merge...")
|
||||
self._run_git(["merge", "--abort"])
|
||||
return False
|
||||
|
||||
if no_commit:
|
||||
# Unstage any files that are gitignored in the main branch
|
||||
# These get staged during merge because they exist in the worktree branch
|
||||
self._unstage_gitignored_files()
|
||||
print(
|
||||
f"Changes from {info.branch} are now staged in your working directory."
|
||||
)
|
||||
print("Review the changes, then commit when ready:")
|
||||
print(" git commit -m 'your commit message'")
|
||||
else:
|
||||
print(f"Successfully merged {info.branch}")
|
||||
|
||||
if delete_after:
|
||||
self.remove_worktree(spec_name, delete_branch=True)
|
||||
|
||||
return True
|
||||
|
||||
def commit_in_worktree(self, spec_name: str, message: str) -> bool:
|
||||
"""Commit all changes in a spec's worktree."""
|
||||
worktree_path = self.get_worktree_path(spec_name)
|
||||
if not worktree_path.exists():
|
||||
return False
|
||||
|
||||
self._run_git(["add", "."], cwd=worktree_path)
|
||||
result = self._run_git(["commit", "-m", message], cwd=worktree_path)
|
||||
|
||||
if result.returncode == 0:
|
||||
return True
|
||||
elif "nothing to commit" in result.stdout + result.stderr:
|
||||
return True
|
||||
else:
|
||||
print(f"Commit failed: {result.stderr}")
|
||||
return False
|
||||
|
||||
# ==================== Listing & Discovery ====================
|
||||
|
||||
def list_all_worktrees(self) -> list[WorktreeInfo]:
|
||||
"""List all spec worktrees."""
|
||||
worktrees = []
|
||||
|
||||
if not self.worktrees_dir.exists():
|
||||
return worktrees
|
||||
|
||||
for item in self.worktrees_dir.iterdir():
|
||||
if item.is_dir():
|
||||
info = self.get_worktree_info(item.name)
|
||||
if info:
|
||||
worktrees.append(info)
|
||||
|
||||
return worktrees
|
||||
|
||||
def list_all_spec_branches(self) -> list[str]:
|
||||
"""List all auto-claude branches (even if worktree removed)."""
|
||||
result = self._run_git(["branch", "--list", "auto-claude/*"])
|
||||
if result.returncode != 0:
|
||||
return []
|
||||
|
||||
branches = []
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
branch = line.strip().lstrip("* ")
|
||||
if branch:
|
||||
branches.append(branch)
|
||||
|
||||
return branches
|
||||
|
||||
def get_changed_files(self, spec_name: str) -> list[tuple[str, str]]:
|
||||
"""Get list of changed files in a spec's worktree."""
|
||||
worktree_path = self.get_worktree_path(spec_name)
|
||||
if not worktree_path.exists():
|
||||
return []
|
||||
|
||||
result = self._run_git(
|
||||
["diff", "--name-status", f"{self.base_branch}...HEAD"], cwd=worktree_path
|
||||
)
|
||||
|
||||
files = []
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
if not line:
|
||||
continue
|
||||
parts = line.split("\t", 1)
|
||||
if len(parts) == 2:
|
||||
files.append((parts[0], parts[1]))
|
||||
|
||||
return files
|
||||
|
||||
def get_change_summary(self, spec_name: str) -> dict:
|
||||
"""Get a summary of changes in a worktree."""
|
||||
files = self.get_changed_files(spec_name)
|
||||
|
||||
new_files = sum(1 for status, _ in files if status == "A")
|
||||
modified_files = sum(1 for status, _ in files if status == "M")
|
||||
deleted_files = sum(1 for status, _ in files if status == "D")
|
||||
|
||||
return {
|
||||
"new_files": new_files,
|
||||
"modified_files": modified_files,
|
||||
"deleted_files": deleted_files,
|
||||
}
|
||||
|
||||
def cleanup_all(self) -> None:
|
||||
"""Remove all worktrees and their branches."""
|
||||
for worktree in self.list_all_worktrees():
|
||||
self.remove_worktree(worktree.spec_name, delete_branch=True)
|
||||
|
||||
def cleanup_stale_worktrees(self) -> None:
|
||||
"""Remove worktrees that aren't registered with git."""
|
||||
if not self.worktrees_dir.exists():
|
||||
return
|
||||
|
||||
# Get list of registered worktrees
|
||||
result = self._run_git(["worktree", "list", "--porcelain"])
|
||||
registered_paths = set()
|
||||
for line in result.stdout.split("\n"):
|
||||
if line.startswith("worktree "):
|
||||
registered_paths.add(Path(line.split(" ", 1)[1]))
|
||||
|
||||
# Remove unregistered directories
|
||||
for item in self.worktrees_dir.iterdir():
|
||||
if item.is_dir() and item not in registered_paths:
|
||||
print(f"Removing stale worktree directory: {item.name}")
|
||||
shutil.rmtree(item, ignore_errors=True)
|
||||
|
||||
self._run_git(["worktree", "prune"])
|
||||
|
||||
def get_test_commands(self, spec_name: str) -> list[str]:
|
||||
"""Detect likely test/run commands for the project."""
|
||||
worktree_path = self.get_worktree_path(spec_name)
|
||||
commands = []
|
||||
|
||||
if (worktree_path / "package.json").exists():
|
||||
commands.append("npm install && npm run dev")
|
||||
commands.append("npm test")
|
||||
|
||||
if (worktree_path / "requirements.txt").exists():
|
||||
commands.append("pip install -r requirements.txt")
|
||||
|
||||
if (worktree_path / "Cargo.toml").exists():
|
||||
commands.append("cargo run")
|
||||
commands.append("cargo test")
|
||||
|
||||
if (worktree_path / "go.mod").exists():
|
||||
commands.append("go run .")
|
||||
commands.append("go test ./...")
|
||||
|
||||
if not commands:
|
||||
commands.append("# Check the project's README for run instructions")
|
||||
|
||||
return commands
|
||||
|
||||
# ==================== Backward Compatibility ====================
|
||||
# These methods provide backward compatibility with the old single-worktree API
|
||||
|
||||
def get_staging_path(self) -> Path | None:
|
||||
"""
|
||||
Backward compatibility: Get path to any existing spec worktree.
|
||||
Prefer using get_worktree_path(spec_name) instead.
|
||||
"""
|
||||
worktrees = self.list_all_worktrees()
|
||||
if worktrees:
|
||||
return worktrees[0].path
|
||||
return None
|
||||
|
||||
def get_staging_info(self) -> WorktreeInfo | None:
|
||||
"""
|
||||
Backward compatibility: Get info about any existing spec worktree.
|
||||
Prefer using get_worktree_info(spec_name) instead.
|
||||
"""
|
||||
worktrees = self.list_all_worktrees()
|
||||
if worktrees:
|
||||
return worktrees[0]
|
||||
return None
|
||||
|
||||
def merge_staging(self, delete_after: bool = True) -> bool:
|
||||
"""
|
||||
Backward compatibility: Merge first found worktree.
|
||||
Prefer using merge_worktree(spec_name) instead.
|
||||
"""
|
||||
worktrees = self.list_all_worktrees()
|
||||
if worktrees:
|
||||
return self.merge_worktree(worktrees[0].spec_name, delete_after)
|
||||
return False
|
||||
|
||||
def remove_staging(self, delete_branch: bool = True) -> None:
|
||||
"""
|
||||
Backward compatibility: Remove first found worktree.
|
||||
Prefer using remove_worktree(spec_name) instead.
|
||||
"""
|
||||
worktrees = self.list_all_worktrees()
|
||||
if worktrees:
|
||||
self.remove_worktree(worktrees[0].spec_name, delete_branch)
|
||||
|
||||
def get_or_create_staging(self, spec_name: str) -> WorktreeInfo:
|
||||
"""
|
||||
Backward compatibility: Alias for get_or_create_worktree.
|
||||
"""
|
||||
return self.get_or_create_worktree(spec_name)
|
||||
|
||||
def staging_exists(self) -> bool:
|
||||
"""
|
||||
Backward compatibility: Check if any spec worktree exists.
|
||||
Prefer using worktree_exists(spec_name) instead.
|
||||
"""
|
||||
return len(self.list_all_worktrees()) > 0
|
||||
|
||||
def commit_in_staging(self, message: str) -> bool:
|
||||
"""
|
||||
Backward compatibility: Commit in first found worktree.
|
||||
Prefer using commit_in_worktree(spec_name, message) instead.
|
||||
"""
|
||||
worktrees = self.list_all_worktrees()
|
||||
if worktrees:
|
||||
return self.commit_in_worktree(worktrees[0].spec_name, message)
|
||||
return False
|
||||
|
||||
def has_uncommitted_changes(self, in_staging: bool = False) -> bool:
|
||||
"""Check if there are uncommitted changes."""
|
||||
worktrees = self.list_all_worktrees()
|
||||
if in_staging and worktrees:
|
||||
cwd = worktrees[0].path
|
||||
else:
|
||||
cwd = None
|
||||
result = self._run_git(["status", "--porcelain"], cwd=cwd)
|
||||
return bool(result.stdout.strip())
|
||||
|
||||
|
||||
# Keep STAGING_WORKTREE_NAME for backward compatibility in imports
|
||||
STAGING_WORKTREE_NAME = "auto-claude"
|
||||
+2
-369
@@ -1,369 +1,2 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Self-Critique System
|
||||
====================
|
||||
|
||||
Implements a self-critique loop that agents must run before marking subtasks complete.
|
||||
This helps catch quality issues early, before verification stage.
|
||||
|
||||
The critique system ensures:
|
||||
- Code follows patterns from reference files
|
||||
- All required files were modified/created
|
||||
- Error handling is present
|
||||
- No debugging artifacts left behind
|
||||
- Implementation matches subtask requirements
|
||||
"""
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class CritiqueResult:
|
||||
"""Result of a self-critique evaluation."""
|
||||
|
||||
passes: bool
|
||||
issues: list[str] = field(default_factory=list)
|
||||
improvements_made: list[str] = field(default_factory=list)
|
||||
recommendations: list[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Convert to dictionary for storage."""
|
||||
return {
|
||||
"passes": self.passes,
|
||||
"issues": self.issues,
|
||||
"improvements_made": self.improvements_made,
|
||||
"recommendations": self.recommendations,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> "CritiqueResult":
|
||||
"""Load from dictionary."""
|
||||
return cls(
|
||||
passes=data.get("passes", False),
|
||||
issues=data.get("issues", []),
|
||||
improvements_made=data.get("improvements_made", []),
|
||||
recommendations=data.get("recommendations", []),
|
||||
)
|
||||
|
||||
|
||||
def generate_critique_prompt(
|
||||
subtask: dict, files_modified: list[str], patterns_from: list[str]
|
||||
) -> str:
|
||||
"""
|
||||
Generate a critique prompt for the agent to self-evaluate.
|
||||
|
||||
Args:
|
||||
subtask: The subtask being implemented
|
||||
files_modified: List of files actually modified
|
||||
patterns_from: List of pattern files to compare against
|
||||
|
||||
Returns:
|
||||
Formatted prompt for self-critique
|
||||
"""
|
||||
subtask_id = subtask.get("id", "unknown")
|
||||
subtask_desc = subtask.get("description", "No description")
|
||||
service = subtask.get("service", "all services")
|
||||
files_to_modify = subtask.get("files_to_modify", [])
|
||||
files_to_create = subtask.get("files_to_create", [])
|
||||
|
||||
prompt = f"""## MANDATORY Self-Critique: {subtask_id}
|
||||
|
||||
**Subtask Description:** {subtask_desc}
|
||||
**Service:** {service}
|
||||
|
||||
Before marking this subtask as complete, you MUST perform a thorough self-critique.
|
||||
This is NOT optional - it's a required quality gate.
|
||||
|
||||
### STEP 1: Code Quality Checklist
|
||||
|
||||
Review your implementation against these criteria:
|
||||
|
||||
**Pattern Adherence:**
|
||||
- [ ] Follows patterns from reference files exactly: {", ".join(patterns_from) if patterns_from else "N/A"}
|
||||
- [ ] Variable naming matches codebase conventions
|
||||
- [ ] Imports organized correctly (grouped, sorted)
|
||||
- [ ] Code style consistent with existing files
|
||||
|
||||
**Error Handling:**
|
||||
- [ ] Try-catch blocks where operations can fail
|
||||
- [ ] Meaningful error messages
|
||||
- [ ] Proper error propagation
|
||||
- [ ] Edge cases considered
|
||||
|
||||
**Code Cleanliness:**
|
||||
- [ ] No console.log/print statements for debugging
|
||||
- [ ] No commented-out code blocks
|
||||
- [ ] No TODO comments without context
|
||||
- [ ] No hardcoded values that should be configurable
|
||||
|
||||
**Best Practices:**
|
||||
- [ ] Functions are focused and single-purpose
|
||||
- [ ] No code duplication
|
||||
- [ ] Appropriate use of constants
|
||||
- [ ] Documentation/comments where needed
|
||||
|
||||
### STEP 2: Implementation Completeness
|
||||
|
||||
**Files Modified:**
|
||||
Expected: {", ".join(files_to_modify) if files_to_modify else "None"}
|
||||
Actual: {", ".join(files_modified) if files_modified else "None"}
|
||||
- [ ] All files_to_modify were actually modified
|
||||
- [ ] No unexpected files were modified
|
||||
|
||||
**Files Created:**
|
||||
Expected: {", ".join(files_to_create) if files_to_create else "None"}
|
||||
- [ ] All files_to_create were actually created
|
||||
- [ ] Files follow naming conventions
|
||||
|
||||
**Requirements:**
|
||||
- [ ] Subtask description requirements fully met
|
||||
- [ ] All acceptance criteria from spec considered
|
||||
- [ ] No scope creep - stayed within subtask boundaries
|
||||
|
||||
### STEP 3: Potential Issues Analysis
|
||||
|
||||
List any concerns, limitations, or potential problems with your implementation:
|
||||
|
||||
1. [Issue 1, or "None identified"]
|
||||
2. [Issue 2, if any]
|
||||
3. [Issue 3, if any]
|
||||
|
||||
Be honest. Finding issues now is better than discovering them during verification.
|
||||
|
||||
### STEP 4: Improvements Made
|
||||
|
||||
If you identified issues in your critique, list what you fixed:
|
||||
|
||||
1. [Improvement 1, or "No fixes needed"]
|
||||
2. [Improvement 2, if applicable]
|
||||
3. [Improvement 3, if applicable]
|
||||
|
||||
### STEP 5: Final Verdict
|
||||
|
||||
**PROCEED:** [YES/NO - Only YES if all critical items pass]
|
||||
|
||||
**REASON:** [Brief explanation of your decision]
|
||||
|
||||
**CONFIDENCE:** [High/Medium/Low - How confident are you in this implementation?]
|
||||
|
||||
---
|
||||
|
||||
## Instructions for Agent
|
||||
|
||||
1. Work through each section methodically
|
||||
2. Check each box honestly - don't skip items
|
||||
3. If you find issues, FIX THEM before continuing
|
||||
4. Re-run this critique after fixes
|
||||
5. Only mark the subtask complete when verdict is YES with High confidence
|
||||
6. Document your critique results in your response
|
||||
|
||||
Remember: The next session has no context. Quality issues you miss now will be harder to fix later.
|
||||
"""
|
||||
|
||||
return prompt
|
||||
|
||||
|
||||
def parse_critique_response(response: str) -> CritiqueResult:
|
||||
"""
|
||||
Parse the agent's critique response into structured data.
|
||||
|
||||
Args:
|
||||
response: The agent's response to the critique prompt
|
||||
|
||||
Returns:
|
||||
CritiqueResult with parsed information
|
||||
"""
|
||||
issues = []
|
||||
improvements = []
|
||||
recommendations = []
|
||||
passes = False
|
||||
|
||||
# Extract PROCEED verdict
|
||||
proceed_match = re.search(
|
||||
r"\*\*PROCEED:\*\*\s*\[?\s*(YES|NO)", response, re.IGNORECASE
|
||||
)
|
||||
if proceed_match:
|
||||
passes = proceed_match.group(1).upper() == "YES"
|
||||
|
||||
# Extract issues from Step 3
|
||||
issues_section = re.search(
|
||||
r"### STEP 3:.*?Potential Issues.*?\n\n(.*?)(?=###|\Z)",
|
||||
response,
|
||||
re.DOTALL | re.IGNORECASE,
|
||||
)
|
||||
if issues_section:
|
||||
issue_lines = issues_section.group(1).strip().split("\n")
|
||||
for line in issue_lines:
|
||||
line = line.strip()
|
||||
if not line or line.startswith("---"):
|
||||
continue
|
||||
# Remove list markers
|
||||
issue = re.sub(r"^\d+\.\s*|\*\s*|-\s*", "", line).strip()
|
||||
# Skip if it's a placeholder or indicates no issues
|
||||
if (
|
||||
issue
|
||||
and issue.lower()
|
||||
not in ["none", "none identified", "no issues", "no concerns"]
|
||||
and issue
|
||||
not in [
|
||||
'[Issue 1, or "None identified"]',
|
||||
"[Issue 2, if any]",
|
||||
"[Issue 3, if any]",
|
||||
]
|
||||
):
|
||||
issues.append(issue)
|
||||
|
||||
# Extract improvements from Step 4
|
||||
improvements_section = re.search(
|
||||
r"### STEP 4:.*?Improvements Made.*?\n\n(.*?)(?=###|\Z)",
|
||||
response,
|
||||
re.DOTALL | re.IGNORECASE,
|
||||
)
|
||||
if improvements_section:
|
||||
improvement_lines = improvements_section.group(1).strip().split("\n")
|
||||
for line in improvement_lines:
|
||||
line = line.strip()
|
||||
if not line or line.startswith("---"):
|
||||
continue
|
||||
# Remove list markers
|
||||
improvement = re.sub(r"^\d+\.\s*|\*\s*|-\s*", "", line).strip()
|
||||
# Skip if it's a placeholder or indicates no improvements
|
||||
if (
|
||||
improvement
|
||||
and improvement.lower()
|
||||
not in ["none", "no fixes needed", "no improvements", "n/a"]
|
||||
and improvement
|
||||
not in [
|
||||
'[Improvement 1, or "No fixes needed"]',
|
||||
"[Improvement 2, if applicable]",
|
||||
"[Improvement 3, if applicable]",
|
||||
]
|
||||
):
|
||||
improvements.append(improvement)
|
||||
|
||||
# Extract confidence level as recommendation
|
||||
confidence_match = re.search(
|
||||
r"\*\*CONFIDENCE:\*\*\s*\[?\s*(High|Medium|Low)", response, re.IGNORECASE
|
||||
)
|
||||
if confidence_match:
|
||||
confidence = confidence_match.group(1)
|
||||
if confidence.lower() != "high":
|
||||
recommendations.append(
|
||||
f"Confidence level: {confidence} - consider additional review"
|
||||
)
|
||||
|
||||
return CritiqueResult(
|
||||
passes=passes,
|
||||
issues=issues,
|
||||
improvements_made=improvements,
|
||||
recommendations=recommendations,
|
||||
)
|
||||
|
||||
|
||||
def should_proceed(result: CritiqueResult) -> bool:
|
||||
"""
|
||||
Determine if the subtask should be marked complete based on critique.
|
||||
|
||||
Args:
|
||||
result: The critique result
|
||||
|
||||
Returns:
|
||||
True if subtask can be marked complete, False otherwise
|
||||
"""
|
||||
# Must pass the critique
|
||||
if not result.passes:
|
||||
return False
|
||||
|
||||
# If there are unresolved issues, don't proceed
|
||||
if result.issues:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def format_critique_summary(result: CritiqueResult) -> str:
|
||||
"""
|
||||
Format a critique result as a human-readable summary.
|
||||
|
||||
Args:
|
||||
result: The critique result
|
||||
|
||||
Returns:
|
||||
Formatted summary string
|
||||
"""
|
||||
lines = ["## Critique Summary"]
|
||||
lines.append("")
|
||||
lines.append(f"**Status:** {'PASSED ✓' if result.passes else 'FAILED ✗'}")
|
||||
lines.append("")
|
||||
|
||||
if result.issues:
|
||||
lines.append("**Issues Identified:**")
|
||||
for i, issue in enumerate(result.issues, 1):
|
||||
lines.append(f"{i}. {issue}")
|
||||
lines.append("")
|
||||
|
||||
if result.improvements_made:
|
||||
lines.append("**Improvements Made:**")
|
||||
for i, improvement in enumerate(result.improvements_made, 1):
|
||||
lines.append(f"{i}. {improvement}")
|
||||
lines.append("")
|
||||
|
||||
if result.recommendations:
|
||||
lines.append("**Recommendations:**")
|
||||
for i, rec in enumerate(result.recommendations, 1):
|
||||
lines.append(f"{i}. {rec}")
|
||||
lines.append("")
|
||||
|
||||
if should_proceed(result):
|
||||
lines.append("**Decision:** Subtask is ready to be marked complete.")
|
||||
else:
|
||||
lines.append("**Decision:** Subtask needs more work before completion.")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# Example usage for testing
|
||||
if __name__ == "__main__":
|
||||
# Demo subtask
|
||||
subtask = {
|
||||
"id": "auth-middleware",
|
||||
"description": "Add JWT authentication middleware",
|
||||
"service": "backend",
|
||||
"files_to_modify": ["app/middleware/auth.py"],
|
||||
"patterns_from": ["app/middleware/cors.py"],
|
||||
}
|
||||
|
||||
files_modified = ["app/middleware/auth.py"]
|
||||
|
||||
# Generate prompt
|
||||
prompt = generate_critique_prompt(subtask, files_modified, subtask["patterns_from"])
|
||||
print(prompt)
|
||||
print("\n" + "=" * 80 + "\n")
|
||||
|
||||
# Simulate a critique response
|
||||
sample_response = """
|
||||
### STEP 3: Potential Issues Analysis
|
||||
|
||||
1. Token expiration edge case not fully tested
|
||||
2. None
|
||||
|
||||
### STEP 4: Improvements Made
|
||||
|
||||
1. Added comprehensive error handling for invalid tokens
|
||||
2. Improved logging for debugging
|
||||
3. Added input validation for JWT format
|
||||
|
||||
### STEP 5: Final Verdict
|
||||
|
||||
**PROCEED:** YES
|
||||
|
||||
**REASON:** All critical items verified, patterns followed, error handling complete
|
||||
|
||||
**CONFIDENCE:** High
|
||||
"""
|
||||
|
||||
# Parse response
|
||||
result = parse_critique_response(sample_response)
|
||||
print(format_critique_summary(result))
|
||||
print(f"\nShould proceed: {should_proceed(result)}")
|
||||
"""Backward compatibility shim - import from spec.critique instead."""
|
||||
from spec.critique import *
|
||||
|
||||
+2
-349
@@ -1,349 +1,2 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Debug Logging Utility
|
||||
=====================
|
||||
|
||||
Centralized debug logging for the Auto-Claude framework.
|
||||
Controlled via environment variables:
|
||||
- DEBUG=true Enable debug mode
|
||||
- DEBUG_LEVEL=1|2|3 Log verbosity (1=basic, 2=detailed, 3=verbose)
|
||||
- DEBUG_LOG_FILE=path Optional file output
|
||||
|
||||
Usage:
|
||||
from debug import debug, debug_detailed, debug_verbose, is_debug_enabled
|
||||
|
||||
debug("run.py", "Starting task execution", task_id="001")
|
||||
debug_detailed("agent", "Agent response received", response_length=1234)
|
||||
debug_verbose("client", "Full request payload", payload=data)
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from functools import wraps
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
# ANSI color codes for terminal output
|
||||
class Colors:
|
||||
RESET = "\033[0m"
|
||||
BOLD = "\033[1m"
|
||||
DIM = "\033[2m"
|
||||
|
||||
# Debug colors
|
||||
DEBUG = "\033[36m" # Cyan
|
||||
DEBUG_DIM = "\033[96m" # Light cyan
|
||||
TIMESTAMP = "\033[90m" # Gray
|
||||
MODULE = "\033[33m" # Yellow
|
||||
KEY = "\033[35m" # Magenta
|
||||
VALUE = "\033[37m" # White
|
||||
SUCCESS = "\033[32m" # Green
|
||||
WARNING = "\033[33m" # Yellow
|
||||
ERROR = "\033[31m" # Red
|
||||
|
||||
|
||||
def _get_debug_enabled() -> bool:
|
||||
"""Check if debug mode is enabled via environment variable."""
|
||||
return os.environ.get("DEBUG", "").lower() in ("true", "1", "yes", "on")
|
||||
|
||||
|
||||
def _get_debug_level() -> int:
|
||||
"""Get debug verbosity level (1-3)."""
|
||||
try:
|
||||
level = int(os.environ.get("DEBUG_LEVEL", "1"))
|
||||
return max(1, min(3, level)) # Clamp to 1-3
|
||||
except ValueError:
|
||||
return 1
|
||||
|
||||
|
||||
def _get_log_file() -> Path | None:
|
||||
"""Get optional log file path."""
|
||||
log_file = os.environ.get("DEBUG_LOG_FILE")
|
||||
if log_file:
|
||||
return Path(log_file)
|
||||
return None
|
||||
|
||||
|
||||
def is_debug_enabled() -> bool:
|
||||
"""Check if debug mode is enabled."""
|
||||
return _get_debug_enabled()
|
||||
|
||||
|
||||
def get_debug_level() -> int:
|
||||
"""Get current debug level."""
|
||||
return _get_debug_level()
|
||||
|
||||
|
||||
def _format_value(value: Any, max_length: int = 200) -> str:
|
||||
"""Format a value for debug output, truncating if necessary."""
|
||||
if value is None:
|
||||
return "None"
|
||||
|
||||
if isinstance(value, (dict, list)):
|
||||
try:
|
||||
formatted = json.dumps(value, indent=2, default=str)
|
||||
if len(formatted) > max_length:
|
||||
formatted = formatted[:max_length] + "..."
|
||||
return formatted
|
||||
except (TypeError, ValueError):
|
||||
return str(value)[:max_length]
|
||||
|
||||
str_value = str(value)
|
||||
if len(str_value) > max_length:
|
||||
return str_value[:max_length] + "..."
|
||||
return str_value
|
||||
|
||||
|
||||
def _write_log(message: str, to_file: bool = True) -> None:
|
||||
"""Write log message to stdout and optionally to file."""
|
||||
print(message, file=sys.stderr)
|
||||
|
||||
if to_file:
|
||||
log_file = _get_log_file()
|
||||
if log_file:
|
||||
try:
|
||||
log_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Strip ANSI codes for file output
|
||||
import re
|
||||
|
||||
clean_message = re.sub(r"\033\[[0-9;]*m", "", message)
|
||||
with open(log_file, "a") as f:
|
||||
f.write(clean_message + "\n")
|
||||
except Exception:
|
||||
pass # Silently fail file logging
|
||||
|
||||
|
||||
def debug(module: str, message: str, level: int = 1, **kwargs) -> None:
|
||||
"""
|
||||
Log a debug message.
|
||||
|
||||
Args:
|
||||
module: Source module name (e.g., "run.py", "ideation_runner")
|
||||
message: Debug message
|
||||
level: Required debug level (1=basic, 2=detailed, 3=verbose)
|
||||
**kwargs: Additional key-value pairs to log
|
||||
"""
|
||||
if not _get_debug_enabled():
|
||||
return
|
||||
|
||||
if _get_debug_level() < level:
|
||||
return
|
||||
|
||||
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
|
||||
|
||||
# Build the log line
|
||||
parts = [
|
||||
f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET}",
|
||||
f"{Colors.DEBUG}[DEBUG]{Colors.RESET}",
|
||||
f"{Colors.MODULE}[{module}]{Colors.RESET}",
|
||||
f"{Colors.DEBUG_DIM}{message}{Colors.RESET}",
|
||||
]
|
||||
|
||||
log_line = " ".join(parts)
|
||||
|
||||
# Add kwargs on separate lines if present
|
||||
if kwargs:
|
||||
for key, value in kwargs.items():
|
||||
formatted_value = _format_value(value)
|
||||
if "\n" in formatted_value:
|
||||
# Multi-line value
|
||||
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}:"
|
||||
for line in formatted_value.split("\n"):
|
||||
log_line += f"\n {Colors.VALUE}{line}{Colors.RESET}"
|
||||
else:
|
||||
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{formatted_value}{Colors.RESET}"
|
||||
|
||||
_write_log(log_line)
|
||||
|
||||
|
||||
def debug_detailed(module: str, message: str, **kwargs) -> None:
|
||||
"""Log a detailed debug message (level 2)."""
|
||||
debug(module, message, level=2, **kwargs)
|
||||
|
||||
|
||||
def debug_verbose(module: str, message: str, **kwargs) -> None:
|
||||
"""Log a verbose debug message (level 3)."""
|
||||
debug(module, message, level=3, **kwargs)
|
||||
|
||||
|
||||
def debug_success(module: str, message: str, **kwargs) -> None:
|
||||
"""Log a success debug message."""
|
||||
if not _get_debug_enabled():
|
||||
return
|
||||
|
||||
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
|
||||
log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.SUCCESS}[OK]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {message}"
|
||||
|
||||
if kwargs:
|
||||
for key, value in kwargs.items():
|
||||
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}"
|
||||
|
||||
_write_log(log_line)
|
||||
|
||||
|
||||
def debug_info(module: str, message: str, **kwargs) -> None:
|
||||
"""Log an info debug message."""
|
||||
if not _get_debug_enabled():
|
||||
return
|
||||
|
||||
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
|
||||
log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.DEBUG}[INFO]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {message}"
|
||||
|
||||
if kwargs:
|
||||
for key, value in kwargs.items():
|
||||
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}"
|
||||
|
||||
_write_log(log_line)
|
||||
|
||||
|
||||
def debug_error(module: str, message: str, **kwargs) -> None:
|
||||
"""Log an error debug message (always shown if debug enabled)."""
|
||||
if not _get_debug_enabled():
|
||||
return
|
||||
|
||||
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
|
||||
log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.ERROR}[ERROR]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {Colors.ERROR}{message}{Colors.RESET}"
|
||||
|
||||
if kwargs:
|
||||
for key, value in kwargs.items():
|
||||
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}"
|
||||
|
||||
_write_log(log_line)
|
||||
|
||||
|
||||
def debug_warning(module: str, message: str, **kwargs) -> None:
|
||||
"""Log a warning debug message."""
|
||||
if not _get_debug_enabled():
|
||||
return
|
||||
|
||||
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
|
||||
log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.WARNING}[WARN]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {Colors.WARNING}{message}{Colors.RESET}"
|
||||
|
||||
if kwargs:
|
||||
for key, value in kwargs.items():
|
||||
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}"
|
||||
|
||||
_write_log(log_line)
|
||||
|
||||
|
||||
def debug_section(module: str, title: str) -> None:
|
||||
"""Log a section header for organizing debug output."""
|
||||
if not _get_debug_enabled():
|
||||
return
|
||||
|
||||
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
|
||||
separator = "─" * 60
|
||||
log_line = f"\n{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.DEBUG}{Colors.BOLD}┌{separator}┐{Colors.RESET}"
|
||||
log_line += f"\n{Colors.TIMESTAMP} {Colors.RESET} {Colors.DEBUG}{Colors.BOLD}│ {module}: {title}{' ' * (58 - len(module) - len(title) - 2)}│{Colors.RESET}"
|
||||
log_line += f"\n{Colors.TIMESTAMP} {Colors.RESET} {Colors.DEBUG}{Colors.BOLD}└{separator}┘{Colors.RESET}"
|
||||
|
||||
_write_log(log_line)
|
||||
|
||||
|
||||
def debug_timer(module: str):
|
||||
"""
|
||||
Decorator to time function execution.
|
||||
|
||||
Usage:
|
||||
@debug_timer("run.py")
|
||||
def my_function():
|
||||
...
|
||||
"""
|
||||
|
||||
def decorator(func):
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
if not _get_debug_enabled():
|
||||
return func(*args, **kwargs)
|
||||
|
||||
start = time.time()
|
||||
debug_detailed(module, f"Starting {func.__name__}()")
|
||||
|
||||
try:
|
||||
result = func(*args, **kwargs)
|
||||
elapsed = time.time() - start
|
||||
debug_success(
|
||||
module,
|
||||
f"Completed {func.__name__}()",
|
||||
elapsed_ms=f"{elapsed * 1000:.1f}ms",
|
||||
)
|
||||
return result
|
||||
except Exception as e:
|
||||
elapsed = time.time() - start
|
||||
debug_error(
|
||||
module,
|
||||
f"Failed {func.__name__}()",
|
||||
error=str(e),
|
||||
elapsed_ms=f"{elapsed * 1000:.1f}ms",
|
||||
)
|
||||
raise
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def debug_async_timer(module: str):
|
||||
"""
|
||||
Decorator to time async function execution.
|
||||
|
||||
Usage:
|
||||
@debug_async_timer("ideation_runner")
|
||||
async def my_async_function():
|
||||
...
|
||||
"""
|
||||
|
||||
def decorator(func):
|
||||
@wraps(func)
|
||||
async def wrapper(*args, **kwargs):
|
||||
if not _get_debug_enabled():
|
||||
return await func(*args, **kwargs)
|
||||
|
||||
start = time.time()
|
||||
debug_detailed(module, f"Starting {func.__name__}()")
|
||||
|
||||
try:
|
||||
result = await func(*args, **kwargs)
|
||||
elapsed = time.time() - start
|
||||
debug_success(
|
||||
module,
|
||||
f"Completed {func.__name__}()",
|
||||
elapsed_ms=f"{elapsed * 1000:.1f}ms",
|
||||
)
|
||||
return result
|
||||
except Exception as e:
|
||||
elapsed = time.time() - start
|
||||
debug_error(
|
||||
module,
|
||||
f"Failed {func.__name__}()",
|
||||
error=str(e),
|
||||
elapsed_ms=f"{elapsed * 1000:.1f}ms",
|
||||
)
|
||||
raise
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def debug_env_status() -> None:
|
||||
"""Print debug environment status on startup."""
|
||||
if not _get_debug_enabled():
|
||||
return
|
||||
|
||||
debug_section("debug", "Debug Mode Enabled")
|
||||
debug(
|
||||
"debug",
|
||||
"Environment configuration",
|
||||
DEBUG=os.environ.get("DEBUG", "not set"),
|
||||
DEBUG_LEVEL=_get_debug_level(),
|
||||
DEBUG_LOG_FILE=os.environ.get("DEBUG_LOG_FILE", "not set"),
|
||||
)
|
||||
|
||||
|
||||
# Print status on import if debug is enabled
|
||||
if _get_debug_enabled():
|
||||
debug_env_status()
|
||||
"""Backward compatibility shim - import from core.debug instead."""
|
||||
from core.debug import *
|
||||
|
||||
@@ -1,529 +1,2 @@
|
||||
"""
|
||||
Graphiti Integration Configuration
|
||||
==================================
|
||||
|
||||
Constants, status mappings, and configuration helpers for Graphiti memory integration.
|
||||
Follows the same patterns as linear_config.py for consistency.
|
||||
|
||||
Multi-Provider Support (V2):
|
||||
- LLM Providers: OpenAI, Anthropic, Azure OpenAI, Ollama
|
||||
- Embedder Providers: OpenAI, Voyage AI, Azure OpenAI, Ollama
|
||||
|
||||
Environment Variables:
|
||||
# Core
|
||||
GRAPHITI_ENABLED: Set to "true" to enable Graphiti integration
|
||||
GRAPHITI_LLM_PROVIDER: openai|anthropic|azure_openai|ollama (default: openai)
|
||||
GRAPHITI_EMBEDDER_PROVIDER: openai|voyage|azure_openai|ollama (default: openai)
|
||||
|
||||
# OpenAI
|
||||
OPENAI_API_KEY: Required for OpenAI provider
|
||||
OPENAI_MODEL: Model for LLM (default: gpt-5-mini)
|
||||
OPENAI_EMBEDDING_MODEL: Model for embeddings (default: text-embedding-3-small)
|
||||
|
||||
# Anthropic (LLM only - needs separate embedder)
|
||||
ANTHROPIC_API_KEY: Required for Anthropic provider
|
||||
GRAPHITI_ANTHROPIC_MODEL: Model for LLM (default: claude-sonnet-4-5-latest)
|
||||
|
||||
# Azure OpenAI
|
||||
AZURE_OPENAI_API_KEY: Required for Azure provider
|
||||
AZURE_OPENAI_BASE_URL: Azure endpoint URL
|
||||
AZURE_OPENAI_LLM_DEPLOYMENT: Deployment name for LLM
|
||||
AZURE_OPENAI_EMBEDDING_DEPLOYMENT: Deployment name for embeddings
|
||||
|
||||
# Voyage AI (embeddings only - commonly used with Anthropic)
|
||||
VOYAGE_API_KEY: Required for Voyage embedder
|
||||
VOYAGE_EMBEDDING_MODEL: Model (default: voyage-3)
|
||||
|
||||
# Ollama (local)
|
||||
OLLAMA_BASE_URL: Ollama server URL (default: http://localhost:11434)
|
||||
OLLAMA_LLM_MODEL: Model for LLM (e.g., deepseek-r1:7b)
|
||||
OLLAMA_EMBEDDING_MODEL: Model for embeddings (e.g., nomic-embed-text)
|
||||
OLLAMA_EMBEDDING_DIM: Embedding dimension (required for Ollama, e.g., 768)
|
||||
|
||||
# FalkorDB
|
||||
GRAPHITI_FALKORDB_HOST: FalkorDB host (default: localhost)
|
||||
GRAPHITI_FALKORDB_PORT: FalkorDB port (default: 6380)
|
||||
GRAPHITI_FALKORDB_PASSWORD: FalkorDB password (default: empty)
|
||||
GRAPHITI_DATABASE: Graph database name (default: auto_claude_memory)
|
||||
GRAPHITI_TELEMETRY_ENABLED: Set to "false" to disable telemetry (default: true)
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
# Default configuration values
|
||||
DEFAULT_FALKORDB_HOST = "localhost"
|
||||
DEFAULT_FALKORDB_PORT = 6380
|
||||
DEFAULT_DATABASE = "auto_claude_memory"
|
||||
DEFAULT_OLLAMA_BASE_URL = "http://localhost:11434"
|
||||
|
||||
# Graphiti state marker file (stores connection info and status)
|
||||
GRAPHITI_STATE_MARKER = ".graphiti_state.json"
|
||||
|
||||
# Episode types for different memory categories
|
||||
EPISODE_TYPE_SESSION_INSIGHT = "session_insight"
|
||||
EPISODE_TYPE_CODEBASE_DISCOVERY = "codebase_discovery"
|
||||
EPISODE_TYPE_PATTERN = "pattern"
|
||||
EPISODE_TYPE_GOTCHA = "gotcha"
|
||||
EPISODE_TYPE_TASK_OUTCOME = "task_outcome"
|
||||
EPISODE_TYPE_QA_RESULT = "qa_result"
|
||||
EPISODE_TYPE_HISTORICAL_CONTEXT = "historical_context"
|
||||
|
||||
|
||||
class LLMProvider(str, Enum):
|
||||
"""Supported LLM providers for Graphiti."""
|
||||
|
||||
OPENAI = "openai"
|
||||
ANTHROPIC = "anthropic"
|
||||
AZURE_OPENAI = "azure_openai"
|
||||
OLLAMA = "ollama"
|
||||
|
||||
|
||||
class EmbedderProvider(str, Enum):
|
||||
"""Supported embedder providers for Graphiti."""
|
||||
|
||||
OPENAI = "openai"
|
||||
VOYAGE = "voyage"
|
||||
AZURE_OPENAI = "azure_openai"
|
||||
OLLAMA = "ollama"
|
||||
|
||||
|
||||
@dataclass
|
||||
class GraphitiConfig:
|
||||
"""Configuration for Graphiti memory integration with multi-provider support."""
|
||||
|
||||
# Core settings
|
||||
enabled: bool = False
|
||||
llm_provider: str = "openai"
|
||||
embedder_provider: str = "openai"
|
||||
|
||||
# FalkorDB connection
|
||||
falkordb_host: str = DEFAULT_FALKORDB_HOST
|
||||
falkordb_port: int = DEFAULT_FALKORDB_PORT
|
||||
falkordb_password: str = ""
|
||||
database: str = DEFAULT_DATABASE
|
||||
telemetry_enabled: bool = True
|
||||
|
||||
# OpenAI settings
|
||||
openai_api_key: str = ""
|
||||
openai_model: str = "gpt-5-mini"
|
||||
openai_embedding_model: str = "text-embedding-3-small"
|
||||
|
||||
# Anthropic settings (LLM only)
|
||||
anthropic_api_key: str = ""
|
||||
anthropic_model: str = "claude-sonnet-4-5-latest"
|
||||
|
||||
# Azure OpenAI settings
|
||||
azure_openai_api_key: str = ""
|
||||
azure_openai_base_url: str = ""
|
||||
azure_openai_llm_deployment: str = ""
|
||||
azure_openai_embedding_deployment: str = ""
|
||||
|
||||
# Voyage AI settings (embeddings only)
|
||||
voyage_api_key: str = ""
|
||||
voyage_embedding_model: str = "voyage-3"
|
||||
|
||||
# Ollama settings (local)
|
||||
ollama_base_url: str = DEFAULT_OLLAMA_BASE_URL
|
||||
ollama_llm_model: str = ""
|
||||
ollama_embedding_model: str = ""
|
||||
ollama_embedding_dim: int = 0 # Required for Ollama embeddings
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "GraphitiConfig":
|
||||
"""Create config from environment variables."""
|
||||
# Check if Graphiti is explicitly enabled
|
||||
enabled_str = os.environ.get("GRAPHITI_ENABLED", "").lower()
|
||||
enabled = enabled_str in ("true", "1", "yes")
|
||||
|
||||
# Provider selection
|
||||
llm_provider = os.environ.get("GRAPHITI_LLM_PROVIDER", "openai").lower()
|
||||
embedder_provider = os.environ.get(
|
||||
"GRAPHITI_EMBEDDER_PROVIDER", "openai"
|
||||
).lower()
|
||||
|
||||
# FalkorDB connection settings
|
||||
falkordb_host = os.environ.get("GRAPHITI_FALKORDB_HOST", DEFAULT_FALKORDB_HOST)
|
||||
|
||||
try:
|
||||
falkordb_port = int(
|
||||
os.environ.get("GRAPHITI_FALKORDB_PORT", str(DEFAULT_FALKORDB_PORT))
|
||||
)
|
||||
except ValueError:
|
||||
falkordb_port = DEFAULT_FALKORDB_PORT
|
||||
|
||||
falkordb_password = os.environ.get("GRAPHITI_FALKORDB_PASSWORD", "")
|
||||
database = os.environ.get("GRAPHITI_DATABASE", DEFAULT_DATABASE)
|
||||
|
||||
# Telemetry setting
|
||||
telemetry_str = os.environ.get("GRAPHITI_TELEMETRY_ENABLED", "true").lower()
|
||||
telemetry_enabled = telemetry_str not in ("false", "0", "no")
|
||||
|
||||
# OpenAI settings
|
||||
openai_api_key = os.environ.get("OPENAI_API_KEY", "")
|
||||
openai_model = os.environ.get("OPENAI_MODEL", "gpt-5-mini")
|
||||
openai_embedding_model = os.environ.get(
|
||||
"OPENAI_EMBEDDING_MODEL", "text-embedding-3-small"
|
||||
)
|
||||
|
||||
# Anthropic settings
|
||||
anthropic_api_key = os.environ.get("ANTHROPIC_API_KEY", "")
|
||||
anthropic_model = os.environ.get(
|
||||
"GRAPHITI_ANTHROPIC_MODEL", "claude-sonnet-4-5-latest"
|
||||
)
|
||||
|
||||
# Azure OpenAI settings
|
||||
azure_openai_api_key = os.environ.get("AZURE_OPENAI_API_KEY", "")
|
||||
azure_openai_base_url = os.environ.get("AZURE_OPENAI_BASE_URL", "")
|
||||
azure_openai_llm_deployment = os.environ.get("AZURE_OPENAI_LLM_DEPLOYMENT", "")
|
||||
azure_openai_embedding_deployment = os.environ.get(
|
||||
"AZURE_OPENAI_EMBEDDING_DEPLOYMENT", ""
|
||||
)
|
||||
|
||||
# Voyage AI settings
|
||||
voyage_api_key = os.environ.get("VOYAGE_API_KEY", "")
|
||||
voyage_embedding_model = os.environ.get("VOYAGE_EMBEDDING_MODEL", "voyage-3")
|
||||
|
||||
# Ollama settings
|
||||
ollama_base_url = os.environ.get("OLLAMA_BASE_URL", DEFAULT_OLLAMA_BASE_URL)
|
||||
ollama_llm_model = os.environ.get("OLLAMA_LLM_MODEL", "")
|
||||
ollama_embedding_model = os.environ.get("OLLAMA_EMBEDDING_MODEL", "")
|
||||
|
||||
# Ollama embedding dimension (required for Ollama)
|
||||
try:
|
||||
ollama_embedding_dim = int(os.environ.get("OLLAMA_EMBEDDING_DIM", "0"))
|
||||
except ValueError:
|
||||
ollama_embedding_dim = 0
|
||||
|
||||
return cls(
|
||||
enabled=enabled,
|
||||
llm_provider=llm_provider,
|
||||
embedder_provider=embedder_provider,
|
||||
falkordb_host=falkordb_host,
|
||||
falkordb_port=falkordb_port,
|
||||
falkordb_password=falkordb_password,
|
||||
database=database,
|
||||
telemetry_enabled=telemetry_enabled,
|
||||
openai_api_key=openai_api_key,
|
||||
openai_model=openai_model,
|
||||
openai_embedding_model=openai_embedding_model,
|
||||
anthropic_api_key=anthropic_api_key,
|
||||
anthropic_model=anthropic_model,
|
||||
azure_openai_api_key=azure_openai_api_key,
|
||||
azure_openai_base_url=azure_openai_base_url,
|
||||
azure_openai_llm_deployment=azure_openai_llm_deployment,
|
||||
azure_openai_embedding_deployment=azure_openai_embedding_deployment,
|
||||
voyage_api_key=voyage_api_key,
|
||||
voyage_embedding_model=voyage_embedding_model,
|
||||
ollama_base_url=ollama_base_url,
|
||||
ollama_llm_model=ollama_llm_model,
|
||||
ollama_embedding_model=ollama_embedding_model,
|
||||
ollama_embedding_dim=ollama_embedding_dim,
|
||||
)
|
||||
|
||||
def is_valid(self) -> bool:
|
||||
"""
|
||||
Check if config has minimum required values for operation.
|
||||
|
||||
Returns True if:
|
||||
- GRAPHITI_ENABLED is true
|
||||
- LLM provider is configured correctly
|
||||
- Embedder provider is configured correctly
|
||||
"""
|
||||
if not self.enabled:
|
||||
return False
|
||||
|
||||
# Validate LLM provider
|
||||
if not self._validate_llm_provider():
|
||||
return False
|
||||
|
||||
# Validate embedder provider
|
||||
if not self._validate_embedder_provider():
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _validate_llm_provider(self) -> bool:
|
||||
"""Validate LLM provider configuration."""
|
||||
if self.llm_provider == "openai":
|
||||
return bool(self.openai_api_key)
|
||||
elif self.llm_provider == "anthropic":
|
||||
return bool(self.anthropic_api_key)
|
||||
elif self.llm_provider == "azure_openai":
|
||||
return bool(
|
||||
self.azure_openai_api_key
|
||||
and self.azure_openai_base_url
|
||||
and self.azure_openai_llm_deployment
|
||||
)
|
||||
elif self.llm_provider == "ollama":
|
||||
return bool(self.ollama_llm_model)
|
||||
return False
|
||||
|
||||
def _validate_embedder_provider(self) -> bool:
|
||||
"""Validate embedder provider configuration."""
|
||||
if self.embedder_provider == "openai":
|
||||
return bool(self.openai_api_key)
|
||||
elif self.embedder_provider == "voyage":
|
||||
return bool(self.voyage_api_key)
|
||||
elif self.embedder_provider == "azure_openai":
|
||||
return bool(
|
||||
self.azure_openai_api_key
|
||||
and self.azure_openai_base_url
|
||||
and self.azure_openai_embedding_deployment
|
||||
)
|
||||
elif self.embedder_provider == "ollama":
|
||||
return bool(self.ollama_embedding_model and self.ollama_embedding_dim)
|
||||
return False
|
||||
|
||||
def get_validation_errors(self) -> list[str]:
|
||||
"""Get list of validation errors for current configuration."""
|
||||
errors = []
|
||||
|
||||
if not self.enabled:
|
||||
errors.append("GRAPHITI_ENABLED must be set to true")
|
||||
return errors
|
||||
|
||||
# LLM provider validation
|
||||
if self.llm_provider == "openai":
|
||||
if not self.openai_api_key:
|
||||
errors.append("OpenAI LLM provider requires OPENAI_API_KEY")
|
||||
elif self.llm_provider == "anthropic":
|
||||
if not self.anthropic_api_key:
|
||||
errors.append("Anthropic LLM provider requires ANTHROPIC_API_KEY")
|
||||
elif self.llm_provider == "azure_openai":
|
||||
if not self.azure_openai_api_key:
|
||||
errors.append("Azure OpenAI LLM provider requires AZURE_OPENAI_API_KEY")
|
||||
if not self.azure_openai_base_url:
|
||||
errors.append(
|
||||
"Azure OpenAI LLM provider requires AZURE_OPENAI_BASE_URL"
|
||||
)
|
||||
if not self.azure_openai_llm_deployment:
|
||||
errors.append(
|
||||
"Azure OpenAI LLM provider requires AZURE_OPENAI_LLM_DEPLOYMENT"
|
||||
)
|
||||
elif self.llm_provider == "ollama":
|
||||
if not self.ollama_llm_model:
|
||||
errors.append("Ollama LLM provider requires OLLAMA_LLM_MODEL")
|
||||
else:
|
||||
errors.append(f"Unknown LLM provider: {self.llm_provider}")
|
||||
|
||||
# Embedder provider validation
|
||||
if self.embedder_provider == "openai":
|
||||
if not self.openai_api_key:
|
||||
errors.append("OpenAI embedder provider requires OPENAI_API_KEY")
|
||||
elif self.embedder_provider == "voyage":
|
||||
if not self.voyage_api_key:
|
||||
errors.append("Voyage embedder provider requires VOYAGE_API_KEY")
|
||||
elif self.embedder_provider == "azure_openai":
|
||||
if not self.azure_openai_api_key:
|
||||
errors.append(
|
||||
"Azure OpenAI embedder provider requires AZURE_OPENAI_API_KEY"
|
||||
)
|
||||
if not self.azure_openai_base_url:
|
||||
errors.append(
|
||||
"Azure OpenAI embedder provider requires AZURE_OPENAI_BASE_URL"
|
||||
)
|
||||
if not self.azure_openai_embedding_deployment:
|
||||
errors.append(
|
||||
"Azure OpenAI embedder provider requires AZURE_OPENAI_EMBEDDING_DEPLOYMENT"
|
||||
)
|
||||
elif self.embedder_provider == "ollama":
|
||||
if not self.ollama_embedding_model:
|
||||
errors.append(
|
||||
"Ollama embedder provider requires OLLAMA_EMBEDDING_MODEL"
|
||||
)
|
||||
if not self.ollama_embedding_dim:
|
||||
errors.append("Ollama embedder provider requires OLLAMA_EMBEDDING_DIM")
|
||||
else:
|
||||
errors.append(f"Unknown embedder provider: {self.embedder_provider}")
|
||||
|
||||
return errors
|
||||
|
||||
def get_connection_uri(self) -> str:
|
||||
"""Get the FalkorDB connection URI."""
|
||||
if self.falkordb_password:
|
||||
return f"redis://:{self.falkordb_password}@{self.falkordb_host}:{self.falkordb_port}"
|
||||
return f"redis://{self.falkordb_host}:{self.falkordb_port}"
|
||||
|
||||
def get_provider_summary(self) -> str:
|
||||
"""Get a summary of configured providers."""
|
||||
return f"LLM: {self.llm_provider}, Embedder: {self.embedder_provider}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class GraphitiState:
|
||||
"""State of Graphiti integration for an auto-claude spec."""
|
||||
|
||||
initialized: bool = False
|
||||
database: str | None = None
|
||||
indices_built: bool = False
|
||||
created_at: str | None = None
|
||||
last_session: int | None = None
|
||||
episode_count: int = 0
|
||||
error_log: list = field(default_factory=list)
|
||||
# V2 additions
|
||||
llm_provider: str | None = None
|
||||
embedder_provider: str | None = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"initialized": self.initialized,
|
||||
"database": self.database,
|
||||
"indices_built": self.indices_built,
|
||||
"created_at": self.created_at,
|
||||
"last_session": self.last_session,
|
||||
"episode_count": self.episode_count,
|
||||
"error_log": self.error_log[-10:], # Keep last 10 errors
|
||||
"llm_provider": self.llm_provider,
|
||||
"embedder_provider": self.embedder_provider,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> "GraphitiState":
|
||||
return cls(
|
||||
initialized=data.get("initialized", False),
|
||||
database=data.get("database"),
|
||||
indices_built=data.get("indices_built", False),
|
||||
created_at=data.get("created_at"),
|
||||
last_session=data.get("last_session"),
|
||||
episode_count=data.get("episode_count", 0),
|
||||
error_log=data.get("error_log", []),
|
||||
llm_provider=data.get("llm_provider"),
|
||||
embedder_provider=data.get("embedder_provider"),
|
||||
)
|
||||
|
||||
def save(self, spec_dir: Path) -> None:
|
||||
"""Save state to the spec directory."""
|
||||
marker_file = spec_dir / GRAPHITI_STATE_MARKER
|
||||
with open(marker_file, "w") as f:
|
||||
json.dump(self.to_dict(), f, indent=2)
|
||||
|
||||
@classmethod
|
||||
def load(cls, spec_dir: Path) -> Optional["GraphitiState"]:
|
||||
"""Load state from the spec directory."""
|
||||
marker_file = spec_dir / GRAPHITI_STATE_MARKER
|
||||
if not marker_file.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(marker_file) as f:
|
||||
return cls.from_dict(json.load(f))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return None
|
||||
|
||||
def record_error(self, error_msg: str) -> None:
|
||||
"""Record an error in the state."""
|
||||
self.error_log.append(
|
||||
{
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"error": error_msg[:500], # Limit error message length
|
||||
}
|
||||
)
|
||||
# Keep only last 10 errors
|
||||
self.error_log = self.error_log[-10:]
|
||||
|
||||
|
||||
def is_graphiti_enabled() -> bool:
|
||||
"""
|
||||
Quick check if Graphiti integration is available.
|
||||
|
||||
Returns True if:
|
||||
- GRAPHITI_ENABLED is set to true/1/yes
|
||||
- Required provider credentials are configured
|
||||
"""
|
||||
config = GraphitiConfig.from_env()
|
||||
return config.is_valid()
|
||||
|
||||
|
||||
def get_graphiti_status() -> dict:
|
||||
"""
|
||||
Get the current Graphiti integration status.
|
||||
|
||||
Returns:
|
||||
Dict with status information:
|
||||
- enabled: bool
|
||||
- available: bool (has required dependencies)
|
||||
- host: str
|
||||
- port: int
|
||||
- database: str
|
||||
- llm_provider: str
|
||||
- embedder_provider: str
|
||||
- reason: str (why unavailable if not available)
|
||||
- errors: list (validation errors if any)
|
||||
"""
|
||||
config = GraphitiConfig.from_env()
|
||||
|
||||
status = {
|
||||
"enabled": config.enabled,
|
||||
"available": False,
|
||||
"host": config.falkordb_host,
|
||||
"port": config.falkordb_port,
|
||||
"database": config.database,
|
||||
"llm_provider": config.llm_provider,
|
||||
"embedder_provider": config.embedder_provider,
|
||||
"reason": "",
|
||||
"errors": [],
|
||||
}
|
||||
|
||||
if not config.enabled:
|
||||
status["reason"] = "GRAPHITI_ENABLED not set to true"
|
||||
return status
|
||||
|
||||
# Get validation errors
|
||||
errors = config.get_validation_errors()
|
||||
if errors:
|
||||
status["errors"] = errors
|
||||
status["reason"] = errors[0] # First error as primary reason
|
||||
return status
|
||||
|
||||
status["available"] = True
|
||||
return status
|
||||
|
||||
|
||||
def get_available_providers() -> dict:
|
||||
"""
|
||||
Get list of available providers based on current environment.
|
||||
|
||||
Returns:
|
||||
Dict with lists of available LLM and embedder providers
|
||||
"""
|
||||
config = GraphitiConfig.from_env()
|
||||
|
||||
available_llm = []
|
||||
available_embedder = []
|
||||
|
||||
# Check OpenAI
|
||||
if config.openai_api_key:
|
||||
available_llm.append("openai")
|
||||
available_embedder.append("openai")
|
||||
|
||||
# Check Anthropic
|
||||
if config.anthropic_api_key:
|
||||
available_llm.append("anthropic")
|
||||
|
||||
# Check Azure OpenAI
|
||||
if config.azure_openai_api_key and config.azure_openai_base_url:
|
||||
if config.azure_openai_llm_deployment:
|
||||
available_llm.append("azure_openai")
|
||||
if config.azure_openai_embedding_deployment:
|
||||
available_embedder.append("azure_openai")
|
||||
|
||||
# Check Voyage
|
||||
if config.voyage_api_key:
|
||||
available_embedder.append("voyage")
|
||||
|
||||
# Check Ollama
|
||||
if config.ollama_llm_model:
|
||||
available_llm.append("ollama")
|
||||
if config.ollama_embedding_model and config.ollama_embedding_dim:
|
||||
available_embedder.append("ollama")
|
||||
|
||||
return {
|
||||
"llm_providers": available_llm,
|
||||
"embedder_providers": available_embedder,
|
||||
}
|
||||
"""Backward compatibility shim - import from integrations.graphiti.config instead."""
|
||||
from integrations.graphiti.config import *
|
||||
|
||||
@@ -1,70 +1,2 @@
|
||||
"""
|
||||
Graphiti Multi-Provider Entry Point
|
||||
====================================
|
||||
|
||||
Main entry point for Graphiti provider functionality.
|
||||
This module re-exports all functionality from the graphiti_providers package.
|
||||
|
||||
The actual implementation has been refactored into a package structure:
|
||||
- graphiti_providers/exceptions.py - Provider exceptions
|
||||
- graphiti_providers/models.py - Embedding dimensions and constants
|
||||
- graphiti_providers/llm_providers/ - LLM provider implementations
|
||||
- graphiti_providers/embedder_providers/ - Embedder provider implementations
|
||||
- graphiti_providers/cross_encoder.py - Cross-encoder/reranker
|
||||
- graphiti_providers/validators.py - Validation and health checks
|
||||
- graphiti_providers/utils.py - Utility functions
|
||||
- graphiti_providers/factory.py - Factory functions
|
||||
|
||||
For backward compatibility, this module re-exports all public APIs.
|
||||
|
||||
Usage:
|
||||
from graphiti_providers import create_llm_client, create_embedder
|
||||
from graphiti_config import GraphitiConfig
|
||||
|
||||
config = GraphitiConfig.from_env()
|
||||
llm_client = create_llm_client(config)
|
||||
embedder = create_embedder(config)
|
||||
"""
|
||||
|
||||
# Re-export all public APIs from the package
|
||||
from graphiti_providers import (
|
||||
# Exceptions
|
||||
ProviderError,
|
||||
ProviderNotInstalled,
|
||||
# Factory functions
|
||||
create_llm_client,
|
||||
create_embedder,
|
||||
create_cross_encoder,
|
||||
# Models
|
||||
EMBEDDING_DIMENSIONS,
|
||||
get_expected_embedding_dim,
|
||||
# Validators
|
||||
validate_embedding_config,
|
||||
test_llm_connection,
|
||||
test_embedder_connection,
|
||||
test_ollama_connection,
|
||||
# Utilities
|
||||
is_graphiti_enabled,
|
||||
get_graph_hints,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# Exceptions
|
||||
"ProviderError",
|
||||
"ProviderNotInstalled",
|
||||
# Factory functions
|
||||
"create_llm_client",
|
||||
"create_embedder",
|
||||
"create_cross_encoder",
|
||||
# Models
|
||||
"EMBEDDING_DIMENSIONS",
|
||||
"get_expected_embedding_dim",
|
||||
# Validators
|
||||
"validate_embedding_config",
|
||||
"test_llm_connection",
|
||||
"test_embedder_connection",
|
||||
"test_ollama_connection",
|
||||
# Utilities
|
||||
"is_graphiti_enabled",
|
||||
"get_graph_hints",
|
||||
]
|
||||
"""Backward compatibility shim - import from integrations.graphiti.providers_pkg instead."""
|
||||
from integrations.graphiti.providers_pkg import *
|
||||
|
||||
@@ -1,705 +0,0 @@
|
||||
"""
|
||||
Graphiti Multi-Provider Factory
|
||||
================================
|
||||
|
||||
Factory functions for creating LLM clients and embedders for Graphiti.
|
||||
Supports multiple providers: OpenAI, Anthropic, Azure OpenAI, and Ollama.
|
||||
|
||||
This module provides:
|
||||
- Lazy imports to avoid ImportError when provider packages not installed
|
||||
- Factory functions that create the correct client based on provider selection
|
||||
- Provider-specific configuration validation
|
||||
- Graceful error handling with helpful messages
|
||||
|
||||
Usage:
|
||||
from graphiti_providers import create_llm_client, create_embedder
|
||||
from graphiti_config import GraphitiConfig
|
||||
|
||||
config = GraphitiConfig.from_env()
|
||||
llm_client = create_llm_client(config)
|
||||
embedder = create_embedder(config)
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, Optional
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
from graphiti_config import GraphitiConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ProviderError(Exception):
|
||||
"""Raised when a provider cannot be initialized."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class ProviderNotInstalled(ProviderError):
|
||||
"""Raised when required packages for a provider are not installed."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# LLM Client Factory
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def create_llm_client(config: "GraphitiConfig") -> Any:
|
||||
"""
|
||||
Create an LLM client based on the configured provider.
|
||||
|
||||
Args:
|
||||
config: GraphitiConfig with provider settings
|
||||
|
||||
Returns:
|
||||
LLM client instance for Graphiti
|
||||
|
||||
Raises:
|
||||
ProviderNotInstalled: If required packages are missing
|
||||
ProviderError: If client creation fails
|
||||
"""
|
||||
provider = config.llm_provider
|
||||
|
||||
logger.info(f"Creating LLM client for provider: {provider}")
|
||||
|
||||
if provider == "openai":
|
||||
return _create_openai_llm_client(config)
|
||||
elif provider == "anthropic":
|
||||
return _create_anthropic_llm_client(config)
|
||||
elif provider == "azure_openai":
|
||||
return _create_azure_openai_llm_client(config)
|
||||
elif provider == "ollama":
|
||||
return _create_ollama_llm_client(config)
|
||||
else:
|
||||
raise ProviderError(f"Unknown LLM provider: {provider}")
|
||||
|
||||
|
||||
def _create_openai_llm_client(config: "GraphitiConfig") -> Any:
|
||||
"""Create OpenAI LLM client."""
|
||||
try:
|
||||
from graphiti_core.llm_client.config import LLMConfig
|
||||
from graphiti_core.llm_client.openai_client import OpenAIClient
|
||||
except ImportError as e:
|
||||
raise ProviderNotInstalled(
|
||||
f"OpenAI provider requires graphiti-core. "
|
||||
f"Install with: pip install graphiti-core\n"
|
||||
f"Error: {e}"
|
||||
)
|
||||
|
||||
if not config.openai_api_key:
|
||||
raise ProviderError("OpenAI provider requires OPENAI_API_KEY")
|
||||
|
||||
llm_config = LLMConfig(
|
||||
api_key=config.openai_api_key,
|
||||
model=config.openai_model,
|
||||
)
|
||||
|
||||
# GPT-5 family and o1/o3 models support reasoning/verbosity params
|
||||
model_lower = config.openai_model.lower()
|
||||
supports_reasoning = (
|
||||
model_lower.startswith("gpt-5")
|
||||
or model_lower.startswith("o1")
|
||||
or model_lower.startswith("o3")
|
||||
)
|
||||
|
||||
if supports_reasoning:
|
||||
# Use defaults for models that support reasoning params
|
||||
return OpenAIClient(config=llm_config)
|
||||
else:
|
||||
# Disable reasoning/verbosity for older models that don't support them
|
||||
return OpenAIClient(config=llm_config, reasoning=None, verbosity=None)
|
||||
|
||||
|
||||
def _create_anthropic_llm_client(config: "GraphitiConfig") -> Any:
|
||||
"""Create Anthropic LLM client."""
|
||||
try:
|
||||
from graphiti_core.llm_client.anthropic_client import AnthropicClient
|
||||
from graphiti_core.llm_client.config import LLMConfig
|
||||
except ImportError as e:
|
||||
raise ProviderNotInstalled(
|
||||
f"Anthropic provider requires graphiti-core[anthropic]. "
|
||||
f"Install with: pip install graphiti-core[anthropic]\n"
|
||||
f"Error: {e}"
|
||||
)
|
||||
|
||||
if not config.anthropic_api_key:
|
||||
raise ProviderError("Anthropic provider requires ANTHROPIC_API_KEY")
|
||||
|
||||
llm_config = LLMConfig(
|
||||
api_key=config.anthropic_api_key,
|
||||
model=config.anthropic_model,
|
||||
)
|
||||
|
||||
return AnthropicClient(config=llm_config)
|
||||
|
||||
|
||||
def _create_azure_openai_llm_client(config: "GraphitiConfig") -> Any:
|
||||
"""Create Azure OpenAI LLM client."""
|
||||
try:
|
||||
from graphiti_core.llm_client.azure_openai_client import AzureOpenAILLMClient
|
||||
from graphiti_core.llm_client.config import LLMConfig
|
||||
from openai import AsyncOpenAI
|
||||
except ImportError as e:
|
||||
raise ProviderNotInstalled(
|
||||
f"Azure OpenAI provider requires graphiti-core and openai. "
|
||||
f"Install with: pip install graphiti-core openai\n"
|
||||
f"Error: {e}"
|
||||
)
|
||||
|
||||
if not config.azure_openai_api_key:
|
||||
raise ProviderError("Azure OpenAI provider requires AZURE_OPENAI_API_KEY")
|
||||
if not config.azure_openai_base_url:
|
||||
raise ProviderError("Azure OpenAI provider requires AZURE_OPENAI_BASE_URL")
|
||||
if not config.azure_openai_llm_deployment:
|
||||
raise ProviderError(
|
||||
"Azure OpenAI provider requires AZURE_OPENAI_LLM_DEPLOYMENT"
|
||||
)
|
||||
|
||||
azure_client = AsyncOpenAI(
|
||||
base_url=config.azure_openai_base_url,
|
||||
api_key=config.azure_openai_api_key,
|
||||
)
|
||||
|
||||
llm_config = LLMConfig(
|
||||
model=config.azure_openai_llm_deployment,
|
||||
small_model=config.azure_openai_llm_deployment,
|
||||
)
|
||||
|
||||
return AzureOpenAILLMClient(azure_client=azure_client, config=llm_config)
|
||||
|
||||
|
||||
def _create_ollama_llm_client(config: "GraphitiConfig") -> Any:
|
||||
"""Create Ollama LLM client (using OpenAI-compatible interface)."""
|
||||
try:
|
||||
from graphiti_core.llm_client.config import LLMConfig
|
||||
from graphiti_core.llm_client.openai_generic_client import OpenAIGenericClient
|
||||
except ImportError as e:
|
||||
raise ProviderNotInstalled(
|
||||
f"Ollama provider requires graphiti-core. "
|
||||
f"Install with: pip install graphiti-core\n"
|
||||
f"Error: {e}"
|
||||
)
|
||||
|
||||
if not config.ollama_llm_model:
|
||||
raise ProviderError("Ollama provider requires OLLAMA_LLM_MODEL")
|
||||
|
||||
# Ensure Ollama base URL ends with /v1 for OpenAI compatibility
|
||||
base_url = config.ollama_base_url
|
||||
if not base_url.endswith("/v1"):
|
||||
base_url = base_url.rstrip("/") + "/v1"
|
||||
|
||||
llm_config = LLMConfig(
|
||||
api_key="ollama", # Ollama requires a dummy API key
|
||||
model=config.ollama_llm_model,
|
||||
small_model=config.ollama_llm_model,
|
||||
base_url=base_url,
|
||||
)
|
||||
|
||||
return OpenAIGenericClient(config=llm_config)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Embedder Factory
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def create_embedder(config: "GraphitiConfig") -> Any:
|
||||
"""
|
||||
Create an embedder based on the configured provider.
|
||||
|
||||
Args:
|
||||
config: GraphitiConfig with provider settings
|
||||
|
||||
Returns:
|
||||
Embedder instance for Graphiti
|
||||
|
||||
Raises:
|
||||
ProviderNotInstalled: If required packages are missing
|
||||
ProviderError: If embedder creation fails
|
||||
"""
|
||||
provider = config.embedder_provider
|
||||
|
||||
logger.info(f"Creating embedder for provider: {provider}")
|
||||
|
||||
if provider == "openai":
|
||||
return _create_openai_embedder(config)
|
||||
elif provider == "voyage":
|
||||
return _create_voyage_embedder(config)
|
||||
elif provider == "azure_openai":
|
||||
return _create_azure_openai_embedder(config)
|
||||
elif provider == "ollama":
|
||||
return _create_ollama_embedder(config)
|
||||
else:
|
||||
raise ProviderError(f"Unknown embedder provider: {provider}")
|
||||
|
||||
|
||||
def _create_openai_embedder(config: "GraphitiConfig") -> Any:
|
||||
"""Create OpenAI embedder."""
|
||||
try:
|
||||
from graphiti_core.embedder.openai import OpenAIEmbedder, OpenAIEmbedderConfig
|
||||
except ImportError as e:
|
||||
raise ProviderNotInstalled(
|
||||
f"OpenAI embedder requires graphiti-core. "
|
||||
f"Install with: pip install graphiti-core\n"
|
||||
f"Error: {e}"
|
||||
)
|
||||
|
||||
if not config.openai_api_key:
|
||||
raise ProviderError("OpenAI embedder requires OPENAI_API_KEY")
|
||||
|
||||
embedder_config = OpenAIEmbedderConfig(
|
||||
api_key=config.openai_api_key,
|
||||
embedding_model=config.openai_embedding_model,
|
||||
)
|
||||
|
||||
return OpenAIEmbedder(config=embedder_config)
|
||||
|
||||
|
||||
def _create_voyage_embedder(config: "GraphitiConfig") -> Any:
|
||||
"""Create Voyage AI embedder (commonly used with Anthropic LLM)."""
|
||||
try:
|
||||
from graphiti_core.embedder.voyage import VoyageAIConfig, VoyageEmbedder
|
||||
except ImportError as e:
|
||||
raise ProviderNotInstalled(
|
||||
f"Voyage embedder requires graphiti-core[voyage]. "
|
||||
f"Install with: pip install graphiti-core[voyage]\n"
|
||||
f"Error: {e}"
|
||||
)
|
||||
|
||||
if not config.voyage_api_key:
|
||||
raise ProviderError("Voyage embedder requires VOYAGE_API_KEY")
|
||||
|
||||
voyage_config = VoyageAIConfig(
|
||||
api_key=config.voyage_api_key,
|
||||
embedding_model=config.voyage_embedding_model,
|
||||
)
|
||||
|
||||
return VoyageEmbedder(config=voyage_config)
|
||||
|
||||
|
||||
def _create_azure_openai_embedder(config: "GraphitiConfig") -> Any:
|
||||
"""Create Azure OpenAI embedder."""
|
||||
try:
|
||||
from graphiti_core.embedder.azure_openai import AzureOpenAIEmbedderClient
|
||||
from openai import AsyncOpenAI
|
||||
except ImportError as e:
|
||||
raise ProviderNotInstalled(
|
||||
f"Azure OpenAI embedder requires graphiti-core and openai. "
|
||||
f"Install with: pip install graphiti-core openai\n"
|
||||
f"Error: {e}"
|
||||
)
|
||||
|
||||
if not config.azure_openai_api_key:
|
||||
raise ProviderError("Azure OpenAI embedder requires AZURE_OPENAI_API_KEY")
|
||||
if not config.azure_openai_base_url:
|
||||
raise ProviderError("Azure OpenAI embedder requires AZURE_OPENAI_BASE_URL")
|
||||
if not config.azure_openai_embedding_deployment:
|
||||
raise ProviderError(
|
||||
"Azure OpenAI embedder requires AZURE_OPENAI_EMBEDDING_DEPLOYMENT"
|
||||
)
|
||||
|
||||
azure_client = AsyncOpenAI(
|
||||
base_url=config.azure_openai_base_url,
|
||||
api_key=config.azure_openai_api_key,
|
||||
)
|
||||
|
||||
return AzureOpenAIEmbedderClient(
|
||||
azure_client=azure_client,
|
||||
model=config.azure_openai_embedding_deployment,
|
||||
)
|
||||
|
||||
|
||||
def _create_ollama_embedder(config: "GraphitiConfig") -> Any:
|
||||
"""Create Ollama embedder (using OpenAI-compatible interface)."""
|
||||
try:
|
||||
from graphiti_core.embedder.openai import OpenAIEmbedder, OpenAIEmbedderConfig
|
||||
except ImportError as e:
|
||||
raise ProviderNotInstalled(
|
||||
f"Ollama embedder requires graphiti-core. "
|
||||
f"Install with: pip install graphiti-core\n"
|
||||
f"Error: {e}"
|
||||
)
|
||||
|
||||
if not config.ollama_embedding_model:
|
||||
raise ProviderError("Ollama embedder requires OLLAMA_EMBEDDING_MODEL")
|
||||
|
||||
# Ensure Ollama base URL ends with /v1 for OpenAI compatibility
|
||||
base_url = config.ollama_base_url
|
||||
if not base_url.endswith("/v1"):
|
||||
base_url = base_url.rstrip("/") + "/v1"
|
||||
|
||||
embedder_config = OpenAIEmbedderConfig(
|
||||
api_key="ollama", # Ollama requires a dummy API key
|
||||
embedding_model=config.ollama_embedding_model,
|
||||
embedding_dim=config.ollama_embedding_dim,
|
||||
base_url=base_url,
|
||||
)
|
||||
|
||||
return OpenAIEmbedder(config=embedder_config)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Cross-Encoder / Reranker Factory (Optional)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def create_cross_encoder(
|
||||
config: "GraphitiConfig", llm_client: Any = None
|
||||
) -> Any | None:
|
||||
"""
|
||||
Create a cross-encoder/reranker for improved search quality.
|
||||
|
||||
This is optional and primarily useful for Ollama setups.
|
||||
Other providers typically have built-in reranking.
|
||||
|
||||
Args:
|
||||
config: GraphitiConfig with provider settings
|
||||
llm_client: Optional LLM client for reranking
|
||||
|
||||
Returns:
|
||||
Cross-encoder instance, or None if not applicable
|
||||
"""
|
||||
# Only create for Ollama provider currently
|
||||
if config.llm_provider != "ollama":
|
||||
return None
|
||||
|
||||
if llm_client is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
from graphiti_core.cross_encoder.openai_reranker_client import (
|
||||
OpenAIRerankerClient,
|
||||
)
|
||||
from graphiti_core.llm_client.config import LLMConfig
|
||||
except ImportError:
|
||||
logger.debug("Cross-encoder not available (optional)")
|
||||
return None
|
||||
|
||||
try:
|
||||
# Create LLM config for reranker
|
||||
base_url = config.ollama_base_url
|
||||
if not base_url.endswith("/v1"):
|
||||
base_url = base_url.rstrip("/") + "/v1"
|
||||
|
||||
llm_config = LLMConfig(
|
||||
api_key="ollama",
|
||||
model=config.ollama_llm_model,
|
||||
base_url=base_url,
|
||||
)
|
||||
|
||||
return OpenAIRerankerClient(client=llm_client, config=llm_config)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not create cross-encoder: {e}")
|
||||
return None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Embedding Dimension Validation
|
||||
# ============================================================================
|
||||
|
||||
# Known embedding dimensions by provider and model
|
||||
EMBEDDING_DIMENSIONS = {
|
||||
# OpenAI
|
||||
"text-embedding-3-small": 1536,
|
||||
"text-embedding-3-large": 3072,
|
||||
"text-embedding-ada-002": 1536,
|
||||
# Voyage AI
|
||||
"voyage-3": 1024,
|
||||
"voyage-3.5": 1024,
|
||||
"voyage-3-lite": 512,
|
||||
"voyage-3.5-lite": 512,
|
||||
"voyage-2": 1024,
|
||||
"voyage-large-2": 1536,
|
||||
# Ollama (common models)
|
||||
"nomic-embed-text": 768,
|
||||
"mxbai-embed-large": 1024,
|
||||
"all-minilm": 384,
|
||||
"snowflake-arctic-embed": 1024,
|
||||
}
|
||||
|
||||
|
||||
def get_expected_embedding_dim(model: str) -> int | None:
|
||||
"""
|
||||
Get the expected embedding dimension for a known model.
|
||||
|
||||
Args:
|
||||
model: Embedding model name
|
||||
|
||||
Returns:
|
||||
Expected dimension, or None if unknown
|
||||
"""
|
||||
# Try exact match first
|
||||
if model in EMBEDDING_DIMENSIONS:
|
||||
return EMBEDDING_DIMENSIONS[model]
|
||||
|
||||
# Try partial match (model name might have version suffix)
|
||||
model_lower = model.lower()
|
||||
for known_model, dim in EMBEDDING_DIMENSIONS.items():
|
||||
if known_model.lower() in model_lower or model_lower in known_model.lower():
|
||||
return dim
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def validate_embedding_config(config: "GraphitiConfig") -> tuple[bool, str]:
|
||||
"""
|
||||
Validate embedding configuration for consistency.
|
||||
|
||||
Checks that embedding dimensions are correctly configured,
|
||||
especially important for Ollama where explicit dimension is required.
|
||||
|
||||
Args:
|
||||
config: GraphitiConfig to validate
|
||||
|
||||
Returns:
|
||||
Tuple of (is_valid, message)
|
||||
"""
|
||||
provider = config.embedder_provider
|
||||
|
||||
if provider == "ollama":
|
||||
# Ollama requires explicit embedding dimension
|
||||
if not config.ollama_embedding_dim:
|
||||
expected = get_expected_embedding_dim(config.ollama_embedding_model)
|
||||
if expected:
|
||||
return False, (
|
||||
f"Ollama embedder requires OLLAMA_EMBEDDING_DIM. "
|
||||
f"For model '{config.ollama_embedding_model}', "
|
||||
f"expected dimension is {expected}."
|
||||
)
|
||||
else:
|
||||
return False, (
|
||||
"Ollama embedder requires OLLAMA_EMBEDDING_DIM. "
|
||||
"Check your model's documentation for the correct dimension."
|
||||
)
|
||||
|
||||
# Check for known dimension mismatches
|
||||
if provider == "openai":
|
||||
expected = get_expected_embedding_dim(config.openai_embedding_model)
|
||||
# OpenAI handles this automatically, just log info
|
||||
if expected:
|
||||
logger.debug(
|
||||
f"OpenAI embedding model '{config.openai_embedding_model}' has dimension {expected}"
|
||||
)
|
||||
|
||||
elif provider == "voyage":
|
||||
expected = get_expected_embedding_dim(config.voyage_embedding_model)
|
||||
if expected:
|
||||
logger.debug(
|
||||
f"Voyage embedding model '{config.voyage_embedding_model}' has dimension {expected}"
|
||||
)
|
||||
|
||||
return True, "Embedding configuration valid"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Provider Health Checks
|
||||
# ============================================================================
|
||||
|
||||
|
||||
async def test_llm_connection(config: "GraphitiConfig") -> tuple[bool, str]:
|
||||
"""
|
||||
Test if LLM provider is reachable.
|
||||
|
||||
Args:
|
||||
config: GraphitiConfig with provider settings
|
||||
|
||||
Returns:
|
||||
Tuple of (success, message)
|
||||
"""
|
||||
try:
|
||||
llm_client = create_llm_client(config)
|
||||
# Most clients don't have a ping method, so just verify creation succeeded
|
||||
return (
|
||||
True,
|
||||
f"LLM client created successfully for provider: {config.llm_provider}",
|
||||
)
|
||||
except ProviderNotInstalled as e:
|
||||
return False, str(e)
|
||||
except ProviderError as e:
|
||||
return False, str(e)
|
||||
except Exception as e:
|
||||
return False, f"Failed to create LLM client: {e}"
|
||||
|
||||
|
||||
async def test_embedder_connection(config: "GraphitiConfig") -> tuple[bool, str]:
|
||||
"""
|
||||
Test if embedder provider is reachable.
|
||||
|
||||
Args:
|
||||
config: GraphitiConfig with provider settings
|
||||
|
||||
Returns:
|
||||
Tuple of (success, message)
|
||||
"""
|
||||
# First validate config
|
||||
valid, msg = validate_embedding_config(config)
|
||||
if not valid:
|
||||
return False, msg
|
||||
|
||||
try:
|
||||
embedder = create_embedder(config)
|
||||
return (
|
||||
True,
|
||||
f"Embedder created successfully for provider: {config.embedder_provider}",
|
||||
)
|
||||
except ProviderNotInstalled as e:
|
||||
return False, str(e)
|
||||
except ProviderError as e:
|
||||
return False, str(e)
|
||||
except Exception as e:
|
||||
return False, f"Failed to create embedder: {e}"
|
||||
|
||||
|
||||
async def test_ollama_connection(
|
||||
base_url: str = "http://localhost:11434",
|
||||
) -> tuple[bool, str]:
|
||||
"""
|
||||
Test if Ollama server is running and reachable.
|
||||
|
||||
Args:
|
||||
base_url: Ollama server URL
|
||||
|
||||
Returns:
|
||||
Tuple of (success, message)
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
try:
|
||||
import aiohttp
|
||||
except ImportError:
|
||||
# Fall back to sync request
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
try:
|
||||
# Normalize URL (remove /v1 suffix if present)
|
||||
url = base_url.rstrip("/")
|
||||
if url.endswith("/v1"):
|
||||
url = url[:-3]
|
||||
|
||||
req = urllib.request.Request(f"{url}/api/tags", method="GET")
|
||||
with urllib.request.urlopen(req, timeout=5) as response:
|
||||
if response.status == 200:
|
||||
return True, f"Ollama is running at {url}"
|
||||
return False, f"Ollama returned status {response.status}"
|
||||
except urllib.error.URLError as e:
|
||||
return False, f"Cannot connect to Ollama at {url}: {e.reason}"
|
||||
except Exception as e:
|
||||
return False, f"Ollama connection error: {e}"
|
||||
|
||||
# Use aiohttp if available
|
||||
try:
|
||||
# Normalize URL
|
||||
url = base_url.rstrip("/")
|
||||
if url.endswith("/v1"):
|
||||
url = url[:-3]
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(
|
||||
f"{url}/api/tags", timeout=aiohttp.ClientTimeout(total=5)
|
||||
) as response:
|
||||
if response.status == 200:
|
||||
return True, f"Ollama is running at {url}"
|
||||
return False, f"Ollama returned status {response.status}"
|
||||
except asyncio.TimeoutError:
|
||||
return False, f"Ollama connection timed out at {url}"
|
||||
except aiohttp.ClientError as e:
|
||||
return False, f"Cannot connect to Ollama at {url}: {e}"
|
||||
except Exception as e:
|
||||
return False, f"Ollama connection error: {e}"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Re-exports and Convenience Functions
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def is_graphiti_enabled() -> bool:
|
||||
"""
|
||||
Check if Graphiti memory integration is available and configured.
|
||||
|
||||
This is a convenience re-export from graphiti_config.
|
||||
Returns True if GRAPHITI_ENABLED=true and provider credentials are valid.
|
||||
"""
|
||||
from graphiti_config import is_graphiti_enabled as _is_graphiti_enabled
|
||||
|
||||
return _is_graphiti_enabled()
|
||||
|
||||
|
||||
async def get_graph_hints(
|
||||
query: str,
|
||||
project_id: str,
|
||||
max_results: int = 10,
|
||||
spec_dir: Optional["Path"] = None,
|
||||
) -> list[dict]:
|
||||
"""
|
||||
Get relevant hints from the Graphiti knowledge graph.
|
||||
|
||||
This is a convenience function for querying historical context
|
||||
from the memory system. Used by spec_runner, ideation_runner,
|
||||
and roadmap_runner to inject historical insights.
|
||||
|
||||
Args:
|
||||
query: Search query (e.g., "authentication patterns", "API design")
|
||||
project_id: Project identifier for scoping results
|
||||
max_results: Maximum number of hints to return
|
||||
spec_dir: Optional spec directory for loading memory instance
|
||||
|
||||
Returns:
|
||||
List of hint dictionaries with keys:
|
||||
- content: str - The hint content
|
||||
- score: float - Relevance score
|
||||
- type: str - Type of hint (pattern, gotcha, outcome, etc.)
|
||||
|
||||
Note:
|
||||
Returns empty list if Graphiti is not enabled or unavailable.
|
||||
This function never raises - it always fails gracefully.
|
||||
"""
|
||||
if not is_graphiti_enabled():
|
||||
logger.debug("Graphiti not enabled, returning empty hints")
|
||||
return []
|
||||
|
||||
try:
|
||||
from pathlib import Path
|
||||
|
||||
from graphiti_memory import GraphitiMemory, GroupIdMode
|
||||
|
||||
# Determine project directory from project_id or use current dir
|
||||
project_dir = Path.cwd()
|
||||
|
||||
# Use spec_dir if provided, otherwise create a temp context
|
||||
if spec_dir is None:
|
||||
# Create a temporary spec dir for the query
|
||||
import tempfile
|
||||
|
||||
spec_dir = Path(tempfile.mkdtemp(prefix="graphiti_query_"))
|
||||
|
||||
# Create memory instance with project-level scope for cross-spec hints
|
||||
memory = GraphitiMemory(
|
||||
spec_dir=spec_dir,
|
||||
project_dir=project_dir,
|
||||
group_id_mode=GroupIdMode.PROJECT,
|
||||
)
|
||||
|
||||
# Query for relevant context
|
||||
hints = await memory.get_relevant_context(
|
||||
query=query,
|
||||
num_results=max_results,
|
||||
include_project_context=True,
|
||||
)
|
||||
|
||||
await memory.close()
|
||||
|
||||
logger.info(f"Retrieved {len(hints)} graph hints for query: {query[:50]}...")
|
||||
return hints
|
||||
|
||||
except ImportError as e:
|
||||
logger.debug(f"Graphiti packages not available: {e}")
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to get graph hints: {e}")
|
||||
return []
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,168 +1,3 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Implementation Plan Manager
|
||||
============================
|
||||
|
||||
DEPRECATED: This module is now a compatibility shim. The implementation has been
|
||||
refactored into the implementation_plan/ package for better modularity.
|
||||
|
||||
Please import from the package directly:
|
||||
from implementation_plan import ImplementationPlan, Subtask, Phase, etc.
|
||||
|
||||
This file re-exports all public APIs for backwards compatibility.
|
||||
|
||||
Core data structures and utilities for subtask-based implementation plans.
|
||||
Replaces the test-centric feature_list.json with implementation_plan.json.
|
||||
|
||||
The key insight: Tests verify outcomes, but SUBTASKS define implementation steps.
|
||||
For complex multi-service features, implementation order matters.
|
||||
|
||||
Workflow Types:
|
||||
- feature: Standard multi-service feature (phases = services)
|
||||
- refactor: Migration/refactor work (phases = stages: add, migrate, remove)
|
||||
- investigation: Bug hunting (phases = investigate, hypothesize, fix)
|
||||
- migration: Data migration (phases = prepare, test, execute, cleanup)
|
||||
- simple: Single-service enhancement (minimal overhead)
|
||||
"""
|
||||
|
||||
# Re-export everything from the implementation_plan package
|
||||
from implementation_plan import (
|
||||
Chunk,
|
||||
ChunkStatus,
|
||||
ImplementationPlan,
|
||||
Phase,
|
||||
PhaseType,
|
||||
Subtask,
|
||||
SubtaskStatus,
|
||||
Verification,
|
||||
VerificationType,
|
||||
WorkflowType,
|
||||
create_feature_plan,
|
||||
create_investigation_plan,
|
||||
create_refactor_plan,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# Enums
|
||||
"WorkflowType",
|
||||
"PhaseType",
|
||||
"SubtaskStatus",
|
||||
"VerificationType",
|
||||
# Models
|
||||
"Verification",
|
||||
"Subtask",
|
||||
"Phase",
|
||||
"ImplementationPlan",
|
||||
# Factories
|
||||
"create_feature_plan",
|
||||
"create_investigation_plan",
|
||||
"create_refactor_plan",
|
||||
# Backwards compatibility
|
||||
"Chunk",
|
||||
"ChunkStatus",
|
||||
]
|
||||
|
||||
|
||||
# CLI for testing
|
||||
if __name__ == "__main__":
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python implementation_plan.py <plan.json>")
|
||||
print(" python implementation_plan.py --demo")
|
||||
sys.exit(1)
|
||||
|
||||
if sys.argv[1] == "--demo":
|
||||
# Create a demo plan
|
||||
plan = create_feature_plan(
|
||||
feature="Avatar Upload with Processing",
|
||||
services=["backend", "worker", "frontend"],
|
||||
phases_config=[
|
||||
{
|
||||
"name": "Backend Foundation",
|
||||
"parallel_safe": True,
|
||||
"subtasks": [
|
||||
{
|
||||
"id": "avatar-model",
|
||||
"service": "backend",
|
||||
"description": "Add avatar fields to User model",
|
||||
"files_to_modify": ["app/models/user.py"],
|
||||
"files_to_create": ["migrations/add_avatar.py"],
|
||||
"verification": {
|
||||
"type": "command",
|
||||
"run": "flask db upgrade",
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "avatar-endpoint",
|
||||
"service": "backend",
|
||||
"description": "POST /api/users/avatar endpoint",
|
||||
"files_to_modify": ["app/routes/users.py"],
|
||||
"patterns_from": ["app/routes/profile.py"],
|
||||
"verification": {
|
||||
"type": "api",
|
||||
"method": "POST",
|
||||
"url": "/api/users/avatar",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "Worker Pipeline",
|
||||
"depends_on": [1],
|
||||
"subtasks": [
|
||||
{
|
||||
"id": "image-task",
|
||||
"service": "worker",
|
||||
"description": "Celery task for image processing",
|
||||
"files_to_create": ["app/tasks/images.py"],
|
||||
"patterns_from": ["app/tasks/reports.py"],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "Frontend",
|
||||
"depends_on": [1],
|
||||
"subtasks": [
|
||||
{
|
||||
"id": "avatar-component",
|
||||
"service": "frontend",
|
||||
"description": "AvatarUpload React component",
|
||||
"files_to_create": ["src/components/AvatarUpload.tsx"],
|
||||
"patterns_from": ["src/components/FileUpload.tsx"],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "Integration",
|
||||
"depends_on": [2, 3],
|
||||
"type": "integration",
|
||||
"subtasks": [
|
||||
{
|
||||
"id": "e2e-wiring",
|
||||
"all_services": True,
|
||||
"description": "Connect frontend → backend → worker",
|
||||
"verification": {
|
||||
"type": "browser",
|
||||
"scenario": "Upload → Process → Display",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
plan.final_acceptance = [
|
||||
"User can upload avatar from profile page",
|
||||
"Avatar is automatically resized",
|
||||
"Large/invalid files show error",
|
||||
]
|
||||
|
||||
print(json.dumps(plan.to_dict(), indent=2))
|
||||
print("\n---\n")
|
||||
print(plan.get_status_summary())
|
||||
else:
|
||||
# Load and display existing plan
|
||||
plan = ImplementationPlan.load(Path(sys.argv[1]))
|
||||
print(plan.get_status_summary())
|
||||
"""Backward compatibility shim - import from implementation_plan package instead."""
|
||||
from implementation_plan.main import *
|
||||
from implementation_plan import *
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Implementation Plan Manager
|
||||
============================
|
||||
|
||||
DEPRECATED: This module is now a compatibility shim. The implementation has been
|
||||
refactored into the implementation_plan/ package for better modularity.
|
||||
|
||||
Please import from the package directly:
|
||||
from implementation_plan import ImplementationPlan, Subtask, Phase, etc.
|
||||
|
||||
This file re-exports all public APIs for backwards compatibility.
|
||||
|
||||
Core data structures and utilities for subtask-based implementation plans.
|
||||
Replaces the test-centric feature_list.json with implementation_plan.json.
|
||||
|
||||
The key insight: Tests verify outcomes, but SUBTASKS define implementation steps.
|
||||
For complex multi-service features, implementation order matters.
|
||||
|
||||
Workflow Types:
|
||||
- feature: Standard multi-service feature (phases = services)
|
||||
- refactor: Migration/refactor work (phases = stages: add, migrate, remove)
|
||||
- investigation: Bug hunting (phases = investigate, hypothesize, fix)
|
||||
- migration: Data migration (phases = prepare, test, execute, cleanup)
|
||||
- simple: Single-service enhancement (minimal overhead)
|
||||
"""
|
||||
|
||||
# Re-export everything from the implementation_plan package
|
||||
from implementation_plan import (
|
||||
Chunk,
|
||||
ChunkStatus,
|
||||
ImplementationPlan,
|
||||
Phase,
|
||||
PhaseType,
|
||||
Subtask,
|
||||
SubtaskStatus,
|
||||
Verification,
|
||||
VerificationType,
|
||||
WorkflowType,
|
||||
create_feature_plan,
|
||||
create_investigation_plan,
|
||||
create_refactor_plan,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# Enums
|
||||
"WorkflowType",
|
||||
"PhaseType",
|
||||
"SubtaskStatus",
|
||||
"VerificationType",
|
||||
# Models
|
||||
"Verification",
|
||||
"Subtask",
|
||||
"Phase",
|
||||
"ImplementationPlan",
|
||||
# Factories
|
||||
"create_feature_plan",
|
||||
"create_investigation_plan",
|
||||
"create_refactor_plan",
|
||||
# Backwards compatibility
|
||||
"Chunk",
|
||||
"ChunkStatus",
|
||||
]
|
||||
|
||||
|
||||
# CLI for testing
|
||||
if __name__ == "__main__":
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python implementation_plan.py <plan.json>")
|
||||
print(" python implementation_plan.py --demo")
|
||||
sys.exit(1)
|
||||
|
||||
if sys.argv[1] == "--demo":
|
||||
# Create a demo plan
|
||||
plan = create_feature_plan(
|
||||
feature="Avatar Upload with Processing",
|
||||
services=["backend", "worker", "frontend"],
|
||||
phases_config=[
|
||||
{
|
||||
"name": "Backend Foundation",
|
||||
"parallel_safe": True,
|
||||
"subtasks": [
|
||||
{
|
||||
"id": "avatar-model",
|
||||
"service": "backend",
|
||||
"description": "Add avatar fields to User model",
|
||||
"files_to_modify": ["app/models/user.py"],
|
||||
"files_to_create": ["migrations/add_avatar.py"],
|
||||
"verification": {
|
||||
"type": "command",
|
||||
"run": "flask db upgrade",
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "avatar-endpoint",
|
||||
"service": "backend",
|
||||
"description": "POST /api/users/avatar endpoint",
|
||||
"files_to_modify": ["app/routes/users.py"],
|
||||
"patterns_from": ["app/routes/profile.py"],
|
||||
"verification": {
|
||||
"type": "api",
|
||||
"method": "POST",
|
||||
"url": "/api/users/avatar",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "Worker Pipeline",
|
||||
"depends_on": [1],
|
||||
"subtasks": [
|
||||
{
|
||||
"id": "image-task",
|
||||
"service": "worker",
|
||||
"description": "Celery task for image processing",
|
||||
"files_to_create": ["app/tasks/images.py"],
|
||||
"patterns_from": ["app/tasks/reports.py"],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "Frontend",
|
||||
"depends_on": [1],
|
||||
"subtasks": [
|
||||
{
|
||||
"id": "avatar-component",
|
||||
"service": "frontend",
|
||||
"description": "AvatarUpload React component",
|
||||
"files_to_create": ["src/components/AvatarUpload.tsx"],
|
||||
"patterns_from": ["src/components/FileUpload.tsx"],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "Integration",
|
||||
"depends_on": [2, 3],
|
||||
"type": "integration",
|
||||
"subtasks": [
|
||||
{
|
||||
"id": "e2e-wiring",
|
||||
"all_services": True,
|
||||
"description": "Connect frontend → backend → worker",
|
||||
"verification": {
|
||||
"type": "browser",
|
||||
"scenario": "Upload → Process → Display",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
plan.final_acceptance = [
|
||||
"User can upload avatar from profile page",
|
||||
"Avatar is automatically resized",
|
||||
"Large/invalid files show error",
|
||||
]
|
||||
|
||||
print(json.dumps(plan.to_dict(), indent=2))
|
||||
print("\n---\n")
|
||||
print(plan.get_status_summary())
|
||||
else:
|
||||
# Load and display existing plan
|
||||
plan = ImplementationPlan.load(Path(sys.argv[1]))
|
||||
print(plan.get_status_summary())
|
||||
@@ -1,584 +1,41 @@
|
||||
"""
|
||||
Insight Extractor
|
||||
=================
|
||||
Insight Extractor Re-export
|
||||
===========================
|
||||
|
||||
Automatically extracts structured insights from completed coding sessions.
|
||||
Runs after each session to capture rich, actionable knowledge for Graphiti memory.
|
||||
|
||||
Uses the Claude Agent SDK (same as the rest of the system) for extraction.
|
||||
Falls back to generic insights if extraction fails (never blocks the build).
|
||||
Re-exports the insight_extractor module from analysis/ for backwards compatibility.
|
||||
Uses importlib to avoid triggering analysis/__init__.py imports.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Check for Claude SDK availability
|
||||
try:
|
||||
from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient
|
||||
SDK_AVAILABLE = True
|
||||
except ImportError:
|
||||
SDK_AVAILABLE = False
|
||||
ClaudeAgentOptions = None
|
||||
ClaudeSDKClient = None
|
||||
|
||||
# Default model for insight extraction (fast and cheap)
|
||||
DEFAULT_EXTRACTION_MODEL = "claude-3-5-haiku-latest"
|
||||
|
||||
# Maximum diff size to send to the LLM (avoid context limits)
|
||||
MAX_DIFF_CHARS = 15000
|
||||
|
||||
# Maximum attempt history entries to include
|
||||
MAX_ATTEMPTS_TO_INCLUDE = 3
|
||||
|
||||
|
||||
def is_extraction_enabled() -> bool:
|
||||
"""Check if insight extraction is enabled."""
|
||||
# Extraction requires Claude SDK and OAuth token
|
||||
if not SDK_AVAILABLE:
|
||||
return False
|
||||
if not os.environ.get("CLAUDE_CODE_OAUTH_TOKEN"):
|
||||
return False
|
||||
enabled_str = os.environ.get("INSIGHT_EXTRACTION_ENABLED", "true").lower()
|
||||
return enabled_str in ("true", "1", "yes")
|
||||
|
||||
|
||||
def get_extraction_model() -> str:
|
||||
"""Get the model to use for insight extraction."""
|
||||
return os.environ.get("INSIGHT_EXTRACTOR_MODEL", DEFAULT_EXTRACTION_MODEL)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Git Helpers
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def get_session_diff(
|
||||
project_dir: Path,
|
||||
commit_before: str | None,
|
||||
commit_after: str | None,
|
||||
) -> str:
|
||||
"""
|
||||
Get the git diff between two commits.
|
||||
|
||||
Args:
|
||||
project_dir: Project root directory
|
||||
commit_before: Commit hash before session (or None)
|
||||
commit_after: Commit hash after session (or None)
|
||||
|
||||
Returns:
|
||||
Diff text (truncated if too large)
|
||||
"""
|
||||
if not commit_before or not commit_after:
|
||||
return "(No commits to diff)"
|
||||
|
||||
if commit_before == commit_after:
|
||||
return "(No changes - same commit)"
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "diff", commit_before, commit_after],
|
||||
cwd=project_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
diff = result.stdout
|
||||
|
||||
if len(diff) > MAX_DIFF_CHARS:
|
||||
# Truncate and add note
|
||||
diff = (
|
||||
diff[:MAX_DIFF_CHARS] + f"\n\n... (truncated, {len(diff)} chars total)"
|
||||
)
|
||||
|
||||
return diff if diff else "(Empty diff)"
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("Git diff timed out")
|
||||
return "(Git diff timed out)"
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to get git diff: {e}")
|
||||
return f"(Failed to get diff: {e})"
|
||||
|
||||
|
||||
def get_changed_files(
|
||||
project_dir: Path,
|
||||
commit_before: str | None,
|
||||
commit_after: str | None,
|
||||
) -> list[str]:
|
||||
"""
|
||||
Get list of files changed between two commits.
|
||||
|
||||
Args:
|
||||
project_dir: Project root directory
|
||||
commit_before: Commit hash before session
|
||||
commit_after: Commit hash after session
|
||||
|
||||
Returns:
|
||||
List of changed file paths
|
||||
"""
|
||||
if not commit_before or not commit_after or commit_before == commit_after:
|
||||
return []
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "diff", "--name-only", commit_before, commit_after],
|
||||
cwd=project_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
files = [f.strip() for f in result.stdout.strip().split("\n") if f.strip()]
|
||||
return files
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to get changed files: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def get_commit_messages(
|
||||
project_dir: Path,
|
||||
commit_before: str | None,
|
||||
commit_after: str | None,
|
||||
) -> str:
|
||||
"""Get commit messages between two commits."""
|
||||
if not commit_before or not commit_after or commit_before == commit_after:
|
||||
return "(No commits)"
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "log", "--oneline", f"{commit_before}..{commit_after}"],
|
||||
cwd=project_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
return result.stdout.strip() if result.stdout.strip() else "(No commits)"
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to get commit messages: {e}")
|
||||
return f"(Failed: {e})"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Input Gathering
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def gather_extraction_inputs(
|
||||
spec_dir: Path,
|
||||
project_dir: Path,
|
||||
subtask_id: str,
|
||||
session_num: int,
|
||||
commit_before: str | None,
|
||||
commit_after: str | None,
|
||||
success: bool,
|
||||
recovery_manager: Any,
|
||||
) -> dict:
|
||||
"""
|
||||
Gather all inputs needed for insight extraction.
|
||||
|
||||
Args:
|
||||
spec_dir: Spec directory
|
||||
project_dir: Project root
|
||||
subtask_id: The subtask that was worked on
|
||||
session_num: Session number
|
||||
commit_before: Commit before session
|
||||
commit_after: Commit after session
|
||||
success: Whether session succeeded
|
||||
recovery_manager: Recovery manager with attempt history
|
||||
|
||||
Returns:
|
||||
Dict with all inputs for the extractor
|
||||
"""
|
||||
# Get subtask description from implementation plan
|
||||
subtask_description = _get_subtask_description(spec_dir, subtask_id)
|
||||
|
||||
# Get git diff
|
||||
diff = get_session_diff(project_dir, commit_before, commit_after)
|
||||
|
||||
# Get changed files
|
||||
changed_files = get_changed_files(project_dir, commit_before, commit_after)
|
||||
|
||||
# Get commit messages
|
||||
commit_messages = get_commit_messages(project_dir, commit_before, commit_after)
|
||||
|
||||
# Get attempt history
|
||||
attempt_history = _get_attempt_history(recovery_manager, subtask_id)
|
||||
|
||||
return {
|
||||
"subtask_id": subtask_id,
|
||||
"subtask_description": subtask_description,
|
||||
"session_num": session_num,
|
||||
"success": success,
|
||||
"diff": diff,
|
||||
"changed_files": changed_files,
|
||||
"commit_messages": commit_messages,
|
||||
"attempt_history": attempt_history,
|
||||
}
|
||||
|
||||
|
||||
def _get_subtask_description(spec_dir: Path, subtask_id: str) -> str:
|
||||
"""Get subtask description from implementation plan."""
|
||||
plan_file = spec_dir / "implementation_plan.json"
|
||||
if not plan_file.exists():
|
||||
return f"Subtask: {subtask_id}"
|
||||
|
||||
try:
|
||||
with open(plan_file) as f:
|
||||
plan = json.load(f)
|
||||
|
||||
# Search through phases for the subtask
|
||||
for phase in plan.get("phases", []):
|
||||
for subtask in phase.get("subtasks", []):
|
||||
if subtask.get("id") == subtask_id:
|
||||
return subtask.get("description", f"Subtask: {subtask_id}")
|
||||
|
||||
return f"Subtask: {subtask_id}"
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load subtask description: {e}")
|
||||
return f"Subtask: {subtask_id}"
|
||||
|
||||
|
||||
def _get_attempt_history(recovery_manager: Any, subtask_id: str) -> list[dict]:
|
||||
"""Get previous attempt history for this subtask."""
|
||||
if not recovery_manager:
|
||||
return []
|
||||
|
||||
try:
|
||||
history = recovery_manager.get_subtask_history(subtask_id)
|
||||
attempts = history.get("attempts", [])
|
||||
|
||||
# Limit to recent attempts
|
||||
return attempts[-MAX_ATTEMPTS_TO_INCLUDE:]
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to get attempt history: {e}")
|
||||
return []
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# LLM Extraction
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def _build_extraction_prompt(inputs: dict) -> str:
|
||||
"""Build the prompt for insight extraction."""
|
||||
prompt_file = Path(__file__).parent / "prompts" / "insight_extractor.md"
|
||||
|
||||
if prompt_file.exists():
|
||||
base_prompt = prompt_file.read_text()
|
||||
else:
|
||||
# Fallback if prompt file missing
|
||||
base_prompt = """Extract structured insights from this coding session.
|
||||
Output ONLY valid JSON with: file_insights, patterns_discovered, gotchas_discovered, approach_outcome, recommendations"""
|
||||
|
||||
# Build session context
|
||||
session_context = f"""
|
||||
---
|
||||
|
||||
## SESSION DATA
|
||||
|
||||
### Subtask
|
||||
- **ID**: {inputs["subtask_id"]}
|
||||
- **Description**: {inputs["subtask_description"]}
|
||||
- **Session Number**: {inputs["session_num"]}
|
||||
- **Outcome**: {"SUCCESS" if inputs["success"] else "FAILED"}
|
||||
|
||||
### Files Changed
|
||||
{chr(10).join(f"- {f}" for f in inputs["changed_files"]) if inputs["changed_files"] else "(No files changed)"}
|
||||
|
||||
### Commit Messages
|
||||
{inputs["commit_messages"]}
|
||||
|
||||
### Git Diff
|
||||
```diff
|
||||
{inputs["diff"]}
|
||||
```
|
||||
|
||||
### Previous Attempts
|
||||
{_format_attempt_history(inputs["attempt_history"])}
|
||||
|
||||
---
|
||||
|
||||
Now analyze this session and output ONLY the JSON object.
|
||||
"""
|
||||
|
||||
return base_prompt + session_context
|
||||
|
||||
|
||||
def _format_attempt_history(attempts: list[dict]) -> str:
|
||||
"""Format attempt history for the prompt."""
|
||||
if not attempts:
|
||||
return "(First attempt - no previous history)"
|
||||
|
||||
lines = []
|
||||
for i, attempt in enumerate(attempts, 1):
|
||||
success = "SUCCESS" if attempt.get("success") else "FAILED"
|
||||
approach = attempt.get("approach", "Unknown approach")
|
||||
error = attempt.get("error", "")
|
||||
lines.append(f"**Attempt {i}** ({success}): {approach}")
|
||||
if error:
|
||||
lines.append(f" Error: {error}")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
async def run_insight_extraction(inputs: dict, project_dir: Path | None = None) -> dict | None:
|
||||
"""
|
||||
Run the insight extraction using Claude Agent SDK.
|
||||
|
||||
Args:
|
||||
inputs: Gathered session inputs
|
||||
project_dir: Project directory for SDK context (optional)
|
||||
|
||||
Returns:
|
||||
Extracted insights dict or None if failed
|
||||
"""
|
||||
if not SDK_AVAILABLE:
|
||||
logger.warning("Claude SDK not available, skipping insight extraction")
|
||||
return None
|
||||
|
||||
oauth_token = os.environ.get("CLAUDE_CODE_OAUTH_TOKEN")
|
||||
if not oauth_token:
|
||||
logger.warning("CLAUDE_CODE_OAUTH_TOKEN not set, skipping insight extraction")
|
||||
return None
|
||||
|
||||
model = get_extraction_model()
|
||||
prompt = _build_extraction_prompt(inputs)
|
||||
|
||||
# Use current directory if project_dir not specified
|
||||
cwd = str(project_dir.resolve()) if project_dir else os.getcwd()
|
||||
|
||||
try:
|
||||
# Create a minimal SDK client for insight extraction
|
||||
# No tools needed - just text generation
|
||||
client = ClaudeSDKClient(
|
||||
options=ClaudeAgentOptions(
|
||||
model=model,
|
||||
system_prompt=(
|
||||
"You are an expert code analyst. You extract structured insights from coding sessions. "
|
||||
"Always respond with valid JSON only, no markdown formatting or explanations."
|
||||
),
|
||||
allowed_tools=[], # No tools needed for extraction
|
||||
max_turns=1, # Single turn extraction
|
||||
cwd=cwd,
|
||||
)
|
||||
)
|
||||
|
||||
# Use async context manager
|
||||
async with client:
|
||||
await client.query(prompt)
|
||||
|
||||
# Collect the response
|
||||
response_text = ""
|
||||
async for msg in client.receive_response():
|
||||
msg_type = type(msg).__name__
|
||||
if msg_type == "AssistantMessage" and hasattr(msg, "content"):
|
||||
for block in msg.content:
|
||||
if hasattr(block, "text"):
|
||||
response_text += block.text
|
||||
|
||||
# Parse JSON from response
|
||||
return parse_insights(response_text)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Insight extraction failed: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def parse_insights(response_text: str) -> dict | None:
|
||||
"""
|
||||
Parse the LLM response into structured insights.
|
||||
|
||||
Args:
|
||||
response_text: Raw LLM response
|
||||
|
||||
Returns:
|
||||
Parsed insights dict or None if parsing failed
|
||||
"""
|
||||
# Try to extract JSON from the response
|
||||
text = response_text.strip()
|
||||
|
||||
# Handle markdown code blocks
|
||||
if text.startswith("```"):
|
||||
# Remove code block markers
|
||||
lines = text.split("\n")
|
||||
# Remove first line (```json or ```)
|
||||
if lines[0].startswith("```"):
|
||||
lines = lines[1:]
|
||||
# Remove last line if it's ``
|
||||
if lines and lines[-1].strip() == "```":
|
||||
lines = lines[:-1]
|
||||
text = "\n".join(lines)
|
||||
|
||||
try:
|
||||
insights = json.loads(text)
|
||||
|
||||
# Validate structure
|
||||
if not isinstance(insights, dict):
|
||||
logger.warning("Insights is not a dict")
|
||||
return None
|
||||
|
||||
# Ensure required keys exist with defaults
|
||||
insights.setdefault("file_insights", [])
|
||||
insights.setdefault("patterns_discovered", [])
|
||||
insights.setdefault("gotchas_discovered", [])
|
||||
insights.setdefault("approach_outcome", {})
|
||||
insights.setdefault("recommendations", [])
|
||||
|
||||
return insights
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning(f"Failed to parse insights JSON: {e}")
|
||||
logger.debug(f"Response text was: {text[:500]}")
|
||||
return None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Main Entry Point
|
||||
# =============================================================================
|
||||
|
||||
|
||||
async def extract_session_insights(
|
||||
spec_dir: Path,
|
||||
project_dir: Path,
|
||||
subtask_id: str,
|
||||
session_num: int,
|
||||
commit_before: str | None,
|
||||
commit_after: str | None,
|
||||
success: bool,
|
||||
recovery_manager: Any,
|
||||
) -> dict:
|
||||
"""
|
||||
Extract insights from a completed coding session.
|
||||
|
||||
This is the main entry point called from post_session_processing().
|
||||
Falls back to generic insights if extraction fails.
|
||||
|
||||
Args:
|
||||
spec_dir: Spec directory
|
||||
project_dir: Project root
|
||||
subtask_id: Subtask that was worked on
|
||||
session_num: Session number
|
||||
commit_before: Commit before session
|
||||
commit_after: Commit after session
|
||||
success: Whether session succeeded
|
||||
recovery_manager: Recovery manager with attempt history
|
||||
|
||||
Returns:
|
||||
Insights dict (rich if extraction succeeded, generic if failed)
|
||||
"""
|
||||
# Check if extraction is enabled
|
||||
if not is_extraction_enabled():
|
||||
logger.info("Insight extraction disabled")
|
||||
return _get_generic_insights(subtask_id, success)
|
||||
|
||||
# Check for no changes
|
||||
if commit_before == commit_after:
|
||||
logger.info("No changes to extract insights from")
|
||||
return _get_generic_insights(subtask_id, success)
|
||||
|
||||
try:
|
||||
# Gather inputs
|
||||
inputs = gather_extraction_inputs(
|
||||
spec_dir=spec_dir,
|
||||
project_dir=project_dir,
|
||||
subtask_id=subtask_id,
|
||||
session_num=session_num,
|
||||
commit_before=commit_before,
|
||||
commit_after=commit_after,
|
||||
success=success,
|
||||
recovery_manager=recovery_manager,
|
||||
)
|
||||
|
||||
# Run extraction
|
||||
extracted = await run_insight_extraction(inputs, project_dir=project_dir)
|
||||
|
||||
if extracted:
|
||||
# Add metadata
|
||||
extracted["subtask_id"] = subtask_id
|
||||
extracted["session_num"] = session_num
|
||||
extracted["success"] = success
|
||||
extracted["changed_files"] = inputs["changed_files"]
|
||||
|
||||
logger.info(
|
||||
f"Extracted insights: {len(extracted.get('file_insights', []))} file insights, "
|
||||
f"{len(extracted.get('patterns_discovered', []))} patterns, "
|
||||
f"{len(extracted.get('gotchas_discovered', []))} gotchas"
|
||||
)
|
||||
return extracted
|
||||
else:
|
||||
logger.warning("Extraction returned no results, using generic insights")
|
||||
return _get_generic_insights(subtask_id, success)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Insight extraction failed: {e}, using generic insights")
|
||||
return _get_generic_insights(subtask_id, success)
|
||||
|
||||
|
||||
def _get_generic_insights(subtask_id: str, success: bool) -> dict:
|
||||
"""Return generic insights when extraction fails or is disabled."""
|
||||
return {
|
||||
"file_insights": [],
|
||||
"patterns_discovered": [],
|
||||
"gotchas_discovered": [],
|
||||
"approach_outcome": {
|
||||
"success": success,
|
||||
"approach_used": f"Implemented subtask: {subtask_id}",
|
||||
"why_it_worked": None,
|
||||
"why_it_failed": None,
|
||||
"alternatives_tried": [],
|
||||
},
|
||||
"recommendations": [],
|
||||
"subtask_id": subtask_id,
|
||||
"success": success,
|
||||
"changed_files": [],
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLI for Testing
|
||||
# =============================================================================
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
import asyncio
|
||||
|
||||
parser = argparse.ArgumentParser(description="Test insight extraction")
|
||||
parser.add_argument("--spec-dir", type=Path, required=True, help="Spec directory")
|
||||
parser.add_argument(
|
||||
"--project-dir", type=Path, required=True, help="Project directory"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--commit-before", type=str, required=True, help="Commit before session"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--commit-after", type=str, required=True, help="Commit after session"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--subtask-id", type=str, default="test-subtask", help="Subtask ID"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
async def main():
|
||||
insights = await extract_session_insights(
|
||||
spec_dir=args.spec_dir,
|
||||
project_dir=args.project_dir,
|
||||
subtask_id=args.subtask_id,
|
||||
session_num=1,
|
||||
commit_before=args.commit_before,
|
||||
commit_after=args.commit_after,
|
||||
success=True,
|
||||
recovery_manager=None,
|
||||
)
|
||||
print(json.dumps(insights, indent=2))
|
||||
|
||||
asyncio.run(main())
|
||||
# Load the module directly without going through the package
|
||||
_module_path = Path(__file__).parent / "analysis" / "insight_extractor.py"
|
||||
_spec = importlib.util.spec_from_file_location("_insight_extractor_impl", _module_path)
|
||||
_module = importlib.util.module_from_spec(_spec)
|
||||
sys.modules["_insight_extractor_impl"] = _module
|
||||
_spec.loader.exec_module(_module)
|
||||
|
||||
# Re-export all public functions
|
||||
extract_session_insights = _module.extract_session_insights
|
||||
gather_extraction_inputs = _module.gather_extraction_inputs
|
||||
get_changed_files = _module.get_changed_files
|
||||
get_commit_messages = _module.get_commit_messages
|
||||
get_extraction_model = _module.get_extraction_model
|
||||
get_session_diff = _module.get_session_diff
|
||||
is_extraction_enabled = _module.is_extraction_enabled
|
||||
parse_insights = _module.parse_insights
|
||||
run_insight_extraction = _module.run_insight_extraction
|
||||
|
||||
__all__ = [
|
||||
"extract_session_insights",
|
||||
"gather_extraction_inputs",
|
||||
"get_changed_files",
|
||||
"get_commit_messages",
|
||||
"get_extraction_model",
|
||||
"get_session_diff",
|
||||
"is_extraction_enabled",
|
||||
"parse_insights",
|
||||
"run_insight_extraction",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
"""
|
||||
Integrations Module
|
||||
===================
|
||||
|
||||
External service integrations for Auto Claude.
|
||||
"""
|
||||
|
||||
__all__ = [
|
||||
"linear",
|
||||
"graphiti",
|
||||
]
|
||||
@@ -0,0 +1,35 @@
|
||||
"""
|
||||
Graphiti Integration
|
||||
====================
|
||||
|
||||
Integration with Graphiti knowledge graph for semantic memory.
|
||||
"""
|
||||
|
||||
# Config imports don't require graphiti package
|
||||
from .config import GraphitiConfig, validate_graphiti_config
|
||||
|
||||
# Lazy imports for components that require graphiti package
|
||||
__all__ = [
|
||||
"GraphitiConfig",
|
||||
"validate_graphiti_config",
|
||||
"GraphitiMemory",
|
||||
"create_llm_client",
|
||||
"create_embedder",
|
||||
]
|
||||
|
||||
|
||||
def __getattr__(name):
|
||||
"""Lazy import to avoid requiring graphiti package for config-only imports."""
|
||||
if name == "GraphitiMemory":
|
||||
from .memory import GraphitiMemory
|
||||
|
||||
return GraphitiMemory
|
||||
elif name == "create_llm_client":
|
||||
from .providers import create_llm_client
|
||||
|
||||
return create_llm_client
|
||||
elif name == "create_embedder":
|
||||
from .providers import create_embedder
|
||||
|
||||
return create_embedder
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
@@ -0,0 +1,547 @@
|
||||
"""
|
||||
Graphiti Integration Configuration
|
||||
==================================
|
||||
|
||||
Constants, status mappings, and configuration helpers for Graphiti memory integration.
|
||||
Follows the same patterns as linear_config.py for consistency.
|
||||
|
||||
Multi-Provider Support (V2):
|
||||
- LLM Providers: OpenAI, Anthropic, Azure OpenAI, Ollama
|
||||
- Embedder Providers: OpenAI, Voyage AI, Azure OpenAI, Ollama
|
||||
|
||||
Environment Variables:
|
||||
# Core
|
||||
GRAPHITI_ENABLED: Set to "true" to enable Graphiti integration
|
||||
GRAPHITI_LLM_PROVIDER: openai|anthropic|azure_openai|ollama (default: openai)
|
||||
GRAPHITI_EMBEDDER_PROVIDER: openai|voyage|azure_openai|ollama (default: openai)
|
||||
|
||||
# OpenAI
|
||||
OPENAI_API_KEY: Required for OpenAI provider
|
||||
OPENAI_MODEL: Model for LLM (default: gpt-5-mini)
|
||||
OPENAI_EMBEDDING_MODEL: Model for embeddings (default: text-embedding-3-small)
|
||||
|
||||
# Anthropic (LLM only - needs separate embedder)
|
||||
ANTHROPIC_API_KEY: Required for Anthropic provider
|
||||
GRAPHITI_ANTHROPIC_MODEL: Model for LLM (default: claude-sonnet-4-5-latest)
|
||||
|
||||
# Azure OpenAI
|
||||
AZURE_OPENAI_API_KEY: Required for Azure provider
|
||||
AZURE_OPENAI_BASE_URL: Azure endpoint URL
|
||||
AZURE_OPENAI_LLM_DEPLOYMENT: Deployment name for LLM
|
||||
AZURE_OPENAI_EMBEDDING_DEPLOYMENT: Deployment name for embeddings
|
||||
|
||||
# Voyage AI (embeddings only - commonly used with Anthropic)
|
||||
VOYAGE_API_KEY: Required for Voyage embedder
|
||||
VOYAGE_EMBEDDING_MODEL: Model (default: voyage-3)
|
||||
|
||||
# Ollama (local)
|
||||
OLLAMA_BASE_URL: Ollama server URL (default: http://localhost:11434)
|
||||
OLLAMA_LLM_MODEL: Model for LLM (e.g., deepseek-r1:7b)
|
||||
OLLAMA_EMBEDDING_MODEL: Model for embeddings (e.g., nomic-embed-text)
|
||||
OLLAMA_EMBEDDING_DIM: Embedding dimension (required for Ollama, e.g., 768)
|
||||
|
||||
# FalkorDB
|
||||
GRAPHITI_FALKORDB_HOST: FalkorDB host (default: localhost)
|
||||
GRAPHITI_FALKORDB_PORT: FalkorDB port (default: 6380)
|
||||
GRAPHITI_FALKORDB_PASSWORD: FalkorDB password (default: empty)
|
||||
GRAPHITI_DATABASE: Graph database name (default: auto_claude_memory)
|
||||
GRAPHITI_TELEMETRY_ENABLED: Set to "false" to disable telemetry (default: true)
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
# Default configuration values
|
||||
DEFAULT_FALKORDB_HOST = "localhost"
|
||||
DEFAULT_FALKORDB_PORT = 6380
|
||||
DEFAULT_DATABASE = "auto_claude_memory"
|
||||
DEFAULT_OLLAMA_BASE_URL = "http://localhost:11434"
|
||||
|
||||
# Graphiti state marker file (stores connection info and status)
|
||||
GRAPHITI_STATE_MARKER = ".graphiti_state.json"
|
||||
|
||||
# Episode types for different memory categories
|
||||
EPISODE_TYPE_SESSION_INSIGHT = "session_insight"
|
||||
EPISODE_TYPE_CODEBASE_DISCOVERY = "codebase_discovery"
|
||||
EPISODE_TYPE_PATTERN = "pattern"
|
||||
EPISODE_TYPE_GOTCHA = "gotcha"
|
||||
EPISODE_TYPE_TASK_OUTCOME = "task_outcome"
|
||||
EPISODE_TYPE_QA_RESULT = "qa_result"
|
||||
EPISODE_TYPE_HISTORICAL_CONTEXT = "historical_context"
|
||||
|
||||
|
||||
class LLMProvider(str, Enum):
|
||||
"""Supported LLM providers for Graphiti."""
|
||||
|
||||
OPENAI = "openai"
|
||||
ANTHROPIC = "anthropic"
|
||||
AZURE_OPENAI = "azure_openai"
|
||||
OLLAMA = "ollama"
|
||||
|
||||
|
||||
class EmbedderProvider(str, Enum):
|
||||
"""Supported embedder providers for Graphiti."""
|
||||
|
||||
OPENAI = "openai"
|
||||
VOYAGE = "voyage"
|
||||
AZURE_OPENAI = "azure_openai"
|
||||
OLLAMA = "ollama"
|
||||
|
||||
|
||||
@dataclass
|
||||
class GraphitiConfig:
|
||||
"""Configuration for Graphiti memory integration with multi-provider support."""
|
||||
|
||||
# Core settings
|
||||
enabled: bool = False
|
||||
llm_provider: str = "openai"
|
||||
embedder_provider: str = "openai"
|
||||
|
||||
# FalkorDB connection
|
||||
falkordb_host: str = DEFAULT_FALKORDB_HOST
|
||||
falkordb_port: int = DEFAULT_FALKORDB_PORT
|
||||
falkordb_password: str = ""
|
||||
database: str = DEFAULT_DATABASE
|
||||
telemetry_enabled: bool = True
|
||||
|
||||
# OpenAI settings
|
||||
openai_api_key: str = ""
|
||||
openai_model: str = "gpt-5-mini"
|
||||
openai_embedding_model: str = "text-embedding-3-small"
|
||||
|
||||
# Anthropic settings (LLM only)
|
||||
anthropic_api_key: str = ""
|
||||
anthropic_model: str = "claude-sonnet-4-5-latest"
|
||||
|
||||
# Azure OpenAI settings
|
||||
azure_openai_api_key: str = ""
|
||||
azure_openai_base_url: str = ""
|
||||
azure_openai_llm_deployment: str = ""
|
||||
azure_openai_embedding_deployment: str = ""
|
||||
|
||||
# Voyage AI settings (embeddings only)
|
||||
voyage_api_key: str = ""
|
||||
voyage_embedding_model: str = "voyage-3"
|
||||
|
||||
# Ollama settings (local)
|
||||
ollama_base_url: str = DEFAULT_OLLAMA_BASE_URL
|
||||
ollama_llm_model: str = ""
|
||||
ollama_embedding_model: str = ""
|
||||
ollama_embedding_dim: int = 0 # Required for Ollama embeddings
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "GraphitiConfig":
|
||||
"""Create config from environment variables."""
|
||||
# Check if Graphiti is explicitly enabled
|
||||
enabled_str = os.environ.get("GRAPHITI_ENABLED", "").lower()
|
||||
enabled = enabled_str in ("true", "1", "yes")
|
||||
|
||||
# Provider selection
|
||||
llm_provider = os.environ.get("GRAPHITI_LLM_PROVIDER", "openai").lower()
|
||||
embedder_provider = os.environ.get(
|
||||
"GRAPHITI_EMBEDDER_PROVIDER", "openai"
|
||||
).lower()
|
||||
|
||||
# FalkorDB connection settings
|
||||
falkordb_host = os.environ.get("GRAPHITI_FALKORDB_HOST", DEFAULT_FALKORDB_HOST)
|
||||
|
||||
try:
|
||||
falkordb_port = int(
|
||||
os.environ.get("GRAPHITI_FALKORDB_PORT", str(DEFAULT_FALKORDB_PORT))
|
||||
)
|
||||
except ValueError:
|
||||
falkordb_port = DEFAULT_FALKORDB_PORT
|
||||
|
||||
falkordb_password = os.environ.get("GRAPHITI_FALKORDB_PASSWORD", "")
|
||||
database = os.environ.get("GRAPHITI_DATABASE", DEFAULT_DATABASE)
|
||||
|
||||
# Telemetry setting
|
||||
telemetry_str = os.environ.get("GRAPHITI_TELEMETRY_ENABLED", "true").lower()
|
||||
telemetry_enabled = telemetry_str not in ("false", "0", "no")
|
||||
|
||||
# OpenAI settings
|
||||
openai_api_key = os.environ.get("OPENAI_API_KEY", "")
|
||||
openai_model = os.environ.get("OPENAI_MODEL", "gpt-5-mini")
|
||||
openai_embedding_model = os.environ.get(
|
||||
"OPENAI_EMBEDDING_MODEL", "text-embedding-3-small"
|
||||
)
|
||||
|
||||
# Anthropic settings
|
||||
anthropic_api_key = os.environ.get("ANTHROPIC_API_KEY", "")
|
||||
anthropic_model = os.environ.get(
|
||||
"GRAPHITI_ANTHROPIC_MODEL", "claude-sonnet-4-5-latest"
|
||||
)
|
||||
|
||||
# Azure OpenAI settings
|
||||
azure_openai_api_key = os.environ.get("AZURE_OPENAI_API_KEY", "")
|
||||
azure_openai_base_url = os.environ.get("AZURE_OPENAI_BASE_URL", "")
|
||||
azure_openai_llm_deployment = os.environ.get("AZURE_OPENAI_LLM_DEPLOYMENT", "")
|
||||
azure_openai_embedding_deployment = os.environ.get(
|
||||
"AZURE_OPENAI_EMBEDDING_DEPLOYMENT", ""
|
||||
)
|
||||
|
||||
# Voyage AI settings
|
||||
voyage_api_key = os.environ.get("VOYAGE_API_KEY", "")
|
||||
voyage_embedding_model = os.environ.get("VOYAGE_EMBEDDING_MODEL", "voyage-3")
|
||||
|
||||
# Ollama settings
|
||||
ollama_base_url = os.environ.get("OLLAMA_BASE_URL", DEFAULT_OLLAMA_BASE_URL)
|
||||
ollama_llm_model = os.environ.get("OLLAMA_LLM_MODEL", "")
|
||||
ollama_embedding_model = os.environ.get("OLLAMA_EMBEDDING_MODEL", "")
|
||||
|
||||
# Ollama embedding dimension (required for Ollama)
|
||||
try:
|
||||
ollama_embedding_dim = int(os.environ.get("OLLAMA_EMBEDDING_DIM", "0"))
|
||||
except ValueError:
|
||||
ollama_embedding_dim = 0
|
||||
|
||||
return cls(
|
||||
enabled=enabled,
|
||||
llm_provider=llm_provider,
|
||||
embedder_provider=embedder_provider,
|
||||
falkordb_host=falkordb_host,
|
||||
falkordb_port=falkordb_port,
|
||||
falkordb_password=falkordb_password,
|
||||
database=database,
|
||||
telemetry_enabled=telemetry_enabled,
|
||||
openai_api_key=openai_api_key,
|
||||
openai_model=openai_model,
|
||||
openai_embedding_model=openai_embedding_model,
|
||||
anthropic_api_key=anthropic_api_key,
|
||||
anthropic_model=anthropic_model,
|
||||
azure_openai_api_key=azure_openai_api_key,
|
||||
azure_openai_base_url=azure_openai_base_url,
|
||||
azure_openai_llm_deployment=azure_openai_llm_deployment,
|
||||
azure_openai_embedding_deployment=azure_openai_embedding_deployment,
|
||||
voyage_api_key=voyage_api_key,
|
||||
voyage_embedding_model=voyage_embedding_model,
|
||||
ollama_base_url=ollama_base_url,
|
||||
ollama_llm_model=ollama_llm_model,
|
||||
ollama_embedding_model=ollama_embedding_model,
|
||||
ollama_embedding_dim=ollama_embedding_dim,
|
||||
)
|
||||
|
||||
def is_valid(self) -> bool:
|
||||
"""
|
||||
Check if config has minimum required values for operation.
|
||||
|
||||
Returns True if:
|
||||
- GRAPHITI_ENABLED is true
|
||||
- LLM provider is configured correctly
|
||||
- Embedder provider is configured correctly
|
||||
"""
|
||||
if not self.enabled:
|
||||
return False
|
||||
|
||||
# Validate LLM provider
|
||||
if not self._validate_llm_provider():
|
||||
return False
|
||||
|
||||
# Validate embedder provider
|
||||
if not self._validate_embedder_provider():
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _validate_llm_provider(self) -> bool:
|
||||
"""Validate LLM provider configuration."""
|
||||
if self.llm_provider == "openai":
|
||||
return bool(self.openai_api_key)
|
||||
elif self.llm_provider == "anthropic":
|
||||
return bool(self.anthropic_api_key)
|
||||
elif self.llm_provider == "azure_openai":
|
||||
return bool(
|
||||
self.azure_openai_api_key
|
||||
and self.azure_openai_base_url
|
||||
and self.azure_openai_llm_deployment
|
||||
)
|
||||
elif self.llm_provider == "ollama":
|
||||
return bool(self.ollama_llm_model)
|
||||
return False
|
||||
|
||||
def _validate_embedder_provider(self) -> bool:
|
||||
"""Validate embedder provider configuration."""
|
||||
if self.embedder_provider == "openai":
|
||||
return bool(self.openai_api_key)
|
||||
elif self.embedder_provider == "voyage":
|
||||
return bool(self.voyage_api_key)
|
||||
elif self.embedder_provider == "azure_openai":
|
||||
return bool(
|
||||
self.azure_openai_api_key
|
||||
and self.azure_openai_base_url
|
||||
and self.azure_openai_embedding_deployment
|
||||
)
|
||||
elif self.embedder_provider == "ollama":
|
||||
return bool(self.ollama_embedding_model and self.ollama_embedding_dim)
|
||||
return False
|
||||
|
||||
def get_validation_errors(self) -> list[str]:
|
||||
"""Get list of validation errors for current configuration."""
|
||||
errors = []
|
||||
|
||||
if not self.enabled:
|
||||
errors.append("GRAPHITI_ENABLED must be set to true")
|
||||
return errors
|
||||
|
||||
# LLM provider validation
|
||||
if self.llm_provider == "openai":
|
||||
if not self.openai_api_key:
|
||||
errors.append("OpenAI LLM provider requires OPENAI_API_KEY")
|
||||
elif self.llm_provider == "anthropic":
|
||||
if not self.anthropic_api_key:
|
||||
errors.append("Anthropic LLM provider requires ANTHROPIC_API_KEY")
|
||||
elif self.llm_provider == "azure_openai":
|
||||
if not self.azure_openai_api_key:
|
||||
errors.append("Azure OpenAI LLM provider requires AZURE_OPENAI_API_KEY")
|
||||
if not self.azure_openai_base_url:
|
||||
errors.append(
|
||||
"Azure OpenAI LLM provider requires AZURE_OPENAI_BASE_URL"
|
||||
)
|
||||
if not self.azure_openai_llm_deployment:
|
||||
errors.append(
|
||||
"Azure OpenAI LLM provider requires AZURE_OPENAI_LLM_DEPLOYMENT"
|
||||
)
|
||||
elif self.llm_provider == "ollama":
|
||||
if not self.ollama_llm_model:
|
||||
errors.append("Ollama LLM provider requires OLLAMA_LLM_MODEL")
|
||||
else:
|
||||
errors.append(f"Unknown LLM provider: {self.llm_provider}")
|
||||
|
||||
# Embedder provider validation
|
||||
if self.embedder_provider == "openai":
|
||||
if not self.openai_api_key:
|
||||
errors.append("OpenAI embedder provider requires OPENAI_API_KEY")
|
||||
elif self.embedder_provider == "voyage":
|
||||
if not self.voyage_api_key:
|
||||
errors.append("Voyage embedder provider requires VOYAGE_API_KEY")
|
||||
elif self.embedder_provider == "azure_openai":
|
||||
if not self.azure_openai_api_key:
|
||||
errors.append(
|
||||
"Azure OpenAI embedder provider requires AZURE_OPENAI_API_KEY"
|
||||
)
|
||||
if not self.azure_openai_base_url:
|
||||
errors.append(
|
||||
"Azure OpenAI embedder provider requires AZURE_OPENAI_BASE_URL"
|
||||
)
|
||||
if not self.azure_openai_embedding_deployment:
|
||||
errors.append(
|
||||
"Azure OpenAI embedder provider requires AZURE_OPENAI_EMBEDDING_DEPLOYMENT"
|
||||
)
|
||||
elif self.embedder_provider == "ollama":
|
||||
if not self.ollama_embedding_model:
|
||||
errors.append(
|
||||
"Ollama embedder provider requires OLLAMA_EMBEDDING_MODEL"
|
||||
)
|
||||
if not self.ollama_embedding_dim:
|
||||
errors.append("Ollama embedder provider requires OLLAMA_EMBEDDING_DIM")
|
||||
else:
|
||||
errors.append(f"Unknown embedder provider: {self.embedder_provider}")
|
||||
|
||||
return errors
|
||||
|
||||
def get_connection_uri(self) -> str:
|
||||
"""Get the FalkorDB connection URI."""
|
||||
if self.falkordb_password:
|
||||
return f"redis://:{self.falkordb_password}@{self.falkordb_host}:{self.falkordb_port}"
|
||||
return f"redis://{self.falkordb_host}:{self.falkordb_port}"
|
||||
|
||||
def get_provider_summary(self) -> str:
|
||||
"""Get a summary of configured providers."""
|
||||
return f"LLM: {self.llm_provider}, Embedder: {self.embedder_provider}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class GraphitiState:
|
||||
"""State of Graphiti integration for an auto-claude spec."""
|
||||
|
||||
initialized: bool = False
|
||||
database: str | None = None
|
||||
indices_built: bool = False
|
||||
created_at: str | None = None
|
||||
last_session: int | None = None
|
||||
episode_count: int = 0
|
||||
error_log: list = field(default_factory=list)
|
||||
# V2 additions
|
||||
llm_provider: str | None = None
|
||||
embedder_provider: str | None = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"initialized": self.initialized,
|
||||
"database": self.database,
|
||||
"indices_built": self.indices_built,
|
||||
"created_at": self.created_at,
|
||||
"last_session": self.last_session,
|
||||
"episode_count": self.episode_count,
|
||||
"error_log": self.error_log[-10:], # Keep last 10 errors
|
||||
"llm_provider": self.llm_provider,
|
||||
"embedder_provider": self.embedder_provider,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> "GraphitiState":
|
||||
return cls(
|
||||
initialized=data.get("initialized", False),
|
||||
database=data.get("database"),
|
||||
indices_built=data.get("indices_built", False),
|
||||
created_at=data.get("created_at"),
|
||||
last_session=data.get("last_session"),
|
||||
episode_count=data.get("episode_count", 0),
|
||||
error_log=data.get("error_log", []),
|
||||
llm_provider=data.get("llm_provider"),
|
||||
embedder_provider=data.get("embedder_provider"),
|
||||
)
|
||||
|
||||
def save(self, spec_dir: Path) -> None:
|
||||
"""Save state to the spec directory."""
|
||||
marker_file = spec_dir / GRAPHITI_STATE_MARKER
|
||||
with open(marker_file, "w") as f:
|
||||
json.dump(self.to_dict(), f, indent=2)
|
||||
|
||||
@classmethod
|
||||
def load(cls, spec_dir: Path) -> Optional["GraphitiState"]:
|
||||
"""Load state from the spec directory."""
|
||||
marker_file = spec_dir / GRAPHITI_STATE_MARKER
|
||||
if not marker_file.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(marker_file) as f:
|
||||
return cls.from_dict(json.load(f))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return None
|
||||
|
||||
def record_error(self, error_msg: str) -> None:
|
||||
"""Record an error in the state."""
|
||||
self.error_log.append(
|
||||
{
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"error": error_msg[:500], # Limit error message length
|
||||
}
|
||||
)
|
||||
# Keep only last 10 errors
|
||||
self.error_log = self.error_log[-10:]
|
||||
|
||||
|
||||
def is_graphiti_enabled() -> bool:
|
||||
"""
|
||||
Quick check if Graphiti integration is available.
|
||||
|
||||
Returns True if:
|
||||
- GRAPHITI_ENABLED is set to true/1/yes
|
||||
- Required provider credentials are configured
|
||||
"""
|
||||
config = GraphitiConfig.from_env()
|
||||
return config.is_valid()
|
||||
|
||||
|
||||
def get_graphiti_status() -> dict:
|
||||
"""
|
||||
Get the current Graphiti integration status.
|
||||
|
||||
Returns:
|
||||
Dict with status information:
|
||||
- enabled: bool
|
||||
- available: bool (has required dependencies)
|
||||
- host: str
|
||||
- port: int
|
||||
- database: str
|
||||
- llm_provider: str
|
||||
- embedder_provider: str
|
||||
- reason: str (why unavailable if not available)
|
||||
- errors: list (validation errors if any)
|
||||
"""
|
||||
config = GraphitiConfig.from_env()
|
||||
|
||||
status = {
|
||||
"enabled": config.enabled,
|
||||
"available": False,
|
||||
"host": config.falkordb_host,
|
||||
"port": config.falkordb_port,
|
||||
"database": config.database,
|
||||
"llm_provider": config.llm_provider,
|
||||
"embedder_provider": config.embedder_provider,
|
||||
"reason": "",
|
||||
"errors": [],
|
||||
}
|
||||
|
||||
if not config.enabled:
|
||||
status["reason"] = "GRAPHITI_ENABLED not set to true"
|
||||
return status
|
||||
|
||||
# Get validation errors
|
||||
errors = config.get_validation_errors()
|
||||
if errors:
|
||||
status["errors"] = errors
|
||||
status["reason"] = errors[0] # First error as primary reason
|
||||
return status
|
||||
|
||||
status["available"] = True
|
||||
return status
|
||||
|
||||
|
||||
def get_available_providers() -> dict:
|
||||
"""
|
||||
Get list of available providers based on current environment.
|
||||
|
||||
Returns:
|
||||
Dict with lists of available LLM and embedder providers
|
||||
"""
|
||||
config = GraphitiConfig.from_env()
|
||||
|
||||
available_llm = []
|
||||
available_embedder = []
|
||||
|
||||
# Check OpenAI
|
||||
if config.openai_api_key:
|
||||
available_llm.append("openai")
|
||||
available_embedder.append("openai")
|
||||
|
||||
# Check Anthropic
|
||||
if config.anthropic_api_key:
|
||||
available_llm.append("anthropic")
|
||||
|
||||
# Check Azure OpenAI
|
||||
if config.azure_openai_api_key and config.azure_openai_base_url:
|
||||
if config.azure_openai_llm_deployment:
|
||||
available_llm.append("azure_openai")
|
||||
if config.azure_openai_embedding_deployment:
|
||||
available_embedder.append("azure_openai")
|
||||
|
||||
# Check Voyage
|
||||
if config.voyage_api_key:
|
||||
available_embedder.append("voyage")
|
||||
|
||||
# Check Ollama
|
||||
if config.ollama_llm_model:
|
||||
available_llm.append("ollama")
|
||||
if config.ollama_embedding_model and config.ollama_embedding_dim:
|
||||
available_embedder.append("ollama")
|
||||
|
||||
return {
|
||||
"llm_providers": available_llm,
|
||||
"embedder_providers": available_embedder,
|
||||
}
|
||||
|
||||
|
||||
def validate_graphiti_config() -> tuple[bool, list[str]]:
|
||||
"""
|
||||
Validate Graphiti configuration from environment.
|
||||
|
||||
Returns:
|
||||
Tuple of (is_valid, error_messages)
|
||||
- is_valid: True if configuration is valid
|
||||
- error_messages: List of validation error messages (empty if valid)
|
||||
"""
|
||||
config = GraphitiConfig.from_env()
|
||||
|
||||
if not config.is_valid():
|
||||
errors = config.get_validation_errors()
|
||||
return False, errors
|
||||
|
||||
return True, []
|
||||
@@ -0,0 +1,70 @@
|
||||
"""
|
||||
Graphiti Multi-Provider Entry Point
|
||||
====================================
|
||||
|
||||
Main entry point for Graphiti provider functionality.
|
||||
This module re-exports all functionality from the graphiti_providers package.
|
||||
|
||||
The actual implementation has been refactored into a package structure:
|
||||
- graphiti_providers/exceptions.py - Provider exceptions
|
||||
- graphiti_providers/models.py - Embedding dimensions and constants
|
||||
- graphiti_providers/llm_providers/ - LLM provider implementations
|
||||
- graphiti_providers/embedder_providers/ - Embedder provider implementations
|
||||
- graphiti_providers/cross_encoder.py - Cross-encoder/reranker
|
||||
- graphiti_providers/validators.py - Validation and health checks
|
||||
- graphiti_providers/utils.py - Utility functions
|
||||
- graphiti_providers/factory.py - Factory functions
|
||||
|
||||
For backward compatibility, this module re-exports all public APIs.
|
||||
|
||||
Usage:
|
||||
from graphiti_providers import create_llm_client, create_embedder
|
||||
from graphiti_config import GraphitiConfig
|
||||
|
||||
config = GraphitiConfig.from_env()
|
||||
llm_client = create_llm_client(config)
|
||||
embedder = create_embedder(config)
|
||||
"""
|
||||
|
||||
# Re-export all public APIs from the package
|
||||
from graphiti_providers import (
|
||||
# Exceptions
|
||||
ProviderError,
|
||||
ProviderNotInstalled,
|
||||
# Factory functions
|
||||
create_llm_client,
|
||||
create_embedder,
|
||||
create_cross_encoder,
|
||||
# Models
|
||||
EMBEDDING_DIMENSIONS,
|
||||
get_expected_embedding_dim,
|
||||
# Validators
|
||||
validate_embedding_config,
|
||||
test_llm_connection,
|
||||
test_embedder_connection,
|
||||
test_ollama_connection,
|
||||
# Utilities
|
||||
is_graphiti_enabled,
|
||||
get_graph_hints,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# Exceptions
|
||||
"ProviderError",
|
||||
"ProviderNotInstalled",
|
||||
# Factory functions
|
||||
"create_llm_client",
|
||||
"create_embedder",
|
||||
"create_cross_encoder",
|
||||
# Models
|
||||
"EMBEDDING_DIMENSIONS",
|
||||
"get_expected_embedding_dim",
|
||||
# Validators
|
||||
"validate_embedding_config",
|
||||
"test_llm_connection",
|
||||
"test_embedder_connection",
|
||||
"test_ollama_connection",
|
||||
# Utilities
|
||||
"is_graphiti_enabled",
|
||||
"get_graph_hints",
|
||||
]
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user