Big backend refactor for upcoming features

This commit is contained in:
AndyMik90
2025-12-16 21:37:12 +01:00
parent 87f353c3af
commit 11fcdf42f3
209 changed files with 12675 additions and 18463 deletions
+8 -5
View File
@@ -60,17 +60,20 @@ python auto-claude/run.py --spec 001 --qa-status
### Testing
```bash
# Run all tests
pytest tests/ -v
# Install test dependencies (required first time)
cd auto-claude && uv pip install -r ../tests/requirements-test.txt
# Run all tests (use virtual environment pytest)
auto-claude/.venv/bin/pytest tests/ -v
# Run single test file
pytest tests/test_security.py -v
auto-claude/.venv/bin/pytest tests/test_security.py -v
# Run specific test
pytest tests/test_security.py::test_bash_command_validation -v
auto-claude/.venv/bin/pytest tests/test_security.py::test_bash_command_validation -v
# Skip slow tests
pytest tests/ -m "not slow"
auto-claude/.venv/bin/pytest tests/ -m "not slow"
```
### Spec Validation
@@ -75,7 +75,7 @@ export function GitHubIssues({ onOpenSettings }: GitHubIssuesProps) {
<div className="flex-1 flex flex-col h-full">
{/* Header */}
<IssueListHeader
repoFullName={syncStatus.repoFullName}
repoFullName={syncStatus.repoFullName ?? ''}
openIssuesCount={getOpenIssuesCount()}
isLoading={isLoading}
searchQuery={searchQuery}
@@ -249,7 +249,7 @@ export function ProjectSettings({ project, open, onOpenChange }: ProjectSettings
envConfig={envConfig}
settings={settings}
onUpdateConfig={updateEnvConfig}
onUpdateSettings={setSettings}
onUpdateSettings={(updates) => setSettings({ ...settings, ...updates })}
infrastructureStatus={infrastructureStatus}
isCheckingInfrastructure={isCheckingInfrastructure}
isStartingFalkorDB={isStartingFalkorDB}
+1 -1
View File
@@ -5,5 +5,5 @@ export { useClaudeAuth } from './useClaudeAuth';
export { useLinearConnection } from './useLinearConnection';
export { useGitHubConnection } from './useGitHubConnection';
export { useInfrastructureStatus } from './useInfrastructureStatus';
export { useIpc } from './useIpc';
export { useIpcListeners } from './useIpc';
export { useVirtualizedTree } from './useVirtualizedTree';
@@ -1,5 +1,5 @@
import { useState, useEffect } from 'react';
import type { ProjectEnvConfig } from '../../../shared/types';
import type { ProjectEnvConfig } from '../../shared/types';
type AuthStatus = 'checking' | 'authenticated' | 'not_authenticated' | 'error';
@@ -1,5 +1,5 @@
import { useState, useEffect } from 'react';
import type { ProjectEnvConfig } from '../../../shared/types';
import type { ProjectEnvConfig } from '../../shared/types';
export function useEnvironmentConfig(projectId: string, autoBuildPath: string | null, open: boolean) {
const [envConfig, setEnvConfig] = useState<ProjectEnvConfig | null>(null);
@@ -1,5 +1,5 @@
import { useState, useEffect } from 'react';
import type { GitHubSyncStatus } from '../../../shared/types';
import type { GitHubSyncStatus } from '../../shared/types';
export function useGitHubConnection(
projectId: string,
@@ -1,5 +1,5 @@
import { useState, useEffect } from 'react';
import type { InfrastructureStatus } from '../../../shared/types';
import type { InfrastructureStatus } from '../../shared/types';
export function useInfrastructureStatus(
graphitiEnabled: boolean | undefined,
@@ -1,5 +1,5 @@
import { useState, useEffect } from 'react';
import type { LinearSyncStatus } from '../../../shared/types';
import type { LinearSyncStatus } from '../../shared/types';
export function useLinearConnection(
projectId: string,
@@ -1,5 +1,5 @@
import { useState, useEffect } from 'react';
import type { Project, ProjectSettings, AutoBuildVersionInfo, ProjectEnvConfig } from '../../../shared/types';
import type { Project, ProjectSettings, AutoBuildVersionInfo, ProjectEnvConfig } from '../../shared/types';
import { checkProjectVersion } from '../stores/project-store';
export function useProjectSettings(project: Project, open: boolean) {
@@ -11,9 +11,16 @@ export const claudeProfileMock = {
}
}),
saveClaudeProfile: async (profile: unknown) => ({
saveClaudeProfile: async (profile: { id: string; name: string; oauthToken?: string; email?: string; isDefault?: boolean; createdAt?: Date }) => ({
success: true,
data: profile
data: {
id: profile.id,
name: profile.name,
oauthToken: profile.oauthToken,
email: profile.email,
isDefault: profile.isDefault ?? false,
createdAt: profile.createdAt ?? new Date(),
}
}),
deleteClaudeProfile: async () => ({ success: true }),
+9
View File
@@ -0,0 +1,9 @@
"""
Auto Claude - Autonomous Coding Framework
==========================================
Multi-agent autonomous coding framework that builds software through
coordinated AI agent sessions.
"""
__version__ = "0.1.0"
+2 -63
View File
@@ -1,63 +1,2 @@
"""
Agent Session Logic
===================
Core agent interaction functions for running autonomous coding sessions.
Uses subtask-based implementation plans with minimal, focused prompts.
Architecture:
- Orchestrator (Python) handles all bookkeeping: memory, commits, progress
- Agent focuses ONLY on implementing code
- Post-session processing updates memory automatically (100% reliable)
Enhanced with status file updates for ccstatusline integration.
Enhanced with Graphiti memory for cross-session context retrieval.
NOTE: This module is now a facade that imports from agents/ submodules.
All logic has been refactored into focused modules for better maintainability.
"""
# Re-export everything from the agents module to maintain backwards compatibility
from agents import (
# Constants
AUTO_CONTINUE_DELAY_SECONDS,
HUMAN_INTERVENTION_FILE,
# Memory functions
debug_memory_system_status,
find_phase_for_subtask,
find_subtask_in_plan,
get_commit_count,
get_graphiti_context,
# Utility functions
get_latest_commit,
load_implementation_plan,
post_session_processing,
# Session management
run_agent_session,
# Main API
run_autonomous_agent,
run_followup_planner,
save_session_memory,
save_session_to_graphiti,
sync_plan_to_source,
)
# Ensure all exports are available at module level
__all__ = [
"run_autonomous_agent",
"run_followup_planner",
"debug_memory_system_status",
"get_graphiti_context",
"save_session_memory",
"save_session_to_graphiti",
"run_agent_session",
"post_session_processing",
"get_latest_commit",
"get_commit_count",
"load_implementation_plan",
"find_subtask_in_plan",
"find_phase_for_subtask",
"sync_plan_to_source",
"AUTO_CONTINUE_DELAY_SECONDS",
"HUMAN_INTERVENTION_FILE",
]
"""Backward compatibility shim - import from core.agent instead."""
from core.agent import *
+1 -1
View File
@@ -21,7 +21,7 @@ from .base import (
from .coder import run_autonomous_agent
# Memory functions
from .memory import (
from .memory_manager import (
debug_memory_system_status,
get_graphiti_context,
save_session_memory,
+62
View File
@@ -0,0 +1,62 @@
"""
Custom MCP Tools for Auto-Claude Agents
========================================
DEPRECATED: This module is now a compatibility shim.
Please import from the tools_pkg package instead:
from agents.tools_pkg import create_auto_claude_mcp_server, get_allowed_tools
This file remains for backward compatibility with existing imports.
All functionality has been moved to the tools_pkg package for better
organization and maintainability.
"""
# Import everything from the package to maintain backward compatibility
# Use try/except to handle both relative and absolute imports
try:
from .tools_pkg import (
ELECTRON_TOOLS,
TOOL_GET_BUILD_PROGRESS,
TOOL_GET_SESSION_CONTEXT,
TOOL_RECORD_DISCOVERY,
TOOL_RECORD_GOTCHA,
TOOL_UPDATE_QA_STATUS,
TOOL_UPDATE_SUBTASK_STATUS,
create_auto_claude_mcp_server,
get_allowed_tools,
is_electron_mcp_enabled,
is_tools_available,
)
except ImportError:
# Fallback for direct execution - import from tools_pkg directly
from tools_pkg import (
ELECTRON_TOOLS,
TOOL_GET_BUILD_PROGRESS,
TOOL_GET_SESSION_CONTEXT,
TOOL_RECORD_DISCOVERY,
TOOL_RECORD_GOTCHA,
TOOL_UPDATE_QA_STATUS,
TOOL_UPDATE_SUBTASK_STATUS,
create_auto_claude_mcp_server,
get_allowed_tools,
is_electron_mcp_enabled,
is_tools_available,
)
__all__ = [
# Main API
"create_auto_claude_mcp_server",
"get_allowed_tools",
"is_tools_available",
# Tool name constants
"TOOL_UPDATE_SUBTASK_STATUS",
"TOOL_GET_BUILD_PROGRESS",
"TOOL_RECORD_DISCOVERY",
"TOOL_RECORD_GOTCHA",
"TOOL_GET_SESSION_CONTEXT",
"TOOL_UPDATE_QA_STATUS",
# Electron MCP
"ELECTRON_TOOLS",
"is_electron_mcp_enabled",
]
+1 -1
View File
@@ -53,7 +53,7 @@ from ui import (
)
from .base import AUTO_CONTINUE_DELAY_SECONDS, HUMAN_INTERVENTION_FILE
from .memory import debug_memory_system_status, get_graphiti_context
from .memory_manager import debug_memory_system_status, get_graphiti_context
from .session import post_session_processing, run_agent_session
from .utils import (
find_phase_for_subtask,
@@ -8,6 +8,7 @@ Handles session memory storage using dual-layer approach:
"""
import logging
import sys
from pathlib import Path
from debug import (
@@ -20,6 +21,9 @@ from debug import (
is_debug_enabled,
)
from graphiti_config import get_graphiti_status, is_graphiti_enabled
# Import from parent memory package
# Now safe since this module is named memory_manager (not memory)
from memory import save_session_insights as save_file_based_memory
logger = logging.getLogger(__name__)
+1 -1
View File
@@ -32,7 +32,7 @@ from ui import (
print_status,
)
from .memory import save_session_memory
from .memory_manager import save_session_memory
from .utils import (
find_subtask_in_plan,
get_commit_count,
+36
View File
@@ -0,0 +1,36 @@
"""
Analysis Module
===============
Code analysis and project scanning tools.
"""
# Import from analyzers subpackage (these are the modular analyzers)
from .analyzers import (
ProjectAnalyzer as ModularProjectAnalyzer,
ServiceAnalyzer,
analyze_project,
analyze_service,
)
# Import from analysis module root (these are other analysis tools)
from .project_analyzer import ProjectAnalyzer
from .risk_classifier import RiskClassifier
from .security_scanner import SecurityScanner
from .ci_discovery import CIDiscovery
from .test_discovery import TestDiscovery
# insight_extractor is a module with functions, not a class, so don't import it here
# Import it directly when needed: from analysis import insight_extractor
__all__ = [
"ProjectAnalyzer",
"ModularProjectAnalyzer",
"ServiceAnalyzer",
"analyze_project",
"analyze_service",
"RiskClassifier",
"SecurityScanner",
"CIDiscovery",
"TestDiscovery",
]
+100
View File
@@ -0,0 +1,100 @@
#!/usr/bin/env python3
"""
Codebase Analyzer
=================
Automatically detects project structure, frameworks, and services.
Supports monorepos with multiple services.
Usage:
# Index entire project (creates project_index.json)
python auto-claude/analyzer.py --index
# Analyze specific service
python auto-claude/analyzer.py --service backend
# Output to specific file
python auto-claude/analyzer.py --index --output path/to/output.json
The analyzer will:
1. Detect if this is a monorepo or single project
2. Find all services/packages and analyze each separately
3. Map interdependencies between services
4. Identify infrastructure (Docker, CI/CD)
5. Document conventions (linting, testing)
This module now serves as a facade to the modular analyzer system in the analyzers/ package.
All actual implementation is in focused submodules for better maintainability.
"""
import json
from pathlib import Path
# Import from the new modular structure
from .analyzers import (
ProjectAnalyzer,
ServiceAnalyzer,
analyze_project,
analyze_service,
)
# Re-export for backward compatibility
__all__ = [
"ServiceAnalyzer",
"ProjectAnalyzer",
"analyze_project",
"analyze_service",
]
def main():
"""CLI entry point."""
import argparse
parser = argparse.ArgumentParser(
description="Analyze project structure, frameworks, and services"
)
parser.add_argument(
"--project-dir",
type=Path,
default=Path.cwd(),
help="Project directory to analyze (default: current directory)",
)
parser.add_argument(
"--index",
action="store_true",
help="Create full project index (default behavior)",
)
parser.add_argument(
"--service",
type=str,
default=None,
help="Analyze a specific service only",
)
parser.add_argument(
"--output",
type=Path,
default=None,
help="Output file for JSON results",
)
parser.add_argument(
"--quiet",
action="store_true",
help="Only output JSON, no status messages",
)
args = parser.parse_args()
# Determine what to analyze
if args.service:
results = analyze_service(args.project_dir, args.service, args.output)
else:
results = analyze_project(args.project_dir, args.output)
# Print results
if not args.quiet or not args.output:
print(json.dumps(results, indent=2))
if __name__ == "__main__":
main()
@@ -0,0 +1,92 @@
"""
Analyzers Package
=================
Modular analyzer system for detecting project structure, frameworks, and services.
Main exports:
- ServiceAnalyzer: Analyzes a single service/package
- ProjectAnalyzer: Analyzes entire projects (single or monorepo)
- analyze_project: Convenience function for project analysis
- analyze_service: Convenience function for service analysis
"""
from pathlib import Path
from typing import Any
from .project_analyzer_module import ProjectAnalyzer
from .service_analyzer import ServiceAnalyzer
# Re-export main classes
__all__ = [
"ServiceAnalyzer",
"ProjectAnalyzer",
"analyze_project",
"analyze_service",
]
def analyze_project(project_dir: Path, output_file: Path | None = None) -> dict:
"""
Analyze a project and optionally save results.
Args:
project_dir: Path to the project root
output_file: Optional path to save JSON output
Returns:
Project index as a dictionary
"""
import json
analyzer = ProjectAnalyzer(project_dir)
results = analyzer.analyze()
if output_file:
output_file.parent.mkdir(parents=True, exist_ok=True)
with open(output_file, "w") as f:
json.dump(results, f, indent=2)
print(f"Project index saved to: {output_file}")
return results
def analyze_service(
project_dir: Path, service_name: str, output_file: Path | None = None
) -> dict:
"""
Analyze a specific service within a project.
Args:
project_dir: Path to the project root
service_name: Name of the service to analyze
output_file: Optional path to save JSON output
Returns:
Service analysis as a dictionary
"""
import json
# Find the service
service_path = project_dir / service_name
if not service_path.exists():
# Check common locations
for parent in ["packages", "apps", "services"]:
candidate = project_dir / parent / service_name
if candidate.exists():
service_path = candidate
break
if not service_path.exists():
raise ValueError(f"Service '{service_name}' not found in {project_dir}")
analyzer = ServiceAnalyzer(service_path, service_name)
results = analyzer.analyze()
if output_file:
output_file.parent.mkdir(parents=True, exist_ok=True)
with open(output_file, "w") as f:
json.dump(results, f, indent=2)
print(f"Service analysis saved to: {output_file}")
return results
+587
View File
@@ -0,0 +1,587 @@
#!/usr/bin/env python3
"""
CI Discovery Module
===================
Parses CI/CD configuration files to extract test commands and workflows.
Supports GitHub Actions, GitLab CI, CircleCI, and Jenkins.
The CI discovery results are used by:
- QA Agent: To understand existing CI test patterns
- Validation Strategy: To match CI commands
- Planner: To align verification with CI
Usage:
from ci_discovery import CIDiscovery
discovery = CIDiscovery()
result = discovery.discover(project_dir)
if result:
print(f"CI System: {result.ci_system}")
print(f"Test Commands: {result.test_commands}")
"""
import json
import re
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
# Try to import yaml, fall back gracefully
try:
import yaml
HAS_YAML = True
except ImportError:
HAS_YAML = False
# =============================================================================
# DATA CLASSES
# =============================================================================
@dataclass
class CIWorkflow:
"""
Represents a CI workflow or job.
Attributes:
name: Name of the workflow/job
trigger: What triggers this workflow (push, pull_request, etc.)
steps: List of step names or commands
test_related: Whether this appears to be test-related
"""
name: str
trigger: list[str] = field(default_factory=list)
steps: list[str] = field(default_factory=list)
test_related: bool = False
@dataclass
class CIConfig:
"""
Result of CI configuration discovery.
Attributes:
ci_system: Name of CI system (github_actions, gitlab, circleci, jenkins)
config_files: List of CI config files found
test_commands: Extracted test commands by type
coverage_command: Coverage command if found
workflows: List of discovered workflows
environment_variables: Environment variables used
"""
ci_system: str
config_files: list[str] = field(default_factory=list)
test_commands: dict[str, str] = field(default_factory=dict)
coverage_command: str | None = None
workflows: list[CIWorkflow] = field(default_factory=list)
environment_variables: list[str] = field(default_factory=list)
# =============================================================================
# CI PARSERS
# =============================================================================
class CIDiscovery:
"""
Discovers CI/CD configurations in a project.
Analyzes:
- GitHub Actions (.github/workflows/*.yml)
- GitLab CI (.gitlab-ci.yml)
- CircleCI (.circleci/config.yml)
- Jenkins (Jenkinsfile)
"""
def __init__(self) -> None:
"""Initialize CI discovery."""
self._cache: dict[str, CIConfig | None] = {}
def discover(self, project_dir: Path) -> CIConfig | None:
"""
Discover CI configuration in the project.
Args:
project_dir: Path to the project root
Returns:
CIConfig if CI found, None otherwise
"""
project_dir = Path(project_dir)
cache_key = str(project_dir.resolve())
if cache_key in self._cache:
return self._cache[cache_key]
# Try each CI system
result = None
# GitHub Actions
github_workflows = project_dir / ".github" / "workflows"
if github_workflows.exists():
result = self._parse_github_actions(github_workflows)
# GitLab CI
if not result:
gitlab_ci = project_dir / ".gitlab-ci.yml"
if gitlab_ci.exists():
result = self._parse_gitlab_ci(gitlab_ci)
# CircleCI
if not result:
circleci = project_dir / ".circleci" / "config.yml"
if circleci.exists():
result = self._parse_circleci(circleci)
# Jenkins
if not result:
jenkinsfile = project_dir / "Jenkinsfile"
if jenkinsfile.exists():
result = self._parse_jenkinsfile(jenkinsfile)
self._cache[cache_key] = result
return result
def _parse_github_actions(self, workflows_dir: Path) -> CIConfig:
"""Parse GitHub Actions workflow files."""
result = CIConfig(ci_system="github_actions")
workflow_files = list(workflows_dir.glob("*.yml")) + list(
workflows_dir.glob("*.yaml")
)
for wf_file in workflow_files:
result.config_files.append(
str(wf_file.relative_to(workflows_dir.parent.parent))
)
try:
content = wf_file.read_text()
workflow_data = self._parse_yaml(content)
if not workflow_data:
continue
# Get workflow name
wf_name = workflow_data.get("name", wf_file.stem)
# Get triggers
triggers = []
on_trigger = workflow_data.get("on", {})
if isinstance(on_trigger, str):
triggers = [on_trigger]
elif isinstance(on_trigger, list):
triggers = on_trigger
elif isinstance(on_trigger, dict):
triggers = list(on_trigger.keys())
# Parse jobs
jobs = workflow_data.get("jobs", {})
for job_name, job_config in jobs.items():
if not isinstance(job_config, dict):
continue
steps = job_config.get("steps", [])
step_commands = []
test_related = False
for step in steps:
if not isinstance(step, dict):
continue
# Get step name or command
step_name = step.get("name", "")
run_cmd = step.get("run", "")
uses = step.get("uses", "")
if step_name:
step_commands.append(step_name)
if run_cmd:
step_commands.append(run_cmd)
# Extract test commands
self._extract_test_commands(run_cmd, result)
if uses:
step_commands.append(f"uses: {uses}")
# Check if test-related
test_keywords = ["test", "pytest", "jest", "vitest", "coverage"]
if any(kw in str(step).lower() for kw in test_keywords):
test_related = True
result.workflows.append(
CIWorkflow(
name=f"{wf_name}/{job_name}",
trigger=triggers,
steps=step_commands,
test_related=test_related,
)
)
# Extract environment variables
env = workflow_data.get("env", {})
if isinstance(env, dict):
result.environment_variables.extend(env.keys())
except Exception:
continue
return result
def _parse_gitlab_ci(self, config_file: Path) -> CIConfig:
"""Parse GitLab CI configuration."""
result = CIConfig(
ci_system="gitlab",
config_files=[".gitlab-ci.yml"],
)
try:
content = config_file.read_text()
data = self._parse_yaml(content)
if not data:
return result
# Parse jobs (top-level keys that aren't special keywords)
special_keys = {
"stages",
"variables",
"image",
"services",
"before_script",
"after_script",
"cache",
"include",
"default",
"workflow",
}
for key, value in data.items():
if key.startswith(".") or key in special_keys:
continue
if not isinstance(value, dict):
continue
job_config = value
script = job_config.get("script", [])
if isinstance(script, str):
script = [script]
test_related = any(
kw in str(script).lower()
for kw in ["test", "pytest", "jest", "vitest", "coverage"]
)
result.workflows.append(
CIWorkflow(
name=key,
trigger=job_config.get("only", [])
or job_config.get("rules", []),
steps=script,
test_related=test_related,
)
)
# Extract test commands
for cmd in script:
if isinstance(cmd, str):
self._extract_test_commands(cmd, result)
# Extract variables
variables = data.get("variables", {})
if isinstance(variables, dict):
result.environment_variables.extend(variables.keys())
except Exception:
pass
return result
def _parse_circleci(self, config_file: Path) -> CIConfig:
"""Parse CircleCI configuration."""
result = CIConfig(
ci_system="circleci",
config_files=[".circleci/config.yml"],
)
try:
content = config_file.read_text()
data = self._parse_yaml(content)
if not data:
return result
# Parse jobs
jobs = data.get("jobs", {})
for job_name, job_config in jobs.items():
if not isinstance(job_config, dict):
continue
steps = job_config.get("steps", [])
step_commands = []
test_related = False
for step in steps:
if isinstance(step, str):
step_commands.append(step)
elif isinstance(step, dict):
if "run" in step:
run = step["run"]
if isinstance(run, str):
step_commands.append(run)
self._extract_test_commands(run, result)
elif isinstance(run, dict):
cmd = run.get("command", "")
step_commands.append(cmd)
self._extract_test_commands(cmd, result)
if any(
kw in str(step).lower()
for kw in ["test", "pytest", "jest", "coverage"]
):
test_related = True
result.workflows.append(
CIWorkflow(
name=job_name,
trigger=[],
steps=step_commands,
test_related=test_related,
)
)
except Exception:
pass
return result
def _parse_jenkinsfile(self, jenkinsfile: Path) -> CIConfig:
"""Parse Jenkinsfile (basic extraction)."""
result = CIConfig(
ci_system="jenkins",
config_files=["Jenkinsfile"],
)
try:
content = jenkinsfile.read_text()
# Extract sh commands using regex
sh_pattern = re.compile(r'sh\s+[\'"]([^\'"]+)[\'"]')
matches = sh_pattern.findall(content)
steps = []
test_related = False
for cmd in matches:
steps.append(cmd)
self._extract_test_commands(cmd, result)
if any(
kw in cmd.lower() for kw in ["test", "pytest", "jest", "coverage"]
):
test_related = True
# Extract stage names
stage_pattern = re.compile(r'stage\s*\([\'"]([^\'"]+)[\'"]\)')
stages = stage_pattern.findall(content)
for stage in stages:
result.workflows.append(
CIWorkflow(
name=stage,
trigger=[],
steps=steps if "test" in stage.lower() else [],
test_related="test" in stage.lower(),
)
)
except Exception:
pass
return result
def _parse_yaml(self, content: str) -> dict | None:
"""Parse YAML content, with fallback to basic parsing if yaml not available."""
if HAS_YAML:
try:
return yaml.safe_load(content)
except Exception:
return None
# Basic fallback for simple YAML (very limited)
# This won't work for complex structures
return None
def _extract_test_commands(self, cmd: str, result: CIConfig) -> None:
"""Extract test commands from a command string."""
cmd_lower = cmd.lower()
# Python pytest
if "pytest" in cmd_lower:
if "pytest" not in result.test_commands:
result.test_commands["unit"] = cmd.strip()
if "--cov" in cmd_lower:
result.coverage_command = cmd.strip()
# Node.js test commands
if (
"npm test" in cmd_lower
or "yarn test" in cmd_lower
or "pnpm test" in cmd_lower
):
if "unit" not in result.test_commands:
result.test_commands["unit"] = cmd.strip()
# Jest/Vitest
if "jest" in cmd_lower or "vitest" in cmd_lower:
if "unit" not in result.test_commands:
result.test_commands["unit"] = cmd.strip()
if "--coverage" in cmd_lower:
result.coverage_command = cmd.strip()
# E2E testing
if "playwright" in cmd_lower:
result.test_commands["e2e"] = cmd.strip()
if "cypress" in cmd_lower:
result.test_commands["e2e"] = cmd.strip()
# Integration tests
if "integration" in cmd_lower:
result.test_commands["integration"] = cmd.strip()
# Go tests
if "go test" in cmd_lower:
if "unit" not in result.test_commands:
result.test_commands["unit"] = cmd.strip()
# Rust tests
if "cargo test" in cmd_lower:
if "unit" not in result.test_commands:
result.test_commands["unit"] = cmd.strip()
def to_dict(self, result: CIConfig) -> dict[str, Any]:
"""Convert result to dictionary for JSON serialization."""
return {
"ci_system": result.ci_system,
"config_files": result.config_files,
"test_commands": result.test_commands,
"coverage_command": result.coverage_command,
"workflows": [
{
"name": w.name,
"trigger": w.trigger,
"steps": w.steps,
"test_related": w.test_related,
}
for w in result.workflows
],
"environment_variables": result.environment_variables,
}
def clear_cache(self) -> None:
"""Clear the internal cache."""
self._cache.clear()
# =============================================================================
# CONVENIENCE FUNCTIONS
# =============================================================================
def discover_ci(project_dir: Path) -> CIConfig | None:
"""
Convenience function to discover CI configuration.
Args:
project_dir: Path to project root
Returns:
CIConfig if found, None otherwise
"""
discovery = CIDiscovery()
return discovery.discover(project_dir)
def get_ci_test_commands(project_dir: Path) -> dict[str, str]:
"""
Get test commands from CI configuration.
Args:
project_dir: Path to project root
Returns:
Dictionary of test type to command
"""
discovery = CIDiscovery()
result = discovery.discover(project_dir)
if result:
return result.test_commands
return {}
def get_ci_system(project_dir: Path) -> str | None:
"""
Get the CI system name if configured.
Args:
project_dir: Path to project root
Returns:
CI system name or None
"""
discovery = CIDiscovery()
result = discovery.discover(project_dir)
if result:
return result.ci_system
return None
# =============================================================================
# CLI
# =============================================================================
def main() -> None:
"""CLI entry point for testing."""
import argparse
parser = argparse.ArgumentParser(description="Discover CI configuration")
parser.add_argument("project_dir", type=Path, help="Path to project root")
parser.add_argument("--json", action="store_true", help="Output as JSON")
args = parser.parse_args()
discovery = CIDiscovery()
result = discovery.discover(args.project_dir)
if not result:
print("No CI configuration found")
return
if args.json:
print(json.dumps(discovery.to_dict(result), indent=2))
else:
print(f"CI System: {result.ci_system}")
print(f"Config Files: {', '.join(result.config_files)}")
print("\nTest Commands:")
for test_type, cmd in result.test_commands.items():
print(f" {test_type}: {cmd}")
if result.coverage_command:
print(f"\nCoverage Command: {result.coverage_command}")
print(f"\nWorkflows ({len(result.workflows)}):")
for w in result.workflows:
marker = "[TEST]" if w.test_related else ""
print(f" - {w.name} {marker}")
if w.trigger:
print(f" Triggers: {', '.join(str(t) for t in w.trigger)}")
if result.environment_variables:
print(f"\nEnvironment Variables: {', '.join(result.environment_variables)}")
if __name__ == "__main__":
main()
+584
View File
@@ -0,0 +1,584 @@
"""
Insight Extractor
=================
Automatically extracts structured insights from completed coding sessions.
Runs after each session to capture rich, actionable knowledge for Graphiti memory.
Uses the Claude Agent SDK (same as the rest of the system) for extraction.
Falls back to generic insights if extraction fails (never blocks the build).
"""
import json
import logging
import os
import subprocess
from pathlib import Path
from typing import Any
logger = logging.getLogger(__name__)
# Check for Claude SDK availability
try:
from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient
SDK_AVAILABLE = True
except ImportError:
SDK_AVAILABLE = False
ClaudeAgentOptions = None
ClaudeSDKClient = None
# Default model for insight extraction (fast and cheap)
DEFAULT_EXTRACTION_MODEL = "claude-3-5-haiku-latest"
# Maximum diff size to send to the LLM (avoid context limits)
MAX_DIFF_CHARS = 15000
# Maximum attempt history entries to include
MAX_ATTEMPTS_TO_INCLUDE = 3
def is_extraction_enabled() -> bool:
"""Check if insight extraction is enabled."""
# Extraction requires Claude SDK and OAuth token
if not SDK_AVAILABLE:
return False
if not os.environ.get("CLAUDE_CODE_OAUTH_TOKEN"):
return False
enabled_str = os.environ.get("INSIGHT_EXTRACTION_ENABLED", "true").lower()
return enabled_str in ("true", "1", "yes")
def get_extraction_model() -> str:
"""Get the model to use for insight extraction."""
return os.environ.get("INSIGHT_EXTRACTOR_MODEL", DEFAULT_EXTRACTION_MODEL)
# =============================================================================
# Git Helpers
# =============================================================================
def get_session_diff(
project_dir: Path,
commit_before: str | None,
commit_after: str | None,
) -> str:
"""
Get the git diff between two commits.
Args:
project_dir: Project root directory
commit_before: Commit hash before session (or None)
commit_after: Commit hash after session (or None)
Returns:
Diff text (truncated if too large)
"""
if not commit_before or not commit_after:
return "(No commits to diff)"
if commit_before == commit_after:
return "(No changes - same commit)"
try:
result = subprocess.run(
["git", "diff", commit_before, commit_after],
cwd=project_dir,
capture_output=True,
text=True,
timeout=30,
)
diff = result.stdout
if len(diff) > MAX_DIFF_CHARS:
# Truncate and add note
diff = (
diff[:MAX_DIFF_CHARS] + f"\n\n... (truncated, {len(diff)} chars total)"
)
return diff if diff else "(Empty diff)"
except subprocess.TimeoutExpired:
logger.warning("Git diff timed out")
return "(Git diff timed out)"
except Exception as e:
logger.warning(f"Failed to get git diff: {e}")
return f"(Failed to get diff: {e})"
def get_changed_files(
project_dir: Path,
commit_before: str | None,
commit_after: str | None,
) -> list[str]:
"""
Get list of files changed between two commits.
Args:
project_dir: Project root directory
commit_before: Commit hash before session
commit_after: Commit hash after session
Returns:
List of changed file paths
"""
if not commit_before or not commit_after or commit_before == commit_after:
return []
try:
result = subprocess.run(
["git", "diff", "--name-only", commit_before, commit_after],
cwd=project_dir,
capture_output=True,
text=True,
timeout=10,
)
files = [f.strip() for f in result.stdout.strip().split("\n") if f.strip()]
return files
except Exception as e:
logger.warning(f"Failed to get changed files: {e}")
return []
def get_commit_messages(
project_dir: Path,
commit_before: str | None,
commit_after: str | None,
) -> str:
"""Get commit messages between two commits."""
if not commit_before or not commit_after or commit_before == commit_after:
return "(No commits)"
try:
result = subprocess.run(
["git", "log", "--oneline", f"{commit_before}..{commit_after}"],
cwd=project_dir,
capture_output=True,
text=True,
timeout=10,
)
return result.stdout.strip() if result.stdout.strip() else "(No commits)"
except Exception as e:
logger.warning(f"Failed to get commit messages: {e}")
return f"(Failed: {e})"
# =============================================================================
# Input Gathering
# =============================================================================
def gather_extraction_inputs(
spec_dir: Path,
project_dir: Path,
subtask_id: str,
session_num: int,
commit_before: str | None,
commit_after: str | None,
success: bool,
recovery_manager: Any,
) -> dict:
"""
Gather all inputs needed for insight extraction.
Args:
spec_dir: Spec directory
project_dir: Project root
subtask_id: The subtask that was worked on
session_num: Session number
commit_before: Commit before session
commit_after: Commit after session
success: Whether session succeeded
recovery_manager: Recovery manager with attempt history
Returns:
Dict with all inputs for the extractor
"""
# Get subtask description from implementation plan
subtask_description = _get_subtask_description(spec_dir, subtask_id)
# Get git diff
diff = get_session_diff(project_dir, commit_before, commit_after)
# Get changed files
changed_files = get_changed_files(project_dir, commit_before, commit_after)
# Get commit messages
commit_messages = get_commit_messages(project_dir, commit_before, commit_after)
# Get attempt history
attempt_history = _get_attempt_history(recovery_manager, subtask_id)
return {
"subtask_id": subtask_id,
"subtask_description": subtask_description,
"session_num": session_num,
"success": success,
"diff": diff,
"changed_files": changed_files,
"commit_messages": commit_messages,
"attempt_history": attempt_history,
}
def _get_subtask_description(spec_dir: Path, subtask_id: str) -> str:
"""Get subtask description from implementation plan."""
plan_file = spec_dir / "implementation_plan.json"
if not plan_file.exists():
return f"Subtask: {subtask_id}"
try:
with open(plan_file) as f:
plan = json.load(f)
# Search through phases for the subtask
for phase in plan.get("phases", []):
for subtask in phase.get("subtasks", []):
if subtask.get("id") == subtask_id:
return subtask.get("description", f"Subtask: {subtask_id}")
return f"Subtask: {subtask_id}"
except Exception as e:
logger.warning(f"Failed to load subtask description: {e}")
return f"Subtask: {subtask_id}"
def _get_attempt_history(recovery_manager: Any, subtask_id: str) -> list[dict]:
"""Get previous attempt history for this subtask."""
if not recovery_manager:
return []
try:
history = recovery_manager.get_subtask_history(subtask_id)
attempts = history.get("attempts", [])
# Limit to recent attempts
return attempts[-MAX_ATTEMPTS_TO_INCLUDE:]
except Exception as e:
logger.warning(f"Failed to get attempt history: {e}")
return []
# =============================================================================
# LLM Extraction
# =============================================================================
def _build_extraction_prompt(inputs: dict) -> str:
"""Build the prompt for insight extraction."""
prompt_file = Path(__file__).parent / "prompts" / "insight_extractor.md"
if prompt_file.exists():
base_prompt = prompt_file.read_text()
else:
# Fallback if prompt file missing
base_prompt = """Extract structured insights from this coding session.
Output ONLY valid JSON with: file_insights, patterns_discovered, gotchas_discovered, approach_outcome, recommendations"""
# Build session context
session_context = f"""
---
## SESSION DATA
### Subtask
- **ID**: {inputs["subtask_id"]}
- **Description**: {inputs["subtask_description"]}
- **Session Number**: {inputs["session_num"]}
- **Outcome**: {"SUCCESS" if inputs["success"] else "FAILED"}
### Files Changed
{chr(10).join(f"- {f}" for f in inputs["changed_files"]) if inputs["changed_files"] else "(No files changed)"}
### Commit Messages
{inputs["commit_messages"]}
### Git Diff
```diff
{inputs["diff"]}
```
### Previous Attempts
{_format_attempt_history(inputs["attempt_history"])}
---
Now analyze this session and output ONLY the JSON object.
"""
return base_prompt + session_context
def _format_attempt_history(attempts: list[dict]) -> str:
"""Format attempt history for the prompt."""
if not attempts:
return "(First attempt - no previous history)"
lines = []
for i, attempt in enumerate(attempts, 1):
success = "SUCCESS" if attempt.get("success") else "FAILED"
approach = attempt.get("approach", "Unknown approach")
error = attempt.get("error", "")
lines.append(f"**Attempt {i}** ({success}): {approach}")
if error:
lines.append(f" Error: {error}")
return "\n".join(lines)
async def run_insight_extraction(inputs: dict, project_dir: Path | None = None) -> dict | None:
"""
Run the insight extraction using Claude Agent SDK.
Args:
inputs: Gathered session inputs
project_dir: Project directory for SDK context (optional)
Returns:
Extracted insights dict or None if failed
"""
if not SDK_AVAILABLE:
logger.warning("Claude SDK not available, skipping insight extraction")
return None
oauth_token = os.environ.get("CLAUDE_CODE_OAUTH_TOKEN")
if not oauth_token:
logger.warning("CLAUDE_CODE_OAUTH_TOKEN not set, skipping insight extraction")
return None
model = get_extraction_model()
prompt = _build_extraction_prompt(inputs)
# Use current directory if project_dir not specified
cwd = str(project_dir.resolve()) if project_dir else os.getcwd()
try:
# Create a minimal SDK client for insight extraction
# No tools needed - just text generation
client = ClaudeSDKClient(
options=ClaudeAgentOptions(
model=model,
system_prompt=(
"You are an expert code analyst. You extract structured insights from coding sessions. "
"Always respond with valid JSON only, no markdown formatting or explanations."
),
allowed_tools=[], # No tools needed for extraction
max_turns=1, # Single turn extraction
cwd=cwd,
)
)
# Use async context manager
async with client:
await client.query(prompt)
# Collect the response
response_text = ""
async for msg in client.receive_response():
msg_type = type(msg).__name__
if msg_type == "AssistantMessage" and hasattr(msg, "content"):
for block in msg.content:
if hasattr(block, "text"):
response_text += block.text
# Parse JSON from response
return parse_insights(response_text)
except Exception as e:
logger.warning(f"Insight extraction failed: {e}")
return None
def parse_insights(response_text: str) -> dict | None:
"""
Parse the LLM response into structured insights.
Args:
response_text: Raw LLM response
Returns:
Parsed insights dict or None if parsing failed
"""
# Try to extract JSON from the response
text = response_text.strip()
# Handle markdown code blocks
if text.startswith("```"):
# Remove code block markers
lines = text.split("\n")
# Remove first line (```json or ```)
if lines[0].startswith("```"):
lines = lines[1:]
# Remove last line if it's ``
if lines and lines[-1].strip() == "```":
lines = lines[:-1]
text = "\n".join(lines)
try:
insights = json.loads(text)
# Validate structure
if not isinstance(insights, dict):
logger.warning("Insights is not a dict")
return None
# Ensure required keys exist with defaults
insights.setdefault("file_insights", [])
insights.setdefault("patterns_discovered", [])
insights.setdefault("gotchas_discovered", [])
insights.setdefault("approach_outcome", {})
insights.setdefault("recommendations", [])
return insights
except json.JSONDecodeError as e:
logger.warning(f"Failed to parse insights JSON: {e}")
logger.debug(f"Response text was: {text[:500]}")
return None
# =============================================================================
# Main Entry Point
# =============================================================================
async def extract_session_insights(
spec_dir: Path,
project_dir: Path,
subtask_id: str,
session_num: int,
commit_before: str | None,
commit_after: str | None,
success: bool,
recovery_manager: Any,
) -> dict:
"""
Extract insights from a completed coding session.
This is the main entry point called from post_session_processing().
Falls back to generic insights if extraction fails.
Args:
spec_dir: Spec directory
project_dir: Project root
subtask_id: Subtask that was worked on
session_num: Session number
commit_before: Commit before session
commit_after: Commit after session
success: Whether session succeeded
recovery_manager: Recovery manager with attempt history
Returns:
Insights dict (rich if extraction succeeded, generic if failed)
"""
# Check if extraction is enabled
if not is_extraction_enabled():
logger.info("Insight extraction disabled")
return _get_generic_insights(subtask_id, success)
# Check for no changes
if commit_before == commit_after:
logger.info("No changes to extract insights from")
return _get_generic_insights(subtask_id, success)
try:
# Gather inputs
inputs = gather_extraction_inputs(
spec_dir=spec_dir,
project_dir=project_dir,
subtask_id=subtask_id,
session_num=session_num,
commit_before=commit_before,
commit_after=commit_after,
success=success,
recovery_manager=recovery_manager,
)
# Run extraction
extracted = await run_insight_extraction(inputs, project_dir=project_dir)
if extracted:
# Add metadata
extracted["subtask_id"] = subtask_id
extracted["session_num"] = session_num
extracted["success"] = success
extracted["changed_files"] = inputs["changed_files"]
logger.info(
f"Extracted insights: {len(extracted.get('file_insights', []))} file insights, "
f"{len(extracted.get('patterns_discovered', []))} patterns, "
f"{len(extracted.get('gotchas_discovered', []))} gotchas"
)
return extracted
else:
logger.warning("Extraction returned no results, using generic insights")
return _get_generic_insights(subtask_id, success)
except Exception as e:
logger.warning(f"Insight extraction failed: {e}, using generic insights")
return _get_generic_insights(subtask_id, success)
def _get_generic_insights(subtask_id: str, success: bool) -> dict:
"""Return generic insights when extraction fails or is disabled."""
return {
"file_insights": [],
"patterns_discovered": [],
"gotchas_discovered": [],
"approach_outcome": {
"success": success,
"approach_used": f"Implemented subtask: {subtask_id}",
"why_it_worked": None,
"why_it_failed": None,
"alternatives_tried": [],
},
"recommendations": [],
"subtask_id": subtask_id,
"success": success,
"changed_files": [],
}
# =============================================================================
# CLI for Testing
# =============================================================================
if __name__ == "__main__":
import argparse
import asyncio
parser = argparse.ArgumentParser(description="Test insight extraction")
parser.add_argument("--spec-dir", type=Path, required=True, help="Spec directory")
parser.add_argument(
"--project-dir", type=Path, required=True, help="Project directory"
)
parser.add_argument(
"--commit-before", type=str, required=True, help="Commit before session"
)
parser.add_argument(
"--commit-after", type=str, required=True, help="Commit after session"
)
parser.add_argument(
"--subtask-id", type=str, default="test-subtask", help="Subtask ID"
)
args = parser.parse_args()
async def main():
insights = await extract_session_insights(
spec_dir=args.spec_dir,
project_dir=args.project_dir,
subtask_id=args.subtask_id,
session_num=1,
commit_before=args.commit_before,
commit_after=args.commit_after,
success=True,
recovery_manager=None,
)
print(json.dumps(insights, indent=2))
asyncio.run(main())
+106
View File
@@ -0,0 +1,106 @@
"""
Smart Project Analyzer for Dynamic Security Profiles
=====================================================
FACADE MODULE: This module re-exports all functionality from the
auto-claude/project/ package for backward compatibility.
The implementation has been refactored into focused modules:
- project/command_registry.py - Command registries
- project/models.py - Data structures
- project/config_parser.py - Config file parsing
- project/stack_detector.py - Stack detection
- project/framework_detector.py - Framework detection
- project/structure_analyzer.py - Project structure analysis
- project/analyzer.py - Main orchestration
This file maintains the original API so existing imports continue to work.
This system:
1. Detects languages, frameworks, databases, and infrastructure
2. Parses package.json scripts, Makefile targets, pyproject.toml scripts
3. Builds a tailored security profile for the specific project
4. Caches the profile for subsequent runs
5. Can re-analyze when project structure changes
The goal: Allow an AI developer to run any command that's legitimately
needed for the detected tech stack, while blocking dangerous operations.
"""
# Re-export all public API from the project module
from project import (
# Command registries
BASE_COMMANDS,
VALIDATED_COMMANDS,
CustomScripts,
# Main classes
ProjectAnalyzer,
SecurityProfile,
TechnologyStack,
# Utility functions
get_or_create_profile,
is_command_allowed,
needs_validation,
)
# Also re-export command registries for backward compatibility
from project.command_registry import (
CLOUD_COMMANDS,
CODE_QUALITY_COMMANDS,
DATABASE_COMMANDS,
FRAMEWORK_COMMANDS,
INFRASTRUCTURE_COMMANDS,
LANGUAGE_COMMANDS,
PACKAGE_MANAGER_COMMANDS,
VERSION_MANAGER_COMMANDS,
)
__all__ = [
# Main classes
"ProjectAnalyzer",
"SecurityProfile",
"TechnologyStack",
"CustomScripts",
# Utility functions
"get_or_create_profile",
"is_command_allowed",
"needs_validation",
# Base command sets
"BASE_COMMANDS",
"VALIDATED_COMMANDS",
# Technology-specific command sets
"LANGUAGE_COMMANDS",
"PACKAGE_MANAGER_COMMANDS",
"FRAMEWORK_COMMANDS",
"DATABASE_COMMANDS",
"INFRASTRUCTURE_COMMANDS",
"CLOUD_COMMANDS",
"CODE_QUALITY_COMMANDS",
"VERSION_MANAGER_COMMANDS",
]
# =============================================================================
# CLI for testing
# =============================================================================
if __name__ == "__main__":
import sys
from pathlib import Path
if len(sys.argv) < 2:
print("Usage: python project_analyzer.py <project_dir> [--force]")
sys.exit(1)
project_dir = Path(sys.argv[1])
force = "--force" in sys.argv
if not project_dir.exists():
print(f"Error: {project_dir} does not exist")
sys.exit(1)
profile = get_or_create_profile(project_dir, force_reanalyze=force)
print("\nAllowed commands:")
for cmd in sorted(profile.get_all_allowed_commands()):
print(f" {cmd}")
+589
View File
@@ -0,0 +1,589 @@
#!/usr/bin/env python3
"""
Risk Classifier Module
======================
Reads the AI-generated complexity_assessment.json and provides programmatic
access to risk classification and validation recommendations.
This module serves as the bridge between the AI complexity assessor prompt
and the rest of the validation system.
Usage:
from risk_classifier import RiskClassifier
classifier = RiskClassifier()
assessment = classifier.load_assessment(spec_dir)
if classifier.should_skip_validation(spec_dir):
print("Validation can be skipped for this task")
test_types = classifier.get_required_test_types(spec_dir)
"""
import json
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
# =============================================================================
# DATA CLASSES
# =============================================================================
@dataclass
class ScopeAnalysis:
"""Analysis of task scope."""
estimated_files: int = 0
estimated_services: int = 0
is_cross_cutting: bool = False
notes: str = ""
@dataclass
class IntegrationAnalysis:
"""Analysis of external integrations."""
external_services: list[str] = field(default_factory=list)
new_dependencies: list[str] = field(default_factory=list)
research_needed: bool = False
notes: str = ""
@dataclass
class InfrastructureAnalysis:
"""Analysis of infrastructure requirements."""
docker_changes: bool = False
database_changes: bool = False
config_changes: bool = False
notes: str = ""
@dataclass
class KnowledgeAnalysis:
"""Analysis of knowledge requirements."""
patterns_exist: bool = True
research_required: bool = False
unfamiliar_tech: list[str] = field(default_factory=list)
notes: str = ""
@dataclass
class RiskAnalysis:
"""Analysis of task risk."""
level: str = "low" # low, medium, high
concerns: list[str] = field(default_factory=list)
notes: str = ""
@dataclass
class ComplexityAnalysis:
"""Full complexity analysis from the AI assessor."""
scope: ScopeAnalysis = field(default_factory=ScopeAnalysis)
integrations: IntegrationAnalysis = field(default_factory=IntegrationAnalysis)
infrastructure: InfrastructureAnalysis = field(
default_factory=InfrastructureAnalysis
)
knowledge: KnowledgeAnalysis = field(default_factory=KnowledgeAnalysis)
risk: RiskAnalysis = field(default_factory=RiskAnalysis)
@dataclass
class ValidationRecommendations:
"""Validation recommendations from the AI assessor."""
risk_level: str = "medium" # trivial, low, medium, high, critical
skip_validation: bool = False
minimal_mode: bool = False
test_types_required: list[str] = field(default_factory=lambda: ["unit"])
security_scan_required: bool = False
staging_deployment_required: bool = False
reasoning: str = ""
@dataclass
class AssessmentFlags:
"""Flags indicating special requirements."""
needs_research: bool = False
needs_self_critique: bool = False
needs_infrastructure_setup: bool = False
@dataclass
class RiskAssessment:
"""Complete risk assessment from complexity_assessment.json."""
complexity: str # simple, standard, complex
workflow_type: str # feature, refactor, investigation, migration, simple
confidence: float
reasoning: str
analysis: ComplexityAnalysis
recommended_phases: list[str]
flags: AssessmentFlags
validation: ValidationRecommendations
created_at: str | None = None
@property
def risk_level(self) -> str:
"""Get the risk level from validation recommendations."""
return self.validation.risk_level
# =============================================================================
# RISK CLASSIFIER
# =============================================================================
class RiskClassifier:
"""
Reads AI-generated complexity_assessment.json and provides risk classification.
The complexity_assessment.json is generated by the AI complexity assessor
agent using the complexity_assessor.md prompt. This module parses that output
and provides programmatic access to the risk classification.
"""
def __init__(self) -> None:
"""Initialize the risk classifier."""
self._cache: dict[str, RiskAssessment] = {}
def load_assessment(self, spec_dir: Path) -> RiskAssessment | None:
"""
Load complexity_assessment.json from spec directory.
Args:
spec_dir: Path to the spec directory containing complexity_assessment.json
Returns:
RiskAssessment object if file exists and is valid, None otherwise
"""
spec_dir = Path(spec_dir)
cache_key = str(spec_dir.resolve())
# Return cached result if available
if cache_key in self._cache:
return self._cache[cache_key]
assessment_file = spec_dir / "complexity_assessment.json"
if not assessment_file.exists():
return None
try:
with open(assessment_file, encoding="utf-8") as f:
data = json.load(f)
assessment = self._parse_assessment(data)
self._cache[cache_key] = assessment
return assessment
except (json.JSONDecodeError, KeyError, TypeError) as e:
# Log error but don't crash - return None to allow fallback behavior
print(f"Warning: Failed to parse complexity_assessment.json: {e}")
return None
def _parse_assessment(self, data: dict[str, Any]) -> RiskAssessment:
"""Parse raw JSON data into a RiskAssessment object."""
# Parse analysis sections
analysis_data = data.get("analysis", {})
analysis = ComplexityAnalysis(
scope=self._parse_scope(analysis_data.get("scope", {})),
integrations=self._parse_integrations(
analysis_data.get("integrations", {})
),
infrastructure=self._parse_infrastructure(
analysis_data.get("infrastructure", {})
),
knowledge=self._parse_knowledge(analysis_data.get("knowledge", {})),
risk=self._parse_risk(analysis_data.get("risk", {})),
)
# Parse flags
flags_data = data.get("flags", {})
flags = AssessmentFlags(
needs_research=flags_data.get("needs_research", False),
needs_self_critique=flags_data.get("needs_self_critique", False),
needs_infrastructure_setup=flags_data.get(
"needs_infrastructure_setup", False
),
)
# Parse validation recommendations
validation_data = data.get("validation_recommendations", {})
validation = self._parse_validation_recommendations(validation_data, analysis)
return RiskAssessment(
complexity=data.get("complexity", "standard"),
workflow_type=data.get("workflow_type", "feature"),
confidence=float(data.get("confidence", 0.5)),
reasoning=data.get("reasoning", ""),
analysis=analysis,
recommended_phases=data.get("recommended_phases", []),
flags=flags,
validation=validation,
created_at=data.get("created_at"),
)
def _parse_scope(self, data: dict[str, Any]) -> ScopeAnalysis:
"""Parse scope analysis section."""
return ScopeAnalysis(
estimated_files=int(data.get("estimated_files", 0)),
estimated_services=int(data.get("estimated_services", 0)),
is_cross_cutting=bool(data.get("is_cross_cutting", False)),
notes=str(data.get("notes", "")),
)
def _parse_integrations(self, data: dict[str, Any]) -> IntegrationAnalysis:
"""Parse integrations analysis section."""
return IntegrationAnalysis(
external_services=list(data.get("external_services", [])),
new_dependencies=list(data.get("new_dependencies", [])),
research_needed=bool(data.get("research_needed", False)),
notes=str(data.get("notes", "")),
)
def _parse_infrastructure(self, data: dict[str, Any]) -> InfrastructureAnalysis:
"""Parse infrastructure analysis section."""
return InfrastructureAnalysis(
docker_changes=bool(data.get("docker_changes", False)),
database_changes=bool(data.get("database_changes", False)),
config_changes=bool(data.get("config_changes", False)),
notes=str(data.get("notes", "")),
)
def _parse_knowledge(self, data: dict[str, Any]) -> KnowledgeAnalysis:
"""Parse knowledge analysis section."""
return KnowledgeAnalysis(
patterns_exist=bool(data.get("patterns_exist", True)),
research_required=bool(data.get("research_required", False)),
unfamiliar_tech=list(data.get("unfamiliar_tech", [])),
notes=str(data.get("notes", "")),
)
def _parse_risk(self, data: dict[str, Any]) -> RiskAnalysis:
"""Parse risk analysis section."""
return RiskAnalysis(
level=str(data.get("level", "low")),
concerns=list(data.get("concerns", [])),
notes=str(data.get("notes", "")),
)
def _parse_validation_recommendations(
self, data: dict[str, Any], analysis: ComplexityAnalysis
) -> ValidationRecommendations:
"""
Parse validation recommendations section.
If validation_recommendations is not present in the JSON (older assessments),
infer appropriate values from the analysis.
"""
if data:
# New format with explicit validation recommendations
return ValidationRecommendations(
risk_level=str(data.get("risk_level", "medium")),
skip_validation=bool(data.get("skip_validation", False)),
minimal_mode=bool(data.get("minimal_mode", False)),
test_types_required=list(data.get("test_types_required", ["unit"])),
security_scan_required=bool(data.get("security_scan_required", False)),
staging_deployment_required=bool(
data.get("staging_deployment_required", False)
),
reasoning=str(data.get("reasoning", "")),
)
else:
# Infer from analysis (backward compatibility)
return self._infer_validation_recommendations(analysis)
def _infer_validation_recommendations(
self, analysis: ComplexityAnalysis
) -> ValidationRecommendations:
"""
Infer validation recommendations from analysis when not explicitly provided.
This provides backward compatibility with older complexity assessments
that don't have the validation_recommendations section.
"""
risk_level = analysis.risk.level
# Map old risk levels to new ones
risk_mapping = {
"low": "low",
"medium": "medium",
"high": "high",
}
normalized_risk = risk_mapping.get(risk_level, "medium")
# Infer test types based on risk
test_types_map = {
"low": ["unit"],
"medium": ["unit", "integration"],
"high": ["unit", "integration", "e2e"],
}
test_types = test_types_map.get(normalized_risk, ["unit", "integration"])
# Security scan for high risk or security-related concerns
security_keywords = [
"security",
"auth",
"password",
"credential",
"token",
"api key",
]
has_security_concerns = any(
kw in str(analysis.risk.concerns).lower() for kw in security_keywords
)
security_scan_required = normalized_risk == "high" or has_security_concerns
# Staging for database or infrastructure changes
staging_required = (
analysis.infrastructure.database_changes
and normalized_risk in ["medium", "high"]
)
# Minimal mode for simple changes
minimal_mode = (
analysis.scope.estimated_files <= 2
and analysis.scope.estimated_services <= 1
and not analysis.integrations.external_services
)
return ValidationRecommendations(
risk_level=normalized_risk,
skip_validation=False, # Never skip by inference
minimal_mode=minimal_mode,
test_types_required=test_types,
security_scan_required=security_scan_required,
staging_deployment_required=staging_required,
reasoning="Inferred from complexity analysis (no explicit recommendations found)",
)
def should_skip_validation(self, spec_dir: Path) -> bool:
"""
Quick check if validation can be skipped entirely.
Args:
spec_dir: Path to the spec directory
Returns:
True if validation can be skipped (trivial changes), False otherwise
"""
assessment = self.load_assessment(spec_dir)
if not assessment:
return False # When in doubt, don't skip
return assessment.validation.skip_validation
def should_use_minimal_mode(self, spec_dir: Path) -> bool:
"""
Check if minimal validation mode should be used.
Args:
spec_dir: Path to the spec directory
Returns:
True if minimal mode is recommended, False otherwise
"""
assessment = self.load_assessment(spec_dir)
if not assessment:
return False
return assessment.validation.minimal_mode
def get_required_test_types(self, spec_dir: Path) -> list[str]:
"""
Get list of required test types based on risk.
Args:
spec_dir: Path to the spec directory
Returns:
List of test types (e.g., ["unit", "integration", "e2e"])
"""
assessment = self.load_assessment(spec_dir)
if not assessment:
return ["unit"] # Default to unit tests
return assessment.validation.test_types_required
def requires_security_scan(self, spec_dir: Path) -> bool:
"""
Check if security scanning is required.
Args:
spec_dir: Path to the spec directory
Returns:
True if security scan is required, False otherwise
"""
assessment = self.load_assessment(spec_dir)
if not assessment:
return False
return assessment.validation.security_scan_required
def requires_staging_deployment(self, spec_dir: Path) -> bool:
"""
Check if staging deployment is required.
Args:
spec_dir: Path to the spec directory
Returns:
True if staging deployment is required, False otherwise
"""
assessment = self.load_assessment(spec_dir)
if not assessment:
return False
return assessment.validation.staging_deployment_required
def get_risk_level(self, spec_dir: Path) -> str:
"""
Get the risk level for the task.
Args:
spec_dir: Path to the spec directory
Returns:
Risk level string (trivial, low, medium, high, critical)
"""
assessment = self.load_assessment(spec_dir)
if not assessment:
return "medium" # Default to medium when unknown
return assessment.validation.risk_level
def get_complexity(self, spec_dir: Path) -> str:
"""
Get the complexity level for the task.
Args:
spec_dir: Path to the spec directory
Returns:
Complexity level string (simple, standard, complex)
"""
assessment = self.load_assessment(spec_dir)
if not assessment:
return "standard" # Default to standard when unknown
return assessment.complexity
def get_validation_summary(self, spec_dir: Path) -> dict[str, Any]:
"""
Get a summary of validation requirements.
Args:
spec_dir: Path to the spec directory
Returns:
Dictionary with validation summary
"""
assessment = self.load_assessment(spec_dir)
if not assessment:
return {
"risk_level": "unknown",
"complexity": "unknown",
"skip_validation": False,
"minimal_mode": False,
"test_types": ["unit"],
"security_scan": False,
"staging_deployment": False,
"confidence": 0.0,
}
return {
"risk_level": assessment.validation.risk_level,
"complexity": assessment.complexity,
"skip_validation": assessment.validation.skip_validation,
"minimal_mode": assessment.validation.minimal_mode,
"test_types": assessment.validation.test_types_required,
"security_scan": assessment.validation.security_scan_required,
"staging_deployment": assessment.validation.staging_deployment_required,
"confidence": assessment.confidence,
"reasoning": assessment.validation.reasoning,
}
def clear_cache(self) -> None:
"""Clear the internal cache of loaded assessments."""
self._cache.clear()
# =============================================================================
# CONVENIENCE FUNCTIONS
# =============================================================================
def load_risk_assessment(spec_dir: Path) -> RiskAssessment | None:
"""
Convenience function to load a risk assessment.
Args:
spec_dir: Path to the spec directory
Returns:
RiskAssessment object or None
"""
classifier = RiskClassifier()
return classifier.load_assessment(spec_dir)
def get_validation_requirements(spec_dir: Path) -> dict[str, Any]:
"""
Convenience function to get validation requirements.
Args:
spec_dir: Path to the spec directory
Returns:
Dictionary with validation requirements
"""
classifier = RiskClassifier()
return classifier.get_validation_summary(spec_dir)
# =============================================================================
# CLI
# =============================================================================
def main() -> None:
"""CLI entry point for testing."""
import argparse
parser = argparse.ArgumentParser(description="Load and display risk assessment")
parser.add_argument(
"spec_dir",
type=Path,
help="Path to spec directory with complexity_assessment.json",
)
parser.add_argument("--json", action="store_true", help="Output as JSON")
args = parser.parse_args()
classifier = RiskClassifier()
summary = classifier.get_validation_summary(args.spec_dir)
if args.json:
print(json.dumps(summary, indent=2))
else:
print(f"Risk Level: {summary['risk_level']}")
print(f"Complexity: {summary['complexity']}")
print(f"Skip Validation: {summary['skip_validation']}")
print(f"Minimal Mode: {summary['minimal_mode']}")
print(f"Test Types: {', '.join(summary['test_types'])}")
print(f"Security Scan: {summary['security_scan']}")
print(f"Staging Deployment: {summary['staging_deployment']}")
print(f"Confidence: {summary['confidence']:.2f}")
if summary.get("reasoning"):
print(f"Reasoning: {summary['reasoning']}")
if __name__ == "__main__":
main()
+597
View File
@@ -0,0 +1,597 @@
#!/usr/bin/env python3
"""
Security Scanner Module
=======================
Consolidates security scanning including secrets detection and SAST tools.
This module integrates the existing scan_secrets.py and provides a unified
interface for all security scanning.
The security scanner is used by:
- QA Agent: To verify no secrets are committed
- Validation Strategy: To run security scans for high-risk changes
Usage:
from analysis.security_scanner import SecurityScanner
scanner = SecurityScanner()
results = scanner.scan(project_dir, spec_dir)
if results.has_critical_issues:
print("Security issues found - blocking QA approval")
"""
import json
import subprocess
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
# Import the existing secrets scanner
try:
from security.scan_secrets import SecretMatch, get_all_tracked_files, scan_files
HAS_SECRETS_SCANNER = True
except ImportError:
HAS_SECRETS_SCANNER = False
SecretMatch = None
# =============================================================================
# DATA CLASSES
# =============================================================================
@dataclass
class SecurityVulnerability:
"""
Represents a security vulnerability found during scanning.
Attributes:
severity: Severity level (critical, high, medium, low, info)
source: Which scanner found this (secrets, bandit, npm_audit, etc.)
title: Short title of the vulnerability
description: Detailed description
file: File where vulnerability was found (if applicable)
line: Line number (if applicable)
cwe: CWE identifier if available
"""
severity: str # critical, high, medium, low, info
source: str # secrets, bandit, npm_audit, semgrep, etc.
title: str
description: str
file: str | None = None
line: int | None = None
cwe: str | None = None
@dataclass
class SecurityScanResult:
"""
Result of a security scan.
Attributes:
secrets: List of detected secrets
vulnerabilities: List of security vulnerabilities
scan_errors: List of errors during scanning
has_critical_issues: Whether any critical issues were found
should_block_qa: Whether these results should block QA approval
"""
secrets: list[dict[str, Any]] = field(default_factory=list)
vulnerabilities: list[SecurityVulnerability] = field(default_factory=list)
scan_errors: list[str] = field(default_factory=list)
has_critical_issues: bool = False
should_block_qa: bool = False
# =============================================================================
# SECURITY SCANNER
# =============================================================================
class SecurityScanner:
"""
Consolidates all security scanning operations.
Integrates:
- scan_secrets.py for secrets detection
- Bandit for Python SAST (if available)
- npm audit for JavaScript vulnerabilities (if applicable)
"""
def __init__(self) -> None:
"""Initialize the security scanner."""
self._bandit_available: bool | None = None
self._npm_available: bool | None = None
def scan(
self,
project_dir: Path,
spec_dir: Path | None = None,
changed_files: list[str] | None = None,
run_secrets: bool = True,
run_sast: bool = True,
run_dependency_audit: bool = True,
) -> SecurityScanResult:
"""
Run all applicable security scans.
Args:
project_dir: Path to the project root
spec_dir: Path to the spec directory (for storing results)
changed_files: Optional list of files to scan (if None, scans all)
run_secrets: Whether to run secrets scanning
run_sast: Whether to run SAST tools
run_dependency_audit: Whether to run dependency audits
Returns:
SecurityScanResult with all findings
"""
project_dir = Path(project_dir)
result = SecurityScanResult()
# Run secrets scan
if run_secrets:
self._run_secrets_scan(project_dir, changed_files, result)
# Run SAST based on project type
if run_sast:
self._run_sast_scans(project_dir, result)
# Run dependency audits
if run_dependency_audit:
self._run_dependency_audits(project_dir, result)
# Determine if should block QA
result.has_critical_issues = (
any(v.severity in ["critical", "high"] for v in result.vulnerabilities)
or len(result.secrets) > 0
)
# Any secrets always block, critical vulnerabilities block
result.should_block_qa = len(result.secrets) > 0 or any(
v.severity == "critical" for v in result.vulnerabilities
)
# Save results if spec_dir provided
if spec_dir:
self._save_results(spec_dir, result)
return result
def _run_secrets_scan(
self,
project_dir: Path,
changed_files: list[str] | None,
result: SecurityScanResult,
) -> None:
"""Run secrets scanning using scan_secrets.py."""
if not HAS_SECRETS_SCANNER:
result.scan_errors.append("scan_secrets module not available")
return
try:
# Get files to scan
if changed_files:
files_to_scan = changed_files
else:
files_to_scan = get_all_tracked_files()
# Run scan
matches = scan_files(files_to_scan, project_dir)
# Convert matches to result format
for match in matches:
result.secrets.append(
{
"file": match.file_path,
"line": match.line_number,
"pattern": match.pattern_name,
"matched_text": self._redact_secret(match.matched_text),
}
)
# Also add as vulnerability
result.vulnerabilities.append(
SecurityVulnerability(
severity="critical",
source="secrets",
title=f"Potential secret: {match.pattern_name}",
description=f"Found potential {match.pattern_name} in file",
file=match.file_path,
line=match.line_number,
)
)
except Exception as e:
result.scan_errors.append(f"Secrets scan error: {str(e)}")
def _run_sast_scans(self, project_dir: Path, result: SecurityScanResult) -> None:
"""Run SAST tools based on project type."""
# Python SAST with Bandit
if self._is_python_project(project_dir):
self._run_bandit(project_dir, result)
# JavaScript/Node.js - npm audit
# (handled in dependency audits for Node projects)
def _run_bandit(self, project_dir: Path, result: SecurityScanResult) -> None:
"""Run Bandit security scanner for Python projects."""
if not self._check_bandit_available():
return
try:
# Find Python source directories
src_dirs = []
for candidate in ["src", "app", project_dir.name, "."]:
candidate_path = project_dir / candidate
if (
candidate_path.exists()
and (candidate_path / "__init__.py").exists()
):
src_dirs.append(str(candidate_path))
if not src_dirs:
# Try to find any Python files
py_files = list(project_dir.glob("**/*.py"))
if not py_files:
return
src_dirs = ["."]
# Run bandit
cmd = [
"bandit",
"-r",
*src_dirs,
"-f",
"json",
"--exit-zero", # Don't fail on findings
]
proc = subprocess.run(
cmd,
cwd=project_dir,
capture_output=True,
text=True,
timeout=120,
)
if proc.stdout:
try:
bandit_output = json.loads(proc.stdout)
for finding in bandit_output.get("results", []):
severity = finding.get("issue_severity", "MEDIUM").lower()
if severity == "high":
severity = "high"
elif severity == "medium":
severity = "medium"
else:
severity = "low"
result.vulnerabilities.append(
SecurityVulnerability(
severity=severity,
source="bandit",
title=finding.get("issue_text", "Unknown issue"),
description=finding.get("issue_text", ""),
file=finding.get("filename"),
line=finding.get("line_number"),
cwe=finding.get("issue_cwe", {}).get("id"),
)
)
except json.JSONDecodeError:
result.scan_errors.append("Failed to parse Bandit output")
except subprocess.TimeoutExpired:
result.scan_errors.append("Bandit scan timed out")
except FileNotFoundError:
result.scan_errors.append("Bandit not found")
except Exception as e:
result.scan_errors.append(f"Bandit error: {str(e)}")
def _run_dependency_audits(
self, project_dir: Path, result: SecurityScanResult
) -> None:
"""Run dependency vulnerability audits."""
# npm audit for JavaScript projects
if (project_dir / "package.json").exists():
self._run_npm_audit(project_dir, result)
# pip-audit for Python projects (if available)
if self._is_python_project(project_dir):
self._run_pip_audit(project_dir, result)
def _run_npm_audit(self, project_dir: Path, result: SecurityScanResult) -> None:
"""Run npm audit for JavaScript projects."""
try:
cmd = ["npm", "audit", "--json"]
proc = subprocess.run(
cmd,
cwd=project_dir,
capture_output=True,
text=True,
timeout=120,
)
if proc.stdout:
try:
audit_output = json.loads(proc.stdout)
# npm audit v2+ format
vulnerabilities = audit_output.get("vulnerabilities", {})
for pkg_name, vuln_info in vulnerabilities.items():
severity = vuln_info.get("severity", "moderate")
if severity == "critical":
severity = "critical"
elif severity == "high":
severity = "high"
elif severity == "moderate":
severity = "medium"
else:
severity = "low"
result.vulnerabilities.append(
SecurityVulnerability(
severity=severity,
source="npm_audit",
title=f"Vulnerable dependency: {pkg_name}",
description=vuln_info.get("via", [{}])[0].get(
"title", ""
)
if isinstance(vuln_info.get("via"), list)
and vuln_info.get("via")
else str(vuln_info.get("via", "")),
file="package.json",
)
)
except json.JSONDecodeError:
pass # npm audit may return invalid JSON on no findings
except subprocess.TimeoutExpired:
result.scan_errors.append("npm audit timed out")
except FileNotFoundError:
pass # npm not available
except Exception as e:
result.scan_errors.append(f"npm audit error: {str(e)}")
def _run_pip_audit(self, project_dir: Path, result: SecurityScanResult) -> None:
"""Run pip-audit for Python projects (if available)."""
try:
cmd = ["pip-audit", "--format", "json"]
proc = subprocess.run(
cmd,
cwd=project_dir,
capture_output=True,
text=True,
timeout=120,
)
if proc.stdout:
try:
audit_output = json.loads(proc.stdout)
for vuln in audit_output:
severity = "high" if vuln.get("fix_versions") else "medium"
result.vulnerabilities.append(
SecurityVulnerability(
severity=severity,
source="pip_audit",
title=f"Vulnerable package: {vuln.get('name')}",
description=vuln.get("description", ""),
cwe=vuln.get("aliases", [""])[0]
if vuln.get("aliases")
else None,
)
)
except json.JSONDecodeError:
pass
except FileNotFoundError:
pass # pip-audit not available
except subprocess.TimeoutExpired:
pass
except Exception:
pass
def _is_python_project(self, project_dir: Path) -> bool:
"""Check if this is a Python project."""
indicators = [
project_dir / "pyproject.toml",
project_dir / "requirements.txt",
project_dir / "setup.py",
project_dir / "setup.cfg",
]
return any(p.exists() for p in indicators)
def _check_bandit_available(self) -> bool:
"""Check if Bandit is available."""
if self._bandit_available is None:
try:
subprocess.run(
["bandit", "--version"],
capture_output=True,
timeout=5,
)
self._bandit_available = True
except (FileNotFoundError, subprocess.TimeoutExpired):
self._bandit_available = False
return self._bandit_available
def _redact_secret(self, text: str) -> str:
"""Redact a secret for safe logging."""
if len(text) <= 8:
return "*" * len(text)
return text[:4] + "*" * (len(text) - 8) + text[-4:]
def _save_results(self, spec_dir: Path, result: SecurityScanResult) -> None:
"""Save scan results to spec directory."""
spec_dir = Path(spec_dir)
spec_dir.mkdir(parents=True, exist_ok=True)
output_file = spec_dir / "security_scan_results.json"
output_data = self.to_dict(result)
with open(output_file, "w", encoding="utf-8") as f:
json.dump(output_data, f, indent=2)
def to_dict(self, result: SecurityScanResult) -> dict[str, Any]:
"""Convert result to dictionary for JSON serialization."""
return {
"secrets": result.secrets,
"vulnerabilities": [
{
"severity": v.severity,
"source": v.source,
"title": v.title,
"description": v.description,
"file": v.file,
"line": v.line,
"cwe": v.cwe,
}
for v in result.vulnerabilities
],
"scan_errors": result.scan_errors,
"has_critical_issues": result.has_critical_issues,
"should_block_qa": result.should_block_qa,
"summary": {
"total_secrets": len(result.secrets),
"total_vulnerabilities": len(result.vulnerabilities),
"critical_count": sum(
1 for v in result.vulnerabilities if v.severity == "critical"
),
"high_count": sum(
1 for v in result.vulnerabilities if v.severity == "high"
),
"medium_count": sum(
1 for v in result.vulnerabilities if v.severity == "medium"
),
"low_count": sum(
1 for v in result.vulnerabilities if v.severity == "low"
),
},
}
# =============================================================================
# CONVENIENCE FUNCTIONS
# =============================================================================
def scan_for_security_issues(
project_dir: Path,
spec_dir: Path | None = None,
changed_files: list[str] | None = None,
) -> SecurityScanResult:
"""
Convenience function to run security scan.
Args:
project_dir: Path to project root
spec_dir: Optional spec directory to save results
changed_files: Optional list of files to scan
Returns:
SecurityScanResult with all findings
"""
scanner = SecurityScanner()
return scanner.scan(project_dir, spec_dir, changed_files)
def has_security_issues(project_dir: Path) -> bool:
"""
Quick check if project has security issues.
Args:
project_dir: Path to project root
Returns:
True if any critical/high issues found
"""
scanner = SecurityScanner()
result = scanner.scan(project_dir, run_sast=False, run_dependency_audit=False)
return result.has_critical_issues
def scan_secrets_only(
project_dir: Path,
changed_files: list[str] | None = None,
) -> list[dict[str, Any]]:
"""
Scan only for secrets (quick scan).
Args:
project_dir: Path to project root
changed_files: Optional list of files to scan
Returns:
List of detected secrets
"""
scanner = SecurityScanner()
result = scanner.scan(
project_dir,
changed_files=changed_files,
run_sast=False,
run_dependency_audit=False,
)
return result.secrets
# =============================================================================
# CLI
# =============================================================================
def main() -> None:
"""CLI entry point for testing."""
import argparse
parser = argparse.ArgumentParser(description="Run security scans")
parser.add_argument("project_dir", type=Path, help="Path to project root")
parser.add_argument("--spec-dir", type=Path, help="Path to spec directory")
parser.add_argument(
"--secrets-only", action="store_true", help="Only scan for secrets"
)
parser.add_argument("--json", action="store_true", help="Output as JSON")
args = parser.parse_args()
scanner = SecurityScanner()
result = scanner.scan(
args.project_dir,
spec_dir=args.spec_dir,
run_sast=not args.secrets_only,
run_dependency_audit=not args.secrets_only,
)
if args.json:
print(json.dumps(scanner.to_dict(result), indent=2))
else:
print(f"Secrets Found: {len(result.secrets)}")
print(f"Vulnerabilities: {len(result.vulnerabilities)}")
print(f"Has Critical Issues: {result.has_critical_issues}")
print(f"Should Block QA: {result.should_block_qa}")
if result.secrets:
print("\nSecrets Detected:")
for secret in result.secrets:
print(f" - {secret['pattern']} in {secret['file']}:{secret['line']}")
if result.vulnerabilities:
print(f"\nVulnerabilities ({len(result.vulnerabilities)}):")
for v in result.vulnerabilities:
print(f" [{v.severity.upper()}] {v.title}")
if v.file:
print(f" File: {v.file}:{v.line or ''}")
if result.scan_errors:
print(f"\nScan Errors ({len(result.scan_errors)}):")
for error in result.scan_errors:
print(f" - {error}")
if __name__ == "__main__":
main()
+688
View File
@@ -0,0 +1,688 @@
#!/usr/bin/env python3
"""
Test Discovery Module
=====================
Detects test frameworks, test commands, and test directories in a project.
This module analyzes project configuration files to discover how tests
should be run.
The test discovery results are used by:
- QA Agent: To determine what test commands to run
- Test Creator: To know what framework to use when creating tests
- Planner: To include correct test commands in verification strategy
Usage:
from test_discovery import TestDiscovery
discovery = TestDiscovery()
result = discovery.discover(project_dir)
print(f"Test frameworks: {result['frameworks']}")
print(f"Test command: {result['test_command']}")
"""
import json
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
# =============================================================================
# DATA CLASSES
# =============================================================================
@dataclass
class TestFramework:
"""
Represents a detected test framework.
Attributes:
name: Name of the framework (e.g., "pytest", "jest", "vitest")
type: Type of testing (unit, integration, e2e, all)
command: Command to run tests
config_file: Configuration file if found
version: Version if detected
coverage_command: Command for coverage if available
"""
__test__ = False # Prevent pytest from collecting this as a test class
name: str
type: str # unit, integration, e2e, all
command: str
config_file: str | None = None
version: str | None = None
coverage_command: str | None = None
@dataclass
class TestDiscoveryResult:
"""
Result of test framework discovery.
Attributes:
frameworks: List of detected test frameworks
test_command: Primary test command to run
test_directories: Discovered test directories
package_manager: Detected package manager
has_tests: Whether any test files were found
coverage_command: Command for coverage if available
"""
__test__ = False # Prevent pytest from collecting this as a test class
frameworks: list[TestFramework] = field(default_factory=list)
test_command: str = ""
test_directories: list[str] = field(default_factory=list)
package_manager: str = ""
has_tests: bool = False
coverage_command: str | None = None
# =============================================================================
# FRAMEWORK DETECTORS
# =============================================================================
# Pattern-based framework detection
FRAMEWORK_PATTERNS = {
# JavaScript/TypeScript
"jest": {
"config_files": [
"jest.config.js",
"jest.config.ts",
"jest.config.mjs",
"jest.config.cjs",
],
"package_key": "jest",
"type": "unit",
"command": "npx jest",
"coverage_command": "npx jest --coverage",
},
"vitest": {
"config_files": ["vitest.config.js", "vitest.config.ts", "vitest.config.mjs"],
"package_key": "vitest",
"type": "unit",
"command": "npx vitest run",
"coverage_command": "npx vitest run --coverage",
},
"mocha": {
"config_files": [
".mocharc.js",
".mocharc.json",
".mocharc.yaml",
".mocharc.yml",
],
"package_key": "mocha",
"type": "unit",
"command": "npx mocha",
"coverage_command": "npx nyc mocha",
},
"playwright": {
"config_files": ["playwright.config.js", "playwright.config.ts"],
"package_key": "@playwright/test",
"type": "e2e",
"command": "npx playwright test",
"coverage_command": None,
},
"cypress": {
"config_files": ["cypress.config.js", "cypress.config.ts", "cypress.json"],
"package_key": "cypress",
"type": "e2e",
"command": "npx cypress run",
"coverage_command": None,
},
# Python
"pytest": {
"config_files": ["pytest.ini", "pyproject.toml", "setup.cfg", "conftest.py"],
"pyproject_key": "pytest",
"requirements_key": "pytest",
"type": "all",
"command": "pytest",
"coverage_command": "pytest --cov",
},
"unittest": {
"config_files": [],
"type": "unit",
"command": "python -m unittest discover",
"coverage_command": "coverage run -m unittest discover",
},
# Rust
"cargo_test": {
"config_files": ["Cargo.toml"],
"type": "all",
"command": "cargo test",
"coverage_command": "cargo tarpaulin",
},
# Go
"go_test": {
"config_files": ["go.mod"],
"type": "all",
"command": "go test ./...",
"coverage_command": "go test -cover ./...",
},
# Ruby
"rspec": {
"config_files": [".rspec", "spec/spec_helper.rb"],
"gemfile_key": "rspec",
"type": "all",
"command": "bundle exec rspec",
"coverage_command": "bundle exec rspec --format documentation",
},
"minitest": {
"config_files": [],
"gemfile_key": "minitest",
"type": "unit",
"command": "bundle exec rake test",
"coverage_command": None,
},
}
# =============================================================================
# TEST DISCOVERY
# =============================================================================
class TestDiscovery:
"""
Discovers test frameworks and configurations in a project.
Analyzes:
- Package files (package.json, pyproject.toml, Cargo.toml, etc.)
- Configuration files (jest.config.js, pytest.ini, etc.)
- Directory structure (tests/, spec/, __tests__/)
"""
__test__ = False # Prevent pytest from collecting this as a test class
def __init__(self) -> None:
"""Initialize the test discovery."""
self._cache: dict[str, TestDiscoveryResult] = {}
def discover(self, project_dir: Path) -> TestDiscoveryResult:
"""
Discover test frameworks and configuration in the project.
Args:
project_dir: Path to the project root
Returns:
TestDiscoveryResult with detected frameworks and commands
"""
project_dir = Path(project_dir)
cache_key = str(project_dir.resolve())
if cache_key in self._cache:
return self._cache[cache_key]
result = TestDiscoveryResult()
# Detect package manager
result.package_manager = self._detect_package_manager(project_dir)
# Discover frameworks based on project type
if (project_dir / "package.json").exists():
self._discover_js_frameworks(project_dir, result)
# Check for Python project indicators
python_indicators = [
project_dir / "pyproject.toml",
project_dir / "requirements.txt",
project_dir / "setup.py",
project_dir / "pytest.ini",
project_dir / "conftest.py",
project_dir / "tests" / "conftest.py",
]
if any(p.exists() for p in python_indicators):
self._discover_python_frameworks(project_dir, result)
if (project_dir / "Cargo.toml").exists():
self._discover_rust_frameworks(project_dir, result)
if (project_dir / "go.mod").exists():
self._discover_go_frameworks(project_dir, result)
if (project_dir / "Gemfile").exists():
self._discover_ruby_frameworks(project_dir, result)
# Find test directories
result.test_directories = self._find_test_directories(project_dir)
# Check if tests exist
result.has_tests = self._has_test_files(project_dir, result.test_directories)
# Set primary test command
if result.frameworks:
result.test_command = result.frameworks[0].command
# Set coverage command from first framework that has one
if not result.coverage_command:
for framework in result.frameworks:
if framework.coverage_command:
result.coverage_command = framework.coverage_command
break
self._cache[cache_key] = result
return result
def _detect_package_manager(self, project_dir: Path) -> str:
"""Detect the package manager used by the project."""
if (project_dir / "pnpm-lock.yaml").exists():
return "pnpm"
if (project_dir / "yarn.lock").exists():
return "yarn"
if (project_dir / "package-lock.json").exists():
return "npm"
if (project_dir / "bun.lockb").exists():
return "bun"
if (project_dir / "uv.lock").exists():
return "uv"
if (project_dir / "poetry.lock").exists():
return "poetry"
if (project_dir / "Pipfile.lock").exists():
return "pipenv"
if (project_dir / "Cargo.lock").exists():
return "cargo"
if (project_dir / "go.sum").exists():
return "go"
if (project_dir / "Gemfile.lock").exists():
return "bundler"
return ""
def _discover_js_frameworks(
self, project_dir: Path, result: TestDiscoveryResult
) -> None:
"""Discover JavaScript/TypeScript test frameworks."""
package_json = project_dir / "package.json"
if not package_json.exists():
return
try:
with open(package_json, encoding="utf-8") as f:
pkg = json.load(f)
except (OSError, json.JSONDecodeError):
return
deps = pkg.get("dependencies", {})
dev_deps = pkg.get("devDependencies", {})
all_deps = {**deps, **dev_deps}
scripts = pkg.get("scripts", {})
# Check for test frameworks in dependencies
for name, pattern in FRAMEWORK_PATTERNS.items():
if "package_key" not in pattern:
continue
if pattern["package_key"] in all_deps:
# Check for config file
config_file = None
for cf in pattern.get("config_files", []):
if (project_dir / cf).exists():
config_file = cf
break
# Get version
version = all_deps.get(pattern["package_key"], "")
if version.startswith("^") or version.startswith("~"):
version = version[1:]
# Determine command - prefer npm scripts if available
command = pattern["command"]
if "test" in scripts and pattern["package_key"] in scripts.get(
"test", ""
):
command = f"{result.package_manager or 'npm'} test"
result.frameworks.append(
TestFramework(
name=name,
type=pattern["type"],
command=command,
config_file=config_file,
version=version,
coverage_command=pattern.get("coverage_command"),
)
)
# Check npm scripts for test commands
if not result.frameworks and "test" in scripts:
test_script = scripts["test"]
if (
test_script
and test_script != 'echo "Error: no test specified" && exit 1'
):
# Try to infer framework from script
framework_name = "npm_test"
framework_type = "unit"
if "jest" in test_script:
framework_name = "jest"
elif "vitest" in test_script:
framework_name = "vitest"
elif "mocha" in test_script:
framework_name = "mocha"
elif "playwright" in test_script:
framework_name = "playwright"
framework_type = "e2e"
elif "cypress" in test_script:
framework_name = "cypress"
framework_type = "e2e"
result.frameworks.append(
TestFramework(
name=framework_name,
type=framework_type,
command=f"{result.package_manager or 'npm'} test",
config_file=None,
)
)
def _discover_python_frameworks(
self, project_dir: Path, result: TestDiscoveryResult
) -> None:
"""Discover Python test frameworks."""
# Check for pytest.ini first (explicit pytest config)
if (project_dir / "pytest.ini").exists():
if not any(f.name == "pytest" for f in result.frameworks):
result.frameworks.append(
TestFramework(
name="pytest",
type="all",
command="pytest",
config_file="pytest.ini",
)
)
# Check pyproject.toml
pyproject = project_dir / "pyproject.toml"
if pyproject.exists():
content = pyproject.read_text()
# Check for pytest
if "pytest" in content:
if not any(f.name == "pytest" for f in result.frameworks):
config_file = (
"pyproject.toml" if "[tool.pytest" in content else None
)
result.frameworks.append(
TestFramework(
name="pytest",
type="all",
command="pytest",
config_file=config_file,
)
)
# Check requirements.txt
requirements = project_dir / "requirements.txt"
if requirements.exists():
content = requirements.read_text().lower()
if "pytest" in content and not any(
f.name == "pytest" for f in result.frameworks
):
result.frameworks.append(
TestFramework(
name="pytest",
type="all",
command="pytest",
config_file=None,
)
)
# Check for conftest.py (pytest marker)
conftest_root = project_dir / "conftest.py"
conftest_tests = project_dir / "tests" / "conftest.py"
if conftest_root.exists() or conftest_tests.exists():
if not any(f.name == "pytest" for f in result.frameworks):
result.frameworks.append(
TestFramework(
name="pytest",
type="all",
command="pytest",
config_file="conftest.py",
)
)
# Fall back to unittest if test files exist but no framework detected
if not result.frameworks:
test_dirs = self._find_test_directories(project_dir)
if test_dirs:
result.frameworks.append(
TestFramework(
name="unittest",
type="unit",
command="python -m unittest discover",
config_file=None,
)
)
def _discover_rust_frameworks(
self, project_dir: Path, result: TestDiscoveryResult
) -> None:
"""Discover Rust test frameworks."""
cargo_toml = project_dir / "Cargo.toml"
if cargo_toml.exists():
result.frameworks.append(
TestFramework(
name="cargo_test",
type="all",
command="cargo test",
config_file="Cargo.toml",
)
)
def _discover_go_frameworks(
self, project_dir: Path, result: TestDiscoveryResult
) -> None:
"""Discover Go test frameworks."""
go_mod = project_dir / "go.mod"
if go_mod.exists():
result.frameworks.append(
TestFramework(
name="go_test",
type="all",
command="go test ./...",
config_file="go.mod",
)
)
def _discover_ruby_frameworks(
self, project_dir: Path, result: TestDiscoveryResult
) -> None:
"""Discover Ruby test frameworks."""
gemfile = project_dir / "Gemfile"
if not gemfile.exists():
return
content = gemfile.read_text().lower()
if "rspec" in content or (project_dir / ".rspec").exists():
result.frameworks.append(
TestFramework(
name="rspec",
type="all",
command="bundle exec rspec",
config_file=".rspec" if (project_dir / ".rspec").exists() else None,
)
)
elif "minitest" in content:
result.frameworks.append(
TestFramework(
name="minitest",
type="unit",
command="bundle exec rake test",
config_file=None,
)
)
def _find_test_directories(self, project_dir: Path) -> list[str]:
"""Find test directories in the project."""
test_dir_patterns = [
"tests",
"test",
"spec",
"__tests__",
"specs",
"test_*",
]
found_dirs = []
for pattern in test_dir_patterns:
if pattern.endswith("*"):
# Glob pattern
for d in project_dir.glob(pattern):
if d.is_dir():
found_dirs.append(str(d.relative_to(project_dir)))
else:
# Exact name
test_dir = project_dir / pattern
if test_dir.is_dir():
found_dirs.append(pattern)
return found_dirs
def _has_test_files(self, project_dir: Path, test_directories: list[str]) -> bool:
"""Check if any test files exist."""
test_file_patterns = [
"**/test_*.py",
"**/*_test.py",
"**/*.test.js",
"**/*.test.ts",
"**/*.test.tsx",
"**/*.spec.js",
"**/*.spec.ts",
"**/*.spec.tsx",
"**/test_*.go",
"**/*_test.go",
"**/*_test.rs",
"**/spec/**/*_spec.rb",
]
# Check in test directories
for test_dir in test_directories:
test_path = project_dir / test_dir
if test_path.exists():
for pattern in test_file_patterns:
if list(test_path.glob(pattern.replace("**/", ""))):
return True
# Check project-wide
for pattern in test_file_patterns:
if list(project_dir.glob(pattern)):
return True
return False
def to_dict(self, result: TestDiscoveryResult) -> dict[str, Any]:
"""Convert result to dictionary for JSON serialization."""
return {
"frameworks": [
{
"name": f.name,
"type": f.type,
"command": f.command,
"config_file": f.config_file,
"version": f.version,
"coverage_command": f.coverage_command,
}
for f in result.frameworks
],
"test_command": result.test_command,
"test_directories": result.test_directories,
"package_manager": result.package_manager,
"has_tests": result.has_tests,
"coverage_command": result.coverage_command,
}
def clear_cache(self) -> None:
"""Clear the internal cache."""
self._cache.clear()
# =============================================================================
# CONVENIENCE FUNCTIONS
# =============================================================================
def discover_tests(project_dir: Path) -> TestDiscoveryResult:
"""
Convenience function to discover tests in a project.
Args:
project_dir: Path to project root
Returns:
TestDiscoveryResult with detected frameworks
"""
discovery = TestDiscovery()
return discovery.discover(project_dir)
def get_test_command(project_dir: Path) -> str:
"""
Get the primary test command for a project.
Args:
project_dir: Path to project root
Returns:
Test command string, or empty string if not found
"""
discovery = TestDiscovery()
result = discovery.discover(project_dir)
return result.test_command
def get_test_frameworks(project_dir: Path) -> list[str]:
"""
Get list of test framework names in a project.
Args:
project_dir: Path to project root
Returns:
List of framework names
"""
discovery = TestDiscovery()
result = discovery.discover(project_dir)
return [f.name for f in result.frameworks]
# =============================================================================
# CLI
# =============================================================================
def main() -> None:
"""CLI entry point for testing."""
import argparse
parser = argparse.ArgumentParser(description="Discover test frameworks")
parser.add_argument("project_dir", type=Path, help="Path to project root")
parser.add_argument("--json", action="store_true", help="Output as JSON")
args = parser.parse_args()
discovery = TestDiscovery()
result = discovery.discover(args.project_dir)
if args.json:
print(json.dumps(discovery.to_dict(result), indent=2))
else:
print(f"Package Manager: {result.package_manager or 'unknown'}")
print(f"Has Tests: {result.has_tests}")
print(f"Test Command: {result.test_command or 'none'}")
print(f"Test Directories: {', '.join(result.test_directories) or 'none'}")
print(f"Coverage Command: {result.coverage_command or 'none'}")
print(f"\nFrameworks ({len(result.frameworks)}):")
for f in result.frameworks:
print(f" - {f.name} ({f.type})")
print(f" Command: {f.command}")
if f.config_file:
print(f" Config: {f.config_file}")
if f.version:
print(f" Version: {f.version}")
if __name__ == "__main__":
main()
+2 -100
View File
@@ -1,100 +1,2 @@
#!/usr/bin/env python3
"""
Codebase Analyzer
=================
Automatically detects project structure, frameworks, and services.
Supports monorepos with multiple services.
Usage:
# Index entire project (creates project_index.json)
python auto-claude/analyzer.py --index
# Analyze specific service
python auto-claude/analyzer.py --service backend
# Output to specific file
python auto-claude/analyzer.py --index --output path/to/output.json
The analyzer will:
1. Detect if this is a monorepo or single project
2. Find all services/packages and analyze each separately
3. Map interdependencies between services
4. Identify infrastructure (Docker, CI/CD)
5. Document conventions (linting, testing)
This module now serves as a facade to the modular analyzer system in the analyzers/ package.
All actual implementation is in focused submodules for better maintainability.
"""
import json
from pathlib import Path
# Import from the new modular structure
from analyzers import (
ProjectAnalyzer,
ServiceAnalyzer,
analyze_project,
analyze_service,
)
# Re-export for backward compatibility
__all__ = [
"ServiceAnalyzer",
"ProjectAnalyzer",
"analyze_project",
"analyze_service",
]
def main():
"""CLI entry point."""
import argparse
parser = argparse.ArgumentParser(
description="Analyze project structure, frameworks, and services"
)
parser.add_argument(
"--project-dir",
type=Path,
default=Path.cwd(),
help="Project directory to analyze (default: current directory)",
)
parser.add_argument(
"--index",
action="store_true",
help="Create full project index (default behavior)",
)
parser.add_argument(
"--service",
type=str,
default=None,
help="Analyze a specific service only",
)
parser.add_argument(
"--output",
type=Path,
default=None,
help="Output file for JSON results",
)
parser.add_argument(
"--quiet",
action="store_true",
help="Only output JSON, no status messages",
)
args = parser.parse_args()
# Determine what to analyze
if args.service:
results = analyze_service(args.project_dir, args.service, args.output)
else:
results = analyze_project(args.project_dir, args.output)
# Print results
if not args.quiet or not args.output:
print(json.dumps(results, indent=2))
if __name__ == "__main__":
main()
"""Backward compatibility shim - import from analysis.analyzer instead."""
from analysis.analyzer import *
+2 -92
View File
@@ -1,92 +1,2 @@
"""
Analyzers Package
=================
Modular analyzer system for detecting project structure, frameworks, and services.
Main exports:
- ServiceAnalyzer: Analyzes a single service/package
- ProjectAnalyzer: Analyzes entire projects (single or monorepo)
- analyze_project: Convenience function for project analysis
- analyze_service: Convenience function for service analysis
"""
from pathlib import Path
from typing import Any
from .project_analyzer_module import ProjectAnalyzer
from .service_analyzer import ServiceAnalyzer
# Re-export main classes
__all__ = [
"ServiceAnalyzer",
"ProjectAnalyzer",
"analyze_project",
"analyze_service",
]
def analyze_project(project_dir: Path, output_file: Path | None = None) -> dict:
"""
Analyze a project and optionally save results.
Args:
project_dir: Path to the project root
output_file: Optional path to save JSON output
Returns:
Project index as a dictionary
"""
import json
analyzer = ProjectAnalyzer(project_dir)
results = analyzer.analyze()
if output_file:
output_file.parent.mkdir(parents=True, exist_ok=True)
with open(output_file, "w") as f:
json.dump(results, f, indent=2)
print(f"Project index saved to: {output_file}")
return results
def analyze_service(
project_dir: Path, service_name: str, output_file: Path | None = None
) -> dict:
"""
Analyze a specific service within a project.
Args:
project_dir: Path to the project root
service_name: Name of the service to analyze
output_file: Optional path to save JSON output
Returns:
Service analysis as a dictionary
"""
import json
# Find the service
service_path = project_dir / service_name
if not service_path.exists():
# Check common locations
for parent in ["packages", "apps", "services"]:
candidate = project_dir / parent / service_name
if candidate.exists():
service_path = candidate
break
if not service_path.exists():
raise ValueError(f"Service '{service_name}' not found in {project_dir}")
analyzer = ServiceAnalyzer(service_path, service_name)
results = analyzer.analyze()
if output_file:
output_file.parent.mkdir(parents=True, exist_ok=True)
with open(output_file, "w") as f:
json.dump(results, f, indent=2)
print(f"Service analysis saved to: {output_file}")
return results
"""Backward compatibility shim - import from analysis.analyzers instead."""
from analysis.analyzers import *
+10 -42
View File
@@ -1,45 +1,13 @@
"""
Custom MCP Tools for Auto-Claude Agents
========================================
"""Backward compatibility shim - import from agents.tools_pkg instead."""
DEPRECATED: This module is now a compatibility shim.
Please import from the auto_claude_tools package instead:
# Direct import to avoid triggering agents.__init__ circular dependencies
import sys
from pathlib import Path
from auto_claude_tools import create_auto_claude_mcp_server, get_allowed_tools
# Add agents directory to path if needed
agents_dir = Path(__file__).parent / "agents"
if str(agents_dir) not in sys.path:
sys.path.insert(0, str(agents_dir))
This file remains for backward compatibility with existing imports.
All functionality has been moved to the auto_claude_tools package for better
organization and maintainability.
"""
# Import everything from the package to maintain backward compatibility
from auto_claude_tools import (
ELECTRON_TOOLS,
TOOL_GET_BUILD_PROGRESS,
TOOL_GET_SESSION_CONTEXT,
TOOL_RECORD_DISCOVERY,
TOOL_RECORD_GOTCHA,
TOOL_UPDATE_QA_STATUS,
TOOL_UPDATE_SUBTASK_STATUS,
create_auto_claude_mcp_server,
get_allowed_tools,
is_electron_mcp_enabled,
is_tools_available,
)
__all__ = [
# Main API
"create_auto_claude_mcp_server",
"get_allowed_tools",
"is_tools_available",
# Tool name constants
"TOOL_UPDATE_SUBTASK_STATUS",
"TOOL_GET_BUILD_PROGRESS",
"TOOL_RECORD_DISCOVERY",
"TOOL_RECORD_GOTCHA",
"TOOL_GET_SESSION_CONTEXT",
"TOOL_UPDATE_QA_STATUS",
# Electron MCP
"ELECTRON_TOOLS",
"is_electron_mcp_enabled",
]
# Import directly from tools_pkg to avoid agents.__init__ circular imports
from tools_pkg import * # noqa: F403, E402
+20 -587
View File
@@ -1,587 +1,20 @@
#!/usr/bin/env python3
"""
CI Discovery Module
===================
Parses CI/CD configuration files to extract test commands and workflows.
Supports GitHub Actions, GitLab CI, CircleCI, and Jenkins.
The CI discovery results are used by:
- QA Agent: To understand existing CI test patterns
- Validation Strategy: To match CI commands
- Planner: To align verification with CI
Usage:
from ci_discovery import CIDiscovery
discovery = CIDiscovery()
result = discovery.discover(project_dir)
if result:
print(f"CI System: {result.ci_system}")
print(f"Test Commands: {result.test_commands}")
"""
import json
import re
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
# Try to import yaml, fall back gracefully
try:
import yaml
HAS_YAML = True
except ImportError:
HAS_YAML = False
# =============================================================================
# DATA CLASSES
# =============================================================================
@dataclass
class CIWorkflow:
"""
Represents a CI workflow or job.
Attributes:
name: Name of the workflow/job
trigger: What triggers this workflow (push, pull_request, etc.)
steps: List of step names or commands
test_related: Whether this appears to be test-related
"""
name: str
trigger: list[str] = field(default_factory=list)
steps: list[str] = field(default_factory=list)
test_related: bool = False
@dataclass
class CIConfig:
"""
Result of CI configuration discovery.
Attributes:
ci_system: Name of CI system (github_actions, gitlab, circleci, jenkins)
config_files: List of CI config files found
test_commands: Extracted test commands by type
coverage_command: Coverage command if found
workflows: List of discovered workflows
environment_variables: Environment variables used
"""
ci_system: str
config_files: list[str] = field(default_factory=list)
test_commands: dict[str, str] = field(default_factory=dict)
coverage_command: str | None = None
workflows: list[CIWorkflow] = field(default_factory=list)
environment_variables: list[str] = field(default_factory=list)
# =============================================================================
# CI PARSERS
# =============================================================================
class CIDiscovery:
"""
Discovers CI/CD configurations in a project.
Analyzes:
- GitHub Actions (.github/workflows/*.yml)
- GitLab CI (.gitlab-ci.yml)
- CircleCI (.circleci/config.yml)
- Jenkins (Jenkinsfile)
"""
def __init__(self) -> None:
"""Initialize CI discovery."""
self._cache: dict[str, CIConfig | None] = {}
def discover(self, project_dir: Path) -> CIConfig | None:
"""
Discover CI configuration in the project.
Args:
project_dir: Path to the project root
Returns:
CIConfig if CI found, None otherwise
"""
project_dir = Path(project_dir)
cache_key = str(project_dir.resolve())
if cache_key in self._cache:
return self._cache[cache_key]
# Try each CI system
result = None
# GitHub Actions
github_workflows = project_dir / ".github" / "workflows"
if github_workflows.exists():
result = self._parse_github_actions(github_workflows)
# GitLab CI
if not result:
gitlab_ci = project_dir / ".gitlab-ci.yml"
if gitlab_ci.exists():
result = self._parse_gitlab_ci(gitlab_ci)
# CircleCI
if not result:
circleci = project_dir / ".circleci" / "config.yml"
if circleci.exists():
result = self._parse_circleci(circleci)
# Jenkins
if not result:
jenkinsfile = project_dir / "Jenkinsfile"
if jenkinsfile.exists():
result = self._parse_jenkinsfile(jenkinsfile)
self._cache[cache_key] = result
return result
def _parse_github_actions(self, workflows_dir: Path) -> CIConfig:
"""Parse GitHub Actions workflow files."""
result = CIConfig(ci_system="github_actions")
workflow_files = list(workflows_dir.glob("*.yml")) + list(
workflows_dir.glob("*.yaml")
)
for wf_file in workflow_files:
result.config_files.append(
str(wf_file.relative_to(workflows_dir.parent.parent))
)
try:
content = wf_file.read_text()
workflow_data = self._parse_yaml(content)
if not workflow_data:
continue
# Get workflow name
wf_name = workflow_data.get("name", wf_file.stem)
# Get triggers
triggers = []
on_trigger = workflow_data.get("on", {})
if isinstance(on_trigger, str):
triggers = [on_trigger]
elif isinstance(on_trigger, list):
triggers = on_trigger
elif isinstance(on_trigger, dict):
triggers = list(on_trigger.keys())
# Parse jobs
jobs = workflow_data.get("jobs", {})
for job_name, job_config in jobs.items():
if not isinstance(job_config, dict):
continue
steps = job_config.get("steps", [])
step_commands = []
test_related = False
for step in steps:
if not isinstance(step, dict):
continue
# Get step name or command
step_name = step.get("name", "")
run_cmd = step.get("run", "")
uses = step.get("uses", "")
if step_name:
step_commands.append(step_name)
if run_cmd:
step_commands.append(run_cmd)
# Extract test commands
self._extract_test_commands(run_cmd, result)
if uses:
step_commands.append(f"uses: {uses}")
# Check if test-related
test_keywords = ["test", "pytest", "jest", "vitest", "coverage"]
if any(kw in str(step).lower() for kw in test_keywords):
test_related = True
result.workflows.append(
CIWorkflow(
name=f"{wf_name}/{job_name}",
trigger=triggers,
steps=step_commands,
test_related=test_related,
)
)
# Extract environment variables
env = workflow_data.get("env", {})
if isinstance(env, dict):
result.environment_variables.extend(env.keys())
except Exception:
continue
return result
def _parse_gitlab_ci(self, config_file: Path) -> CIConfig:
"""Parse GitLab CI configuration."""
result = CIConfig(
ci_system="gitlab",
config_files=[".gitlab-ci.yml"],
)
try:
content = config_file.read_text()
data = self._parse_yaml(content)
if not data:
return result
# Parse jobs (top-level keys that aren't special keywords)
special_keys = {
"stages",
"variables",
"image",
"services",
"before_script",
"after_script",
"cache",
"include",
"default",
"workflow",
}
for key, value in data.items():
if key.startswith(".") or key in special_keys:
continue
if not isinstance(value, dict):
continue
job_config = value
script = job_config.get("script", [])
if isinstance(script, str):
script = [script]
test_related = any(
kw in str(script).lower()
for kw in ["test", "pytest", "jest", "vitest", "coverage"]
)
result.workflows.append(
CIWorkflow(
name=key,
trigger=job_config.get("only", [])
or job_config.get("rules", []),
steps=script,
test_related=test_related,
)
)
# Extract test commands
for cmd in script:
if isinstance(cmd, str):
self._extract_test_commands(cmd, result)
# Extract variables
variables = data.get("variables", {})
if isinstance(variables, dict):
result.environment_variables.extend(variables.keys())
except Exception:
pass
return result
def _parse_circleci(self, config_file: Path) -> CIConfig:
"""Parse CircleCI configuration."""
result = CIConfig(
ci_system="circleci",
config_files=[".circleci/config.yml"],
)
try:
content = config_file.read_text()
data = self._parse_yaml(content)
if not data:
return result
# Parse jobs
jobs = data.get("jobs", {})
for job_name, job_config in jobs.items():
if not isinstance(job_config, dict):
continue
steps = job_config.get("steps", [])
step_commands = []
test_related = False
for step in steps:
if isinstance(step, str):
step_commands.append(step)
elif isinstance(step, dict):
if "run" in step:
run = step["run"]
if isinstance(run, str):
step_commands.append(run)
self._extract_test_commands(run, result)
elif isinstance(run, dict):
cmd = run.get("command", "")
step_commands.append(cmd)
self._extract_test_commands(cmd, result)
if any(
kw in str(step).lower()
for kw in ["test", "pytest", "jest", "coverage"]
):
test_related = True
result.workflows.append(
CIWorkflow(
name=job_name,
trigger=[],
steps=step_commands,
test_related=test_related,
)
)
except Exception:
pass
return result
def _parse_jenkinsfile(self, jenkinsfile: Path) -> CIConfig:
"""Parse Jenkinsfile (basic extraction)."""
result = CIConfig(
ci_system="jenkins",
config_files=["Jenkinsfile"],
)
try:
content = jenkinsfile.read_text()
# Extract sh commands using regex
sh_pattern = re.compile(r'sh\s+[\'"]([^\'"]+)[\'"]')
matches = sh_pattern.findall(content)
steps = []
test_related = False
for cmd in matches:
steps.append(cmd)
self._extract_test_commands(cmd, result)
if any(
kw in cmd.lower() for kw in ["test", "pytest", "jest", "coverage"]
):
test_related = True
# Extract stage names
stage_pattern = re.compile(r'stage\s*\([\'"]([^\'"]+)[\'"]\)')
stages = stage_pattern.findall(content)
for stage in stages:
result.workflows.append(
CIWorkflow(
name=stage,
trigger=[],
steps=steps if "test" in stage.lower() else [],
test_related="test" in stage.lower(),
)
)
except Exception:
pass
return result
def _parse_yaml(self, content: str) -> dict | None:
"""Parse YAML content, with fallback to basic parsing if yaml not available."""
if HAS_YAML:
try:
return yaml.safe_load(content)
except Exception:
return None
# Basic fallback for simple YAML (very limited)
# This won't work for complex structures
return None
def _extract_test_commands(self, cmd: str, result: CIConfig) -> None:
"""Extract test commands from a command string."""
cmd_lower = cmd.lower()
# Python pytest
if "pytest" in cmd_lower:
if "pytest" not in result.test_commands:
result.test_commands["unit"] = cmd.strip()
if "--cov" in cmd_lower:
result.coverage_command = cmd.strip()
# Node.js test commands
if (
"npm test" in cmd_lower
or "yarn test" in cmd_lower
or "pnpm test" in cmd_lower
):
if "unit" not in result.test_commands:
result.test_commands["unit"] = cmd.strip()
# Jest/Vitest
if "jest" in cmd_lower or "vitest" in cmd_lower:
if "unit" not in result.test_commands:
result.test_commands["unit"] = cmd.strip()
if "--coverage" in cmd_lower:
result.coverage_command = cmd.strip()
# E2E testing
if "playwright" in cmd_lower:
result.test_commands["e2e"] = cmd.strip()
if "cypress" in cmd_lower:
result.test_commands["e2e"] = cmd.strip()
# Integration tests
if "integration" in cmd_lower:
result.test_commands["integration"] = cmd.strip()
# Go tests
if "go test" in cmd_lower:
if "unit" not in result.test_commands:
result.test_commands["unit"] = cmd.strip()
# Rust tests
if "cargo test" in cmd_lower:
if "unit" not in result.test_commands:
result.test_commands["unit"] = cmd.strip()
def to_dict(self, result: CIConfig) -> dict[str, Any]:
"""Convert result to dictionary for JSON serialization."""
return {
"ci_system": result.ci_system,
"config_files": result.config_files,
"test_commands": result.test_commands,
"coverage_command": result.coverage_command,
"workflows": [
{
"name": w.name,
"trigger": w.trigger,
"steps": w.steps,
"test_related": w.test_related,
}
for w in result.workflows
],
"environment_variables": result.environment_variables,
}
def clear_cache(self) -> None:
"""Clear the internal cache."""
self._cache.clear()
# =============================================================================
# CONVENIENCE FUNCTIONS
# =============================================================================
def discover_ci(project_dir: Path) -> CIConfig | None:
"""
Convenience function to discover CI configuration.
Args:
project_dir: Path to project root
Returns:
CIConfig if found, None otherwise
"""
discovery = CIDiscovery()
return discovery.discover(project_dir)
def get_ci_test_commands(project_dir: Path) -> dict[str, str]:
"""
Get test commands from CI configuration.
Args:
project_dir: Path to project root
Returns:
Dictionary of test type to command
"""
discovery = CIDiscovery()
result = discovery.discover(project_dir)
if result:
return result.test_commands
return {}
def get_ci_system(project_dir: Path) -> str | None:
"""
Get the CI system name if configured.
Args:
project_dir: Path to project root
Returns:
CI system name or None
"""
discovery = CIDiscovery()
result = discovery.discover(project_dir)
if result:
return result.ci_system
return None
# =============================================================================
# CLI
# =============================================================================
def main() -> None:
"""CLI entry point for testing."""
import argparse
parser = argparse.ArgumentParser(description="Discover CI configuration")
parser.add_argument("project_dir", type=Path, help="Path to project root")
parser.add_argument("--json", action="store_true", help="Output as JSON")
args = parser.parse_args()
discovery = CIDiscovery()
result = discovery.discover(args.project_dir)
if not result:
print("No CI configuration found")
return
if args.json:
print(json.dumps(discovery.to_dict(result), indent=2))
else:
print(f"CI System: {result.ci_system}")
print(f"Config Files: {', '.join(result.config_files)}")
print("\nTest Commands:")
for test_type, cmd in result.test_commands.items():
print(f" {test_type}: {cmd}")
if result.coverage_command:
print(f"\nCoverage Command: {result.coverage_command}")
print(f"\nWorkflows ({len(result.workflows)}):")
for w in result.workflows:
marker = "[TEST]" if w.test_related else ""
print(f" - {w.name} {marker}")
if w.trigger:
print(f" Triggers: {', '.join(str(t) for t in w.trigger)}")
if result.environment_variables:
print(f"\nEnvironment Variables: {', '.join(result.environment_variables)}")
if __name__ == "__main__":
main()
"""Backward compatibility shim - import from analysis.ci_discovery instead."""
from analysis.ci_discovery import (
CIConfig,
CIWorkflow,
CIDiscovery,
discover_ci,
get_ci_test_commands,
get_ci_system,
HAS_YAML,
)
__all__ = [
"CIConfig",
"CIWorkflow",
"CIDiscovery",
"discover_ci",
"get_ci_test_commands",
"get_ci_system",
"HAS_YAML",
]
+19 -310
View File
@@ -1,315 +1,24 @@
"""
Claude SDK Client Configuration
===============================
Functions for creating and configuring the Claude Agent SDK client.
"""
import json
"""Backward compatibility shim - import from core.client instead."""
import sys
import os
from pathlib import Path
from auto_claude_tools import (
create_auto_claude_mcp_server,
is_tools_available,
)
from auto_claude_tools import (
get_allowed_tools as get_agent_allowed_tools,
)
from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient
from claude_agent_sdk.types import HookMatcher
from linear_updater import is_linear_enabled
from security import bash_security_hook
# Add auto-claude to path if not present
_auto_claude_dir = os.path.dirname(os.path.abspath(__file__))
if _auto_claude_dir not in sys.path:
sys.path.insert(0, _auto_claude_dir)
# Use lazy imports to avoid circular dependency
def __getattr__(name):
"""Lazy import to avoid circular imports with auto_claude_tools."""
from core import client as _client
return getattr(_client, name)
def is_graphiti_mcp_enabled() -> bool:
"""
Check if Graphiti MCP server integration is enabled.
Requires GRAPHITI_MCP_URL to be set (e.g., http://localhost:8000/mcp/)
This is separate from GRAPHITI_ENABLED which controls the Python library integration.
"""
return bool(os.environ.get("GRAPHITI_MCP_URL"))
def get_graphiti_mcp_url() -> str:
"""Get the Graphiti MCP server URL."""
return os.environ.get("GRAPHITI_MCP_URL", "http://localhost:8000/mcp/")
def is_electron_mcp_enabled() -> bool:
"""
Check if Electron MCP server integration is enabled.
Requires ELECTRON_MCP_ENABLED to be set to 'true'.
When enabled, QA agents can use Puppeteer MCP tools to connect to Electron apps
via Chrome DevTools Protocol on the configured debug port.
"""
return os.environ.get("ELECTRON_MCP_ENABLED", "").lower() == "true"
def get_electron_debug_port() -> int:
"""Get the Electron remote debugging port (default: 9222)."""
return int(os.environ.get("ELECTRON_DEBUG_PORT", "9222"))
# Puppeteer MCP tools for browser automation
PUPPETEER_TOOLS = [
"mcp__puppeteer__puppeteer_connect_active_tab",
"mcp__puppeteer__puppeteer_navigate",
"mcp__puppeteer__puppeteer_screenshot",
"mcp__puppeteer__puppeteer_click",
"mcp__puppeteer__puppeteer_fill",
"mcp__puppeteer__puppeteer_select",
"mcp__puppeteer__puppeteer_hover",
"mcp__puppeteer__puppeteer_evaluate",
# For wildcard imports, define __all__
__all__ = [
"create_claude_client",
"ClaudeClient",
"is_graphiti_mcp_enabled",
"get_graphiti_mcp_url",
"is_electron_mcp_enabled",
"get_electron_debug_port",
]
# Linear MCP tools for project management (when LINEAR_API_KEY is set)
LINEAR_TOOLS = [
"mcp__linear-server__list_teams",
"mcp__linear-server__get_team",
"mcp__linear-server__list_projects",
"mcp__linear-server__get_project",
"mcp__linear-server__create_project",
"mcp__linear-server__update_project",
"mcp__linear-server__list_issues",
"mcp__linear-server__get_issue",
"mcp__linear-server__create_issue",
"mcp__linear-server__update_issue",
"mcp__linear-server__list_comments",
"mcp__linear-server__create_comment",
"mcp__linear-server__list_issue_statuses",
"mcp__linear-server__list_issue_labels",
"mcp__linear-server__list_users",
"mcp__linear-server__get_user",
]
# Context7 MCP tools for documentation lookup (always enabled)
CONTEXT7_TOOLS = [
"mcp__context7__resolve-library-id",
"mcp__context7__get-library-docs",
]
# Graphiti MCP tools for knowledge graph memory (when GRAPHITI_MCP_ENABLED is set)
# See: https://docs.falkordb.com/agentic-memory/graphiti-mcp-server.html
GRAPHITI_MCP_TOOLS = [
"mcp__graphiti-memory__search_nodes", # Search entity summaries
"mcp__graphiti-memory__search_facts", # Search relationships between entities
"mcp__graphiti-memory__add_episode", # Add data to knowledge graph
"mcp__graphiti-memory__get_episodes", # Retrieve recent episodes
"mcp__graphiti-memory__get_entity_edge", # Get specific entity/relationship
]
# Electron MCP tools for desktop app automation (when ELECTRON_MCP_ENABLED is set)
# Uses puppeteer-mcp-server to connect to Electron apps via Chrome DevTools Protocol.
# Electron app must be started with --remote-debugging-port=9222 (or ELECTRON_DEBUG_PORT).
# These tools are only available to QA agents (qa_reviewer, qa_fixer), not Coder/Planner.
ELECTRON_TOOLS = [
"mcp__electron__electron_connect", # Connect to Electron app via DevTools
"mcp__electron__electron_screenshot", # Take screenshot of Electron window
"mcp__electron__electron_click", # Click element in Electron app
"mcp__electron__electron_fill", # Fill input field in Electron app
"mcp__electron__electron_evaluate", # Execute JS in Electron renderer
"mcp__electron__electron_get_window_info", # Get window state/bounds
"mcp__electron__electron_get_console", # Get console logs from renderer
]
# Built-in tools
BUILTIN_TOOLS = [
"Read",
"Write",
"Edit",
"Glob",
"Grep",
"Bash",
]
def create_client(
project_dir: Path,
spec_dir: Path,
model: str,
agent_type: str = "coder",
) -> ClaudeSDKClient:
"""
Create a Claude Agent SDK client with multi-layered security.
Args:
project_dir: Root directory for the project (working directory)
spec_dir: Directory containing the spec (for settings file)
model: Claude model to use
agent_type: Type of agent - 'planner', 'coder', 'qa_reviewer', or 'qa_fixer'
This determines which custom auto-claude tools are available.
Returns:
Configured ClaudeSDKClient
Security layers (defense in depth):
1. Sandbox - OS-level bash command isolation prevents filesystem escape
2. Permissions - File operations restricted to project_dir only
3. Security hooks - Bash commands validated against an allowlist
(see security.py for ALLOWED_COMMANDS)
4. Tool filtering - Each agent type only sees relevant tools (prevents misuse)
"""
oauth_token = os.environ.get("CLAUDE_CODE_OAUTH_TOKEN")
if not oauth_token:
raise ValueError(
"CLAUDE_CODE_OAUTH_TOKEN environment variable not set.\n"
"Get your token by running: claude setup-token"
)
# Check if Linear integration is enabled
linear_enabled = is_linear_enabled()
linear_api_key = os.environ.get("LINEAR_API_KEY", "")
# Check if custom auto-claude tools are available
auto_claude_tools_enabled = is_tools_available()
# Build the list of allowed tools
# Start with agent-specific tools (includes base tools + auto-claude tools)
if auto_claude_tools_enabled:
allowed_tools_list = get_agent_allowed_tools(agent_type)
else:
allowed_tools_list = [*BUILTIN_TOOLS]
# Check if Graphiti MCP is enabled
graphiti_mcp_enabled = is_graphiti_mcp_enabled()
# Check if Electron MCP is enabled (for QA agents testing Electron apps)
electron_mcp_enabled = is_electron_mcp_enabled()
# Add external MCP tools
allowed_tools_list.extend(PUPPETEER_TOOLS)
allowed_tools_list.extend(CONTEXT7_TOOLS)
if linear_enabled:
allowed_tools_list.extend(LINEAR_TOOLS)
if graphiti_mcp_enabled:
allowed_tools_list.extend(GRAPHITI_MCP_TOOLS)
# Add Electron MCP tools only for QA agents (qa_reviewer, qa_fixer) when enabled
# This prevents context bloat for coder/planner agents who don't need desktop automation
if electron_mcp_enabled and agent_type in ("qa_reviewer", "qa_fixer"):
allowed_tools_list.extend(ELECTRON_TOOLS)
# Create comprehensive security settings
# Note: Using relative paths ("./**") restricts access to project directory
# since cwd is set to project_dir
security_settings = {
"sandbox": {"enabled": True, "autoAllowBashIfSandboxed": True},
"permissions": {
"defaultMode": "acceptEdits", # Auto-approve edits within allowed directories
"allow": [
# Allow all file operations within the project directory
"Read(./**)",
"Write(./**)",
"Edit(./**)",
"Glob(./**)",
"Grep(./**)",
# Bash permission granted here, but actual commands are validated
# by the bash_security_hook (see security.py for allowed commands)
"Bash(*)",
# Allow Puppeteer MCP tools for browser automation
*PUPPETEER_TOOLS,
# Allow Context7 MCP tools for documentation lookup
*CONTEXT7_TOOLS,
# Allow Linear MCP tools for project management (if enabled)
*(LINEAR_TOOLS if linear_enabled else []),
# Allow Graphiti MCP tools for knowledge graph memory (if enabled)
*(GRAPHITI_MCP_TOOLS if graphiti_mcp_enabled else []),
# Allow Electron MCP tools for QA agents only (if enabled)
*(ELECTRON_TOOLS if electron_mcp_enabled and agent_type in ("qa_reviewer", "qa_fixer") else []),
],
},
}
# Write settings to a file in the project directory
settings_file = project_dir / ".claude_settings.json"
with open(settings_file, "w") as f:
json.dump(security_settings, f, indent=2)
print(f"Security settings: {settings_file}")
print(" - Sandbox enabled (OS-level bash isolation)")
print(f" - Filesystem restricted to: {project_dir.resolve()}")
print(" - Bash commands restricted to allowlist")
mcp_servers_list = ["puppeteer (browser automation)", "context7 (documentation)"]
if linear_enabled:
mcp_servers_list.append("linear (project management)")
if graphiti_mcp_enabled:
mcp_servers_list.append("graphiti-memory (knowledge graph)")
if electron_mcp_enabled:
mcp_servers_list.append(f"electron (desktop automation, port {get_electron_debug_port()})")
if auto_claude_tools_enabled:
mcp_servers_list.append(f"auto-claude ({agent_type} tools)")
print(f" - MCP servers: {', '.join(mcp_servers_list)}")
print()
# Configure MCP servers
mcp_servers = {
"puppeteer": {"command": "npx", "args": ["puppeteer-mcp-server"]},
"context7": {"command": "npx", "args": ["-y", "@upstash/context7-mcp"]},
}
# Add Linear MCP server if enabled
if linear_enabled:
mcp_servers["linear"] = {
"type": "http",
"url": "https://mcp.linear.app/mcp",
"headers": {"Authorization": f"Bearer {linear_api_key}"},
}
# Add Graphiti MCP server if enabled
# Requires running: docker run -d -p 8000:8000 falkordb/graphiti-knowledge-graph-mcp
if graphiti_mcp_enabled:
mcp_servers["graphiti-memory"] = {
"type": "http",
"url": get_graphiti_mcp_url(),
}
# Add Electron MCP server if enabled
# Electron app must be started with --remote-debugging-port=<port> for connection
# Uses puppeteer-mcp-server to connect to Electron via Chrome DevTools Protocol
if electron_mcp_enabled:
electron_port = get_electron_debug_port()
mcp_servers["electron"] = {
"command": "npx",
"args": [
"-y",
"@anthropic/puppeteer-mcp-server",
"--debug-port",
str(electron_port),
],
}
# Add custom auto-claude MCP server if available
auto_claude_mcp_server = None
if auto_claude_tools_enabled:
auto_claude_mcp_server = create_auto_claude_mcp_server(spec_dir, project_dir)
if auto_claude_mcp_server:
mcp_servers["auto-claude"] = auto_claude_mcp_server
return ClaudeSDKClient(
options=ClaudeAgentOptions(
model=model,
system_prompt=(
f"You are an expert full-stack developer building production-quality software. "
f"Your working directory is: {project_dir.resolve()}\n"
f"Your filesystem access is RESTRICTED to this directory only. "
f"Use relative paths (starting with ./) for all file operations. "
f"Never use absolute paths or try to access files outside your working directory.\n\n"
f"You follow existing code patterns, write clean maintainable code, and verify "
f"your work through thorough testing. You communicate progress through Git commits "
f"and build-progress.txt updates."
),
allowed_tools=allowed_tools_list,
mcp_servers=mcp_servers,
hooks={
"PreToolUse": [
HookMatcher(matcher="Bash", hooks=[bash_security_hook]),
],
},
max_turns=1000,
cwd=str(project_dir.resolve()),
settings=str(settings_file.resolve()),
)
)
+36
View File
@@ -0,0 +1,36 @@
"""
Core Framework Module
=====================
Core components for the Auto Claude autonomous coding framework.
"""
# Note: We use lazy imports here because the full agent module has many dependencies
# that may not be needed for basic operations like workspace management.
__all__ = [
"run_autonomous_agent",
"run_followup_planner",
"WorkspaceManager",
"WorktreeManager",
"ProgressTracker",
]
def __getattr__(name):
"""Lazy imports to avoid circular dependencies and heavy imports."""
if name in ("run_autonomous_agent", "run_followup_planner"):
from .agent import run_autonomous_agent, run_followup_planner
return locals()[name]
elif name == "WorkspaceManager":
from .workspace import WorkspaceManager
return WorkspaceManager
elif name == "WorktreeManager":
from .worktree import WorktreeManager
return WorktreeManager
elif name == "ProgressTracker":
from .progress import ProgressTracker
return ProgressTracker
elif name in ("create_claude_client", "ClaudeClient"):
from . import client as _client
return getattr(_client, name)
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+63
View File
@@ -0,0 +1,63 @@
"""
Agent Session Logic
===================
Core agent interaction functions for running autonomous coding sessions.
Uses subtask-based implementation plans with minimal, focused prompts.
Architecture:
- Orchestrator (Python) handles all bookkeeping: memory, commits, progress
- Agent focuses ONLY on implementing code
- Post-session processing updates memory automatically (100% reliable)
Enhanced with status file updates for ccstatusline integration.
Enhanced with Graphiti memory for cross-session context retrieval.
NOTE: This module is now a facade that imports from agents/ submodules.
All logic has been refactored into focused modules for better maintainability.
"""
# Re-export everything from the agents module to maintain backwards compatibility
from agents import (
# Constants
AUTO_CONTINUE_DELAY_SECONDS,
HUMAN_INTERVENTION_FILE,
# Memory functions
debug_memory_system_status,
find_phase_for_subtask,
find_subtask_in_plan,
get_commit_count,
get_graphiti_context,
# Utility functions
get_latest_commit,
load_implementation_plan,
post_session_processing,
# Session management
run_agent_session,
# Main API
run_autonomous_agent,
run_followup_planner,
save_session_memory,
save_session_to_graphiti,
sync_plan_to_source,
)
# Ensure all exports are available at module level
__all__ = [
"run_autonomous_agent",
"run_followup_planner",
"debug_memory_system_status",
"get_graphiti_context",
"save_session_memory",
"save_session_to_graphiti",
"run_agent_session",
"post_session_processing",
"get_latest_commit",
"get_commit_count",
"load_implementation_plan",
"find_subtask_in_plan",
"find_phase_for_subtask",
"sync_plan_to_source",
"AUTO_CONTINUE_DELAY_SECONDS",
"HUMAN_INTERVENTION_FILE",
]
+315
View File
@@ -0,0 +1,315 @@
"""
Claude SDK Client Configuration
===============================
Functions for creating and configuring the Claude Agent SDK client.
"""
import json
import os
from pathlib import Path
from auto_claude_tools import (
create_auto_claude_mcp_server,
is_tools_available,
)
from auto_claude_tools import (
get_allowed_tools as get_agent_allowed_tools,
)
from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient
from claude_agent_sdk.types import HookMatcher
from linear_updater import is_linear_enabled
from security import bash_security_hook
def is_graphiti_mcp_enabled() -> bool:
"""
Check if Graphiti MCP server integration is enabled.
Requires GRAPHITI_MCP_URL to be set (e.g., http://localhost:8000/mcp/)
This is separate from GRAPHITI_ENABLED which controls the Python library integration.
"""
return bool(os.environ.get("GRAPHITI_MCP_URL"))
def get_graphiti_mcp_url() -> str:
"""Get the Graphiti MCP server URL."""
return os.environ.get("GRAPHITI_MCP_URL", "http://localhost:8000/mcp/")
def is_electron_mcp_enabled() -> bool:
"""
Check if Electron MCP server integration is enabled.
Requires ELECTRON_MCP_ENABLED to be set to 'true'.
When enabled, QA agents can use Puppeteer MCP tools to connect to Electron apps
via Chrome DevTools Protocol on the configured debug port.
"""
return os.environ.get("ELECTRON_MCP_ENABLED", "").lower() == "true"
def get_electron_debug_port() -> int:
"""Get the Electron remote debugging port (default: 9222)."""
return int(os.environ.get("ELECTRON_DEBUG_PORT", "9222"))
# Puppeteer MCP tools for browser automation
PUPPETEER_TOOLS = [
"mcp__puppeteer__puppeteer_connect_active_tab",
"mcp__puppeteer__puppeteer_navigate",
"mcp__puppeteer__puppeteer_screenshot",
"mcp__puppeteer__puppeteer_click",
"mcp__puppeteer__puppeteer_fill",
"mcp__puppeteer__puppeteer_select",
"mcp__puppeteer__puppeteer_hover",
"mcp__puppeteer__puppeteer_evaluate",
]
# Linear MCP tools for project management (when LINEAR_API_KEY is set)
LINEAR_TOOLS = [
"mcp__linear-server__list_teams",
"mcp__linear-server__get_team",
"mcp__linear-server__list_projects",
"mcp__linear-server__get_project",
"mcp__linear-server__create_project",
"mcp__linear-server__update_project",
"mcp__linear-server__list_issues",
"mcp__linear-server__get_issue",
"mcp__linear-server__create_issue",
"mcp__linear-server__update_issue",
"mcp__linear-server__list_comments",
"mcp__linear-server__create_comment",
"mcp__linear-server__list_issue_statuses",
"mcp__linear-server__list_issue_labels",
"mcp__linear-server__list_users",
"mcp__linear-server__get_user",
]
# Context7 MCP tools for documentation lookup (always enabled)
CONTEXT7_TOOLS = [
"mcp__context7__resolve-library-id",
"mcp__context7__get-library-docs",
]
# Graphiti MCP tools for knowledge graph memory (when GRAPHITI_MCP_ENABLED is set)
# See: https://docs.falkordb.com/agentic-memory/graphiti-mcp-server.html
GRAPHITI_MCP_TOOLS = [
"mcp__graphiti-memory__search_nodes", # Search entity summaries
"mcp__graphiti-memory__search_facts", # Search relationships between entities
"mcp__graphiti-memory__add_episode", # Add data to knowledge graph
"mcp__graphiti-memory__get_episodes", # Retrieve recent episodes
"mcp__graphiti-memory__get_entity_edge", # Get specific entity/relationship
]
# Electron MCP tools for desktop app automation (when ELECTRON_MCP_ENABLED is set)
# Uses puppeteer-mcp-server to connect to Electron apps via Chrome DevTools Protocol.
# Electron app must be started with --remote-debugging-port=9222 (or ELECTRON_DEBUG_PORT).
# These tools are only available to QA agents (qa_reviewer, qa_fixer), not Coder/Planner.
ELECTRON_TOOLS = [
"mcp__electron__electron_connect", # Connect to Electron app via DevTools
"mcp__electron__electron_screenshot", # Take screenshot of Electron window
"mcp__electron__electron_click", # Click element in Electron app
"mcp__electron__electron_fill", # Fill input field in Electron app
"mcp__electron__electron_evaluate", # Execute JS in Electron renderer
"mcp__electron__electron_get_window_info", # Get window state/bounds
"mcp__electron__electron_get_console", # Get console logs from renderer
]
# Built-in tools
BUILTIN_TOOLS = [
"Read",
"Write",
"Edit",
"Glob",
"Grep",
"Bash",
]
def create_client(
project_dir: Path,
spec_dir: Path,
model: str,
agent_type: str = "coder",
) -> ClaudeSDKClient:
"""
Create a Claude Agent SDK client with multi-layered security.
Args:
project_dir: Root directory for the project (working directory)
spec_dir: Directory containing the spec (for settings file)
model: Claude model to use
agent_type: Type of agent - 'planner', 'coder', 'qa_reviewer', or 'qa_fixer'
This determines which custom auto-claude tools are available.
Returns:
Configured ClaudeSDKClient
Security layers (defense in depth):
1. Sandbox - OS-level bash command isolation prevents filesystem escape
2. Permissions - File operations restricted to project_dir only
3. Security hooks - Bash commands validated against an allowlist
(see security.py for ALLOWED_COMMANDS)
4. Tool filtering - Each agent type only sees relevant tools (prevents misuse)
"""
oauth_token = os.environ.get("CLAUDE_CODE_OAUTH_TOKEN")
if not oauth_token:
raise ValueError(
"CLAUDE_CODE_OAUTH_TOKEN environment variable not set.\n"
"Get your token by running: claude setup-token"
)
# Check if Linear integration is enabled
linear_enabled = is_linear_enabled()
linear_api_key = os.environ.get("LINEAR_API_KEY", "")
# Check if custom auto-claude tools are available
auto_claude_tools_enabled = is_tools_available()
# Build the list of allowed tools
# Start with agent-specific tools (includes base tools + auto-claude tools)
if auto_claude_tools_enabled:
allowed_tools_list = get_agent_allowed_tools(agent_type)
else:
allowed_tools_list = [*BUILTIN_TOOLS]
# Check if Graphiti MCP is enabled
graphiti_mcp_enabled = is_graphiti_mcp_enabled()
# Check if Electron MCP is enabled (for QA agents testing Electron apps)
electron_mcp_enabled = is_electron_mcp_enabled()
# Add external MCP tools
allowed_tools_list.extend(PUPPETEER_TOOLS)
allowed_tools_list.extend(CONTEXT7_TOOLS)
if linear_enabled:
allowed_tools_list.extend(LINEAR_TOOLS)
if graphiti_mcp_enabled:
allowed_tools_list.extend(GRAPHITI_MCP_TOOLS)
# Add Electron MCP tools only for QA agents (qa_reviewer, qa_fixer) when enabled
# This prevents context bloat for coder/planner agents who don't need desktop automation
if electron_mcp_enabled and agent_type in ("qa_reviewer", "qa_fixer"):
allowed_tools_list.extend(ELECTRON_TOOLS)
# Create comprehensive security settings
# Note: Using relative paths ("./**") restricts access to project directory
# since cwd is set to project_dir
security_settings = {
"sandbox": {"enabled": True, "autoAllowBashIfSandboxed": True},
"permissions": {
"defaultMode": "acceptEdits", # Auto-approve edits within allowed directories
"allow": [
# Allow all file operations within the project directory
"Read(./**)",
"Write(./**)",
"Edit(./**)",
"Glob(./**)",
"Grep(./**)",
# Bash permission granted here, but actual commands are validated
# by the bash_security_hook (see security.py for allowed commands)
"Bash(*)",
# Allow Puppeteer MCP tools for browser automation
*PUPPETEER_TOOLS,
# Allow Context7 MCP tools for documentation lookup
*CONTEXT7_TOOLS,
# Allow Linear MCP tools for project management (if enabled)
*(LINEAR_TOOLS if linear_enabled else []),
# Allow Graphiti MCP tools for knowledge graph memory (if enabled)
*(GRAPHITI_MCP_TOOLS if graphiti_mcp_enabled else []),
# Allow Electron MCP tools for QA agents only (if enabled)
*(ELECTRON_TOOLS if electron_mcp_enabled and agent_type in ("qa_reviewer", "qa_fixer") else []),
],
},
}
# Write settings to a file in the project directory
settings_file = project_dir / ".claude_settings.json"
with open(settings_file, "w") as f:
json.dump(security_settings, f, indent=2)
print(f"Security settings: {settings_file}")
print(" - Sandbox enabled (OS-level bash isolation)")
print(f" - Filesystem restricted to: {project_dir.resolve()}")
print(" - Bash commands restricted to allowlist")
mcp_servers_list = ["puppeteer (browser automation)", "context7 (documentation)"]
if linear_enabled:
mcp_servers_list.append("linear (project management)")
if graphiti_mcp_enabled:
mcp_servers_list.append("graphiti-memory (knowledge graph)")
if electron_mcp_enabled:
mcp_servers_list.append(f"electron (desktop automation, port {get_electron_debug_port()})")
if auto_claude_tools_enabled:
mcp_servers_list.append(f"auto-claude ({agent_type} tools)")
print(f" - MCP servers: {', '.join(mcp_servers_list)}")
print()
# Configure MCP servers
mcp_servers = {
"puppeteer": {"command": "npx", "args": ["puppeteer-mcp-server"]},
"context7": {"command": "npx", "args": ["-y", "@upstash/context7-mcp"]},
}
# Add Linear MCP server if enabled
if linear_enabled:
mcp_servers["linear"] = {
"type": "http",
"url": "https://mcp.linear.app/mcp",
"headers": {"Authorization": f"Bearer {linear_api_key}"},
}
# Add Graphiti MCP server if enabled
# Requires running: docker run -d -p 8000:8000 falkordb/graphiti-knowledge-graph-mcp
if graphiti_mcp_enabled:
mcp_servers["graphiti-memory"] = {
"type": "http",
"url": get_graphiti_mcp_url(),
}
# Add Electron MCP server if enabled
# Electron app must be started with --remote-debugging-port=<port> for connection
# Uses puppeteer-mcp-server to connect to Electron via Chrome DevTools Protocol
if electron_mcp_enabled:
electron_port = get_electron_debug_port()
mcp_servers["electron"] = {
"command": "npx",
"args": [
"-y",
"@anthropic/puppeteer-mcp-server",
"--debug-port",
str(electron_port),
],
}
# Add custom auto-claude MCP server if available
auto_claude_mcp_server = None
if auto_claude_tools_enabled:
auto_claude_mcp_server = create_auto_claude_mcp_server(spec_dir, project_dir)
if auto_claude_mcp_server:
mcp_servers["auto-claude"] = auto_claude_mcp_server
return ClaudeSDKClient(
options=ClaudeAgentOptions(
model=model,
system_prompt=(
f"You are an expert full-stack developer building production-quality software. "
f"Your working directory is: {project_dir.resolve()}\n"
f"Your filesystem access is RESTRICTED to this directory only. "
f"Use relative paths (starting with ./) for all file operations. "
f"Never use absolute paths or try to access files outside your working directory.\n\n"
f"You follow existing code patterns, write clean maintainable code, and verify "
f"your work through thorough testing. You communicate progress through Git commits "
f"and build-progress.txt updates."
),
allowed_tools=allowed_tools_list,
mcp_servers=mcp_servers,
hooks={
"PreToolUse": [
HookMatcher(matcher="Bash", hooks=[bash_security_hook]),
],
},
max_turns=1000,
cwd=str(project_dir.resolve()),
settings=str(settings_file.resolve()),
)
)
+349
View File
@@ -0,0 +1,349 @@
#!/usr/bin/env python3
"""
Debug Logging Utility
=====================
Centralized debug logging for the Auto-Claude framework.
Controlled via environment variables:
- DEBUG=true Enable debug mode
- DEBUG_LEVEL=1|2|3 Log verbosity (1=basic, 2=detailed, 3=verbose)
- DEBUG_LOG_FILE=path Optional file output
Usage:
from debug import debug, debug_detailed, debug_verbose, is_debug_enabled
debug("run.py", "Starting task execution", task_id="001")
debug_detailed("agent", "Agent response received", response_length=1234)
debug_verbose("client", "Full request payload", payload=data)
"""
import json
import os
import sys
import time
from datetime import datetime
from functools import wraps
from pathlib import Path
from typing import Any
# ANSI color codes for terminal output
class Colors:
RESET = "\033[0m"
BOLD = "\033[1m"
DIM = "\033[2m"
# Debug colors
DEBUG = "\033[36m" # Cyan
DEBUG_DIM = "\033[96m" # Light cyan
TIMESTAMP = "\033[90m" # Gray
MODULE = "\033[33m" # Yellow
KEY = "\033[35m" # Magenta
VALUE = "\033[37m" # White
SUCCESS = "\033[32m" # Green
WARNING = "\033[33m" # Yellow
ERROR = "\033[31m" # Red
def _get_debug_enabled() -> bool:
"""Check if debug mode is enabled via environment variable."""
return os.environ.get("DEBUG", "").lower() in ("true", "1", "yes", "on")
def _get_debug_level() -> int:
"""Get debug verbosity level (1-3)."""
try:
level = int(os.environ.get("DEBUG_LEVEL", "1"))
return max(1, min(3, level)) # Clamp to 1-3
except ValueError:
return 1
def _get_log_file() -> Path | None:
"""Get optional log file path."""
log_file = os.environ.get("DEBUG_LOG_FILE")
if log_file:
return Path(log_file)
return None
def is_debug_enabled() -> bool:
"""Check if debug mode is enabled."""
return _get_debug_enabled()
def get_debug_level() -> int:
"""Get current debug level."""
return _get_debug_level()
def _format_value(value: Any, max_length: int = 200) -> str:
"""Format a value for debug output, truncating if necessary."""
if value is None:
return "None"
if isinstance(value, (dict, list)):
try:
formatted = json.dumps(value, indent=2, default=str)
if len(formatted) > max_length:
formatted = formatted[:max_length] + "..."
return formatted
except (TypeError, ValueError):
return str(value)[:max_length]
str_value = str(value)
if len(str_value) > max_length:
return str_value[:max_length] + "..."
return str_value
def _write_log(message: str, to_file: bool = True) -> None:
"""Write log message to stdout and optionally to file."""
print(message, file=sys.stderr)
if to_file:
log_file = _get_log_file()
if log_file:
try:
log_file.parent.mkdir(parents=True, exist_ok=True)
# Strip ANSI codes for file output
import re
clean_message = re.sub(r"\033\[[0-9;]*m", "", message)
with open(log_file, "a") as f:
f.write(clean_message + "\n")
except Exception:
pass # Silently fail file logging
def debug(module: str, message: str, level: int = 1, **kwargs) -> None:
"""
Log a debug message.
Args:
module: Source module name (e.g., "run.py", "ideation_runner")
message: Debug message
level: Required debug level (1=basic, 2=detailed, 3=verbose)
**kwargs: Additional key-value pairs to log
"""
if not _get_debug_enabled():
return
if _get_debug_level() < level:
return
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
# Build the log line
parts = [
f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET}",
f"{Colors.DEBUG}[DEBUG]{Colors.RESET}",
f"{Colors.MODULE}[{module}]{Colors.RESET}",
f"{Colors.DEBUG_DIM}{message}{Colors.RESET}",
]
log_line = " ".join(parts)
# Add kwargs on separate lines if present
if kwargs:
for key, value in kwargs.items():
formatted_value = _format_value(value)
if "\n" in formatted_value:
# Multi-line value
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}:"
for line in formatted_value.split("\n"):
log_line += f"\n {Colors.VALUE}{line}{Colors.RESET}"
else:
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{formatted_value}{Colors.RESET}"
_write_log(log_line)
def debug_detailed(module: str, message: str, **kwargs) -> None:
"""Log a detailed debug message (level 2)."""
debug(module, message, level=2, **kwargs)
def debug_verbose(module: str, message: str, **kwargs) -> None:
"""Log a verbose debug message (level 3)."""
debug(module, message, level=3, **kwargs)
def debug_success(module: str, message: str, **kwargs) -> None:
"""Log a success debug message."""
if not _get_debug_enabled():
return
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.SUCCESS}[OK]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {message}"
if kwargs:
for key, value in kwargs.items():
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}"
_write_log(log_line)
def debug_info(module: str, message: str, **kwargs) -> None:
"""Log an info debug message."""
if not _get_debug_enabled():
return
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.DEBUG}[INFO]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {message}"
if kwargs:
for key, value in kwargs.items():
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}"
_write_log(log_line)
def debug_error(module: str, message: str, **kwargs) -> None:
"""Log an error debug message (always shown if debug enabled)."""
if not _get_debug_enabled():
return
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.ERROR}[ERROR]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {Colors.ERROR}{message}{Colors.RESET}"
if kwargs:
for key, value in kwargs.items():
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}"
_write_log(log_line)
def debug_warning(module: str, message: str, **kwargs) -> None:
"""Log a warning debug message."""
if not _get_debug_enabled():
return
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.WARNING}[WARN]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {Colors.WARNING}{message}{Colors.RESET}"
if kwargs:
for key, value in kwargs.items():
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}"
_write_log(log_line)
def debug_section(module: str, title: str) -> None:
"""Log a section header for organizing debug output."""
if not _get_debug_enabled():
return
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
separator = "" * 60
log_line = f"\n{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.DEBUG}{Colors.BOLD}{separator}{Colors.RESET}"
log_line += f"\n{Colors.TIMESTAMP} {Colors.RESET} {Colors.DEBUG}{Colors.BOLD}{module}: {title}{' ' * (58 - len(module) - len(title) - 2)}{Colors.RESET}"
log_line += f"\n{Colors.TIMESTAMP} {Colors.RESET} {Colors.DEBUG}{Colors.BOLD}{separator}{Colors.RESET}"
_write_log(log_line)
def debug_timer(module: str):
"""
Decorator to time function execution.
Usage:
@debug_timer("run.py")
def my_function():
...
"""
def decorator(func):
@wraps(func)
def wrapper(*args, **kwargs):
if not _get_debug_enabled():
return func(*args, **kwargs)
start = time.time()
debug_detailed(module, f"Starting {func.__name__}()")
try:
result = func(*args, **kwargs)
elapsed = time.time() - start
debug_success(
module,
f"Completed {func.__name__}()",
elapsed_ms=f"{elapsed * 1000:.1f}ms",
)
return result
except Exception as e:
elapsed = time.time() - start
debug_error(
module,
f"Failed {func.__name__}()",
error=str(e),
elapsed_ms=f"{elapsed * 1000:.1f}ms",
)
raise
return wrapper
return decorator
def debug_async_timer(module: str):
"""
Decorator to time async function execution.
Usage:
@debug_async_timer("ideation_runner")
async def my_async_function():
...
"""
def decorator(func):
@wraps(func)
async def wrapper(*args, **kwargs):
if not _get_debug_enabled():
return await func(*args, **kwargs)
start = time.time()
debug_detailed(module, f"Starting {func.__name__}()")
try:
result = await func(*args, **kwargs)
elapsed = time.time() - start
debug_success(
module,
f"Completed {func.__name__}()",
elapsed_ms=f"{elapsed * 1000:.1f}ms",
)
return result
except Exception as e:
elapsed = time.time() - start
debug_error(
module,
f"Failed {func.__name__}()",
error=str(e),
elapsed_ms=f"{elapsed * 1000:.1f}ms",
)
raise
return wrapper
return decorator
def debug_env_status() -> None:
"""Print debug environment status on startup."""
if not _get_debug_enabled():
return
debug_section("debug", "Debug Mode Enabled")
debug(
"debug",
"Environment configuration",
DEBUG=os.environ.get("DEBUG", "not set"),
DEBUG_LEVEL=_get_debug_level(),
DEBUG_LOG_FILE=os.environ.get("DEBUG_LOG_FILE", "not set"),
)
# Print status on import if debug is enabled
if _get_debug_enabled():
debug_env_status()
+467
View File
@@ -0,0 +1,467 @@
"""
Progress Tracking Utilities
===========================
Functions for tracking and displaying progress of the autonomous coding agent.
Uses subtask-based implementation plans (implementation_plan.json).
Enhanced with colored output, icons, and better visual formatting.
"""
import json
from pathlib import Path
from ui import (
Icons,
bold,
box,
highlight,
icon,
muted,
print_phase_status,
print_status,
progress_bar,
success,
warning,
)
def count_subtasks(spec_dir: Path) -> tuple[int, int]:
"""
Count completed and total subtasks in implementation_plan.json.
Args:
spec_dir: Directory containing implementation_plan.json
Returns:
(completed_count, total_count)
"""
plan_file = spec_dir / "implementation_plan.json"
if not plan_file.exists():
return 0, 0
try:
with open(plan_file) as f:
plan = json.load(f)
total = 0
completed = 0
for phase in plan.get("phases", []):
for subtask in phase.get("subtasks", []):
total += 1
if subtask.get("status") == "completed":
completed += 1
return completed, total
except (OSError, json.JSONDecodeError):
return 0, 0
def count_subtasks_detailed(spec_dir: Path) -> dict:
"""
Count subtasks by status.
Returns:
Dict with completed, in_progress, pending, failed counts
"""
plan_file = spec_dir / "implementation_plan.json"
result = {
"completed": 0,
"in_progress": 0,
"pending": 0,
"failed": 0,
"total": 0,
}
if not plan_file.exists():
return result
try:
with open(plan_file) as f:
plan = json.load(f)
for phase in plan.get("phases", []):
for subtask in phase.get("subtasks", []):
result["total"] += 1
status = subtask.get("status", "pending")
if status in result:
result[status] += 1
else:
result["pending"] += 1
return result
except (OSError, json.JSONDecodeError):
return result
def is_build_complete(spec_dir: Path) -> bool:
"""
Check if all subtasks are completed.
Args:
spec_dir: Directory containing implementation_plan.json
Returns:
True if all subtasks complete, False otherwise
"""
completed, total = count_subtasks(spec_dir)
return total > 0 and completed == total
def get_progress_percentage(spec_dir: Path) -> float:
"""
Get the progress as a percentage.
Args:
spec_dir: Directory containing implementation_plan.json
Returns:
Percentage of subtasks completed (0-100)
"""
completed, total = count_subtasks(spec_dir)
if total == 0:
return 0.0
return (completed / total) * 100
def print_session_header(
session_num: int,
is_planner: bool,
subtask_id: str = None,
subtask_desc: str = None,
phase_name: str = None,
attempt: int = 1,
) -> None:
"""Print a formatted header for the session."""
session_type = "PLANNER AGENT" if is_planner else "CODING AGENT"
session_icon = Icons.GEAR if is_planner else Icons.LIGHTNING
content = [
bold(f"{icon(session_icon)} SESSION {session_num}: {session_type}"),
]
if subtask_id:
content.append("")
subtask_line = f"{icon(Icons.SUBTASK)} Subtask: {highlight(subtask_id)}"
if subtask_desc:
# Truncate long descriptions
desc = subtask_desc[:50] + "..." if len(subtask_desc) > 50 else subtask_desc
subtask_line += f" - {desc}"
content.append(subtask_line)
if phase_name:
content.append(f"{icon(Icons.PHASE)} Phase: {phase_name}")
if attempt > 1:
content.append(warning(f"{icon(Icons.WARNING)} Attempt: {attempt}"))
print()
print(box(content, width=70, style="heavy"))
print()
def print_progress_summary(spec_dir: Path, show_next: bool = True) -> None:
"""Print a summary of current progress with enhanced formatting."""
completed, total = count_subtasks(spec_dir)
if total > 0:
print()
# Progress bar
print(f"Progress: {progress_bar(completed, total, width=40)}")
# Status message
if completed == total:
print_status("BUILD COMPLETE - All subtasks completed!", "success")
else:
remaining = total - completed
print_status(f"{remaining} subtasks remaining", "info")
# Phase summary
try:
with open(spec_dir / "implementation_plan.json") as f:
plan = json.load(f)
print("\nPhases:")
for phase in plan.get("phases", []):
phase_subtasks = phase.get("subtasks", [])
phase_completed = sum(
1 for s in phase_subtasks if s.get("status") == "completed"
)
phase_total = len(phase_subtasks)
phase_name = phase.get("name", phase.get("id", "Unknown"))
if phase_completed == phase_total:
status = "complete"
elif phase_completed > 0 or any(
s.get("status") == "in_progress" for s in phase_subtasks
):
status = "in_progress"
else:
# Check if blocked by dependencies
deps = phase.get("depends_on", [])
all_deps_complete = True
for dep_id in deps:
for p in plan.get("phases", []):
if p.get("id") == dep_id or p.get("phase") == dep_id:
p_subtasks = p.get("subtasks", [])
if not all(
s.get("status") == "completed" for s in p_subtasks
):
all_deps_complete = False
break
status = "pending" if all_deps_complete else "blocked"
print_phase_status(phase_name, phase_completed, phase_total, status)
# Show next subtask if requested
if show_next and completed < total:
next_subtask = get_next_subtask(spec_dir)
if next_subtask:
print()
next_id = next_subtask.get("id", "unknown")
next_desc = next_subtask.get("description", "")
if len(next_desc) > 60:
next_desc = next_desc[:57] + "..."
print(
f" {icon(Icons.ARROW_RIGHT)} Next: {highlight(next_id)} - {next_desc}"
)
except (OSError, json.JSONDecodeError):
pass
else:
print()
print_status("No implementation subtasks yet - planner needs to run", "pending")
def print_build_complete_banner(spec_dir: Path) -> None:
"""Print a completion banner."""
content = [
success(f"{icon(Icons.SUCCESS)} BUILD COMPLETE!"),
"",
"All subtasks have been implemented successfully.",
"",
muted("Next steps:"),
f" 1. Review the {highlight('auto-claude/*')} branch",
" 2. Run manual tests",
" 3. Create a PR and merge to main",
]
print()
print(box(content, width=70, style="heavy"))
print()
def print_paused_banner(
spec_dir: Path,
spec_name: str,
has_worktree: bool = False,
) -> None:
"""Print a paused banner with resume instructions."""
completed, total = count_subtasks(spec_dir)
content = [
warning(f"{icon(Icons.PAUSE)} BUILD PAUSED"),
"",
f"Progress saved: {completed}/{total} subtasks complete",
]
if has_worktree:
content.append("")
content.append(muted("Your build is in a separate workspace and is safe."))
print()
print(box(content, width=70, style="heavy"))
def get_plan_summary(spec_dir: Path) -> dict:
"""
Get a detailed summary of implementation plan status.
Args:
spec_dir: Directory containing implementation_plan.json
Returns:
Dictionary with plan statistics
"""
plan_file = spec_dir / "implementation_plan.json"
if not plan_file.exists():
return {
"workflow_type": None,
"total_phases": 0,
"total_subtasks": 0,
"completed_subtasks": 0,
"pending_subtasks": 0,
"in_progress_subtasks": 0,
"failed_subtasks": 0,
"phases": [],
}
try:
with open(plan_file) as f:
plan = json.load(f)
summary = {
"workflow_type": plan.get("workflow_type"),
"total_phases": len(plan.get("phases", [])),
"total_subtasks": 0,
"completed_subtasks": 0,
"pending_subtasks": 0,
"in_progress_subtasks": 0,
"failed_subtasks": 0,
"phases": [],
}
for phase in plan.get("phases", []):
phase_info = {
"id": phase.get("id"),
"phase": phase.get("phase"),
"name": phase.get("name"),
"depends_on": phase.get("depends_on", []),
"subtasks": [],
"completed": 0,
"total": 0,
}
for subtask in phase.get("subtasks", []):
status = subtask.get("status", "pending")
summary["total_subtasks"] += 1
phase_info["total"] += 1
if status == "completed":
summary["completed_subtasks"] += 1
phase_info["completed"] += 1
elif status == "in_progress":
summary["in_progress_subtasks"] += 1
elif status == "failed":
summary["failed_subtasks"] += 1
else:
summary["pending_subtasks"] += 1
phase_info["subtasks"].append(
{
"id": subtask.get("id"),
"description": subtask.get("description"),
"status": status,
"service": subtask.get("service"),
}
)
summary["phases"].append(phase_info)
return summary
except (OSError, json.JSONDecodeError):
return {
"workflow_type": None,
"total_phases": 0,
"total_subtasks": 0,
"completed_subtasks": 0,
"pending_subtasks": 0,
"in_progress_subtasks": 0,
"failed_subtasks": 0,
"phases": [],
}
def get_current_phase(spec_dir: Path) -> dict | None:
"""Get the current phase being worked on."""
plan_file = spec_dir / "implementation_plan.json"
if not plan_file.exists():
return None
try:
with open(plan_file) as f:
plan = json.load(f)
for phase in plan.get("phases", []):
subtasks = phase.get("subtasks", [])
# Phase is current if it has incomplete subtasks and dependencies are met
has_incomplete = any(s.get("status") != "completed" for s in subtasks)
if has_incomplete:
return {
"id": phase.get("id"),
"phase": phase.get("phase"),
"name": phase.get("name"),
"completed": sum(
1 for s in subtasks if s.get("status") == "completed"
),
"total": len(subtasks),
}
return None
except (OSError, json.JSONDecodeError):
return None
def get_next_subtask(spec_dir: Path) -> dict | None:
"""
Find the next subtask to work on, respecting phase dependencies.
Args:
spec_dir: Directory containing implementation_plan.json
Returns:
The next subtask dict to work on, or None if all complete
"""
plan_file = spec_dir / "implementation_plan.json"
if not plan_file.exists():
return None
try:
with open(plan_file) as f:
plan = json.load(f)
phases = plan.get("phases", [])
# Build a map of phase completion
phase_complete = {}
for phase in phases:
phase_id = phase.get("id") or phase.get("phase")
subtasks = phase.get("subtasks", [])
phase_complete[phase_id] = all(
s.get("status") == "completed" for s in subtasks
)
# Find next available subtask
for phase in phases:
phase_id = phase.get("id") or phase.get("phase")
depends_on = phase.get("depends_on", [])
# Check if dependencies are satisfied
deps_satisfied = all(phase_complete.get(dep, False) for dep in depends_on)
if not deps_satisfied:
continue
# Find first pending subtask in this phase
for subtask in phase.get("subtasks", []):
if subtask.get("status") == "pending":
return {
"phase_id": phase_id,
"phase_name": phase.get("name"),
"phase_num": phase.get("phase"),
**subtask,
}
return None
except (OSError, json.JSONDecodeError):
return None
def format_duration(seconds: float) -> str:
"""Format a duration in human-readable form."""
if seconds < 60:
return f"{seconds:.0f}s"
elif seconds < 3600:
minutes = seconds / 60
return f"{minutes:.1f}m"
else:
hours = seconds / 3600
return f"{hours:.1f}h"
@@ -6,19 +6,15 @@ Workspace Management - Per-Spec Architecture
Handles workspace isolation through Git worktrees, where each spec
gets its own isolated worktree in .worktrees/{spec-name}/.
Key changes from old design:
- Each spec has its own worktree (not shared)
- Worktree path: .worktrees/{spec-name}/
- Branch name: auto-claude/{spec-name}
- Fixed: get_existing_build_worktree() now properly checks spec_name
- Fixed: finalize_workspace() skips prompts in auto_continue mode
This module has been refactored for better maintainability:
- Models and enums: workspace/models.py
- Git utilities: workspace/git_utils.py
- Setup functions: workspace/setup.py
- Display functions: workspace/display.py
- Finalization: workspace/finalization.py
- Complex merge operations: remain here (workspace.py)
Terminology mapping (technical -> user-friendly):
- worktree -> "separate workspace"
- branch -> "version of your project"
- uncommitted changes -> "unsaved work"
- merge -> "add to your project"
- working directory -> "your project"
Public API is exported via workspace/__init__.py for backward compatibility.
"""
import asyncio
@@ -27,7 +23,6 @@ import shutil
import subprocess
import sys
from dataclasses import dataclass
from enum import Enum
from pathlib import Path
from typing import Optional
@@ -69,645 +64,81 @@ from merge import (
FileTimelineTracker,
)
# Track if we've already tried to install the git hook this session
_git_hook_check_done = False
# Import from refactored modules in core/workspace/
from core.workspace.models import (
WorkspaceMode,
WorkspaceChoice,
ParallelMergeTask,
ParallelMergeResult,
MergeLock,
MergeLockError,
)
from core.workspace.git_utils import (
has_uncommitted_changes,
get_current_branch,
get_existing_build_worktree,
get_file_content_from_ref as _get_file_content_from_ref,
get_changed_files_from_branch as _get_changed_files_from_branch,
is_process_running as _is_process_running,
is_binary_file as _is_binary_file,
is_lock_file as _is_lock_file,
validate_merged_syntax as _validate_merged_syntax,
create_conflict_file_with_git as _create_conflict_file_with_git,
MAX_FILE_LINES_FOR_AI,
MAX_PARALLEL_AI_MERGES,
MAX_SYNTAX_FIX_RETRIES,
BINARY_EXTENSIONS,
LOCK_FILES,
MERGE_LOCK_TIMEOUT,
)
from core.workspace.setup import (
choose_workspace,
copy_spec_to_worktree,
setup_workspace,
ensure_timeline_hook_installed as _ensure_timeline_hook_installed,
initialize_timeline_tracking as _initialize_timeline_tracking,
)
from core.workspace.display import (
show_build_summary,
show_changed_files,
print_merge_success as _print_merge_success,
print_conflict_info as _print_conflict_info,
)
from core.workspace.finalization import (
finalize_workspace,
handle_workspace_choice,
review_existing_build,
discard_existing_build,
check_existing_build,
list_all_worktrees,
cleanup_all_worktrees,
)
MODULE = "workspace"
class WorkspaceMode(Enum):
"""How auto-claude should work."""
ISOLATED = "isolated" # Work in a separate worktree (safe)
DIRECT = "direct" # Work directly in user's project
class WorkspaceChoice(Enum):
"""User's choice after build completes."""
MERGE = "merge" # Add changes to project
REVIEW = "review" # Show what changed
TEST = "test" # Test the feature in the staging worktree
LATER = "later" # Decide later
def has_uncommitted_changes(project_dir: Path) -> bool:
"""Check if user has unsaved work."""
result = subprocess.run(
["git", "status", "--porcelain"],
cwd=project_dir,
capture_output=True,
text=True,
)
return bool(result.stdout.strip())
def get_current_branch(project_dir: Path) -> str:
"""Get the current branch name."""
result = subprocess.run(
["git", "rev-parse", "--abbrev-ref", "HEAD"],
cwd=project_dir,
capture_output=True,
text=True,
)
return result.stdout.strip()
def get_existing_build_worktree(project_dir: Path, spec_name: str) -> Path | None:
"""
Check if there's an existing worktree for this specific spec.
Args:
project_dir: The main project directory
spec_name: The spec folder name (e.g., "001-feature-name")
Returns:
Path to the worktree if it exists for this spec, None otherwise
"""
# Per-spec worktree path: .worktrees/{spec-name}/
worktree_path = project_dir / ".worktrees" / spec_name
if worktree_path.exists():
return worktree_path
return None
def choose_workspace(
project_dir: Path,
spec_name: str,
force_isolated: bool = False,
force_direct: bool = False,
auto_continue: bool = False,
) -> WorkspaceMode:
"""
Let user choose where auto-claude should work.
Uses simple, non-technical language. Safe defaults.
Args:
project_dir: The project directory
spec_name: Name of the spec being built
force_isolated: Skip prompts and use isolated mode
force_direct: Skip prompts and use direct mode
auto_continue: Non-interactive mode (for UI integration) - skip all prompts
Returns:
WorkspaceMode indicating where to work
"""
# Handle forced modes
if force_isolated:
return WorkspaceMode.ISOLATED
if force_direct:
return WorkspaceMode.DIRECT
# Non-interactive mode: default to isolated for safety
if auto_continue:
print("Auto-continue: Using isolated workspace for safety.")
return WorkspaceMode.ISOLATED
# Check for unsaved work
has_unsaved = has_uncommitted_changes(project_dir)
if has_unsaved:
# Unsaved work detected - use isolated mode for safety
content = [
success(f"{icon(Icons.SHIELD)} YOUR WORK IS PROTECTED"),
"",
"You have unsaved work in your project.",
"",
"To keep your work safe, the AI will build in a",
"separate workspace. Your current files won't be",
"touched until you're ready.",
]
print()
print(box(content, width=60, style="heavy"))
print()
try:
input("Press Enter to continue...")
except KeyboardInterrupt:
print()
print_status("Cancelled.", "info")
sys.exit(0)
return WorkspaceMode.ISOLATED
# Clean working directory - give choice with enhanced menu
options = [
MenuOption(
key="isolated",
label="Separate workspace (Recommended)",
icon=Icons.SHIELD,
description="Your current files stay untouched. Easy to review and undo.",
),
MenuOption(
key="direct",
label="Right here in your project",
icon=Icons.LIGHTNING,
description="Changes happen directly. Best if you're not working on anything else.",
),
]
choice = select_menu(
title="Where should the AI build your feature?",
options=options,
allow_quit=True,
)
if choice is None:
print()
print_status("Cancelled.", "info")
sys.exit(0)
if choice == "direct":
print()
print_status("Working directly in your project.", "info")
return WorkspaceMode.DIRECT
else:
print()
print_status("Using a separate workspace for safety.", "success")
return WorkspaceMode.ISOLATED
def copy_spec_to_worktree(
source_spec_dir: Path,
worktree_path: Path,
spec_name: str,
) -> Path:
"""
Copy spec files into the worktree so the AI can access them.
The AI's filesystem is restricted to the worktree, so spec files
must be copied inside for access.
Args:
source_spec_dir: Original spec directory (may be outside worktree)
worktree_path: Path to the worktree
spec_name: Name of the spec folder
Returns:
Path to the spec directory inside the worktree
"""
# Determine target location inside worktree
# Use .auto-claude/specs/{spec_name}/ as the standard location
# Note: auto-claude/ is source code, .auto-claude/ is the installed instance
target_spec_dir = worktree_path / ".auto-claude" / "specs" / spec_name
# Create parent directories if needed
target_spec_dir.parent.mkdir(parents=True, exist_ok=True)
# Copy spec files (overwrite if exists to get latest)
if target_spec_dir.exists():
shutil.rmtree(target_spec_dir)
shutil.copytree(source_spec_dir, target_spec_dir)
return target_spec_dir
def setup_workspace(
project_dir: Path,
spec_name: str,
mode: WorkspaceMode,
source_spec_dir: Path | None = None,
) -> tuple[Path, WorktreeManager | None, Path | None]:
"""
Set up the workspace based on user's choice.
Uses per-spec worktrees - each spec gets its own isolated worktree.
Args:
project_dir: The project directory
spec_name: Name of the spec being built (e.g., "001-feature-name")
mode: The workspace mode to use
source_spec_dir: Optional source spec directory to copy to worktree
Returns:
Tuple of (working_directory, worktree_manager or None, localized_spec_dir or None)
When using isolated mode with source_spec_dir:
- working_directory: Path to the worktree
- worktree_manager: Manager for the worktree
- localized_spec_dir: Path to spec files INSIDE the worktree (accessible to AI)
"""
if mode == WorkspaceMode.DIRECT:
# Work directly in project - spec_dir stays as-is
return project_dir, None, source_spec_dir
# Create isolated workspace using per-spec worktree
print()
print_status("Setting up separate workspace...", "progress")
# Ensure timeline tracking hook is installed (once per session)
_ensure_timeline_hook_installed(project_dir)
manager = WorktreeManager(project_dir)
manager.setup()
# Get or create worktree for THIS SPECIFIC SPEC
worktree_info = manager.get_or_create_worktree(spec_name)
# Copy spec files to worktree if provided
localized_spec_dir = None
if source_spec_dir and source_spec_dir.exists():
localized_spec_dir = copy_spec_to_worktree(
source_spec_dir, worktree_info.path, spec_name
)
print_status("Spec files copied to workspace", "success")
print_status(f"Workspace ready: {worktree_info.path.name}", "success")
print()
# Initialize FileTimelineTracker for this task
_initialize_timeline_tracking(
project_dir=project_dir,
spec_name=spec_name,
worktree_path=worktree_info.path,
source_spec_dir=localized_spec_dir or source_spec_dir,
)
return worktree_info.path, manager, localized_spec_dir
def _ensure_timeline_hook_installed(project_dir: Path) -> None:
"""
Ensure the FileTimelineTracker git post-commit hook is installed.
This enables tracking human commits to main branch for drift detection.
Called once per session during first workspace setup.
"""
global _git_hook_check_done
if _git_hook_check_done:
return
_git_hook_check_done = True
try:
git_dir = project_dir / ".git"
if not git_dir.exists():
return # Not a git repo
# Handle worktrees (where .git is a file, not directory)
if git_dir.is_file():
content = git_dir.read_text().strip()
if content.startswith("gitdir:"):
git_dir = Path(content.split(":", 1)[1].strip())
else:
return
hook_path = git_dir / "hooks" / "post-commit"
# Check if hook already installed
if hook_path.exists():
if "FileTimelineTracker" in hook_path.read_text():
debug(MODULE, "FileTimelineTracker hook already installed")
return
# Auto-install the hook (silent, non-intrusive)
from merge.install_hook import install_hook
install_hook(project_dir)
debug(MODULE, "Auto-installed FileTimelineTracker git hook")
except Exception as e:
# Non-fatal - hook installation is optional
debug_warning(MODULE, f"Could not auto-install timeline hook: {e}")
def _initialize_timeline_tracking(
project_dir: Path,
spec_name: str,
worktree_path: Path,
source_spec_dir: Path | None = None,
) -> None:
"""
Initialize FileTimelineTracker for a new task.
This registers the task's branch point and the files it intends to modify,
enabling intent-aware merge conflict resolution later.
"""
try:
tracker = FileTimelineTracker(project_dir)
# Get task intent from implementation plan
task_intent = ""
task_title = spec_name
files_to_modify = []
if source_spec_dir:
plan_path = source_spec_dir / "implementation_plan.json"
if plan_path.exists():
import json
with open(plan_path) as f:
plan = json.load(f)
task_title = plan.get("title", spec_name)
task_intent = plan.get("description", "")
# Extract files from phases/subtasks
for phase in plan.get("phases", []):
for subtask in phase.get("subtasks", []):
files_to_modify.extend(subtask.get("files", []))
# Get the current branch point commit
import subprocess
result = subprocess.run(
["git", "rev-parse", "HEAD"],
cwd=project_dir,
capture_output=True,
text=True,
)
branch_point = result.stdout.strip() if result.returncode == 0 else None
if files_to_modify and branch_point:
# Register the task with known files
tracker.on_task_start(
task_id=spec_name,
files_to_modify=list(set(files_to_modify)), # Dedupe
branch_point_commit=branch_point,
task_intent=task_intent,
task_title=task_title,
)
debug(MODULE, f"Timeline tracking initialized for {spec_name}",
files_tracked=len(files_to_modify),
branch_point=branch_point[:8] if branch_point else None)
else:
# Initialize retroactively from worktree if no plan
tracker.initialize_from_worktree(
task_id=spec_name,
worktree_path=worktree_path,
task_intent=task_intent,
task_title=task_title,
)
except Exception as e:
# Non-fatal - timeline tracking is supplementary
debug_warning(MODULE, f"Could not initialize timeline tracking: {e}")
print(muted(f" Note: Timeline tracking could not be initialized: {e}"))
def show_build_summary(manager: WorktreeManager, spec_name: str) -> None:
"""Show a summary of what was built."""
summary = manager.get_change_summary(spec_name)
files = manager.get_changed_files(spec_name)
total = summary["new_files"] + summary["modified_files"] + summary["deleted_files"]
if total == 0:
print_status("No changes were made.", "info")
return
print()
print(bold("What was built:"))
if summary["new_files"] > 0:
print(
success(
f" + {summary['new_files']} new file{'s' if summary['new_files'] != 1 else ''}"
)
)
if summary["modified_files"] > 0:
print(
info(
f" ~ {summary['modified_files']} modified file{'s' if summary['modified_files'] != 1 else ''}"
)
)
if summary["deleted_files"] > 0:
print(
error(
f" - {summary['deleted_files']} deleted file{'s' if summary['deleted_files'] != 1 else ''}"
)
)
def show_changed_files(manager: WorktreeManager, spec_name: str) -> None:
"""Show detailed list of changed files."""
files = manager.get_changed_files(spec_name)
if not files:
print_status("No changes.", "info")
return
print()
print(bold("Changed files:"))
for status, filepath in files:
if status == "A":
print(success(f" + {filepath}"))
elif status == "M":
print(info(f" ~ {filepath}"))
elif status == "D":
print(error(f" - {filepath}"))
else:
print(f" {status} {filepath}")
def finalize_workspace(
project_dir: Path,
spec_name: str,
manager: WorktreeManager | None,
auto_continue: bool = False,
) -> WorkspaceChoice:
"""
Handle post-build workflow - let user decide what to do with changes.
Safe design:
- No "discard" option (requires separate --discard command)
- Default is "test" - encourages testing before merging
- Everything is preserved until user explicitly merges or discards
Args:
project_dir: The project directory
spec_name: Name of the spec that was built
manager: The worktree manager (None if direct mode was used)
auto_continue: If True, skip interactive prompts (UI mode)
Returns:
WorkspaceChoice indicating what user wants to do
"""
if manager is None:
# Direct mode - nothing to finalize
content = [
success(f"{icon(Icons.SUCCESS)} BUILD COMPLETE!"),
"",
"Changes were made directly to your project.",
muted("Use 'git status' to see what changed."),
]
print()
print(box(content, width=60, style="heavy"))
return WorkspaceChoice.MERGE # Already merged
# In auto_continue mode (UI), skip interactive prompts
# The worktree stays for the UI to manage
if auto_continue:
worktree_info = manager.get_worktree_info(spec_name)
if worktree_info:
print()
print(success(f"Build complete in worktree: {worktree_info.path}"))
print(muted("Worktree preserved for UI review."))
return WorkspaceChoice.LATER
# Isolated mode - show options with testing as the recommended path
content = [
success(f"{icon(Icons.SUCCESS)} BUILD COMPLETE!"),
"",
"The AI built your feature in a separate workspace.",
]
print()
print(box(content, width=60, style="heavy"))
show_build_summary(manager, spec_name)
# Get the worktree path for test instructions
worktree_info = manager.get_worktree_info(spec_name)
staging_path = worktree_info.path if worktree_info else None
# Enhanced menu for post-build options
options = [
MenuOption(
key="test",
label="Test the feature (Recommended)",
icon=Icons.PLAY,
description="Run the app and try it out before adding to your project",
),
MenuOption(
key="merge",
label="Add to my project now",
icon=Icons.SUCCESS,
description="Merge the changes into your files immediately",
),
MenuOption(
key="review",
label="Review what changed",
icon=Icons.FILE,
description="See exactly what files were modified",
),
MenuOption(
key="later",
label="Decide later",
icon=Icons.PAUSE,
description="Your build is saved - you can come back anytime",
),
]
print()
choice = select_menu(
title="What would you like to do?",
options=options,
allow_quit=False,
)
if choice == "test":
return WorkspaceChoice.TEST
elif choice == "merge":
return WorkspaceChoice.MERGE
elif choice == "review":
return WorkspaceChoice.REVIEW
else:
return WorkspaceChoice.LATER
def handle_workspace_choice(
choice: WorkspaceChoice,
project_dir: Path,
spec_name: str,
manager: WorktreeManager,
) -> None:
"""
Execute the user's choice.
Args:
choice: What the user wants to do
project_dir: The project directory
spec_name: Name of the spec
manager: The worktree manager
"""
worktree_info = manager.get_worktree_info(spec_name)
staging_path = worktree_info.path if worktree_info else None
if choice == WorkspaceChoice.TEST:
# Show testing instructions
content = [
bold(f"{icon(Icons.PLAY)} TEST YOUR FEATURE"),
"",
"Your feature is ready to test in a separate workspace.",
]
print()
print(box(content, width=60, style="heavy"))
print()
print("To test it, open a NEW terminal and run:")
print()
if staging_path:
print(highlight(f" cd {staging_path}"))
else:
print(highlight(f" cd {project_dir}/.worktrees/{spec_name}"))
# Show likely test/run commands
if staging_path:
commands = manager.get_test_commands(spec_name)
print()
print("Then run your project:")
for cmd in commands[:2]: # Show top 2 commands
print(f" {cmd}")
print()
print(muted("-" * 60))
print()
print("When you're done testing:")
print(highlight(f" python auto-claude/run.py --spec {spec_name} --merge"))
print()
print("To discard (if you don't like it):")
print(muted(f" python auto-claude/run.py --spec {spec_name} --discard"))
print()
elif choice == WorkspaceChoice.MERGE:
print()
print_status("Adding changes to your project...", "progress")
success_result = manager.merge_worktree(spec_name, delete_after=True)
if success_result:
print()
print_status("Your feature has been added to your project.", "success")
else:
print()
print_status("There was a conflict merging the changes.", "error")
print(muted("Your build is still saved in the separate workspace."))
print(muted("You may need to merge manually or ask for help."))
elif choice == WorkspaceChoice.REVIEW:
show_changed_files(manager, spec_name)
print()
print(muted("-" * 60))
print()
print("To see full details of changes:")
if worktree_info:
print(
muted(
f" git diff {worktree_info.base_branch}...{worktree_info.branch}"
)
)
print()
print("To test the feature:")
if staging_path:
print(highlight(f" cd {staging_path}"))
print()
print("To add these changes to your project:")
print(highlight(f" python auto-claude/run.py --spec {spec_name} --merge"))
print()
else: # LATER
print()
print_status("No problem! Your build is saved.", "success")
print()
print("To test the feature:")
if staging_path:
print(highlight(f" cd {staging_path}"))
else:
print(highlight(f" cd {project_dir}/.worktrees/{spec_name}"))
print()
print("When you're ready to add it:")
print(highlight(f" python auto-claude/run.py --spec {spec_name} --merge"))
print()
print("To see what was built:")
print(muted(f" python auto-claude/run.py --spec {spec_name} --review"))
print()
# The following functions are now imported from refactored modules above.
# They are kept here only to avoid breaking the existing code that still needs
# the complex merge operations below.
# Remaining complex merge operations that reference each other:
# - merge_existing_build
# - _try_smart_merge
# - _try_smart_merge_inner
# - _check_git_conflicts
# - _resolve_git_conflicts_with_ai
# - _create_async_claude_client
# - _async_ai_call
# - _merge_file_with_ai_async
# - _run_parallel_merges
# - _record_merge_completion
# - _get_task_intent
# - _get_recent_merges_context
# - _merge_file_with_ai
# - _heuristic_merge
def merge_existing_build(
project_dir: Path,
@@ -1236,15 +667,22 @@ def _resolve_git_conflicts_with_ai(
simple_merges.append((file_path, None)) # None = delete
debug(MODULE, f" {file_path}: deleted (no AI needed)")
else:
# File exists in both - needs AI merge
files_needing_ai_merge.append(ParallelMergeTask(
file_path=file_path,
main_content=main_content,
worktree_content=worktree_content,
base_content=base_content,
spec_name=spec_name,
))
debug(MODULE, f" {file_path}: needs AI merge")
# File exists in both - check if it's a lock file
if _is_lock_file(file_path):
# Lock files should never go through AI - just take worktree version
# User can run package manager install to regenerate if needed
simple_merges.append((file_path, worktree_content))
debug(MODULE, f" {file_path}: lock file (taking worktree version)")
else:
# Regular file - needs AI merge
files_needing_ai_merge.append(ParallelMergeTask(
file_path=file_path,
main_content=main_content,
worktree_content=worktree_content,
base_content=base_content,
spec_name=spec_name,
))
debug(MODULE, f" {file_path}: needs AI merge")
except Exception as e:
print(error(f" ✗ Failed to categorize {file_path}: {e}"))
@@ -1265,7 +703,11 @@ def _resolve_git_conflicts_with_ai(
target_path.write_text(merged_content, encoding="utf-8")
subprocess.run(["git", "add", file_path], cwd=project_dir, capture_output=True)
resolved_files.append(file_path)
print(success(f"{file_path} (new file)"))
# Determine the type for display
if _is_lock_file(file_path):
print(success(f"{file_path} (lock file - took worktree version)"))
else:
print(success(f"{file_path} (new file)"))
else:
# Delete the file
target_path = project_dir / file_path
@@ -1330,14 +772,8 @@ def _resolve_git_conflicts_with_ai(
if remaining_conflicts:
print(muted(f" Failed: {len(remaining_conflicts)}"))
if remaining_conflicts:
return {
"success": False,
"resolved_files": resolved_files,
"remaining_conflicts": remaining_conflicts,
}
# All conflicts resolved - now merge remaining non-conflicting files
# ALWAYS process non-conflicting files, even if some conflicts failed
# This ensures we get as much of the build as possible
# (New files were already copied at the start)
print(muted(" Merging remaining files..."))
@@ -1371,18 +807,31 @@ def _resolve_git_conflicts_with_ai(
if resolved_files:
_record_merge_completion(project_dir, spec_name, resolved_files)
return {
"success": True,
# Build result - partial success if some files failed but we got others
result = {
"success": len(remaining_conflicts) == 0,
"resolved_files": resolved_files,
"stats": {
"files_merged": len(resolved_files),
"conflicts_resolved": len(conflicting_files),
"conflicts_resolved": len(conflicting_files) - len(remaining_conflicts),
"ai_assisted": ai_merged_count,
"auto_merged": auto_merged_count,
"parallel_ai_merges": len(files_needing_ai_merge),
},
}
# Add remaining conflicts if any (for UI to show what needs manual attention)
if remaining_conflicts:
result["remaining_conflicts"] = remaining_conflicts
result["partial_success"] = len(resolved_files) > 0
print()
print(warning(f"{len(remaining_conflicts)} file(s) could not be auto-merged:"))
for conflict in remaining_conflicts:
print(muted(f" - {conflict['file']}: {conflict['reason']}"))
print(muted(" These files may need manual review."))
return result
def _get_file_content_from_ref(project_dir: Path, ref: str, file_path: str) -> Optional[str]:
"""Get file content from a git ref (branch, commit, etc.)."""
@@ -1702,14 +1151,55 @@ async def _merge_file_with_ai_async(
resolutions = extract_conflict_resolutions(response, conflicts, language)
if resolutions:
merged = reassemble_with_resolutions(merged_content, conflicts, resolutions)
is_valid, _ = _validate_merged_syntax(file_path, merged, project_dir)
is_valid, error_msg = _validate_merged_syntax(file_path, merged, project_dir)
# Retry loop: if syntax is invalid, give AI feedback to fix it
retry_count = 0
while not is_valid and retry_count < MAX_SYNTAX_FIX_RETRIES:
retry_count += 1
debug(MODULE, f"[PARALLEL] Syntax error in merge, retry {retry_count}/{MAX_SYNTAX_FIX_RETRIES}: {file_path}")
# Build fix prompt with error feedback
fix_prompt = f"""The merged code you produced has a syntax error:
{error_msg}
Here is your merged code that has the error:
```{language}
{merged}
```
Please fix the syntax error and output the corrected code. Make sure:
1. All lines are properly separated (no concatenated lines)
2. All brackets/braces are properly matched
3. All statements end correctly
Output ONLY the fixed code, no explanations."""
fix_response = await _async_ai_call(
client,
"You are an expert code fixer. Fix the syntax error in the code.",
fix_prompt,
)
if fix_response:
fixed = resolver._extract_code_block(fix_response, language)
if not fixed and resolver._looks_like_code(fix_response, language):
fixed = fix_response.strip()
if fixed:
merged = fixed
is_valid, error_msg = _validate_merged_syntax(file_path, merged, project_dir)
if is_valid:
debug_success(MODULE, f"[PARALLEL] Conflict-only merge succeeded: {file_path}")
debug_success(MODULE, f"[PARALLEL] Conflict-only merge succeeded: {file_path}" +
(f" (after {retry_count} fix retries)" if retry_count > 0 else ""))
return ParallelMergeResult(
file_path=file_path,
merged_content=merged,
success=True
)
else:
debug(MODULE, f"[PARALLEL] Conflict-only merge failed validation after {retry_count} retries, falling back to full-file: {file_path}")
# Case 3: Full-file AI merge (fallback)
debug(MODULE, f"[PARALLEL] Full-file merge for: {file_path}")
@@ -1736,21 +1226,62 @@ async def _merge_file_with_ai_async(
merged = response.strip()
if merged:
is_valid, _ = _validate_merged_syntax(file_path, merged, project_dir)
is_valid, error_msg = _validate_merged_syntax(file_path, merged, project_dir)
# Retry loop: if syntax is invalid, give AI feedback to fix it
retry_count = 0
while not is_valid and retry_count < MAX_SYNTAX_FIX_RETRIES:
retry_count += 1
debug(MODULE, f"[PARALLEL] Syntax error in full-file merge, retry {retry_count}/{MAX_SYNTAX_FIX_RETRIES}: {file_path}")
# Build fix prompt with error feedback
fix_prompt = f"""The merged code you produced has a syntax error:
{error_msg}
Here is your merged code that has the error:
```{language}
{merged}
```
Please fix the syntax error and output the corrected code. Make sure:
1. All lines are properly separated (no concatenated lines)
2. All brackets/braces are properly matched
3. All statements end correctly
Output ONLY the fixed code, no explanations."""
fix_response = await _async_ai_call(
client,
"You are an expert code fixer. Fix the syntax error in the code.",
fix_prompt,
)
if fix_response:
fixed = resolver._extract_code_block(fix_response, language)
if not fixed and resolver._looks_like_code(fix_response, language):
fixed = fix_response.strip()
if fixed:
merged = fixed
is_valid, error_msg = _validate_merged_syntax(file_path, merged, project_dir)
if is_valid:
debug_success(MODULE, f"[PARALLEL] Full-file merge succeeded: {file_path}")
debug_success(MODULE, f"[PARALLEL] Full-file merge succeeded: {file_path}" +
(f" (after {retry_count} fix retries)" if retry_count > 0 else ""))
return ParallelMergeResult(
file_path=file_path,
merged_content=merged,
success=True
)
else:
debug_error(MODULE, f"[PARALLEL] Full-file merge failed validation after {retry_count} retries: {file_path}")
# AI couldn't merge
return ParallelMergeResult(
file_path=file_path,
merged_content=None,
success=False,
error="AI could not merge file"
error="AI could not merge file - syntax validation failed after retries"
)
except Exception as e:
@@ -104,6 +104,7 @@ from .finalization import (
__all__ = [
# Merge Operations (from workspace.py)
'merge_existing_build',
'_run_parallel_merges', # Private but used by tests
# Models
'WorkspaceMode',
'WorkspaceChoice',
+561
View File
@@ -0,0 +1,561 @@
#!/usr/bin/env python3
"""
Git Worktree Manager - Per-Spec Architecture
=============================================
Each spec gets its own worktree:
- Worktree path: .worktrees/{spec-name}/
- Branch name: auto-claude/{spec-name}
This allows:
1. Multiple specs to be worked on simultaneously
2. Each spec's changes are isolated
3. Branches persist until explicitly merged
4. Clear 1:1:1 mapping: spec → worktree → branch
"""
import asyncio
import re
import shutil
import subprocess
from dataclasses import dataclass
from pathlib import Path
class WorktreeError(Exception):
"""Error during worktree operations."""
pass
@dataclass
class WorktreeInfo:
"""Information about a spec's worktree."""
path: Path
branch: str
spec_name: str
base_branch: str
is_active: bool = True
commit_count: int = 0
files_changed: int = 0
additions: int = 0
deletions: int = 0
class WorktreeManager:
"""
Manages per-spec Git worktrees.
Each spec gets its own worktree in .worktrees/{spec-name}/ with
a corresponding branch auto-claude/{spec-name}.
"""
def __init__(self, project_dir: Path, base_branch: str | None = None):
self.project_dir = project_dir
self.base_branch = base_branch or self._get_current_branch()
self.worktrees_dir = project_dir / ".worktrees"
self._merge_lock = asyncio.Lock()
def _get_current_branch(self) -> str:
"""Get the current git branch."""
result = subprocess.run(
["git", "rev-parse", "--abbrev-ref", "HEAD"],
cwd=self.project_dir,
capture_output=True,
text=True,
)
if result.returncode != 0:
raise WorktreeError(f"Failed to get current branch: {result.stderr}")
return result.stdout.strip()
def _run_git(
self, args: list[str], cwd: Path | None = None
) -> subprocess.CompletedProcess:
"""Run a git command and return the result."""
return subprocess.run(
["git"] + args,
cwd=cwd or self.project_dir,
capture_output=True,
text=True,
)
def _unstage_gitignored_files(self) -> None:
"""
Unstage any staged files that are gitignored in the current branch.
This is needed after a --no-commit merge because files that exist in the
source branch (like spec files in auto-claude/specs/) get staged even if
they're gitignored in the target branch.
"""
# Get list of staged files
result = self._run_git(["diff", "--cached", "--name-only"])
if result.returncode != 0 or not result.stdout.strip():
return
staged_files = result.stdout.strip().split("\n")
# Check which staged files are gitignored
# git check-ignore returns the files that ARE ignored
result = self._run_git(["check-ignore", "--stdin"], cwd=self.project_dir)
# We need to pass the files via stdin
result = subprocess.run(
["git", "check-ignore", "--stdin"],
cwd=self.project_dir,
input="\n".join(staged_files),
capture_output=True,
text=True,
)
if not result.stdout.strip():
return
ignored_files = result.stdout.strip().split("\n")
if ignored_files:
print(f"Unstaging {len(ignored_files)} gitignored file(s)...")
# Unstage each ignored file
for file in ignored_files:
if file.strip():
self._run_git(["reset", "HEAD", "--", file.strip()])
def setup(self) -> None:
"""Create worktrees directory if needed."""
self.worktrees_dir.mkdir(exist_ok=True)
# ==================== Per-Spec Worktree Methods ====================
def get_worktree_path(self, spec_name: str) -> Path:
"""Get the worktree path for a spec."""
return self.worktrees_dir / spec_name
def get_branch_name(self, spec_name: str) -> str:
"""Get the branch name for a spec."""
return f"auto-claude/{spec_name}"
def worktree_exists(self, spec_name: str) -> bool:
"""Check if a worktree exists for a spec."""
return self.get_worktree_path(spec_name).exists()
def get_worktree_info(self, spec_name: str) -> WorktreeInfo | None:
"""Get info about a spec's worktree."""
worktree_path = self.get_worktree_path(spec_name)
if not worktree_path.exists():
return None
# Verify the branch exists in the worktree
result = self._run_git(["rev-parse", "--abbrev-ref", "HEAD"], cwd=worktree_path)
if result.returncode != 0:
return None
actual_branch = result.stdout.strip()
# Get statistics
stats = self._get_worktree_stats(spec_name)
return WorktreeInfo(
path=worktree_path,
branch=actual_branch,
spec_name=spec_name,
base_branch=self.base_branch,
is_active=True,
**stats,
)
def _get_worktree_stats(self, spec_name: str) -> dict:
"""Get diff statistics for a worktree."""
worktree_path = self.get_worktree_path(spec_name)
stats = {
"commit_count": 0,
"files_changed": 0,
"additions": 0,
"deletions": 0,
}
if not worktree_path.exists():
return stats
# Commit count
result = self._run_git(
["rev-list", "--count", f"{self.base_branch}..HEAD"], cwd=worktree_path
)
if result.returncode == 0:
stats["commit_count"] = int(result.stdout.strip() or "0")
# Diff stats
result = self._run_git(
["diff", "--shortstat", f"{self.base_branch}...HEAD"], cwd=worktree_path
)
if result.returncode == 0 and result.stdout.strip():
# Parse: "3 files changed, 50 insertions(+), 10 deletions(-)"
match = re.search(r"(\d+) files? changed", result.stdout)
if match:
stats["files_changed"] = int(match.group(1))
match = re.search(r"(\d+) insertions?", result.stdout)
if match:
stats["additions"] = int(match.group(1))
match = re.search(r"(\d+) deletions?", result.stdout)
if match:
stats["deletions"] = int(match.group(1))
return stats
def create_worktree(self, spec_name: str) -> WorktreeInfo:
"""
Create a worktree for a spec.
Args:
spec_name: The spec folder name (e.g., "002-implement-memory")
Returns:
WorktreeInfo for the created worktree
"""
worktree_path = self.get_worktree_path(spec_name)
branch_name = self.get_branch_name(spec_name)
# Remove existing if present (from crashed previous run)
if worktree_path.exists():
self._run_git(["worktree", "remove", "--force", str(worktree_path)])
# Delete branch if it exists (from previous attempt)
self._run_git(["branch", "-D", branch_name])
# Create worktree with new branch from base
result = self._run_git(
["worktree", "add", "-b", branch_name, str(worktree_path), self.base_branch]
)
if result.returncode != 0:
raise WorktreeError(
f"Failed to create worktree for {spec_name}: {result.stderr}"
)
print(f"Created worktree: {worktree_path.name} on branch {branch_name}")
return WorktreeInfo(
path=worktree_path,
branch=branch_name,
spec_name=spec_name,
base_branch=self.base_branch,
is_active=True,
)
def get_or_create_worktree(self, spec_name: str) -> WorktreeInfo:
"""
Get existing worktree or create a new one for a spec.
Args:
spec_name: The spec folder name
Returns:
WorktreeInfo for the worktree
"""
existing = self.get_worktree_info(spec_name)
if existing:
print(f"Using existing worktree: {existing.path}")
return existing
return self.create_worktree(spec_name)
def remove_worktree(self, spec_name: str, delete_branch: bool = False) -> None:
"""
Remove a spec's worktree.
Args:
spec_name: The spec folder name
delete_branch: Whether to also delete the branch
"""
worktree_path = self.get_worktree_path(spec_name)
branch_name = self.get_branch_name(spec_name)
if worktree_path.exists():
result = self._run_git(
["worktree", "remove", "--force", str(worktree_path)]
)
if result.returncode == 0:
print(f"Removed worktree: {worktree_path.name}")
else:
print(f"Warning: Could not remove worktree: {result.stderr}")
shutil.rmtree(worktree_path, ignore_errors=True)
if delete_branch:
self._run_git(["branch", "-D", branch_name])
print(f"Deleted branch: {branch_name}")
self._run_git(["worktree", "prune"])
def merge_worktree(
self, spec_name: str, delete_after: bool = False, no_commit: bool = False
) -> bool:
"""
Merge a spec's worktree branch back to base branch.
Args:
spec_name: The spec folder name
delete_after: Whether to remove worktree and branch after merge
no_commit: If True, merge changes but don't commit (stage only for review)
Returns:
True if merge succeeded
"""
info = self.get_worktree_info(spec_name)
if not info:
print(f"No worktree found for spec: {spec_name}")
return False
if no_commit:
print(
f"Merging {info.branch} into {self.base_branch} (staged, not committed)..."
)
else:
print(f"Merging {info.branch} into {self.base_branch}...")
# Switch to base branch in main project
result = self._run_git(["checkout", self.base_branch])
if result.returncode != 0:
print(f"Error: Could not checkout base branch: {result.stderr}")
return False
# Merge the spec branch
merge_args = ["merge", "--no-ff", info.branch]
if no_commit:
# --no-commit stages the merge but doesn't create the commit
merge_args.append("--no-commit")
else:
merge_args.extend(["-m", f"auto-claude: Merge {info.branch}"])
result = self._run_git(merge_args)
if result.returncode != 0:
print("Merge conflict! Aborting merge...")
self._run_git(["merge", "--abort"])
return False
if no_commit:
# Unstage any files that are gitignored in the main branch
# These get staged during merge because they exist in the worktree branch
self._unstage_gitignored_files()
print(
f"Changes from {info.branch} are now staged in your working directory."
)
print("Review the changes, then commit when ready:")
print(" git commit -m 'your commit message'")
else:
print(f"Successfully merged {info.branch}")
if delete_after:
self.remove_worktree(spec_name, delete_branch=True)
return True
def commit_in_worktree(self, spec_name: str, message: str) -> bool:
"""Commit all changes in a spec's worktree."""
worktree_path = self.get_worktree_path(spec_name)
if not worktree_path.exists():
return False
self._run_git(["add", "."], cwd=worktree_path)
result = self._run_git(["commit", "-m", message], cwd=worktree_path)
if result.returncode == 0:
return True
elif "nothing to commit" in result.stdout + result.stderr:
return True
else:
print(f"Commit failed: {result.stderr}")
return False
# ==================== Listing & Discovery ====================
def list_all_worktrees(self) -> list[WorktreeInfo]:
"""List all spec worktrees."""
worktrees = []
if not self.worktrees_dir.exists():
return worktrees
for item in self.worktrees_dir.iterdir():
if item.is_dir():
info = self.get_worktree_info(item.name)
if info:
worktrees.append(info)
return worktrees
def list_all_spec_branches(self) -> list[str]:
"""List all auto-claude branches (even if worktree removed)."""
result = self._run_git(["branch", "--list", "auto-claude/*"])
if result.returncode != 0:
return []
branches = []
for line in result.stdout.strip().split("\n"):
branch = line.strip().lstrip("* ")
if branch:
branches.append(branch)
return branches
def get_changed_files(self, spec_name: str) -> list[tuple[str, str]]:
"""Get list of changed files in a spec's worktree."""
worktree_path = self.get_worktree_path(spec_name)
if not worktree_path.exists():
return []
result = self._run_git(
["diff", "--name-status", f"{self.base_branch}...HEAD"], cwd=worktree_path
)
files = []
for line in result.stdout.strip().split("\n"):
if not line:
continue
parts = line.split("\t", 1)
if len(parts) == 2:
files.append((parts[0], parts[1]))
return files
def get_change_summary(self, spec_name: str) -> dict:
"""Get a summary of changes in a worktree."""
files = self.get_changed_files(spec_name)
new_files = sum(1 for status, _ in files if status == "A")
modified_files = sum(1 for status, _ in files if status == "M")
deleted_files = sum(1 for status, _ in files if status == "D")
return {
"new_files": new_files,
"modified_files": modified_files,
"deleted_files": deleted_files,
}
def cleanup_all(self) -> None:
"""Remove all worktrees and their branches."""
for worktree in self.list_all_worktrees():
self.remove_worktree(worktree.spec_name, delete_branch=True)
def cleanup_stale_worktrees(self) -> None:
"""Remove worktrees that aren't registered with git."""
if not self.worktrees_dir.exists():
return
# Get list of registered worktrees
result = self._run_git(["worktree", "list", "--porcelain"])
registered_paths = set()
for line in result.stdout.split("\n"):
if line.startswith("worktree "):
registered_paths.add(Path(line.split(" ", 1)[1]))
# Remove unregistered directories
for item in self.worktrees_dir.iterdir():
if item.is_dir() and item not in registered_paths:
print(f"Removing stale worktree directory: {item.name}")
shutil.rmtree(item, ignore_errors=True)
self._run_git(["worktree", "prune"])
def get_test_commands(self, spec_name: str) -> list[str]:
"""Detect likely test/run commands for the project."""
worktree_path = self.get_worktree_path(spec_name)
commands = []
if (worktree_path / "package.json").exists():
commands.append("npm install && npm run dev")
commands.append("npm test")
if (worktree_path / "requirements.txt").exists():
commands.append("pip install -r requirements.txt")
if (worktree_path / "Cargo.toml").exists():
commands.append("cargo run")
commands.append("cargo test")
if (worktree_path / "go.mod").exists():
commands.append("go run .")
commands.append("go test ./...")
if not commands:
commands.append("# Check the project's README for run instructions")
return commands
# ==================== Backward Compatibility ====================
# These methods provide backward compatibility with the old single-worktree API
def get_staging_path(self) -> Path | None:
"""
Backward compatibility: Get path to any existing spec worktree.
Prefer using get_worktree_path(spec_name) instead.
"""
worktrees = self.list_all_worktrees()
if worktrees:
return worktrees[0].path
return None
def get_staging_info(self) -> WorktreeInfo | None:
"""
Backward compatibility: Get info about any existing spec worktree.
Prefer using get_worktree_info(spec_name) instead.
"""
worktrees = self.list_all_worktrees()
if worktrees:
return worktrees[0]
return None
def merge_staging(self, delete_after: bool = True) -> bool:
"""
Backward compatibility: Merge first found worktree.
Prefer using merge_worktree(spec_name) instead.
"""
worktrees = self.list_all_worktrees()
if worktrees:
return self.merge_worktree(worktrees[0].spec_name, delete_after)
return False
def remove_staging(self, delete_branch: bool = True) -> None:
"""
Backward compatibility: Remove first found worktree.
Prefer using remove_worktree(spec_name) instead.
"""
worktrees = self.list_all_worktrees()
if worktrees:
self.remove_worktree(worktrees[0].spec_name, delete_branch)
def get_or_create_staging(self, spec_name: str) -> WorktreeInfo:
"""
Backward compatibility: Alias for get_or_create_worktree.
"""
return self.get_or_create_worktree(spec_name)
def staging_exists(self) -> bool:
"""
Backward compatibility: Check if any spec worktree exists.
Prefer using worktree_exists(spec_name) instead.
"""
return len(self.list_all_worktrees()) > 0
def commit_in_staging(self, message: str) -> bool:
"""
Backward compatibility: Commit in first found worktree.
Prefer using commit_in_worktree(spec_name, message) instead.
"""
worktrees = self.list_all_worktrees()
if worktrees:
return self.commit_in_worktree(worktrees[0].spec_name, message)
return False
def has_uncommitted_changes(self, in_staging: bool = False) -> bool:
"""Check if there are uncommitted changes."""
worktrees = self.list_all_worktrees()
if in_staging and worktrees:
cwd = worktrees[0].path
else:
cwd = None
result = self._run_git(["status", "--porcelain"], cwd=cwd)
return bool(result.stdout.strip())
# Keep STAGING_WORKTREE_NAME for backward compatibility in imports
STAGING_WORKTREE_NAME = "auto-claude"
+2 -369
View File
@@ -1,369 +1,2 @@
#!/usr/bin/env python3
"""
Self-Critique System
====================
Implements a self-critique loop that agents must run before marking subtasks complete.
This helps catch quality issues early, before verification stage.
The critique system ensures:
- Code follows patterns from reference files
- All required files were modified/created
- Error handling is present
- No debugging artifacts left behind
- Implementation matches subtask requirements
"""
import re
from dataclasses import dataclass, field
@dataclass
class CritiqueResult:
"""Result of a self-critique evaluation."""
passes: bool
issues: list[str] = field(default_factory=list)
improvements_made: list[str] = field(default_factory=list)
recommendations: list[str] = field(default_factory=list)
def to_dict(self) -> dict:
"""Convert to dictionary for storage."""
return {
"passes": self.passes,
"issues": self.issues,
"improvements_made": self.improvements_made,
"recommendations": self.recommendations,
}
@classmethod
def from_dict(cls, data: dict) -> "CritiqueResult":
"""Load from dictionary."""
return cls(
passes=data.get("passes", False),
issues=data.get("issues", []),
improvements_made=data.get("improvements_made", []),
recommendations=data.get("recommendations", []),
)
def generate_critique_prompt(
subtask: dict, files_modified: list[str], patterns_from: list[str]
) -> str:
"""
Generate a critique prompt for the agent to self-evaluate.
Args:
subtask: The subtask being implemented
files_modified: List of files actually modified
patterns_from: List of pattern files to compare against
Returns:
Formatted prompt for self-critique
"""
subtask_id = subtask.get("id", "unknown")
subtask_desc = subtask.get("description", "No description")
service = subtask.get("service", "all services")
files_to_modify = subtask.get("files_to_modify", [])
files_to_create = subtask.get("files_to_create", [])
prompt = f"""## MANDATORY Self-Critique: {subtask_id}
**Subtask Description:** {subtask_desc}
**Service:** {service}
Before marking this subtask as complete, you MUST perform a thorough self-critique.
This is NOT optional - it's a required quality gate.
### STEP 1: Code Quality Checklist
Review your implementation against these criteria:
**Pattern Adherence:**
- [ ] Follows patterns from reference files exactly: {", ".join(patterns_from) if patterns_from else "N/A"}
- [ ] Variable naming matches codebase conventions
- [ ] Imports organized correctly (grouped, sorted)
- [ ] Code style consistent with existing files
**Error Handling:**
- [ ] Try-catch blocks where operations can fail
- [ ] Meaningful error messages
- [ ] Proper error propagation
- [ ] Edge cases considered
**Code Cleanliness:**
- [ ] No console.log/print statements for debugging
- [ ] No commented-out code blocks
- [ ] No TODO comments without context
- [ ] No hardcoded values that should be configurable
**Best Practices:**
- [ ] Functions are focused and single-purpose
- [ ] No code duplication
- [ ] Appropriate use of constants
- [ ] Documentation/comments where needed
### STEP 2: Implementation Completeness
**Files Modified:**
Expected: {", ".join(files_to_modify) if files_to_modify else "None"}
Actual: {", ".join(files_modified) if files_modified else "None"}
- [ ] All files_to_modify were actually modified
- [ ] No unexpected files were modified
**Files Created:**
Expected: {", ".join(files_to_create) if files_to_create else "None"}
- [ ] All files_to_create were actually created
- [ ] Files follow naming conventions
**Requirements:**
- [ ] Subtask description requirements fully met
- [ ] All acceptance criteria from spec considered
- [ ] No scope creep - stayed within subtask boundaries
### STEP 3: Potential Issues Analysis
List any concerns, limitations, or potential problems with your implementation:
1. [Issue 1, or "None identified"]
2. [Issue 2, if any]
3. [Issue 3, if any]
Be honest. Finding issues now is better than discovering them during verification.
### STEP 4: Improvements Made
If you identified issues in your critique, list what you fixed:
1. [Improvement 1, or "No fixes needed"]
2. [Improvement 2, if applicable]
3. [Improvement 3, if applicable]
### STEP 5: Final Verdict
**PROCEED:** [YES/NO - Only YES if all critical items pass]
**REASON:** [Brief explanation of your decision]
**CONFIDENCE:** [High/Medium/Low - How confident are you in this implementation?]
---
## Instructions for Agent
1. Work through each section methodically
2. Check each box honestly - don't skip items
3. If you find issues, FIX THEM before continuing
4. Re-run this critique after fixes
5. Only mark the subtask complete when verdict is YES with High confidence
6. Document your critique results in your response
Remember: The next session has no context. Quality issues you miss now will be harder to fix later.
"""
return prompt
def parse_critique_response(response: str) -> CritiqueResult:
"""
Parse the agent's critique response into structured data.
Args:
response: The agent's response to the critique prompt
Returns:
CritiqueResult with parsed information
"""
issues = []
improvements = []
recommendations = []
passes = False
# Extract PROCEED verdict
proceed_match = re.search(
r"\*\*PROCEED:\*\*\s*\[?\s*(YES|NO)", response, re.IGNORECASE
)
if proceed_match:
passes = proceed_match.group(1).upper() == "YES"
# Extract issues from Step 3
issues_section = re.search(
r"### STEP 3:.*?Potential Issues.*?\n\n(.*?)(?=###|\Z)",
response,
re.DOTALL | re.IGNORECASE,
)
if issues_section:
issue_lines = issues_section.group(1).strip().split("\n")
for line in issue_lines:
line = line.strip()
if not line or line.startswith("---"):
continue
# Remove list markers
issue = re.sub(r"^\d+\.\s*|\*\s*|-\s*", "", line).strip()
# Skip if it's a placeholder or indicates no issues
if (
issue
and issue.lower()
not in ["none", "none identified", "no issues", "no concerns"]
and issue
not in [
'[Issue 1, or "None identified"]',
"[Issue 2, if any]",
"[Issue 3, if any]",
]
):
issues.append(issue)
# Extract improvements from Step 4
improvements_section = re.search(
r"### STEP 4:.*?Improvements Made.*?\n\n(.*?)(?=###|\Z)",
response,
re.DOTALL | re.IGNORECASE,
)
if improvements_section:
improvement_lines = improvements_section.group(1).strip().split("\n")
for line in improvement_lines:
line = line.strip()
if not line or line.startswith("---"):
continue
# Remove list markers
improvement = re.sub(r"^\d+\.\s*|\*\s*|-\s*", "", line).strip()
# Skip if it's a placeholder or indicates no improvements
if (
improvement
and improvement.lower()
not in ["none", "no fixes needed", "no improvements", "n/a"]
and improvement
not in [
'[Improvement 1, or "No fixes needed"]',
"[Improvement 2, if applicable]",
"[Improvement 3, if applicable]",
]
):
improvements.append(improvement)
# Extract confidence level as recommendation
confidence_match = re.search(
r"\*\*CONFIDENCE:\*\*\s*\[?\s*(High|Medium|Low)", response, re.IGNORECASE
)
if confidence_match:
confidence = confidence_match.group(1)
if confidence.lower() != "high":
recommendations.append(
f"Confidence level: {confidence} - consider additional review"
)
return CritiqueResult(
passes=passes,
issues=issues,
improvements_made=improvements,
recommendations=recommendations,
)
def should_proceed(result: CritiqueResult) -> bool:
"""
Determine if the subtask should be marked complete based on critique.
Args:
result: The critique result
Returns:
True if subtask can be marked complete, False otherwise
"""
# Must pass the critique
if not result.passes:
return False
# If there are unresolved issues, don't proceed
if result.issues:
return False
return True
def format_critique_summary(result: CritiqueResult) -> str:
"""
Format a critique result as a human-readable summary.
Args:
result: The critique result
Returns:
Formatted summary string
"""
lines = ["## Critique Summary"]
lines.append("")
lines.append(f"**Status:** {'PASSED ✓' if result.passes else 'FAILED ✗'}")
lines.append("")
if result.issues:
lines.append("**Issues Identified:**")
for i, issue in enumerate(result.issues, 1):
lines.append(f"{i}. {issue}")
lines.append("")
if result.improvements_made:
lines.append("**Improvements Made:**")
for i, improvement in enumerate(result.improvements_made, 1):
lines.append(f"{i}. {improvement}")
lines.append("")
if result.recommendations:
lines.append("**Recommendations:**")
for i, rec in enumerate(result.recommendations, 1):
lines.append(f"{i}. {rec}")
lines.append("")
if should_proceed(result):
lines.append("**Decision:** Subtask is ready to be marked complete.")
else:
lines.append("**Decision:** Subtask needs more work before completion.")
return "\n".join(lines)
# Example usage for testing
if __name__ == "__main__":
# Demo subtask
subtask = {
"id": "auth-middleware",
"description": "Add JWT authentication middleware",
"service": "backend",
"files_to_modify": ["app/middleware/auth.py"],
"patterns_from": ["app/middleware/cors.py"],
}
files_modified = ["app/middleware/auth.py"]
# Generate prompt
prompt = generate_critique_prompt(subtask, files_modified, subtask["patterns_from"])
print(prompt)
print("\n" + "=" * 80 + "\n")
# Simulate a critique response
sample_response = """
### STEP 3: Potential Issues Analysis
1. Token expiration edge case not fully tested
2. None
### STEP 4: Improvements Made
1. Added comprehensive error handling for invalid tokens
2. Improved logging for debugging
3. Added input validation for JWT format
### STEP 5: Final Verdict
**PROCEED:** YES
**REASON:** All critical items verified, patterns followed, error handling complete
**CONFIDENCE:** High
"""
# Parse response
result = parse_critique_response(sample_response)
print(format_critique_summary(result))
print(f"\nShould proceed: {should_proceed(result)}")
"""Backward compatibility shim - import from spec.critique instead."""
from spec.critique import *
+2 -349
View File
@@ -1,349 +1,2 @@
#!/usr/bin/env python3
"""
Debug Logging Utility
=====================
Centralized debug logging for the Auto-Claude framework.
Controlled via environment variables:
- DEBUG=true Enable debug mode
- DEBUG_LEVEL=1|2|3 Log verbosity (1=basic, 2=detailed, 3=verbose)
- DEBUG_LOG_FILE=path Optional file output
Usage:
from debug import debug, debug_detailed, debug_verbose, is_debug_enabled
debug("run.py", "Starting task execution", task_id="001")
debug_detailed("agent", "Agent response received", response_length=1234)
debug_verbose("client", "Full request payload", payload=data)
"""
import json
import os
import sys
import time
from datetime import datetime
from functools import wraps
from pathlib import Path
from typing import Any
# ANSI color codes for terminal output
class Colors:
RESET = "\033[0m"
BOLD = "\033[1m"
DIM = "\033[2m"
# Debug colors
DEBUG = "\033[36m" # Cyan
DEBUG_DIM = "\033[96m" # Light cyan
TIMESTAMP = "\033[90m" # Gray
MODULE = "\033[33m" # Yellow
KEY = "\033[35m" # Magenta
VALUE = "\033[37m" # White
SUCCESS = "\033[32m" # Green
WARNING = "\033[33m" # Yellow
ERROR = "\033[31m" # Red
def _get_debug_enabled() -> bool:
"""Check if debug mode is enabled via environment variable."""
return os.environ.get("DEBUG", "").lower() in ("true", "1", "yes", "on")
def _get_debug_level() -> int:
"""Get debug verbosity level (1-3)."""
try:
level = int(os.environ.get("DEBUG_LEVEL", "1"))
return max(1, min(3, level)) # Clamp to 1-3
except ValueError:
return 1
def _get_log_file() -> Path | None:
"""Get optional log file path."""
log_file = os.environ.get("DEBUG_LOG_FILE")
if log_file:
return Path(log_file)
return None
def is_debug_enabled() -> bool:
"""Check if debug mode is enabled."""
return _get_debug_enabled()
def get_debug_level() -> int:
"""Get current debug level."""
return _get_debug_level()
def _format_value(value: Any, max_length: int = 200) -> str:
"""Format a value for debug output, truncating if necessary."""
if value is None:
return "None"
if isinstance(value, (dict, list)):
try:
formatted = json.dumps(value, indent=2, default=str)
if len(formatted) > max_length:
formatted = formatted[:max_length] + "..."
return formatted
except (TypeError, ValueError):
return str(value)[:max_length]
str_value = str(value)
if len(str_value) > max_length:
return str_value[:max_length] + "..."
return str_value
def _write_log(message: str, to_file: bool = True) -> None:
"""Write log message to stdout and optionally to file."""
print(message, file=sys.stderr)
if to_file:
log_file = _get_log_file()
if log_file:
try:
log_file.parent.mkdir(parents=True, exist_ok=True)
# Strip ANSI codes for file output
import re
clean_message = re.sub(r"\033\[[0-9;]*m", "", message)
with open(log_file, "a") as f:
f.write(clean_message + "\n")
except Exception:
pass # Silently fail file logging
def debug(module: str, message: str, level: int = 1, **kwargs) -> None:
"""
Log a debug message.
Args:
module: Source module name (e.g., "run.py", "ideation_runner")
message: Debug message
level: Required debug level (1=basic, 2=detailed, 3=verbose)
**kwargs: Additional key-value pairs to log
"""
if not _get_debug_enabled():
return
if _get_debug_level() < level:
return
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
# Build the log line
parts = [
f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET}",
f"{Colors.DEBUG}[DEBUG]{Colors.RESET}",
f"{Colors.MODULE}[{module}]{Colors.RESET}",
f"{Colors.DEBUG_DIM}{message}{Colors.RESET}",
]
log_line = " ".join(parts)
# Add kwargs on separate lines if present
if kwargs:
for key, value in kwargs.items():
formatted_value = _format_value(value)
if "\n" in formatted_value:
# Multi-line value
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}:"
for line in formatted_value.split("\n"):
log_line += f"\n {Colors.VALUE}{line}{Colors.RESET}"
else:
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{formatted_value}{Colors.RESET}"
_write_log(log_line)
def debug_detailed(module: str, message: str, **kwargs) -> None:
"""Log a detailed debug message (level 2)."""
debug(module, message, level=2, **kwargs)
def debug_verbose(module: str, message: str, **kwargs) -> None:
"""Log a verbose debug message (level 3)."""
debug(module, message, level=3, **kwargs)
def debug_success(module: str, message: str, **kwargs) -> None:
"""Log a success debug message."""
if not _get_debug_enabled():
return
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.SUCCESS}[OK]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {message}"
if kwargs:
for key, value in kwargs.items():
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}"
_write_log(log_line)
def debug_info(module: str, message: str, **kwargs) -> None:
"""Log an info debug message."""
if not _get_debug_enabled():
return
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.DEBUG}[INFO]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {message}"
if kwargs:
for key, value in kwargs.items():
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}"
_write_log(log_line)
def debug_error(module: str, message: str, **kwargs) -> None:
"""Log an error debug message (always shown if debug enabled)."""
if not _get_debug_enabled():
return
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.ERROR}[ERROR]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {Colors.ERROR}{message}{Colors.RESET}"
if kwargs:
for key, value in kwargs.items():
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}"
_write_log(log_line)
def debug_warning(module: str, message: str, **kwargs) -> None:
"""Log a warning debug message."""
if not _get_debug_enabled():
return
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.WARNING}[WARN]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {Colors.WARNING}{message}{Colors.RESET}"
if kwargs:
for key, value in kwargs.items():
log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}"
_write_log(log_line)
def debug_section(module: str, title: str) -> None:
"""Log a section header for organizing debug output."""
if not _get_debug_enabled():
return
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
separator = "" * 60
log_line = f"\n{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.DEBUG}{Colors.BOLD}{separator}{Colors.RESET}"
log_line += f"\n{Colors.TIMESTAMP} {Colors.RESET} {Colors.DEBUG}{Colors.BOLD}{module}: {title}{' ' * (58 - len(module) - len(title) - 2)}{Colors.RESET}"
log_line += f"\n{Colors.TIMESTAMP} {Colors.RESET} {Colors.DEBUG}{Colors.BOLD}{separator}{Colors.RESET}"
_write_log(log_line)
def debug_timer(module: str):
"""
Decorator to time function execution.
Usage:
@debug_timer("run.py")
def my_function():
...
"""
def decorator(func):
@wraps(func)
def wrapper(*args, **kwargs):
if not _get_debug_enabled():
return func(*args, **kwargs)
start = time.time()
debug_detailed(module, f"Starting {func.__name__}()")
try:
result = func(*args, **kwargs)
elapsed = time.time() - start
debug_success(
module,
f"Completed {func.__name__}()",
elapsed_ms=f"{elapsed * 1000:.1f}ms",
)
return result
except Exception as e:
elapsed = time.time() - start
debug_error(
module,
f"Failed {func.__name__}()",
error=str(e),
elapsed_ms=f"{elapsed * 1000:.1f}ms",
)
raise
return wrapper
return decorator
def debug_async_timer(module: str):
"""
Decorator to time async function execution.
Usage:
@debug_async_timer("ideation_runner")
async def my_async_function():
...
"""
def decorator(func):
@wraps(func)
async def wrapper(*args, **kwargs):
if not _get_debug_enabled():
return await func(*args, **kwargs)
start = time.time()
debug_detailed(module, f"Starting {func.__name__}()")
try:
result = await func(*args, **kwargs)
elapsed = time.time() - start
debug_success(
module,
f"Completed {func.__name__}()",
elapsed_ms=f"{elapsed * 1000:.1f}ms",
)
return result
except Exception as e:
elapsed = time.time() - start
debug_error(
module,
f"Failed {func.__name__}()",
error=str(e),
elapsed_ms=f"{elapsed * 1000:.1f}ms",
)
raise
return wrapper
return decorator
def debug_env_status() -> None:
"""Print debug environment status on startup."""
if not _get_debug_enabled():
return
debug_section("debug", "Debug Mode Enabled")
debug(
"debug",
"Environment configuration",
DEBUG=os.environ.get("DEBUG", "not set"),
DEBUG_LEVEL=_get_debug_level(),
DEBUG_LOG_FILE=os.environ.get("DEBUG_LOG_FILE", "not set"),
)
# Print status on import if debug is enabled
if _get_debug_enabled():
debug_env_status()
"""Backward compatibility shim - import from core.debug instead."""
from core.debug import *
+2 -529
View File
@@ -1,529 +1,2 @@
"""
Graphiti Integration Configuration
==================================
Constants, status mappings, and configuration helpers for Graphiti memory integration.
Follows the same patterns as linear_config.py for consistency.
Multi-Provider Support (V2):
- LLM Providers: OpenAI, Anthropic, Azure OpenAI, Ollama
- Embedder Providers: OpenAI, Voyage AI, Azure OpenAI, Ollama
Environment Variables:
# Core
GRAPHITI_ENABLED: Set to "true" to enable Graphiti integration
GRAPHITI_LLM_PROVIDER: openai|anthropic|azure_openai|ollama (default: openai)
GRAPHITI_EMBEDDER_PROVIDER: openai|voyage|azure_openai|ollama (default: openai)
# OpenAI
OPENAI_API_KEY: Required for OpenAI provider
OPENAI_MODEL: Model for LLM (default: gpt-5-mini)
OPENAI_EMBEDDING_MODEL: Model for embeddings (default: text-embedding-3-small)
# Anthropic (LLM only - needs separate embedder)
ANTHROPIC_API_KEY: Required for Anthropic provider
GRAPHITI_ANTHROPIC_MODEL: Model for LLM (default: claude-sonnet-4-5-latest)
# Azure OpenAI
AZURE_OPENAI_API_KEY: Required for Azure provider
AZURE_OPENAI_BASE_URL: Azure endpoint URL
AZURE_OPENAI_LLM_DEPLOYMENT: Deployment name for LLM
AZURE_OPENAI_EMBEDDING_DEPLOYMENT: Deployment name for embeddings
# Voyage AI (embeddings only - commonly used with Anthropic)
VOYAGE_API_KEY: Required for Voyage embedder
VOYAGE_EMBEDDING_MODEL: Model (default: voyage-3)
# Ollama (local)
OLLAMA_BASE_URL: Ollama server URL (default: http://localhost:11434)
OLLAMA_LLM_MODEL: Model for LLM (e.g., deepseek-r1:7b)
OLLAMA_EMBEDDING_MODEL: Model for embeddings (e.g., nomic-embed-text)
OLLAMA_EMBEDDING_DIM: Embedding dimension (required for Ollama, e.g., 768)
# FalkorDB
GRAPHITI_FALKORDB_HOST: FalkorDB host (default: localhost)
GRAPHITI_FALKORDB_PORT: FalkorDB port (default: 6380)
GRAPHITI_FALKORDB_PASSWORD: FalkorDB password (default: empty)
GRAPHITI_DATABASE: Graph database name (default: auto_claude_memory)
GRAPHITI_TELEMETRY_ENABLED: Set to "false" to disable telemetry (default: true)
"""
import json
import os
from dataclasses import dataclass, field
from datetime import datetime
from enum import Enum
from pathlib import Path
from typing import Optional
# Default configuration values
DEFAULT_FALKORDB_HOST = "localhost"
DEFAULT_FALKORDB_PORT = 6380
DEFAULT_DATABASE = "auto_claude_memory"
DEFAULT_OLLAMA_BASE_URL = "http://localhost:11434"
# Graphiti state marker file (stores connection info and status)
GRAPHITI_STATE_MARKER = ".graphiti_state.json"
# Episode types for different memory categories
EPISODE_TYPE_SESSION_INSIGHT = "session_insight"
EPISODE_TYPE_CODEBASE_DISCOVERY = "codebase_discovery"
EPISODE_TYPE_PATTERN = "pattern"
EPISODE_TYPE_GOTCHA = "gotcha"
EPISODE_TYPE_TASK_OUTCOME = "task_outcome"
EPISODE_TYPE_QA_RESULT = "qa_result"
EPISODE_TYPE_HISTORICAL_CONTEXT = "historical_context"
class LLMProvider(str, Enum):
"""Supported LLM providers for Graphiti."""
OPENAI = "openai"
ANTHROPIC = "anthropic"
AZURE_OPENAI = "azure_openai"
OLLAMA = "ollama"
class EmbedderProvider(str, Enum):
"""Supported embedder providers for Graphiti."""
OPENAI = "openai"
VOYAGE = "voyage"
AZURE_OPENAI = "azure_openai"
OLLAMA = "ollama"
@dataclass
class GraphitiConfig:
"""Configuration for Graphiti memory integration with multi-provider support."""
# Core settings
enabled: bool = False
llm_provider: str = "openai"
embedder_provider: str = "openai"
# FalkorDB connection
falkordb_host: str = DEFAULT_FALKORDB_HOST
falkordb_port: int = DEFAULT_FALKORDB_PORT
falkordb_password: str = ""
database: str = DEFAULT_DATABASE
telemetry_enabled: bool = True
# OpenAI settings
openai_api_key: str = ""
openai_model: str = "gpt-5-mini"
openai_embedding_model: str = "text-embedding-3-small"
# Anthropic settings (LLM only)
anthropic_api_key: str = ""
anthropic_model: str = "claude-sonnet-4-5-latest"
# Azure OpenAI settings
azure_openai_api_key: str = ""
azure_openai_base_url: str = ""
azure_openai_llm_deployment: str = ""
azure_openai_embedding_deployment: str = ""
# Voyage AI settings (embeddings only)
voyage_api_key: str = ""
voyage_embedding_model: str = "voyage-3"
# Ollama settings (local)
ollama_base_url: str = DEFAULT_OLLAMA_BASE_URL
ollama_llm_model: str = ""
ollama_embedding_model: str = ""
ollama_embedding_dim: int = 0 # Required for Ollama embeddings
@classmethod
def from_env(cls) -> "GraphitiConfig":
"""Create config from environment variables."""
# Check if Graphiti is explicitly enabled
enabled_str = os.environ.get("GRAPHITI_ENABLED", "").lower()
enabled = enabled_str in ("true", "1", "yes")
# Provider selection
llm_provider = os.environ.get("GRAPHITI_LLM_PROVIDER", "openai").lower()
embedder_provider = os.environ.get(
"GRAPHITI_EMBEDDER_PROVIDER", "openai"
).lower()
# FalkorDB connection settings
falkordb_host = os.environ.get("GRAPHITI_FALKORDB_HOST", DEFAULT_FALKORDB_HOST)
try:
falkordb_port = int(
os.environ.get("GRAPHITI_FALKORDB_PORT", str(DEFAULT_FALKORDB_PORT))
)
except ValueError:
falkordb_port = DEFAULT_FALKORDB_PORT
falkordb_password = os.environ.get("GRAPHITI_FALKORDB_PASSWORD", "")
database = os.environ.get("GRAPHITI_DATABASE", DEFAULT_DATABASE)
# Telemetry setting
telemetry_str = os.environ.get("GRAPHITI_TELEMETRY_ENABLED", "true").lower()
telemetry_enabled = telemetry_str not in ("false", "0", "no")
# OpenAI settings
openai_api_key = os.environ.get("OPENAI_API_KEY", "")
openai_model = os.environ.get("OPENAI_MODEL", "gpt-5-mini")
openai_embedding_model = os.environ.get(
"OPENAI_EMBEDDING_MODEL", "text-embedding-3-small"
)
# Anthropic settings
anthropic_api_key = os.environ.get("ANTHROPIC_API_KEY", "")
anthropic_model = os.environ.get(
"GRAPHITI_ANTHROPIC_MODEL", "claude-sonnet-4-5-latest"
)
# Azure OpenAI settings
azure_openai_api_key = os.environ.get("AZURE_OPENAI_API_KEY", "")
azure_openai_base_url = os.environ.get("AZURE_OPENAI_BASE_URL", "")
azure_openai_llm_deployment = os.environ.get("AZURE_OPENAI_LLM_DEPLOYMENT", "")
azure_openai_embedding_deployment = os.environ.get(
"AZURE_OPENAI_EMBEDDING_DEPLOYMENT", ""
)
# Voyage AI settings
voyage_api_key = os.environ.get("VOYAGE_API_KEY", "")
voyage_embedding_model = os.environ.get("VOYAGE_EMBEDDING_MODEL", "voyage-3")
# Ollama settings
ollama_base_url = os.environ.get("OLLAMA_BASE_URL", DEFAULT_OLLAMA_BASE_URL)
ollama_llm_model = os.environ.get("OLLAMA_LLM_MODEL", "")
ollama_embedding_model = os.environ.get("OLLAMA_EMBEDDING_MODEL", "")
# Ollama embedding dimension (required for Ollama)
try:
ollama_embedding_dim = int(os.environ.get("OLLAMA_EMBEDDING_DIM", "0"))
except ValueError:
ollama_embedding_dim = 0
return cls(
enabled=enabled,
llm_provider=llm_provider,
embedder_provider=embedder_provider,
falkordb_host=falkordb_host,
falkordb_port=falkordb_port,
falkordb_password=falkordb_password,
database=database,
telemetry_enabled=telemetry_enabled,
openai_api_key=openai_api_key,
openai_model=openai_model,
openai_embedding_model=openai_embedding_model,
anthropic_api_key=anthropic_api_key,
anthropic_model=anthropic_model,
azure_openai_api_key=azure_openai_api_key,
azure_openai_base_url=azure_openai_base_url,
azure_openai_llm_deployment=azure_openai_llm_deployment,
azure_openai_embedding_deployment=azure_openai_embedding_deployment,
voyage_api_key=voyage_api_key,
voyage_embedding_model=voyage_embedding_model,
ollama_base_url=ollama_base_url,
ollama_llm_model=ollama_llm_model,
ollama_embedding_model=ollama_embedding_model,
ollama_embedding_dim=ollama_embedding_dim,
)
def is_valid(self) -> bool:
"""
Check if config has minimum required values for operation.
Returns True if:
- GRAPHITI_ENABLED is true
- LLM provider is configured correctly
- Embedder provider is configured correctly
"""
if not self.enabled:
return False
# Validate LLM provider
if not self._validate_llm_provider():
return False
# Validate embedder provider
if not self._validate_embedder_provider():
return False
return True
def _validate_llm_provider(self) -> bool:
"""Validate LLM provider configuration."""
if self.llm_provider == "openai":
return bool(self.openai_api_key)
elif self.llm_provider == "anthropic":
return bool(self.anthropic_api_key)
elif self.llm_provider == "azure_openai":
return bool(
self.azure_openai_api_key
and self.azure_openai_base_url
and self.azure_openai_llm_deployment
)
elif self.llm_provider == "ollama":
return bool(self.ollama_llm_model)
return False
def _validate_embedder_provider(self) -> bool:
"""Validate embedder provider configuration."""
if self.embedder_provider == "openai":
return bool(self.openai_api_key)
elif self.embedder_provider == "voyage":
return bool(self.voyage_api_key)
elif self.embedder_provider == "azure_openai":
return bool(
self.azure_openai_api_key
and self.azure_openai_base_url
and self.azure_openai_embedding_deployment
)
elif self.embedder_provider == "ollama":
return bool(self.ollama_embedding_model and self.ollama_embedding_dim)
return False
def get_validation_errors(self) -> list[str]:
"""Get list of validation errors for current configuration."""
errors = []
if not self.enabled:
errors.append("GRAPHITI_ENABLED must be set to true")
return errors
# LLM provider validation
if self.llm_provider == "openai":
if not self.openai_api_key:
errors.append("OpenAI LLM provider requires OPENAI_API_KEY")
elif self.llm_provider == "anthropic":
if not self.anthropic_api_key:
errors.append("Anthropic LLM provider requires ANTHROPIC_API_KEY")
elif self.llm_provider == "azure_openai":
if not self.azure_openai_api_key:
errors.append("Azure OpenAI LLM provider requires AZURE_OPENAI_API_KEY")
if not self.azure_openai_base_url:
errors.append(
"Azure OpenAI LLM provider requires AZURE_OPENAI_BASE_URL"
)
if not self.azure_openai_llm_deployment:
errors.append(
"Azure OpenAI LLM provider requires AZURE_OPENAI_LLM_DEPLOYMENT"
)
elif self.llm_provider == "ollama":
if not self.ollama_llm_model:
errors.append("Ollama LLM provider requires OLLAMA_LLM_MODEL")
else:
errors.append(f"Unknown LLM provider: {self.llm_provider}")
# Embedder provider validation
if self.embedder_provider == "openai":
if not self.openai_api_key:
errors.append("OpenAI embedder provider requires OPENAI_API_KEY")
elif self.embedder_provider == "voyage":
if not self.voyage_api_key:
errors.append("Voyage embedder provider requires VOYAGE_API_KEY")
elif self.embedder_provider == "azure_openai":
if not self.azure_openai_api_key:
errors.append(
"Azure OpenAI embedder provider requires AZURE_OPENAI_API_KEY"
)
if not self.azure_openai_base_url:
errors.append(
"Azure OpenAI embedder provider requires AZURE_OPENAI_BASE_URL"
)
if not self.azure_openai_embedding_deployment:
errors.append(
"Azure OpenAI embedder provider requires AZURE_OPENAI_EMBEDDING_DEPLOYMENT"
)
elif self.embedder_provider == "ollama":
if not self.ollama_embedding_model:
errors.append(
"Ollama embedder provider requires OLLAMA_EMBEDDING_MODEL"
)
if not self.ollama_embedding_dim:
errors.append("Ollama embedder provider requires OLLAMA_EMBEDDING_DIM")
else:
errors.append(f"Unknown embedder provider: {self.embedder_provider}")
return errors
def get_connection_uri(self) -> str:
"""Get the FalkorDB connection URI."""
if self.falkordb_password:
return f"redis://:{self.falkordb_password}@{self.falkordb_host}:{self.falkordb_port}"
return f"redis://{self.falkordb_host}:{self.falkordb_port}"
def get_provider_summary(self) -> str:
"""Get a summary of configured providers."""
return f"LLM: {self.llm_provider}, Embedder: {self.embedder_provider}"
@dataclass
class GraphitiState:
"""State of Graphiti integration for an auto-claude spec."""
initialized: bool = False
database: str | None = None
indices_built: bool = False
created_at: str | None = None
last_session: int | None = None
episode_count: int = 0
error_log: list = field(default_factory=list)
# V2 additions
llm_provider: str | None = None
embedder_provider: str | None = None
def to_dict(self) -> dict:
return {
"initialized": self.initialized,
"database": self.database,
"indices_built": self.indices_built,
"created_at": self.created_at,
"last_session": self.last_session,
"episode_count": self.episode_count,
"error_log": self.error_log[-10:], # Keep last 10 errors
"llm_provider": self.llm_provider,
"embedder_provider": self.embedder_provider,
}
@classmethod
def from_dict(cls, data: dict) -> "GraphitiState":
return cls(
initialized=data.get("initialized", False),
database=data.get("database"),
indices_built=data.get("indices_built", False),
created_at=data.get("created_at"),
last_session=data.get("last_session"),
episode_count=data.get("episode_count", 0),
error_log=data.get("error_log", []),
llm_provider=data.get("llm_provider"),
embedder_provider=data.get("embedder_provider"),
)
def save(self, spec_dir: Path) -> None:
"""Save state to the spec directory."""
marker_file = spec_dir / GRAPHITI_STATE_MARKER
with open(marker_file, "w") as f:
json.dump(self.to_dict(), f, indent=2)
@classmethod
def load(cls, spec_dir: Path) -> Optional["GraphitiState"]:
"""Load state from the spec directory."""
marker_file = spec_dir / GRAPHITI_STATE_MARKER
if not marker_file.exists():
return None
try:
with open(marker_file) as f:
return cls.from_dict(json.load(f))
except (OSError, json.JSONDecodeError):
return None
def record_error(self, error_msg: str) -> None:
"""Record an error in the state."""
self.error_log.append(
{
"timestamp": datetime.now().isoformat(),
"error": error_msg[:500], # Limit error message length
}
)
# Keep only last 10 errors
self.error_log = self.error_log[-10:]
def is_graphiti_enabled() -> bool:
"""
Quick check if Graphiti integration is available.
Returns True if:
- GRAPHITI_ENABLED is set to true/1/yes
- Required provider credentials are configured
"""
config = GraphitiConfig.from_env()
return config.is_valid()
def get_graphiti_status() -> dict:
"""
Get the current Graphiti integration status.
Returns:
Dict with status information:
- enabled: bool
- available: bool (has required dependencies)
- host: str
- port: int
- database: str
- llm_provider: str
- embedder_provider: str
- reason: str (why unavailable if not available)
- errors: list (validation errors if any)
"""
config = GraphitiConfig.from_env()
status = {
"enabled": config.enabled,
"available": False,
"host": config.falkordb_host,
"port": config.falkordb_port,
"database": config.database,
"llm_provider": config.llm_provider,
"embedder_provider": config.embedder_provider,
"reason": "",
"errors": [],
}
if not config.enabled:
status["reason"] = "GRAPHITI_ENABLED not set to true"
return status
# Get validation errors
errors = config.get_validation_errors()
if errors:
status["errors"] = errors
status["reason"] = errors[0] # First error as primary reason
return status
status["available"] = True
return status
def get_available_providers() -> dict:
"""
Get list of available providers based on current environment.
Returns:
Dict with lists of available LLM and embedder providers
"""
config = GraphitiConfig.from_env()
available_llm = []
available_embedder = []
# Check OpenAI
if config.openai_api_key:
available_llm.append("openai")
available_embedder.append("openai")
# Check Anthropic
if config.anthropic_api_key:
available_llm.append("anthropic")
# Check Azure OpenAI
if config.azure_openai_api_key and config.azure_openai_base_url:
if config.azure_openai_llm_deployment:
available_llm.append("azure_openai")
if config.azure_openai_embedding_deployment:
available_embedder.append("azure_openai")
# Check Voyage
if config.voyage_api_key:
available_embedder.append("voyage")
# Check Ollama
if config.ollama_llm_model:
available_llm.append("ollama")
if config.ollama_embedding_model and config.ollama_embedding_dim:
available_embedder.append("ollama")
return {
"llm_providers": available_llm,
"embedder_providers": available_embedder,
}
"""Backward compatibility shim - import from integrations.graphiti.config instead."""
from integrations.graphiti.config import *
+2 -70
View File
@@ -1,70 +1,2 @@
"""
Graphiti Multi-Provider Entry Point
====================================
Main entry point for Graphiti provider functionality.
This module re-exports all functionality from the graphiti_providers package.
The actual implementation has been refactored into a package structure:
- graphiti_providers/exceptions.py - Provider exceptions
- graphiti_providers/models.py - Embedding dimensions and constants
- graphiti_providers/llm_providers/ - LLM provider implementations
- graphiti_providers/embedder_providers/ - Embedder provider implementations
- graphiti_providers/cross_encoder.py - Cross-encoder/reranker
- graphiti_providers/validators.py - Validation and health checks
- graphiti_providers/utils.py - Utility functions
- graphiti_providers/factory.py - Factory functions
For backward compatibility, this module re-exports all public APIs.
Usage:
from graphiti_providers import create_llm_client, create_embedder
from graphiti_config import GraphitiConfig
config = GraphitiConfig.from_env()
llm_client = create_llm_client(config)
embedder = create_embedder(config)
"""
# Re-export all public APIs from the package
from graphiti_providers import (
# Exceptions
ProviderError,
ProviderNotInstalled,
# Factory functions
create_llm_client,
create_embedder,
create_cross_encoder,
# Models
EMBEDDING_DIMENSIONS,
get_expected_embedding_dim,
# Validators
validate_embedding_config,
test_llm_connection,
test_embedder_connection,
test_ollama_connection,
# Utilities
is_graphiti_enabled,
get_graph_hints,
)
__all__ = [
# Exceptions
"ProviderError",
"ProviderNotInstalled",
# Factory functions
"create_llm_client",
"create_embedder",
"create_cross_encoder",
# Models
"EMBEDDING_DIMENSIONS",
"get_expected_embedding_dim",
# Validators
"validate_embedding_config",
"test_llm_connection",
"test_embedder_connection",
"test_ollama_connection",
# Utilities
"is_graphiti_enabled",
"get_graph_hints",
]
"""Backward compatibility shim - import from integrations.graphiti.providers_pkg instead."""
from integrations.graphiti.providers_pkg import *
-705
View File
@@ -1,705 +0,0 @@
"""
Graphiti Multi-Provider Factory
================================
Factory functions for creating LLM clients and embedders for Graphiti.
Supports multiple providers: OpenAI, Anthropic, Azure OpenAI, and Ollama.
This module provides:
- Lazy imports to avoid ImportError when provider packages not installed
- Factory functions that create the correct client based on provider selection
- Provider-specific configuration validation
- Graceful error handling with helpful messages
Usage:
from graphiti_providers import create_llm_client, create_embedder
from graphiti_config import GraphitiConfig
config = GraphitiConfig.from_env()
llm_client = create_llm_client(config)
embedder = create_embedder(config)
"""
import logging
from typing import TYPE_CHECKING, Any, Optional
if TYPE_CHECKING:
from pathlib import Path
from graphiti_config import GraphitiConfig
logger = logging.getLogger(__name__)
class ProviderError(Exception):
"""Raised when a provider cannot be initialized."""
pass
class ProviderNotInstalled(ProviderError):
"""Raised when required packages for a provider are not installed."""
pass
# ============================================================================
# LLM Client Factory
# ============================================================================
def create_llm_client(config: "GraphitiConfig") -> Any:
"""
Create an LLM client based on the configured provider.
Args:
config: GraphitiConfig with provider settings
Returns:
LLM client instance for Graphiti
Raises:
ProviderNotInstalled: If required packages are missing
ProviderError: If client creation fails
"""
provider = config.llm_provider
logger.info(f"Creating LLM client for provider: {provider}")
if provider == "openai":
return _create_openai_llm_client(config)
elif provider == "anthropic":
return _create_anthropic_llm_client(config)
elif provider == "azure_openai":
return _create_azure_openai_llm_client(config)
elif provider == "ollama":
return _create_ollama_llm_client(config)
else:
raise ProviderError(f"Unknown LLM provider: {provider}")
def _create_openai_llm_client(config: "GraphitiConfig") -> Any:
"""Create OpenAI LLM client."""
try:
from graphiti_core.llm_client.config import LLMConfig
from graphiti_core.llm_client.openai_client import OpenAIClient
except ImportError as e:
raise ProviderNotInstalled(
f"OpenAI provider requires graphiti-core. "
f"Install with: pip install graphiti-core\n"
f"Error: {e}"
)
if not config.openai_api_key:
raise ProviderError("OpenAI provider requires OPENAI_API_KEY")
llm_config = LLMConfig(
api_key=config.openai_api_key,
model=config.openai_model,
)
# GPT-5 family and o1/o3 models support reasoning/verbosity params
model_lower = config.openai_model.lower()
supports_reasoning = (
model_lower.startswith("gpt-5")
or model_lower.startswith("o1")
or model_lower.startswith("o3")
)
if supports_reasoning:
# Use defaults for models that support reasoning params
return OpenAIClient(config=llm_config)
else:
# Disable reasoning/verbosity for older models that don't support them
return OpenAIClient(config=llm_config, reasoning=None, verbosity=None)
def _create_anthropic_llm_client(config: "GraphitiConfig") -> Any:
"""Create Anthropic LLM client."""
try:
from graphiti_core.llm_client.anthropic_client import AnthropicClient
from graphiti_core.llm_client.config import LLMConfig
except ImportError as e:
raise ProviderNotInstalled(
f"Anthropic provider requires graphiti-core[anthropic]. "
f"Install with: pip install graphiti-core[anthropic]\n"
f"Error: {e}"
)
if not config.anthropic_api_key:
raise ProviderError("Anthropic provider requires ANTHROPIC_API_KEY")
llm_config = LLMConfig(
api_key=config.anthropic_api_key,
model=config.anthropic_model,
)
return AnthropicClient(config=llm_config)
def _create_azure_openai_llm_client(config: "GraphitiConfig") -> Any:
"""Create Azure OpenAI LLM client."""
try:
from graphiti_core.llm_client.azure_openai_client import AzureOpenAILLMClient
from graphiti_core.llm_client.config import LLMConfig
from openai import AsyncOpenAI
except ImportError as e:
raise ProviderNotInstalled(
f"Azure OpenAI provider requires graphiti-core and openai. "
f"Install with: pip install graphiti-core openai\n"
f"Error: {e}"
)
if not config.azure_openai_api_key:
raise ProviderError("Azure OpenAI provider requires AZURE_OPENAI_API_KEY")
if not config.azure_openai_base_url:
raise ProviderError("Azure OpenAI provider requires AZURE_OPENAI_BASE_URL")
if not config.azure_openai_llm_deployment:
raise ProviderError(
"Azure OpenAI provider requires AZURE_OPENAI_LLM_DEPLOYMENT"
)
azure_client = AsyncOpenAI(
base_url=config.azure_openai_base_url,
api_key=config.azure_openai_api_key,
)
llm_config = LLMConfig(
model=config.azure_openai_llm_deployment,
small_model=config.azure_openai_llm_deployment,
)
return AzureOpenAILLMClient(azure_client=azure_client, config=llm_config)
def _create_ollama_llm_client(config: "GraphitiConfig") -> Any:
"""Create Ollama LLM client (using OpenAI-compatible interface)."""
try:
from graphiti_core.llm_client.config import LLMConfig
from graphiti_core.llm_client.openai_generic_client import OpenAIGenericClient
except ImportError as e:
raise ProviderNotInstalled(
f"Ollama provider requires graphiti-core. "
f"Install with: pip install graphiti-core\n"
f"Error: {e}"
)
if not config.ollama_llm_model:
raise ProviderError("Ollama provider requires OLLAMA_LLM_MODEL")
# Ensure Ollama base URL ends with /v1 for OpenAI compatibility
base_url = config.ollama_base_url
if not base_url.endswith("/v1"):
base_url = base_url.rstrip("/") + "/v1"
llm_config = LLMConfig(
api_key="ollama", # Ollama requires a dummy API key
model=config.ollama_llm_model,
small_model=config.ollama_llm_model,
base_url=base_url,
)
return OpenAIGenericClient(config=llm_config)
# ============================================================================
# Embedder Factory
# ============================================================================
def create_embedder(config: "GraphitiConfig") -> Any:
"""
Create an embedder based on the configured provider.
Args:
config: GraphitiConfig with provider settings
Returns:
Embedder instance for Graphiti
Raises:
ProviderNotInstalled: If required packages are missing
ProviderError: If embedder creation fails
"""
provider = config.embedder_provider
logger.info(f"Creating embedder for provider: {provider}")
if provider == "openai":
return _create_openai_embedder(config)
elif provider == "voyage":
return _create_voyage_embedder(config)
elif provider == "azure_openai":
return _create_azure_openai_embedder(config)
elif provider == "ollama":
return _create_ollama_embedder(config)
else:
raise ProviderError(f"Unknown embedder provider: {provider}")
def _create_openai_embedder(config: "GraphitiConfig") -> Any:
"""Create OpenAI embedder."""
try:
from graphiti_core.embedder.openai import OpenAIEmbedder, OpenAIEmbedderConfig
except ImportError as e:
raise ProviderNotInstalled(
f"OpenAI embedder requires graphiti-core. "
f"Install with: pip install graphiti-core\n"
f"Error: {e}"
)
if not config.openai_api_key:
raise ProviderError("OpenAI embedder requires OPENAI_API_KEY")
embedder_config = OpenAIEmbedderConfig(
api_key=config.openai_api_key,
embedding_model=config.openai_embedding_model,
)
return OpenAIEmbedder(config=embedder_config)
def _create_voyage_embedder(config: "GraphitiConfig") -> Any:
"""Create Voyage AI embedder (commonly used with Anthropic LLM)."""
try:
from graphiti_core.embedder.voyage import VoyageAIConfig, VoyageEmbedder
except ImportError as e:
raise ProviderNotInstalled(
f"Voyage embedder requires graphiti-core[voyage]. "
f"Install with: pip install graphiti-core[voyage]\n"
f"Error: {e}"
)
if not config.voyage_api_key:
raise ProviderError("Voyage embedder requires VOYAGE_API_KEY")
voyage_config = VoyageAIConfig(
api_key=config.voyage_api_key,
embedding_model=config.voyage_embedding_model,
)
return VoyageEmbedder(config=voyage_config)
def _create_azure_openai_embedder(config: "GraphitiConfig") -> Any:
"""Create Azure OpenAI embedder."""
try:
from graphiti_core.embedder.azure_openai import AzureOpenAIEmbedderClient
from openai import AsyncOpenAI
except ImportError as e:
raise ProviderNotInstalled(
f"Azure OpenAI embedder requires graphiti-core and openai. "
f"Install with: pip install graphiti-core openai\n"
f"Error: {e}"
)
if not config.azure_openai_api_key:
raise ProviderError("Azure OpenAI embedder requires AZURE_OPENAI_API_KEY")
if not config.azure_openai_base_url:
raise ProviderError("Azure OpenAI embedder requires AZURE_OPENAI_BASE_URL")
if not config.azure_openai_embedding_deployment:
raise ProviderError(
"Azure OpenAI embedder requires AZURE_OPENAI_EMBEDDING_DEPLOYMENT"
)
azure_client = AsyncOpenAI(
base_url=config.azure_openai_base_url,
api_key=config.azure_openai_api_key,
)
return AzureOpenAIEmbedderClient(
azure_client=azure_client,
model=config.azure_openai_embedding_deployment,
)
def _create_ollama_embedder(config: "GraphitiConfig") -> Any:
"""Create Ollama embedder (using OpenAI-compatible interface)."""
try:
from graphiti_core.embedder.openai import OpenAIEmbedder, OpenAIEmbedderConfig
except ImportError as e:
raise ProviderNotInstalled(
f"Ollama embedder requires graphiti-core. "
f"Install with: pip install graphiti-core\n"
f"Error: {e}"
)
if not config.ollama_embedding_model:
raise ProviderError("Ollama embedder requires OLLAMA_EMBEDDING_MODEL")
# Ensure Ollama base URL ends with /v1 for OpenAI compatibility
base_url = config.ollama_base_url
if not base_url.endswith("/v1"):
base_url = base_url.rstrip("/") + "/v1"
embedder_config = OpenAIEmbedderConfig(
api_key="ollama", # Ollama requires a dummy API key
embedding_model=config.ollama_embedding_model,
embedding_dim=config.ollama_embedding_dim,
base_url=base_url,
)
return OpenAIEmbedder(config=embedder_config)
# ============================================================================
# Cross-Encoder / Reranker Factory (Optional)
# ============================================================================
def create_cross_encoder(
config: "GraphitiConfig", llm_client: Any = None
) -> Any | None:
"""
Create a cross-encoder/reranker for improved search quality.
This is optional and primarily useful for Ollama setups.
Other providers typically have built-in reranking.
Args:
config: GraphitiConfig with provider settings
llm_client: Optional LLM client for reranking
Returns:
Cross-encoder instance, or None if not applicable
"""
# Only create for Ollama provider currently
if config.llm_provider != "ollama":
return None
if llm_client is None:
return None
try:
from graphiti_core.cross_encoder.openai_reranker_client import (
OpenAIRerankerClient,
)
from graphiti_core.llm_client.config import LLMConfig
except ImportError:
logger.debug("Cross-encoder not available (optional)")
return None
try:
# Create LLM config for reranker
base_url = config.ollama_base_url
if not base_url.endswith("/v1"):
base_url = base_url.rstrip("/") + "/v1"
llm_config = LLMConfig(
api_key="ollama",
model=config.ollama_llm_model,
base_url=base_url,
)
return OpenAIRerankerClient(client=llm_client, config=llm_config)
except Exception as e:
logger.warning(f"Could not create cross-encoder: {e}")
return None
# ============================================================================
# Embedding Dimension Validation
# ============================================================================
# Known embedding dimensions by provider and model
EMBEDDING_DIMENSIONS = {
# OpenAI
"text-embedding-3-small": 1536,
"text-embedding-3-large": 3072,
"text-embedding-ada-002": 1536,
# Voyage AI
"voyage-3": 1024,
"voyage-3.5": 1024,
"voyage-3-lite": 512,
"voyage-3.5-lite": 512,
"voyage-2": 1024,
"voyage-large-2": 1536,
# Ollama (common models)
"nomic-embed-text": 768,
"mxbai-embed-large": 1024,
"all-minilm": 384,
"snowflake-arctic-embed": 1024,
}
def get_expected_embedding_dim(model: str) -> int | None:
"""
Get the expected embedding dimension for a known model.
Args:
model: Embedding model name
Returns:
Expected dimension, or None if unknown
"""
# Try exact match first
if model in EMBEDDING_DIMENSIONS:
return EMBEDDING_DIMENSIONS[model]
# Try partial match (model name might have version suffix)
model_lower = model.lower()
for known_model, dim in EMBEDDING_DIMENSIONS.items():
if known_model.lower() in model_lower or model_lower in known_model.lower():
return dim
return None
def validate_embedding_config(config: "GraphitiConfig") -> tuple[bool, str]:
"""
Validate embedding configuration for consistency.
Checks that embedding dimensions are correctly configured,
especially important for Ollama where explicit dimension is required.
Args:
config: GraphitiConfig to validate
Returns:
Tuple of (is_valid, message)
"""
provider = config.embedder_provider
if provider == "ollama":
# Ollama requires explicit embedding dimension
if not config.ollama_embedding_dim:
expected = get_expected_embedding_dim(config.ollama_embedding_model)
if expected:
return False, (
f"Ollama embedder requires OLLAMA_EMBEDDING_DIM. "
f"For model '{config.ollama_embedding_model}', "
f"expected dimension is {expected}."
)
else:
return False, (
"Ollama embedder requires OLLAMA_EMBEDDING_DIM. "
"Check your model's documentation for the correct dimension."
)
# Check for known dimension mismatches
if provider == "openai":
expected = get_expected_embedding_dim(config.openai_embedding_model)
# OpenAI handles this automatically, just log info
if expected:
logger.debug(
f"OpenAI embedding model '{config.openai_embedding_model}' has dimension {expected}"
)
elif provider == "voyage":
expected = get_expected_embedding_dim(config.voyage_embedding_model)
if expected:
logger.debug(
f"Voyage embedding model '{config.voyage_embedding_model}' has dimension {expected}"
)
return True, "Embedding configuration valid"
# ============================================================================
# Provider Health Checks
# ============================================================================
async def test_llm_connection(config: "GraphitiConfig") -> tuple[bool, str]:
"""
Test if LLM provider is reachable.
Args:
config: GraphitiConfig with provider settings
Returns:
Tuple of (success, message)
"""
try:
llm_client = create_llm_client(config)
# Most clients don't have a ping method, so just verify creation succeeded
return (
True,
f"LLM client created successfully for provider: {config.llm_provider}",
)
except ProviderNotInstalled as e:
return False, str(e)
except ProviderError as e:
return False, str(e)
except Exception as e:
return False, f"Failed to create LLM client: {e}"
async def test_embedder_connection(config: "GraphitiConfig") -> tuple[bool, str]:
"""
Test if embedder provider is reachable.
Args:
config: GraphitiConfig with provider settings
Returns:
Tuple of (success, message)
"""
# First validate config
valid, msg = validate_embedding_config(config)
if not valid:
return False, msg
try:
embedder = create_embedder(config)
return (
True,
f"Embedder created successfully for provider: {config.embedder_provider}",
)
except ProviderNotInstalled as e:
return False, str(e)
except ProviderError as e:
return False, str(e)
except Exception as e:
return False, f"Failed to create embedder: {e}"
async def test_ollama_connection(
base_url: str = "http://localhost:11434",
) -> tuple[bool, str]:
"""
Test if Ollama server is running and reachable.
Args:
base_url: Ollama server URL
Returns:
Tuple of (success, message)
"""
import asyncio
try:
import aiohttp
except ImportError:
# Fall back to sync request
import urllib.error
import urllib.request
try:
# Normalize URL (remove /v1 suffix if present)
url = base_url.rstrip("/")
if url.endswith("/v1"):
url = url[:-3]
req = urllib.request.Request(f"{url}/api/tags", method="GET")
with urllib.request.urlopen(req, timeout=5) as response:
if response.status == 200:
return True, f"Ollama is running at {url}"
return False, f"Ollama returned status {response.status}"
except urllib.error.URLError as e:
return False, f"Cannot connect to Ollama at {url}: {e.reason}"
except Exception as e:
return False, f"Ollama connection error: {e}"
# Use aiohttp if available
try:
# Normalize URL
url = base_url.rstrip("/")
if url.endswith("/v1"):
url = url[:-3]
async with aiohttp.ClientSession() as session:
async with session.get(
f"{url}/api/tags", timeout=aiohttp.ClientTimeout(total=5)
) as response:
if response.status == 200:
return True, f"Ollama is running at {url}"
return False, f"Ollama returned status {response.status}"
except asyncio.TimeoutError:
return False, f"Ollama connection timed out at {url}"
except aiohttp.ClientError as e:
return False, f"Cannot connect to Ollama at {url}: {e}"
except Exception as e:
return False, f"Ollama connection error: {e}"
# ============================================================================
# Re-exports and Convenience Functions
# ============================================================================
def is_graphiti_enabled() -> bool:
"""
Check if Graphiti memory integration is available and configured.
This is a convenience re-export from graphiti_config.
Returns True if GRAPHITI_ENABLED=true and provider credentials are valid.
"""
from graphiti_config import is_graphiti_enabled as _is_graphiti_enabled
return _is_graphiti_enabled()
async def get_graph_hints(
query: str,
project_id: str,
max_results: int = 10,
spec_dir: Optional["Path"] = None,
) -> list[dict]:
"""
Get relevant hints from the Graphiti knowledge graph.
This is a convenience function for querying historical context
from the memory system. Used by spec_runner, ideation_runner,
and roadmap_runner to inject historical insights.
Args:
query: Search query (e.g., "authentication patterns", "API design")
project_id: Project identifier for scoping results
max_results: Maximum number of hints to return
spec_dir: Optional spec directory for loading memory instance
Returns:
List of hint dictionaries with keys:
- content: str - The hint content
- score: float - Relevance score
- type: str - Type of hint (pattern, gotcha, outcome, etc.)
Note:
Returns empty list if Graphiti is not enabled or unavailable.
This function never raises - it always fails gracefully.
"""
if not is_graphiti_enabled():
logger.debug("Graphiti not enabled, returning empty hints")
return []
try:
from pathlib import Path
from graphiti_memory import GraphitiMemory, GroupIdMode
# Determine project directory from project_id or use current dir
project_dir = Path.cwd()
# Use spec_dir if provided, otherwise create a temp context
if spec_dir is None:
# Create a temporary spec dir for the query
import tempfile
spec_dir = Path(tempfile.mkdtemp(prefix="graphiti_query_"))
# Create memory instance with project-level scope for cross-spec hints
memory = GraphitiMemory(
spec_dir=spec_dir,
project_dir=project_dir,
group_id_mode=GroupIdMode.PROJECT,
)
# Query for relevant context
hints = await memory.get_relevant_context(
query=query,
num_results=max_results,
include_project_context=True,
)
await memory.close()
logger.info(f"Retrieved {len(hints)} graph hints for query: {query[:50]}...")
return hints
except ImportError as e:
logger.debug(f"Graphiti packages not available: {e}")
return []
except Exception as e:
logger.warning(f"Failed to get graph hints: {e}")
return []
File diff suppressed because it is too large Load Diff
+3 -168
View File
@@ -1,168 +1,3 @@
#!/usr/bin/env python3
"""
Implementation Plan Manager
============================
DEPRECATED: This module is now a compatibility shim. The implementation has been
refactored into the implementation_plan/ package for better modularity.
Please import from the package directly:
from implementation_plan import ImplementationPlan, Subtask, Phase, etc.
This file re-exports all public APIs for backwards compatibility.
Core data structures and utilities for subtask-based implementation plans.
Replaces the test-centric feature_list.json with implementation_plan.json.
The key insight: Tests verify outcomes, but SUBTASKS define implementation steps.
For complex multi-service features, implementation order matters.
Workflow Types:
- feature: Standard multi-service feature (phases = services)
- refactor: Migration/refactor work (phases = stages: add, migrate, remove)
- investigation: Bug hunting (phases = investigate, hypothesize, fix)
- migration: Data migration (phases = prepare, test, execute, cleanup)
- simple: Single-service enhancement (minimal overhead)
"""
# Re-export everything from the implementation_plan package
from implementation_plan import (
Chunk,
ChunkStatus,
ImplementationPlan,
Phase,
PhaseType,
Subtask,
SubtaskStatus,
Verification,
VerificationType,
WorkflowType,
create_feature_plan,
create_investigation_plan,
create_refactor_plan,
)
__all__ = [
# Enums
"WorkflowType",
"PhaseType",
"SubtaskStatus",
"VerificationType",
# Models
"Verification",
"Subtask",
"Phase",
"ImplementationPlan",
# Factories
"create_feature_plan",
"create_investigation_plan",
"create_refactor_plan",
# Backwards compatibility
"Chunk",
"ChunkStatus",
]
# CLI for testing
if __name__ == "__main__":
import json
import sys
from pathlib import Path
if len(sys.argv) < 2:
print("Usage: python implementation_plan.py <plan.json>")
print(" python implementation_plan.py --demo")
sys.exit(1)
if sys.argv[1] == "--demo":
# Create a demo plan
plan = create_feature_plan(
feature="Avatar Upload with Processing",
services=["backend", "worker", "frontend"],
phases_config=[
{
"name": "Backend Foundation",
"parallel_safe": True,
"subtasks": [
{
"id": "avatar-model",
"service": "backend",
"description": "Add avatar fields to User model",
"files_to_modify": ["app/models/user.py"],
"files_to_create": ["migrations/add_avatar.py"],
"verification": {
"type": "command",
"run": "flask db upgrade",
},
},
{
"id": "avatar-endpoint",
"service": "backend",
"description": "POST /api/users/avatar endpoint",
"files_to_modify": ["app/routes/users.py"],
"patterns_from": ["app/routes/profile.py"],
"verification": {
"type": "api",
"method": "POST",
"url": "/api/users/avatar",
},
},
],
},
{
"name": "Worker Pipeline",
"depends_on": [1],
"subtasks": [
{
"id": "image-task",
"service": "worker",
"description": "Celery task for image processing",
"files_to_create": ["app/tasks/images.py"],
"patterns_from": ["app/tasks/reports.py"],
},
],
},
{
"name": "Frontend",
"depends_on": [1],
"subtasks": [
{
"id": "avatar-component",
"service": "frontend",
"description": "AvatarUpload React component",
"files_to_create": ["src/components/AvatarUpload.tsx"],
"patterns_from": ["src/components/FileUpload.tsx"],
},
],
},
{
"name": "Integration",
"depends_on": [2, 3],
"type": "integration",
"subtasks": [
{
"id": "e2e-wiring",
"all_services": True,
"description": "Connect frontend → backend → worker",
"verification": {
"type": "browser",
"scenario": "Upload → Process → Display",
},
},
],
},
],
)
plan.final_acceptance = [
"User can upload avatar from profile page",
"Avatar is automatically resized",
"Large/invalid files show error",
]
print(json.dumps(plan.to_dict(), indent=2))
print("\n---\n")
print(plan.get_status_summary())
else:
# Load and display existing plan
plan = ImplementationPlan.load(Path(sys.argv[1]))
print(plan.get_status_summary())
"""Backward compatibility shim - import from implementation_plan package instead."""
from implementation_plan.main import *
from implementation_plan import *
+168
View File
@@ -0,0 +1,168 @@
#!/usr/bin/env python3
"""
Implementation Plan Manager
============================
DEPRECATED: This module is now a compatibility shim. The implementation has been
refactored into the implementation_plan/ package for better modularity.
Please import from the package directly:
from implementation_plan import ImplementationPlan, Subtask, Phase, etc.
This file re-exports all public APIs for backwards compatibility.
Core data structures and utilities for subtask-based implementation plans.
Replaces the test-centric feature_list.json with implementation_plan.json.
The key insight: Tests verify outcomes, but SUBTASKS define implementation steps.
For complex multi-service features, implementation order matters.
Workflow Types:
- feature: Standard multi-service feature (phases = services)
- refactor: Migration/refactor work (phases = stages: add, migrate, remove)
- investigation: Bug hunting (phases = investigate, hypothesize, fix)
- migration: Data migration (phases = prepare, test, execute, cleanup)
- simple: Single-service enhancement (minimal overhead)
"""
# Re-export everything from the implementation_plan package
from implementation_plan import (
Chunk,
ChunkStatus,
ImplementationPlan,
Phase,
PhaseType,
Subtask,
SubtaskStatus,
Verification,
VerificationType,
WorkflowType,
create_feature_plan,
create_investigation_plan,
create_refactor_plan,
)
__all__ = [
# Enums
"WorkflowType",
"PhaseType",
"SubtaskStatus",
"VerificationType",
# Models
"Verification",
"Subtask",
"Phase",
"ImplementationPlan",
# Factories
"create_feature_plan",
"create_investigation_plan",
"create_refactor_plan",
# Backwards compatibility
"Chunk",
"ChunkStatus",
]
# CLI for testing
if __name__ == "__main__":
import json
import sys
from pathlib import Path
if len(sys.argv) < 2:
print("Usage: python implementation_plan.py <plan.json>")
print(" python implementation_plan.py --demo")
sys.exit(1)
if sys.argv[1] == "--demo":
# Create a demo plan
plan = create_feature_plan(
feature="Avatar Upload with Processing",
services=["backend", "worker", "frontend"],
phases_config=[
{
"name": "Backend Foundation",
"parallel_safe": True,
"subtasks": [
{
"id": "avatar-model",
"service": "backend",
"description": "Add avatar fields to User model",
"files_to_modify": ["app/models/user.py"],
"files_to_create": ["migrations/add_avatar.py"],
"verification": {
"type": "command",
"run": "flask db upgrade",
},
},
{
"id": "avatar-endpoint",
"service": "backend",
"description": "POST /api/users/avatar endpoint",
"files_to_modify": ["app/routes/users.py"],
"patterns_from": ["app/routes/profile.py"],
"verification": {
"type": "api",
"method": "POST",
"url": "/api/users/avatar",
},
},
],
},
{
"name": "Worker Pipeline",
"depends_on": [1],
"subtasks": [
{
"id": "image-task",
"service": "worker",
"description": "Celery task for image processing",
"files_to_create": ["app/tasks/images.py"],
"patterns_from": ["app/tasks/reports.py"],
},
],
},
{
"name": "Frontend",
"depends_on": [1],
"subtasks": [
{
"id": "avatar-component",
"service": "frontend",
"description": "AvatarUpload React component",
"files_to_create": ["src/components/AvatarUpload.tsx"],
"patterns_from": ["src/components/FileUpload.tsx"],
},
],
},
{
"name": "Integration",
"depends_on": [2, 3],
"type": "integration",
"subtasks": [
{
"id": "e2e-wiring",
"all_services": True,
"description": "Connect frontend → backend → worker",
"verification": {
"type": "browser",
"scenario": "Upload → Process → Display",
},
},
],
},
],
)
plan.final_acceptance = [
"User can upload avatar from profile page",
"Avatar is automatically resized",
"Large/invalid files show error",
]
print(json.dumps(plan.to_dict(), indent=2))
print("\n---\n")
print(plan.get_status_summary())
else:
# Load and display existing plan
plan = ImplementationPlan.load(Path(sys.argv[1]))
print(plan.get_status_summary())
+35 -578
View File
@@ -1,584 +1,41 @@
"""
Insight Extractor
=================
Insight Extractor Re-export
===========================
Automatically extracts structured insights from completed coding sessions.
Runs after each session to capture rich, actionable knowledge for Graphiti memory.
Uses the Claude Agent SDK (same as the rest of the system) for extraction.
Falls back to generic insights if extraction fails (never blocks the build).
Re-exports the insight_extractor module from analysis/ for backwards compatibility.
Uses importlib to avoid triggering analysis/__init__.py imports.
"""
import json
import logging
import os
import subprocess
import importlib.util
import sys
from pathlib import Path
from typing import Any
logger = logging.getLogger(__name__)
# Check for Claude SDK availability
try:
from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient
SDK_AVAILABLE = True
except ImportError:
SDK_AVAILABLE = False
ClaudeAgentOptions = None
ClaudeSDKClient = None
# Default model for insight extraction (fast and cheap)
DEFAULT_EXTRACTION_MODEL = "claude-3-5-haiku-latest"
# Maximum diff size to send to the LLM (avoid context limits)
MAX_DIFF_CHARS = 15000
# Maximum attempt history entries to include
MAX_ATTEMPTS_TO_INCLUDE = 3
def is_extraction_enabled() -> bool:
"""Check if insight extraction is enabled."""
# Extraction requires Claude SDK and OAuth token
if not SDK_AVAILABLE:
return False
if not os.environ.get("CLAUDE_CODE_OAUTH_TOKEN"):
return False
enabled_str = os.environ.get("INSIGHT_EXTRACTION_ENABLED", "true").lower()
return enabled_str in ("true", "1", "yes")
def get_extraction_model() -> str:
"""Get the model to use for insight extraction."""
return os.environ.get("INSIGHT_EXTRACTOR_MODEL", DEFAULT_EXTRACTION_MODEL)
# =============================================================================
# Git Helpers
# =============================================================================
def get_session_diff(
project_dir: Path,
commit_before: str | None,
commit_after: str | None,
) -> str:
"""
Get the git diff between two commits.
Args:
project_dir: Project root directory
commit_before: Commit hash before session (or None)
commit_after: Commit hash after session (or None)
Returns:
Diff text (truncated if too large)
"""
if not commit_before or not commit_after:
return "(No commits to diff)"
if commit_before == commit_after:
return "(No changes - same commit)"
try:
result = subprocess.run(
["git", "diff", commit_before, commit_after],
cwd=project_dir,
capture_output=True,
text=True,
timeout=30,
)
diff = result.stdout
if len(diff) > MAX_DIFF_CHARS:
# Truncate and add note
diff = (
diff[:MAX_DIFF_CHARS] + f"\n\n... (truncated, {len(diff)} chars total)"
)
return diff if diff else "(Empty diff)"
except subprocess.TimeoutExpired:
logger.warning("Git diff timed out")
return "(Git diff timed out)"
except Exception as e:
logger.warning(f"Failed to get git diff: {e}")
return f"(Failed to get diff: {e})"
def get_changed_files(
project_dir: Path,
commit_before: str | None,
commit_after: str | None,
) -> list[str]:
"""
Get list of files changed between two commits.
Args:
project_dir: Project root directory
commit_before: Commit hash before session
commit_after: Commit hash after session
Returns:
List of changed file paths
"""
if not commit_before or not commit_after or commit_before == commit_after:
return []
try:
result = subprocess.run(
["git", "diff", "--name-only", commit_before, commit_after],
cwd=project_dir,
capture_output=True,
text=True,
timeout=10,
)
files = [f.strip() for f in result.stdout.strip().split("\n") if f.strip()]
return files
except Exception as e:
logger.warning(f"Failed to get changed files: {e}")
return []
def get_commit_messages(
project_dir: Path,
commit_before: str | None,
commit_after: str | None,
) -> str:
"""Get commit messages between two commits."""
if not commit_before or not commit_after or commit_before == commit_after:
return "(No commits)"
try:
result = subprocess.run(
["git", "log", "--oneline", f"{commit_before}..{commit_after}"],
cwd=project_dir,
capture_output=True,
text=True,
timeout=10,
)
return result.stdout.strip() if result.stdout.strip() else "(No commits)"
except Exception as e:
logger.warning(f"Failed to get commit messages: {e}")
return f"(Failed: {e})"
# =============================================================================
# Input Gathering
# =============================================================================
def gather_extraction_inputs(
spec_dir: Path,
project_dir: Path,
subtask_id: str,
session_num: int,
commit_before: str | None,
commit_after: str | None,
success: bool,
recovery_manager: Any,
) -> dict:
"""
Gather all inputs needed for insight extraction.
Args:
spec_dir: Spec directory
project_dir: Project root
subtask_id: The subtask that was worked on
session_num: Session number
commit_before: Commit before session
commit_after: Commit after session
success: Whether session succeeded
recovery_manager: Recovery manager with attempt history
Returns:
Dict with all inputs for the extractor
"""
# Get subtask description from implementation plan
subtask_description = _get_subtask_description(spec_dir, subtask_id)
# Get git diff
diff = get_session_diff(project_dir, commit_before, commit_after)
# Get changed files
changed_files = get_changed_files(project_dir, commit_before, commit_after)
# Get commit messages
commit_messages = get_commit_messages(project_dir, commit_before, commit_after)
# Get attempt history
attempt_history = _get_attempt_history(recovery_manager, subtask_id)
return {
"subtask_id": subtask_id,
"subtask_description": subtask_description,
"session_num": session_num,
"success": success,
"diff": diff,
"changed_files": changed_files,
"commit_messages": commit_messages,
"attempt_history": attempt_history,
}
def _get_subtask_description(spec_dir: Path, subtask_id: str) -> str:
"""Get subtask description from implementation plan."""
plan_file = spec_dir / "implementation_plan.json"
if not plan_file.exists():
return f"Subtask: {subtask_id}"
try:
with open(plan_file) as f:
plan = json.load(f)
# Search through phases for the subtask
for phase in plan.get("phases", []):
for subtask in phase.get("subtasks", []):
if subtask.get("id") == subtask_id:
return subtask.get("description", f"Subtask: {subtask_id}")
return f"Subtask: {subtask_id}"
except Exception as e:
logger.warning(f"Failed to load subtask description: {e}")
return f"Subtask: {subtask_id}"
def _get_attempt_history(recovery_manager: Any, subtask_id: str) -> list[dict]:
"""Get previous attempt history for this subtask."""
if not recovery_manager:
return []
try:
history = recovery_manager.get_subtask_history(subtask_id)
attempts = history.get("attempts", [])
# Limit to recent attempts
return attempts[-MAX_ATTEMPTS_TO_INCLUDE:]
except Exception as e:
logger.warning(f"Failed to get attempt history: {e}")
return []
# =============================================================================
# LLM Extraction
# =============================================================================
def _build_extraction_prompt(inputs: dict) -> str:
"""Build the prompt for insight extraction."""
prompt_file = Path(__file__).parent / "prompts" / "insight_extractor.md"
if prompt_file.exists():
base_prompt = prompt_file.read_text()
else:
# Fallback if prompt file missing
base_prompt = """Extract structured insights from this coding session.
Output ONLY valid JSON with: file_insights, patterns_discovered, gotchas_discovered, approach_outcome, recommendations"""
# Build session context
session_context = f"""
---
## SESSION DATA
### Subtask
- **ID**: {inputs["subtask_id"]}
- **Description**: {inputs["subtask_description"]}
- **Session Number**: {inputs["session_num"]}
- **Outcome**: {"SUCCESS" if inputs["success"] else "FAILED"}
### Files Changed
{chr(10).join(f"- {f}" for f in inputs["changed_files"]) if inputs["changed_files"] else "(No files changed)"}
### Commit Messages
{inputs["commit_messages"]}
### Git Diff
```diff
{inputs["diff"]}
```
### Previous Attempts
{_format_attempt_history(inputs["attempt_history"])}
---
Now analyze this session and output ONLY the JSON object.
"""
return base_prompt + session_context
def _format_attempt_history(attempts: list[dict]) -> str:
"""Format attempt history for the prompt."""
if not attempts:
return "(First attempt - no previous history)"
lines = []
for i, attempt in enumerate(attempts, 1):
success = "SUCCESS" if attempt.get("success") else "FAILED"
approach = attempt.get("approach", "Unknown approach")
error = attempt.get("error", "")
lines.append(f"**Attempt {i}** ({success}): {approach}")
if error:
lines.append(f" Error: {error}")
return "\n".join(lines)
async def run_insight_extraction(inputs: dict, project_dir: Path | None = None) -> dict | None:
"""
Run the insight extraction using Claude Agent SDK.
Args:
inputs: Gathered session inputs
project_dir: Project directory for SDK context (optional)
Returns:
Extracted insights dict or None if failed
"""
if not SDK_AVAILABLE:
logger.warning("Claude SDK not available, skipping insight extraction")
return None
oauth_token = os.environ.get("CLAUDE_CODE_OAUTH_TOKEN")
if not oauth_token:
logger.warning("CLAUDE_CODE_OAUTH_TOKEN not set, skipping insight extraction")
return None
model = get_extraction_model()
prompt = _build_extraction_prompt(inputs)
# Use current directory if project_dir not specified
cwd = str(project_dir.resolve()) if project_dir else os.getcwd()
try:
# Create a minimal SDK client for insight extraction
# No tools needed - just text generation
client = ClaudeSDKClient(
options=ClaudeAgentOptions(
model=model,
system_prompt=(
"You are an expert code analyst. You extract structured insights from coding sessions. "
"Always respond with valid JSON only, no markdown formatting or explanations."
),
allowed_tools=[], # No tools needed for extraction
max_turns=1, # Single turn extraction
cwd=cwd,
)
)
# Use async context manager
async with client:
await client.query(prompt)
# Collect the response
response_text = ""
async for msg in client.receive_response():
msg_type = type(msg).__name__
if msg_type == "AssistantMessage" and hasattr(msg, "content"):
for block in msg.content:
if hasattr(block, "text"):
response_text += block.text
# Parse JSON from response
return parse_insights(response_text)
except Exception as e:
logger.warning(f"Insight extraction failed: {e}")
return None
def parse_insights(response_text: str) -> dict | None:
"""
Parse the LLM response into structured insights.
Args:
response_text: Raw LLM response
Returns:
Parsed insights dict or None if parsing failed
"""
# Try to extract JSON from the response
text = response_text.strip()
# Handle markdown code blocks
if text.startswith("```"):
# Remove code block markers
lines = text.split("\n")
# Remove first line (```json or ```)
if lines[0].startswith("```"):
lines = lines[1:]
# Remove last line if it's ``
if lines and lines[-1].strip() == "```":
lines = lines[:-1]
text = "\n".join(lines)
try:
insights = json.loads(text)
# Validate structure
if not isinstance(insights, dict):
logger.warning("Insights is not a dict")
return None
# Ensure required keys exist with defaults
insights.setdefault("file_insights", [])
insights.setdefault("patterns_discovered", [])
insights.setdefault("gotchas_discovered", [])
insights.setdefault("approach_outcome", {})
insights.setdefault("recommendations", [])
return insights
except json.JSONDecodeError as e:
logger.warning(f"Failed to parse insights JSON: {e}")
logger.debug(f"Response text was: {text[:500]}")
return None
# =============================================================================
# Main Entry Point
# =============================================================================
async def extract_session_insights(
spec_dir: Path,
project_dir: Path,
subtask_id: str,
session_num: int,
commit_before: str | None,
commit_after: str | None,
success: bool,
recovery_manager: Any,
) -> dict:
"""
Extract insights from a completed coding session.
This is the main entry point called from post_session_processing().
Falls back to generic insights if extraction fails.
Args:
spec_dir: Spec directory
project_dir: Project root
subtask_id: Subtask that was worked on
session_num: Session number
commit_before: Commit before session
commit_after: Commit after session
success: Whether session succeeded
recovery_manager: Recovery manager with attempt history
Returns:
Insights dict (rich if extraction succeeded, generic if failed)
"""
# Check if extraction is enabled
if not is_extraction_enabled():
logger.info("Insight extraction disabled")
return _get_generic_insights(subtask_id, success)
# Check for no changes
if commit_before == commit_after:
logger.info("No changes to extract insights from")
return _get_generic_insights(subtask_id, success)
try:
# Gather inputs
inputs = gather_extraction_inputs(
spec_dir=spec_dir,
project_dir=project_dir,
subtask_id=subtask_id,
session_num=session_num,
commit_before=commit_before,
commit_after=commit_after,
success=success,
recovery_manager=recovery_manager,
)
# Run extraction
extracted = await run_insight_extraction(inputs, project_dir=project_dir)
if extracted:
# Add metadata
extracted["subtask_id"] = subtask_id
extracted["session_num"] = session_num
extracted["success"] = success
extracted["changed_files"] = inputs["changed_files"]
logger.info(
f"Extracted insights: {len(extracted.get('file_insights', []))} file insights, "
f"{len(extracted.get('patterns_discovered', []))} patterns, "
f"{len(extracted.get('gotchas_discovered', []))} gotchas"
)
return extracted
else:
logger.warning("Extraction returned no results, using generic insights")
return _get_generic_insights(subtask_id, success)
except Exception as e:
logger.warning(f"Insight extraction failed: {e}, using generic insights")
return _get_generic_insights(subtask_id, success)
def _get_generic_insights(subtask_id: str, success: bool) -> dict:
"""Return generic insights when extraction fails or is disabled."""
return {
"file_insights": [],
"patterns_discovered": [],
"gotchas_discovered": [],
"approach_outcome": {
"success": success,
"approach_used": f"Implemented subtask: {subtask_id}",
"why_it_worked": None,
"why_it_failed": None,
"alternatives_tried": [],
},
"recommendations": [],
"subtask_id": subtask_id,
"success": success,
"changed_files": [],
}
# =============================================================================
# CLI for Testing
# =============================================================================
if __name__ == "__main__":
import argparse
import asyncio
parser = argparse.ArgumentParser(description="Test insight extraction")
parser.add_argument("--spec-dir", type=Path, required=True, help="Spec directory")
parser.add_argument(
"--project-dir", type=Path, required=True, help="Project directory"
)
parser.add_argument(
"--commit-before", type=str, required=True, help="Commit before session"
)
parser.add_argument(
"--commit-after", type=str, required=True, help="Commit after session"
)
parser.add_argument(
"--subtask-id", type=str, default="test-subtask", help="Subtask ID"
)
args = parser.parse_args()
async def main():
insights = await extract_session_insights(
spec_dir=args.spec_dir,
project_dir=args.project_dir,
subtask_id=args.subtask_id,
session_num=1,
commit_before=args.commit_before,
commit_after=args.commit_after,
success=True,
recovery_manager=None,
)
print(json.dumps(insights, indent=2))
asyncio.run(main())
# Load the module directly without going through the package
_module_path = Path(__file__).parent / "analysis" / "insight_extractor.py"
_spec = importlib.util.spec_from_file_location("_insight_extractor_impl", _module_path)
_module = importlib.util.module_from_spec(_spec)
sys.modules["_insight_extractor_impl"] = _module
_spec.loader.exec_module(_module)
# Re-export all public functions
extract_session_insights = _module.extract_session_insights
gather_extraction_inputs = _module.gather_extraction_inputs
get_changed_files = _module.get_changed_files
get_commit_messages = _module.get_commit_messages
get_extraction_model = _module.get_extraction_model
get_session_diff = _module.get_session_diff
is_extraction_enabled = _module.is_extraction_enabled
parse_insights = _module.parse_insights
run_insight_extraction = _module.run_insight_extraction
__all__ = [
"extract_session_insights",
"gather_extraction_inputs",
"get_changed_files",
"get_commit_messages",
"get_extraction_model",
"get_session_diff",
"is_extraction_enabled",
"parse_insights",
"run_insight_extraction",
]
+11
View File
@@ -0,0 +1,11 @@
"""
Integrations Module
===================
External service integrations for Auto Claude.
"""
__all__ = [
"linear",
"graphiti",
]
@@ -0,0 +1,35 @@
"""
Graphiti Integration
====================
Integration with Graphiti knowledge graph for semantic memory.
"""
# Config imports don't require graphiti package
from .config import GraphitiConfig, validate_graphiti_config
# Lazy imports for components that require graphiti package
__all__ = [
"GraphitiConfig",
"validate_graphiti_config",
"GraphitiMemory",
"create_llm_client",
"create_embedder",
]
def __getattr__(name):
"""Lazy import to avoid requiring graphiti package for config-only imports."""
if name == "GraphitiMemory":
from .memory import GraphitiMemory
return GraphitiMemory
elif name == "create_llm_client":
from .providers import create_llm_client
return create_llm_client
elif name == "create_embedder":
from .providers import create_embedder
return create_embedder
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+547
View File
@@ -0,0 +1,547 @@
"""
Graphiti Integration Configuration
==================================
Constants, status mappings, and configuration helpers for Graphiti memory integration.
Follows the same patterns as linear_config.py for consistency.
Multi-Provider Support (V2):
- LLM Providers: OpenAI, Anthropic, Azure OpenAI, Ollama
- Embedder Providers: OpenAI, Voyage AI, Azure OpenAI, Ollama
Environment Variables:
# Core
GRAPHITI_ENABLED: Set to "true" to enable Graphiti integration
GRAPHITI_LLM_PROVIDER: openai|anthropic|azure_openai|ollama (default: openai)
GRAPHITI_EMBEDDER_PROVIDER: openai|voyage|azure_openai|ollama (default: openai)
# OpenAI
OPENAI_API_KEY: Required for OpenAI provider
OPENAI_MODEL: Model for LLM (default: gpt-5-mini)
OPENAI_EMBEDDING_MODEL: Model for embeddings (default: text-embedding-3-small)
# Anthropic (LLM only - needs separate embedder)
ANTHROPIC_API_KEY: Required for Anthropic provider
GRAPHITI_ANTHROPIC_MODEL: Model for LLM (default: claude-sonnet-4-5-latest)
# Azure OpenAI
AZURE_OPENAI_API_KEY: Required for Azure provider
AZURE_OPENAI_BASE_URL: Azure endpoint URL
AZURE_OPENAI_LLM_DEPLOYMENT: Deployment name for LLM
AZURE_OPENAI_EMBEDDING_DEPLOYMENT: Deployment name for embeddings
# Voyage AI (embeddings only - commonly used with Anthropic)
VOYAGE_API_KEY: Required for Voyage embedder
VOYAGE_EMBEDDING_MODEL: Model (default: voyage-3)
# Ollama (local)
OLLAMA_BASE_URL: Ollama server URL (default: http://localhost:11434)
OLLAMA_LLM_MODEL: Model for LLM (e.g., deepseek-r1:7b)
OLLAMA_EMBEDDING_MODEL: Model for embeddings (e.g., nomic-embed-text)
OLLAMA_EMBEDDING_DIM: Embedding dimension (required for Ollama, e.g., 768)
# FalkorDB
GRAPHITI_FALKORDB_HOST: FalkorDB host (default: localhost)
GRAPHITI_FALKORDB_PORT: FalkorDB port (default: 6380)
GRAPHITI_FALKORDB_PASSWORD: FalkorDB password (default: empty)
GRAPHITI_DATABASE: Graph database name (default: auto_claude_memory)
GRAPHITI_TELEMETRY_ENABLED: Set to "false" to disable telemetry (default: true)
"""
import json
import os
from dataclasses import dataclass, field
from datetime import datetime
from enum import Enum
from pathlib import Path
from typing import Optional
# Default configuration values
DEFAULT_FALKORDB_HOST = "localhost"
DEFAULT_FALKORDB_PORT = 6380
DEFAULT_DATABASE = "auto_claude_memory"
DEFAULT_OLLAMA_BASE_URL = "http://localhost:11434"
# Graphiti state marker file (stores connection info and status)
GRAPHITI_STATE_MARKER = ".graphiti_state.json"
# Episode types for different memory categories
EPISODE_TYPE_SESSION_INSIGHT = "session_insight"
EPISODE_TYPE_CODEBASE_DISCOVERY = "codebase_discovery"
EPISODE_TYPE_PATTERN = "pattern"
EPISODE_TYPE_GOTCHA = "gotcha"
EPISODE_TYPE_TASK_OUTCOME = "task_outcome"
EPISODE_TYPE_QA_RESULT = "qa_result"
EPISODE_TYPE_HISTORICAL_CONTEXT = "historical_context"
class LLMProvider(str, Enum):
"""Supported LLM providers for Graphiti."""
OPENAI = "openai"
ANTHROPIC = "anthropic"
AZURE_OPENAI = "azure_openai"
OLLAMA = "ollama"
class EmbedderProvider(str, Enum):
"""Supported embedder providers for Graphiti."""
OPENAI = "openai"
VOYAGE = "voyage"
AZURE_OPENAI = "azure_openai"
OLLAMA = "ollama"
@dataclass
class GraphitiConfig:
"""Configuration for Graphiti memory integration with multi-provider support."""
# Core settings
enabled: bool = False
llm_provider: str = "openai"
embedder_provider: str = "openai"
# FalkorDB connection
falkordb_host: str = DEFAULT_FALKORDB_HOST
falkordb_port: int = DEFAULT_FALKORDB_PORT
falkordb_password: str = ""
database: str = DEFAULT_DATABASE
telemetry_enabled: bool = True
# OpenAI settings
openai_api_key: str = ""
openai_model: str = "gpt-5-mini"
openai_embedding_model: str = "text-embedding-3-small"
# Anthropic settings (LLM only)
anthropic_api_key: str = ""
anthropic_model: str = "claude-sonnet-4-5-latest"
# Azure OpenAI settings
azure_openai_api_key: str = ""
azure_openai_base_url: str = ""
azure_openai_llm_deployment: str = ""
azure_openai_embedding_deployment: str = ""
# Voyage AI settings (embeddings only)
voyage_api_key: str = ""
voyage_embedding_model: str = "voyage-3"
# Ollama settings (local)
ollama_base_url: str = DEFAULT_OLLAMA_BASE_URL
ollama_llm_model: str = ""
ollama_embedding_model: str = ""
ollama_embedding_dim: int = 0 # Required for Ollama embeddings
@classmethod
def from_env(cls) -> "GraphitiConfig":
"""Create config from environment variables."""
# Check if Graphiti is explicitly enabled
enabled_str = os.environ.get("GRAPHITI_ENABLED", "").lower()
enabled = enabled_str in ("true", "1", "yes")
# Provider selection
llm_provider = os.environ.get("GRAPHITI_LLM_PROVIDER", "openai").lower()
embedder_provider = os.environ.get(
"GRAPHITI_EMBEDDER_PROVIDER", "openai"
).lower()
# FalkorDB connection settings
falkordb_host = os.environ.get("GRAPHITI_FALKORDB_HOST", DEFAULT_FALKORDB_HOST)
try:
falkordb_port = int(
os.environ.get("GRAPHITI_FALKORDB_PORT", str(DEFAULT_FALKORDB_PORT))
)
except ValueError:
falkordb_port = DEFAULT_FALKORDB_PORT
falkordb_password = os.environ.get("GRAPHITI_FALKORDB_PASSWORD", "")
database = os.environ.get("GRAPHITI_DATABASE", DEFAULT_DATABASE)
# Telemetry setting
telemetry_str = os.environ.get("GRAPHITI_TELEMETRY_ENABLED", "true").lower()
telemetry_enabled = telemetry_str not in ("false", "0", "no")
# OpenAI settings
openai_api_key = os.environ.get("OPENAI_API_KEY", "")
openai_model = os.environ.get("OPENAI_MODEL", "gpt-5-mini")
openai_embedding_model = os.environ.get(
"OPENAI_EMBEDDING_MODEL", "text-embedding-3-small"
)
# Anthropic settings
anthropic_api_key = os.environ.get("ANTHROPIC_API_KEY", "")
anthropic_model = os.environ.get(
"GRAPHITI_ANTHROPIC_MODEL", "claude-sonnet-4-5-latest"
)
# Azure OpenAI settings
azure_openai_api_key = os.environ.get("AZURE_OPENAI_API_KEY", "")
azure_openai_base_url = os.environ.get("AZURE_OPENAI_BASE_URL", "")
azure_openai_llm_deployment = os.environ.get("AZURE_OPENAI_LLM_DEPLOYMENT", "")
azure_openai_embedding_deployment = os.environ.get(
"AZURE_OPENAI_EMBEDDING_DEPLOYMENT", ""
)
# Voyage AI settings
voyage_api_key = os.environ.get("VOYAGE_API_KEY", "")
voyage_embedding_model = os.environ.get("VOYAGE_EMBEDDING_MODEL", "voyage-3")
# Ollama settings
ollama_base_url = os.environ.get("OLLAMA_BASE_URL", DEFAULT_OLLAMA_BASE_URL)
ollama_llm_model = os.environ.get("OLLAMA_LLM_MODEL", "")
ollama_embedding_model = os.environ.get("OLLAMA_EMBEDDING_MODEL", "")
# Ollama embedding dimension (required for Ollama)
try:
ollama_embedding_dim = int(os.environ.get("OLLAMA_EMBEDDING_DIM", "0"))
except ValueError:
ollama_embedding_dim = 0
return cls(
enabled=enabled,
llm_provider=llm_provider,
embedder_provider=embedder_provider,
falkordb_host=falkordb_host,
falkordb_port=falkordb_port,
falkordb_password=falkordb_password,
database=database,
telemetry_enabled=telemetry_enabled,
openai_api_key=openai_api_key,
openai_model=openai_model,
openai_embedding_model=openai_embedding_model,
anthropic_api_key=anthropic_api_key,
anthropic_model=anthropic_model,
azure_openai_api_key=azure_openai_api_key,
azure_openai_base_url=azure_openai_base_url,
azure_openai_llm_deployment=azure_openai_llm_deployment,
azure_openai_embedding_deployment=azure_openai_embedding_deployment,
voyage_api_key=voyage_api_key,
voyage_embedding_model=voyage_embedding_model,
ollama_base_url=ollama_base_url,
ollama_llm_model=ollama_llm_model,
ollama_embedding_model=ollama_embedding_model,
ollama_embedding_dim=ollama_embedding_dim,
)
def is_valid(self) -> bool:
"""
Check if config has minimum required values for operation.
Returns True if:
- GRAPHITI_ENABLED is true
- LLM provider is configured correctly
- Embedder provider is configured correctly
"""
if not self.enabled:
return False
# Validate LLM provider
if not self._validate_llm_provider():
return False
# Validate embedder provider
if not self._validate_embedder_provider():
return False
return True
def _validate_llm_provider(self) -> bool:
"""Validate LLM provider configuration."""
if self.llm_provider == "openai":
return bool(self.openai_api_key)
elif self.llm_provider == "anthropic":
return bool(self.anthropic_api_key)
elif self.llm_provider == "azure_openai":
return bool(
self.azure_openai_api_key
and self.azure_openai_base_url
and self.azure_openai_llm_deployment
)
elif self.llm_provider == "ollama":
return bool(self.ollama_llm_model)
return False
def _validate_embedder_provider(self) -> bool:
"""Validate embedder provider configuration."""
if self.embedder_provider == "openai":
return bool(self.openai_api_key)
elif self.embedder_provider == "voyage":
return bool(self.voyage_api_key)
elif self.embedder_provider == "azure_openai":
return bool(
self.azure_openai_api_key
and self.azure_openai_base_url
and self.azure_openai_embedding_deployment
)
elif self.embedder_provider == "ollama":
return bool(self.ollama_embedding_model and self.ollama_embedding_dim)
return False
def get_validation_errors(self) -> list[str]:
"""Get list of validation errors for current configuration."""
errors = []
if not self.enabled:
errors.append("GRAPHITI_ENABLED must be set to true")
return errors
# LLM provider validation
if self.llm_provider == "openai":
if not self.openai_api_key:
errors.append("OpenAI LLM provider requires OPENAI_API_KEY")
elif self.llm_provider == "anthropic":
if not self.anthropic_api_key:
errors.append("Anthropic LLM provider requires ANTHROPIC_API_KEY")
elif self.llm_provider == "azure_openai":
if not self.azure_openai_api_key:
errors.append("Azure OpenAI LLM provider requires AZURE_OPENAI_API_KEY")
if not self.azure_openai_base_url:
errors.append(
"Azure OpenAI LLM provider requires AZURE_OPENAI_BASE_URL"
)
if not self.azure_openai_llm_deployment:
errors.append(
"Azure OpenAI LLM provider requires AZURE_OPENAI_LLM_DEPLOYMENT"
)
elif self.llm_provider == "ollama":
if not self.ollama_llm_model:
errors.append("Ollama LLM provider requires OLLAMA_LLM_MODEL")
else:
errors.append(f"Unknown LLM provider: {self.llm_provider}")
# Embedder provider validation
if self.embedder_provider == "openai":
if not self.openai_api_key:
errors.append("OpenAI embedder provider requires OPENAI_API_KEY")
elif self.embedder_provider == "voyage":
if not self.voyage_api_key:
errors.append("Voyage embedder provider requires VOYAGE_API_KEY")
elif self.embedder_provider == "azure_openai":
if not self.azure_openai_api_key:
errors.append(
"Azure OpenAI embedder provider requires AZURE_OPENAI_API_KEY"
)
if not self.azure_openai_base_url:
errors.append(
"Azure OpenAI embedder provider requires AZURE_OPENAI_BASE_URL"
)
if not self.azure_openai_embedding_deployment:
errors.append(
"Azure OpenAI embedder provider requires AZURE_OPENAI_EMBEDDING_DEPLOYMENT"
)
elif self.embedder_provider == "ollama":
if not self.ollama_embedding_model:
errors.append(
"Ollama embedder provider requires OLLAMA_EMBEDDING_MODEL"
)
if not self.ollama_embedding_dim:
errors.append("Ollama embedder provider requires OLLAMA_EMBEDDING_DIM")
else:
errors.append(f"Unknown embedder provider: {self.embedder_provider}")
return errors
def get_connection_uri(self) -> str:
"""Get the FalkorDB connection URI."""
if self.falkordb_password:
return f"redis://:{self.falkordb_password}@{self.falkordb_host}:{self.falkordb_port}"
return f"redis://{self.falkordb_host}:{self.falkordb_port}"
def get_provider_summary(self) -> str:
"""Get a summary of configured providers."""
return f"LLM: {self.llm_provider}, Embedder: {self.embedder_provider}"
@dataclass
class GraphitiState:
"""State of Graphiti integration for an auto-claude spec."""
initialized: bool = False
database: str | None = None
indices_built: bool = False
created_at: str | None = None
last_session: int | None = None
episode_count: int = 0
error_log: list = field(default_factory=list)
# V2 additions
llm_provider: str | None = None
embedder_provider: str | None = None
def to_dict(self) -> dict:
return {
"initialized": self.initialized,
"database": self.database,
"indices_built": self.indices_built,
"created_at": self.created_at,
"last_session": self.last_session,
"episode_count": self.episode_count,
"error_log": self.error_log[-10:], # Keep last 10 errors
"llm_provider": self.llm_provider,
"embedder_provider": self.embedder_provider,
}
@classmethod
def from_dict(cls, data: dict) -> "GraphitiState":
return cls(
initialized=data.get("initialized", False),
database=data.get("database"),
indices_built=data.get("indices_built", False),
created_at=data.get("created_at"),
last_session=data.get("last_session"),
episode_count=data.get("episode_count", 0),
error_log=data.get("error_log", []),
llm_provider=data.get("llm_provider"),
embedder_provider=data.get("embedder_provider"),
)
def save(self, spec_dir: Path) -> None:
"""Save state to the spec directory."""
marker_file = spec_dir / GRAPHITI_STATE_MARKER
with open(marker_file, "w") as f:
json.dump(self.to_dict(), f, indent=2)
@classmethod
def load(cls, spec_dir: Path) -> Optional["GraphitiState"]:
"""Load state from the spec directory."""
marker_file = spec_dir / GRAPHITI_STATE_MARKER
if not marker_file.exists():
return None
try:
with open(marker_file) as f:
return cls.from_dict(json.load(f))
except (OSError, json.JSONDecodeError):
return None
def record_error(self, error_msg: str) -> None:
"""Record an error in the state."""
self.error_log.append(
{
"timestamp": datetime.now().isoformat(),
"error": error_msg[:500], # Limit error message length
}
)
# Keep only last 10 errors
self.error_log = self.error_log[-10:]
def is_graphiti_enabled() -> bool:
"""
Quick check if Graphiti integration is available.
Returns True if:
- GRAPHITI_ENABLED is set to true/1/yes
- Required provider credentials are configured
"""
config = GraphitiConfig.from_env()
return config.is_valid()
def get_graphiti_status() -> dict:
"""
Get the current Graphiti integration status.
Returns:
Dict with status information:
- enabled: bool
- available: bool (has required dependencies)
- host: str
- port: int
- database: str
- llm_provider: str
- embedder_provider: str
- reason: str (why unavailable if not available)
- errors: list (validation errors if any)
"""
config = GraphitiConfig.from_env()
status = {
"enabled": config.enabled,
"available": False,
"host": config.falkordb_host,
"port": config.falkordb_port,
"database": config.database,
"llm_provider": config.llm_provider,
"embedder_provider": config.embedder_provider,
"reason": "",
"errors": [],
}
if not config.enabled:
status["reason"] = "GRAPHITI_ENABLED not set to true"
return status
# Get validation errors
errors = config.get_validation_errors()
if errors:
status["errors"] = errors
status["reason"] = errors[0] # First error as primary reason
return status
status["available"] = True
return status
def get_available_providers() -> dict:
"""
Get list of available providers based on current environment.
Returns:
Dict with lists of available LLM and embedder providers
"""
config = GraphitiConfig.from_env()
available_llm = []
available_embedder = []
# Check OpenAI
if config.openai_api_key:
available_llm.append("openai")
available_embedder.append("openai")
# Check Anthropic
if config.anthropic_api_key:
available_llm.append("anthropic")
# Check Azure OpenAI
if config.azure_openai_api_key and config.azure_openai_base_url:
if config.azure_openai_llm_deployment:
available_llm.append("azure_openai")
if config.azure_openai_embedding_deployment:
available_embedder.append("azure_openai")
# Check Voyage
if config.voyage_api_key:
available_embedder.append("voyage")
# Check Ollama
if config.ollama_llm_model:
available_llm.append("ollama")
if config.ollama_embedding_model and config.ollama_embedding_dim:
available_embedder.append("ollama")
return {
"llm_providers": available_llm,
"embedder_providers": available_embedder,
}
def validate_graphiti_config() -> tuple[bool, list[str]]:
"""
Validate Graphiti configuration from environment.
Returns:
Tuple of (is_valid, error_messages)
- is_valid: True if configuration is valid
- error_messages: List of validation error messages (empty if valid)
"""
config = GraphitiConfig.from_env()
if not config.is_valid():
errors = config.get_validation_errors()
return False, errors
return True, []
@@ -0,0 +1,70 @@
"""
Graphiti Multi-Provider Entry Point
====================================
Main entry point for Graphiti provider functionality.
This module re-exports all functionality from the graphiti_providers package.
The actual implementation has been refactored into a package structure:
- graphiti_providers/exceptions.py - Provider exceptions
- graphiti_providers/models.py - Embedding dimensions and constants
- graphiti_providers/llm_providers/ - LLM provider implementations
- graphiti_providers/embedder_providers/ - Embedder provider implementations
- graphiti_providers/cross_encoder.py - Cross-encoder/reranker
- graphiti_providers/validators.py - Validation and health checks
- graphiti_providers/utils.py - Utility functions
- graphiti_providers/factory.py - Factory functions
For backward compatibility, this module re-exports all public APIs.
Usage:
from graphiti_providers import create_llm_client, create_embedder
from graphiti_config import GraphitiConfig
config = GraphitiConfig.from_env()
llm_client = create_llm_client(config)
embedder = create_embedder(config)
"""
# Re-export all public APIs from the package
from graphiti_providers import (
# Exceptions
ProviderError,
ProviderNotInstalled,
# Factory functions
create_llm_client,
create_embedder,
create_cross_encoder,
# Models
EMBEDDING_DIMENSIONS,
get_expected_embedding_dim,
# Validators
validate_embedding_config,
test_llm_connection,
test_embedder_connection,
test_ollama_connection,
# Utilities
is_graphiti_enabled,
get_graph_hints,
)
__all__ = [
# Exceptions
"ProviderError",
"ProviderNotInstalled",
# Factory functions
"create_llm_client",
"create_embedder",
"create_cross_encoder",
# Models
"EMBEDDING_DIMENSIONS",
"get_expected_embedding_dim",
# Validators
"validate_embedding_config",
"test_llm_connection",
"test_embedder_connection",
"test_ollama_connection",
# Utilities
"is_graphiti_enabled",
"get_graph_hints",
]

Some files were not shown because too many files have changed in this diff Show More