From 6a6247bbf2d4dfe295f14b7549426726785fbb4d Mon Sep 17 00:00:00 2001 From: TamerineSky <168569051+TamerineSky@users.noreply.github.com> Date: Mon, 19 Jan 2026 14:22:55 -0700 Subject: [PATCH] Fix Windows UTF-8 encoding errors across entire backend (251 instances) (#782) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix UTF-8 encoding for Priorities 1-2 (Core & Agents - 18 instances) Add encoding="utf-8" to file operations in: - Priority 1: Core Infrastructure (8 instances) - core/progress.py (6 read operations) - core/debug.py (1 append operation) - core/workspace/setup.py (1 read operation) - Priority 2: Agent System (10 instances) - agents/utils.py (1 read) - agents/tools_pkg/tools/subtask.py (1 read, 1 write) - agents/tools_pkg/tools/memory.py (2 read, 1 write, 1 append) - agents/tools_pkg/tools/qa.py (1 read, 1 write) - agents/tools_pkg/tools/progress.py (1 read) All changes use double quotes for ruff format compliance. * Fix UTF-8 encoding for Priorities 3-4 (Spec & Project - 26 instances) Add encoding="utf-8" to file operations in: - Priority 3: Spec Pipeline (21 instances) - spec/context.py (4: 2 read, 2 write) - spec/complexity.py (3: 2 read, 1 write) - spec/requirements.py (3: 2 read, 1 write) - spec/validator.py (3 write operations) - spec/writer.py (2: 1 read, 1 write) - spec/discovery.py (1 read) - spec/pipeline/orchestrator.py (2 read) - spec/phases/requirements_phases.py (1 write) - spec/validate_pkg/auto_fix.py (2: 1 read, 1 write) - Priority 4: Project Analyzer (5 instances) - project/analyzer.py (2: 1 read, 1 write) - project/config_parser.py (2 read operations) - project/stack_detector.py (1 read) All changes use double quotes for ruff format compliance. * Fix UTF-8 encoding for Priorities 5-7 (Services, Analysis, Ideation - 43 instances) Add encoding="utf-8" to file operations in: - Priority 5: Services (12 instances) - services/recovery.py (8: 4 read, 4 write) - services/context.py (4 read operations) - Priority 6: Analysis & QA (6 instances) - analysis/analyzers/__init__.py (2 write) - analysis/insight_extractor.py (1 read) - qa/criteria.py (2: 1 read, 1 write) - qa/report.py (1 read) - Priority 7: Ideation & Roadmap (25 instances) - ideation/analyzer.py (3 read) - ideation/formatter.py (4 read, 1 write) - ideation/phase_executor.py (5: 3 read, 2 write) - ideation/runner.py (1 read) - runners/roadmap/competitor_analyzer.py (3: 1 read, 2 write) - runners/roadmap/graph_integration.py (3 write) - runners/roadmap/orchestrator.py (1 read) - runners/roadmap/phases.py (2 read) - runners/insights_runner.py (3 read) All changes use double quotes for ruff format compliance. * Fix UTF-8 encoding for Priorities 8-14 (All remaining - 85+ instances) Add encoding="utf-8" to file operations across all remaining modules: Priorities 8-10 (Merge, Memory, Integrations - 26 instances): - merge/ (4 files) - memory/ (3 files) - context/ (3 files) - integrations/ (4 files) Priorities 11-14 (GitHub, GitLab, AI, Other - 59 instances): - runners/github/ (19 files) - runners/gitlab/ (3 files) - runners/ai_analyzer/ (1 file) All changes use double quotes for ruff format compliance. Applied using Python regex script for efficiency. * Fix UTF-8 encoding for missed instances (23 instances) Fix remaining instances missed by batch script: - cli/batch_commands.py (3 instances) - cli/followup_commands.py (1 instance) - core/client.py (1 instance) - phase_config.py (1 instance) - planner_lib/context.py (4 instances) - prediction/main.py (1 instance) - prediction/memory_loader.py (1 instance) - prompts_pkg/prompts.py (2 instances) - review/formatters.py (1 instance) - review/state.py (2 instances) - spec/phases/spec_phases.py (1 instance) - spec/pipeline/models.py (1 instance) - spec/validate_pkg/validators/context_validator.py (1 instance) - spec/validate_pkg/validators/implementation_plan_validator.py (1 instance) - ui/status.py (2 instances) All encoding parameters use double quotes for ruff format compliance. Verified: 0 instances without encoding remain in source code. * Fix missed os.fdopen() calls and duplicate encoding bug Thorough verification found 3 additional issues: - runners/github/file_lock.py:462 - os.fdopen missing encoding - runners/github/trust.py:442 - os.fdopen missing encoding - runners/insights_runner.py:372 - duplicate encoding parameter All fixed. Final count: 251 instances with encoding="utf-8" * Fix missed Path.read_text() and Path.write_text() encoding (99 instances) Gemini Code Assist review found instances we missed: - Path.read_text() without encoding: 77 instances → fixed - Path.write_text() without encoding: 22 instances → fixed Total UTF-8 encoding fixes: 350 instances across codebase - open() operations: 251 instances - Path.read_text(): 98 instances - Path.write_text(): 30 instances All text file operations now explicitly use encoding="utf-8". Addresses feedback from PR #782 review. * Fix critical syntax errors from CodeRabbit review - Fix os.getpid() syntax error in core/workspace/models.py (2 instances) Changed: os.getpid(, encoding="utf-8") -> str(os.getpid()) - Fix json.dumps invalid encoding parameter (3 instances) json.dumps() doesn't accept encoding parameter Changed: json.dumps(data, encoding="utf-8") -> json.dumps(data) Files: runners/ai_analyzer/cache_manager.py, runners/github/test_file_lock.py - Fix tempfile.NamedTemporaryFile missing encoding Added encoding="utf-8" to spec/requirements.py:22 - Fix subprocess.run text=True to encoding Changed: text=True -> encoding="utf-8" in core/workspace/setup.py:375 All critical syntax errors from CodeRabbit review resolved. * Fix critical syntax errors in test_context_gatherer.py - Line 78: Move encoding="utf-8" outside of JS string content Changed: write_text("...encoding="utf-8"...") To: write_text("...", encoding="utf-8") - Line 102: Move encoding="utf-8" outside of JS string content Changed: write_text("...encoding="utf-8"...") To: write_text("...", encoding="utf-8") Fixes syntax errors where encoding parameter was incorrectly placed inside the JavaScript code string instead of as write_text() parameter. * Fix CodeRabbit issues: UnicodeDecodeError handling and trailing newlines - Add UnicodeDecodeError to exception handling in agents/utils.py and spec/validate_pkg/auto_fix.py - Fix trailing newline preservation in merge/file_merger.py (2 locations) - Add encoding parameter to atomic_write() in runners/github/file_lock.py These fixes ensure robust error handling for malformed UTF-8 files and preserve file formatting during merge operations. * Fix test fixture to use UTF-8 encoding consistently Update spec_file fixture in tests/conftest.py to write spec file with encoding="utf-8" to match how it's read in validators. This ensures consistency between test fixtures and production code. * Fix linting errors and security vulnerabilities from merge - Remove unused tree-sitter methods in semantic_analyzer.py that caused F821 undefined name errors - Fix regex injection vulnerability in bump-version.js by properly escaping all regex special characters - Add escapeRegex() function to prevent security issues when version string is used in RegExp constructor Resolves ruff linting failures and CodeQL security alerts. * Fix code formatting for ruff compliance Apply formatting fixes to meet line length requirements: - context/builder.py: Split long line with array slicing - planner_lib/context.py: Split long ternary expression - spec/requirements.py: Split long tempfile.NamedTemporaryFile call Resolves ruff format check failures. * Fix missing UTF-8 encoding in init.py gitignore operations Found by pre-commit hook testing in PR #795: - Line 96: Path.read_text() without encoding - Line 122: Path.write_text() without encoding These handle .gitignore file operations and could fail on Windows with special characters in gitignore comments or entries. Total fixes in PR #782: 253 instances (was 251, +2 from init.py) * Add pre-commit hook for UTF-8 encoding enforcement 1. Encoding Check Script (scripts/check_encoding.py): - Validates all file operations have encoding="utf-8" - Checks open(), Path.read_text(), Path.write_text() - Checks json.load/dump with open() - Allows binary mode without encoding - Windows-compatible emoji output with UTF-8 reconfiguration 2. Pre-commit Config (.pre-commit-config.yaml): - Added check-file-encoding hook for apps/backend/ - Runs automatically before commits - Scoped to backend Python files only 3. Tests (tests/test_check_encoding.py): - Comprehensive test coverage (10 tests, all passing) - Tests detection of missing encoding - Tests allowlist for binary files - Tests multiple issues in single file - Tests file type filtering Purpose: - Prevent regression of 251 UTF-8 encoding fixes from PR #782 - Catch missing encoding in new code during development - Fast feedback loop for developers Implementation Notes: - Hook scoped to apps/backend/ to avoid false positives in test code - Uses simple regex matching for speed - Compatible with existing pre-commit infrastructure - Already caught 6 real issues in apps/backend/core/progress.py Related: PR #782 - Fix Windows UTF-8 encoding errors * Address CodeRabbit and Gemini review feedback Fixes based on automated review comments: 1. Binary Mode Detection (Critical Fix): - Replaced brittle regex with robust pattern: r'["'][rwax+]*b[rwax+]*["']' - Now correctly detects all binary modes: rb, wb, ab, r+b, w+b, etc. - Prevents false positives on text mode 'w' without 'b' - Added comprehensive tests for wb, ab, and text w modes 2. Encoding Detection Robustness (Critical Fix): - Changed from 'encoding=' string match to word boundary regex: r'\bencoding\s*=' - Now handles encoding with spaces: encoding = "utf-8" - Prevents false matches of substrings containing 'encoding=' - Applied across all checks (open, read_text, write_text, json.load, json.dump) - Added test for spaces around equals sign 3. Test Coverage Improvements: - Added json.dump() with encoding test (passing case) - Added json.dump() without encoding test (failing case) - Fixed test assertions to match actual behavior (== 1 not == 2) - Added 6 new tests for improved binary/text mode coverage - Total tests increased from 10 to 16, all passing ✅ 4. Code Cleanup: - Removed unused pytest import (CodeQL warning) - Simplified check_files() to remove unused variable tracking All changes validated with comprehensive test suite (16/16 passing). Related: PR #795 review feedback from CodeRabbit and Gemini Code Assist * docs: Add UTF-8 encoding guidelines and Windows development guide 1. CONTRIBUTING.md: - Added concise file encoding section after Code Style - DO/DON'T examples for common file operations - Covers open(), Path methods, json operations - References PR #782 and windows-development.md 2. guides/windows-development.md (NEW): - Comprehensive Windows development guide - File encoding (cp1252 vs UTF-8 issue) - Line endings, path separators, shell commands - Development environment recommendations - Common pitfalls and solutions - Testing guidelines 3. .github/PULL_REQUEST_TEMPLATE.md: - Added encoding checklist item for Python PRs - Helps catch missing encoding during review 4. guides/README.md: - Added windows-development.md to guide index - Organized with CLI-USAGE and linux guides Purpose: Educate developers about UTF-8 encoding requirements to prevent regressions of the 251 encoding issues fixed in PR #782. Automated checking via pre-commit hooks (PR #795) + developer education ensures long-term Windows compatibility. Related: - PR #782: Fix Windows UTF-8 encoding errors (251 instances) - PR #795: Add pre-commit hooks for encoding enforcement * Address review comments from CodeRabbit and Gemini 1. Fix CONTRIBUTING.md markdown linting issues - Add blank lines around code blocks (MD031) - Add JSON write example with ensure_ascii=False (Gemini suggestion) 2. Fix guides/windows-development.md markdown linting (39 violations) - Rename duplicate headings: "The Problem"/"The Solution" → "Problem"/"Solution" (MD024) - Add blank lines around all code blocks (MD031) - Add language specifiers to code blocks (MD040) - Add blank lines before/after headings (MD022) - Wrap long lines to <=80 characters (MD013) - Add blank line before list (MD032) - Use Gemini's idiomatic line ending normalization pattern 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 * Fix additional UTF-8 encoding issues and improve encoding check script - Add encoding="utf-8" to 5 files that were missing it: - cli/workspace_commands.py: read_text for worktree config - context/pattern_discovery.py: read_text with errors param - context/search.py: read_text with errors param - core/sentry.py: open for package.json version detection - core/workspace/setup.py: open for security profile JSON - Improve check_encoding.py script to reduce false positives: - Use negative lookbehind to exclude os.open(), urlopen(), etc. - Handle nested parentheses correctly when checking args - Skip self.method.read_text() calls (custom methods, not Path) Co-Authored-By: Claude Opus 4.5 * Fix missing UTF-8 encoding in locked_write() function Add encoding parameter to locked_write() async context manager and use it in os.fdopen() call. This fixes HIGH priority issue from PR review where locked_write() was missing UTF-8 encoding support, which could cause encoding errors on Windows when writing files with non-ASCII content. Co-Authored-By: Claude Opus 4.5 * Add UnicodeDecodeError handling for file loading resilience Address CodeRabbit review feedback: - runner.py: Add UnicodeDecodeError to exception handling when loading batch files - trust.py: Add exception handling in get_state() and get_all_states() to gracefully handle corrupted state files instead of failing completely Co-Authored-By: Claude Opus 4.5 * Fix atomic_write to handle binary mode correctly The atomic_write function was unconditionally passing encoding to os.fdopen, which would crash with ValueError if called with binary mode (e.g., 'wb'). Apply the same fix used in locked_write: only pass encoding for text modes. Co-Authored-By: Claude Opus 4.5 * Fix run_git() call with invalid parameters in setup.py Remove capture_output and encoding kwargs from run_git() call - these parameters are already handled internally by run_git() and passing them causes TypeError since the function doesn't accept them. Co-Authored-By: Claude Opus 4.5 * Fix CodeQL warnings and potential double-newline bug - Remove unused is_path_call variables in check_encoding.py - Remove unused failed_count variable in check_encoding.py - Remove unused escapeRegex function in bump-version.js - Fix potential double-newline when adding imports in file_merger.py (strip trailing newlines from content_after before inserting) Co-Authored-By: Claude Opus 4.5 * Fix Ruff formatting: wrap long line in file_merger.py Co-Authored-By: Claude Opus 4.5 * Add UnicodeDecodeError handling to all JSON file loading Comprehensively add UnicodeDecodeError to exception handlers across the codebase to handle legacy-encoded or corrupted files gracefully: - 32+ locations now catch UnicodeDecodeError alongside OSError and json.JSONDecodeError - context/builder.py: Regenerate index on decode failure - planner_lib/context.py: Use empty dicts on decode failure - check_encoding.py: Handle OSError for unreadable files - cleanup.py: Handle decode errors in index pruning This ensures the codebase is robust against non-UTF-8 files that may exist from previous Windows runs with cp1252 encoding. Co-Authored-By: Claude Opus 4.5 * Add explanatory comments to empty except clauses Address CodeQL notices about empty except clauses with just 'pass' by adding explanatory comments describing the intent. Co-Authored-By: Claude Opus 4.5 * Fix review issues from Andy's Auto Claude PR Review 1. [HIGH] Fix double-close bug in trust.py:449 - Remove try/except around os.fdopen since it takes ownership of fd - The with statement handles closing, no need for explicit os.close() 2. [LOW] Fix dead code in file_merger.py:87,159 - Simplify endswith check to just '\n' since content is already normalized to LF at that point 3. [LOW] Fix escaped backslash-n in test_context_gatherer.py:150 - Change "\n" (literal backslash-n) to "\n" (actual newline) 4. [LOW] Fix coder.md examples missing encoding parameter - Add encoding="utf-8" to read_text() and open() calls in examples Co-Authored-By: Claude Opus 4.5 --------- Co-authored-by: TamerineSky Co-authored-by: Claude Opus 4.5 --- .github/PULL_REQUEST_TEMPLATE.md | 1 + .pre-commit-config.yaml | 11 + CONTRIBUTING.md | 58 +++ apps/backend/agents/coder.py | 2 +- apps/backend/agents/tools_pkg/tools/memory.py | 12 +- .../agents/tools_pkg/tools/progress.py | 2 +- apps/backend/agents/tools_pkg/tools/qa.py | 2 +- .../backend/agents/tools_pkg/tools/subtask.py | 2 +- apps/backend/agents/utils.py | 4 +- apps/backend/analysis/analyzers/__init__.py | 4 +- apps/backend/analysis/analyzers/base.py | 2 +- .../analyzers/context/auth_detector.py | 2 +- .../analyzers/context/jobs_detector.py | 2 +- .../analyzers/context/monitoring_detector.py | 2 +- .../analysis/analyzers/database_detector.py | 12 +- .../analyzers/project_analyzer_module.py | 2 +- .../analysis/analyzers/route_detector.py | 14 +- apps/backend/analysis/ci_discovery.py | 8 +- apps/backend/analysis/insight_extractor.py | 4 +- apps/backend/analysis/test_discovery.py | 8 +- apps/backend/cli/batch_commands.py | 6 +- apps/backend/cli/build_commands.py | 2 +- apps/backend/cli/followup_commands.py | 6 +- apps/backend/cli/input_handlers.py | 2 +- apps/backend/cli/workspace_commands.py | 2 +- apps/backend/context/builder.py | 12 +- apps/backend/context/main.py | 2 +- apps/backend/context/pattern_discovery.py | 2 +- apps/backend/context/search.py | 2 +- apps/backend/context/serialization.py | 4 +- apps/backend/core/client.py | 2 +- apps/backend/core/debug.py | 2 +- apps/backend/core/progress.py | 24 +- apps/backend/core/sentry.py | 2 +- apps/backend/core/workspace/models.py | 8 +- apps/backend/core/workspace/setup.py | 11 +- apps/backend/ideation/analyzer.py | 8 +- apps/backend/ideation/formatter.py | 8 +- apps/backend/ideation/generator.py | 2 +- apps/backend/ideation/phase_executor.py | 14 +- apps/backend/ideation/prioritizer.py | 4 +- apps/backend/ideation/runner.py | 2 +- apps/backend/init.py | 10 +- apps/backend/integrations/graphiti/config.py | 6 +- apps/backend/integrations/linear/config.py | 6 +- .../integrations/linear/integration.py | 4 +- apps/backend/integrations/linear/updater.py | 6 +- apps/backend/memory/codebase_map.py | 10 +- apps/backend/memory/patterns.py | 12 +- apps/backend/memory/sessions.py | 6 +- apps/backend/merge/file_evolution/storage.py | 4 +- apps/backend/merge/file_merger.py | 20 +- apps/backend/merge/install_hook.py | 12 +- apps/backend/merge/models.py | 2 +- apps/backend/merge/timeline_persistence.py | 8 +- apps/backend/phase_config.py | 2 +- apps/backend/planner_lib/context.py | 22 +- apps/backend/prediction/main.py | 2 +- apps/backend/prediction/memory_loader.py | 8 +- apps/backend/project/analyzer.py | 4 +- apps/backend/project/config_parser.py | 4 +- apps/backend/project/stack_detector.py | 2 +- apps/backend/prompts/coder.md | 8 +- apps/backend/prompts_pkg/prompt_generator.py | 4 +- apps/backend/prompts_pkg/prompts.py | 16 +- apps/backend/qa/criteria.py | 6 +- apps/backend/qa/fixer.py | 2 +- apps/backend/qa/report.py | 10 +- apps/backend/review/formatters.py | 2 +- apps/backend/review/state.py | 6 +- .../runners/ai_analyzer/cache_manager.py | 4 +- .../runners/ai_analyzer/claude_client.py | 2 +- apps/backend/runners/ai_analyzer_runner.py | 2 +- apps/backend/runners/github/audit.py | 4 +- apps/backend/runners/github/batch_issues.py | 8 +- .../backend/runners/github/batch_validator.py | 2 +- apps/backend/runners/github/bot_detection.py | 2 +- apps/backend/runners/github/cleanup.py | 12 +- .../runners/github/context_gatherer.py | 2 +- apps/backend/runners/github/duplicates.py | 4 +- apps/backend/runners/github/file_lock.py | 23 +- apps/backend/runners/github/learning.py | 2 +- apps/backend/runners/github/lifecycle.py | 8 +- .../runners/github/memory_integration.py | 4 +- apps/backend/runners/github/models.py | 10 +- apps/backend/runners/github/multi_repo.py | 4 +- apps/backend/runners/github/onboarding.py | 4 +- apps/backend/runners/github/override.py | 6 +- apps/backend/runners/github/purge_strategy.py | 2 +- apps/backend/runners/github/runner.py | 4 +- .../runners/github/test_context_gatherer.py | 16 +- apps/backend/runners/github/test_file_lock.py | 14 +- apps/backend/runners/github/trust.py | 29 +- apps/backend/runners/gitlab/glab_client.py | 2 +- apps/backend/runners/gitlab/models.py | 4 +- apps/backend/runners/gitlab/runner.py | 2 +- apps/backend/runners/insights_runner.py | 4 +- .../runners/roadmap/competitor_analyzer.py | 6 +- apps/backend/runners/roadmap/executor.py | 2 +- .../runners/roadmap/graph_integration.py | 6 +- apps/backend/runners/roadmap/orchestrator.py | 2 +- apps/backend/runners/roadmap/phases.py | 4 +- apps/backend/runners/spec_runner.py | 2 +- apps/backend/security/scan_secrets.py | 2 +- apps/backend/services/context.py | 22 +- apps/backend/services/orchestrator.py | 2 +- apps/backend/services/recovery.py | 20 +- apps/backend/spec/compaction.py | 2 +- apps/backend/spec/complexity.py | 6 +- apps/backend/spec/context.py | 10 +- apps/backend/spec/discovery.py | 2 +- .../spec/phases/requirements_phases.py | 2 +- apps/backend/spec/phases/spec_phases.py | 2 +- apps/backend/spec/pipeline/agent_runner.py | 2 +- apps/backend/spec/pipeline/models.py | 2 +- apps/backend/spec/pipeline/orchestrator.py | 4 +- apps/backend/spec/requirements.py | 10 +- apps/backend/spec/validate_pkg/auto_fix.py | 2 +- .../validators/context_validator.py | 2 +- .../implementation_plan_validator.py | 2 +- .../validators/spec_document_validator.py | 2 +- apps/backend/spec/validation_strategy.py | 6 +- apps/backend/spec/validator.py | 6 +- apps/backend/spec/writer.py | 4 +- apps/backend/task_logger/storage.py | 4 +- apps/backend/ui/status.py | 6 +- guides/README.md | 2 + guides/windows-development.md | 337 +++++++++++++++++ scripts/check_encoding.py | 251 +++++++++++++ tests/conftest.py | 2 +- tests/test_check_encoding.py | 355 ++++++++++++++++++ 131 files changed, 1407 insertions(+), 349 deletions(-) create mode 100644 guides/windows-development.md create mode 100644 scripts/check_encoding.py create mode 100644 tests/test_check_encoding.py diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c9db1818..4ea39d02 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -39,6 +39,7 @@ Follow conventional commits: `: ` - [ ] I've tested my changes locally - [ ] I've followed the code principles (SOLID, DRY, KISS) - [ ] My PR is small and focused (< 400 lines ideally) +- [ ] **(Python only)** All file operations specify `encoding="utf-8"` for text files ## Platform Testing Checklist diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 64dd628d..78c2b699 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -76,6 +76,17 @@ repos: files: ^package\.json$ pass_filenames: false + # Python encoding check - prevent regression of UTF-8 encoding fixes (PR #782) + - repo: local + hooks: + - id: check-file-encoding + name: Check file encoding parameters + entry: python scripts/check_encoding.py + language: system + types: [python] + files: ^apps/backend/ + description: Ensures all file operations specify encoding="utf-8" + # Python linting (apps/backend/) - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.14.10 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ae90bf43..353dedda 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -357,6 +357,64 @@ export default function(props) { - End files with a newline - Keep line length under 100 characters when practical +### File Encoding (Python) + +**Always specify `encoding="utf-8"` for text file operations** to ensure Windows compatibility. + +Windows Python defaults to `cp1252` encoding instead of UTF-8, causing errors with: +- Emoji (🚀, ✅, ❌) +- International characters (ñ, é, 中文, العربية) +- Special symbols (™, ©, ®) + +**DO:** + +```python +# Reading files +with open(path, encoding="utf-8") as f: + content = f.read() + +# Writing files +with open(path, "w", encoding="utf-8") as f: + f.write(content) + +# Path methods +from pathlib import Path +content = Path(file).read_text(encoding="utf-8") +Path(file).write_text(content, encoding="utf-8") + +# JSON files - reading +import json +with open(path, encoding="utf-8") as f: + data = json.load(f) + +# JSON files - writing +with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False, indent=2) +``` + +**DON'T:** + +```python +# Wrong - platform-dependent encoding +with open(path) as f: + content = f.read() + +# Wrong - Path methods without encoding +content = Path(file).read_text() + +# Wrong - encoding on json.dump (not open!) +json.dump(data, f, encoding="utf-8") # ERROR +``` + +**Binary files - NO encoding:** + +```python +with open(path, "rb") as f: # Correct + data = f.read() +``` + +Our pre-commit hooks automatically check for missing encoding parameters. See [PR #782](https://github.com/AndyMik90/Auto-Claude/pull/782) for the comprehensive encoding fix and [guides/windows-development.md](guides/windows-development.md) for Windows-specific development guidance. + ## Testing ### Python Tests diff --git a/apps/backend/agents/coder.py b/apps/backend/agents/coder.py index caf7e80a..b498ff86 100644 --- a/apps/backend/agents/coder.py +++ b/apps/backend/agents/coder.py @@ -226,7 +226,7 @@ async def run_autonomous_agent( print(" PAUSED BY HUMAN") print("=" * 70) - pause_content = pause_file.read_text().strip() + pause_content = pause_file.read_text(encoding="utf-8").strip() if pause_content: print(f"\nMessage: {pause_content}") diff --git a/apps/backend/agents/tools_pkg/tools/memory.py b/apps/backend/agents/tools_pkg/tools/memory.py index 52457a59..4a9868a1 100644 --- a/apps/backend/agents/tools_pkg/tools/memory.py +++ b/apps/backend/agents/tools_pkg/tools/memory.py @@ -171,7 +171,7 @@ def create_memory_tools(spec_dir: Path, project_dir: Path) -> list: # PRIMARY: Save to file-based storage (always works) # Load existing map or create new if codebase_map_file.exists(): - with open(codebase_map_file) as f: + with open(codebase_map_file, encoding="utf-8") as f: codebase_map = json.load(f) else: codebase_map = { @@ -187,7 +187,7 @@ def create_memory_tools(spec_dir: Path, project_dir: Path) -> list: } codebase_map["last_updated"] = datetime.now(timezone.utc).isoformat() - with open(codebase_map_file, "w") as f: + with open(codebase_map_file, "w", encoding="utf-8") as f: json.dump(codebase_map, f, indent=2) # SECONDARY: Also save to Graphiti/LadybugDB (for Memory UI) @@ -246,7 +246,7 @@ def create_memory_tools(spec_dir: Path, project_dir: Path) -> list: entry += f"\n\n_Context: {context}_" entry += "\n" - with open(gotchas_file, "a") as f: + with open(gotchas_file, "a", encoding="utf-8") as f: if not gotchas_file.exists() or gotchas_file.stat().st_size == 0: f.write( "# Gotchas & Pitfalls\n\nThings to watch out for in this codebase.\n" @@ -303,7 +303,7 @@ def create_memory_tools(spec_dir: Path, project_dir: Path) -> list: codebase_map_file = memory_dir / "codebase_map.json" if codebase_map_file.exists(): try: - with open(codebase_map_file) as f: + with open(codebase_map_file, encoding="utf-8") as f: codebase_map = json.load(f) discoveries = codebase_map.get("discovered_files", {}) @@ -319,7 +319,7 @@ def create_memory_tools(spec_dir: Path, project_dir: Path) -> list: gotchas_file = memory_dir / "gotchas.md" if gotchas_file.exists(): try: - content = gotchas_file.read_text() + content = gotchas_file.read_text(encoding="utf-8") if content.strip(): result_parts.append("\n## Gotchas") # Take last 1000 chars to avoid too much context @@ -333,7 +333,7 @@ def create_memory_tools(spec_dir: Path, project_dir: Path) -> list: patterns_file = memory_dir / "patterns.md" if patterns_file.exists(): try: - content = patterns_file.read_text() + content = patterns_file.read_text(encoding="utf-8") if content.strip(): result_parts.append("\n## Patterns") result_parts.append( diff --git a/apps/backend/agents/tools_pkg/tools/progress.py b/apps/backend/agents/tools_pkg/tools/progress.py index 387d8265..d30292b2 100644 --- a/apps/backend/agents/tools_pkg/tools/progress.py +++ b/apps/backend/agents/tools_pkg/tools/progress.py @@ -57,7 +57,7 @@ def create_progress_tools(spec_dir: Path, project_dir: Path) -> list: } try: - with open(plan_file) as f: + with open(plan_file, encoding="utf-8") as f: plan = json.load(f) stats = { diff --git a/apps/backend/agents/tools_pkg/tools/qa.py b/apps/backend/agents/tools_pkg/tools/qa.py index d0448c55..1f9bcdeb 100644 --- a/apps/backend/agents/tools_pkg/tools/qa.py +++ b/apps/backend/agents/tools_pkg/tools/qa.py @@ -140,7 +140,7 @@ def create_qa_tools(spec_dir: Path, project_dir: Path) -> list: except json.JSONDecodeError: tests_passed = {} - with open(plan_file) as f: + with open(plan_file, encoding="utf-8") as f: plan = json.load(f) qa_session = _apply_qa_update(plan, status, issues, tests_passed) diff --git a/apps/backend/agents/tools_pkg/tools/subtask.py b/apps/backend/agents/tools_pkg/tools/subtask.py index ac79be98..2c2660ad 100644 --- a/apps/backend/agents/tools_pkg/tools/subtask.py +++ b/apps/backend/agents/tools_pkg/tools/subtask.py @@ -113,7 +113,7 @@ def create_subtask_tools(spec_dir: Path, project_dir: Path) -> list: } try: - with open(plan_file) as f: + with open(plan_file, encoding="utf-8") as f: plan = json.load(f) subtask_found = _update_subtask_in_plan(plan, subtask_id, status, notes) diff --git a/apps/backend/agents/utils.py b/apps/backend/agents/utils.py index 614cdb79..840f08f9 100644 --- a/apps/backend/agents/utils.py +++ b/apps/backend/agents/utils.py @@ -48,9 +48,9 @@ def load_implementation_plan(spec_dir: Path) -> dict | None: if not plan_file.exists(): return None try: - with open(plan_file) as f: + with open(plan_file, encoding="utf-8") as f: return json.load(f) - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): return None diff --git a/apps/backend/analysis/analyzers/__init__.py b/apps/backend/analysis/analyzers/__init__.py index a04b2310..816a4d32 100644 --- a/apps/backend/analysis/analyzers/__init__.py +++ b/apps/backend/analysis/analyzers/__init__.py @@ -46,7 +46,7 @@ def analyze_project(project_dir: Path, output_file: Path | None = None) -> dict: if output_file: output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as f: + with open(output_file, "w", encoding="utf-8") as f: json.dump(results, f, indent=2) print(f"Project index saved to: {output_file}") @@ -87,7 +87,7 @@ def analyze_service( if output_file: output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as f: + with open(output_file, "w", encoding="utf-8") as f: json.dump(results, f, indent=2) print(f"Service analysis saved to: {output_file}") diff --git a/apps/backend/analysis/analyzers/base.py b/apps/backend/analysis/analyzers/base.py index 5bb604fc..0a7dd4c2 100644 --- a/apps/backend/analysis/analyzers/base.py +++ b/apps/backend/analysis/analyzers/base.py @@ -99,7 +99,7 @@ class BaseAnalyzer: def _read_file(self, path: str) -> str: """Read a file relative to the analyzer's path.""" try: - return (self.path / path).read_text() + return (self.path / path).read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): return "" diff --git a/apps/backend/analysis/analyzers/context/auth_detector.py b/apps/backend/analysis/analyzers/context/auth_detector.py index 65151764..2cf356d7 100644 --- a/apps/backend/analysis/analyzers/context/auth_detector.py +++ b/apps/backend/analysis/analyzers/context/auth_detector.py @@ -126,7 +126,7 @@ class AuthDetector(BaseAnalyzer): for py_file in all_py_files: try: - content = py_file.read_text() + content = py_file.read_text(encoding="utf-8") # Find custom decorators if ( "@require" in content diff --git a/apps/backend/analysis/analyzers/context/jobs_detector.py b/apps/backend/analysis/analyzers/context/jobs_detector.py index 05aba889..282e6cbb 100644 --- a/apps/backend/analysis/analyzers/context/jobs_detector.py +++ b/apps/backend/analysis/analyzers/context/jobs_detector.py @@ -52,7 +52,7 @@ class JobsDetector(BaseAnalyzer): tasks = [] for task_file in celery_files: try: - content = task_file.read_text() + content = task_file.read_text(encoding="utf-8") # Find @celery.task or @shared_task decorators task_pattern = r"@(?:celery\.task|shared_task|app\.task)\s*(?:\([^)]*\))?\s*def\s+(\w+)" task_matches = re.findall(task_pattern, content) diff --git a/apps/backend/analysis/analyzers/context/monitoring_detector.py b/apps/backend/analysis/analyzers/context/monitoring_detector.py index 0175547a..f04d6838 100644 --- a/apps/backend/analysis/analyzers/context/monitoring_detector.py +++ b/apps/backend/analysis/analyzers/context/monitoring_detector.py @@ -77,7 +77,7 @@ class MonitoringDetector(BaseAnalyzer): continue try: - content = file_path.read_text() + content = file_path.read_text(encoding="utf-8") # Look for actual Prometheus imports or usage patterns prometheus_patterns = [ "from prometheus_client import", diff --git a/apps/backend/analysis/analyzers/database_detector.py b/apps/backend/analysis/analyzers/database_detector.py index f4380b9c..21b53479 100644 --- a/apps/backend/analysis/analyzers/database_detector.py +++ b/apps/backend/analysis/analyzers/database_detector.py @@ -52,7 +52,7 @@ class DatabaseDetector(BaseAnalyzer): for file_path in py_files: try: - content = file_path.read_text() + content = file_path.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): continue @@ -119,7 +119,7 @@ class DatabaseDetector(BaseAnalyzer): for file_path in model_files: try: - content = file_path.read_text() + content = file_path.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): continue @@ -168,7 +168,7 @@ class DatabaseDetector(BaseAnalyzer): return models try: - content = schema_file.read_text() + content = schema_file.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): return models @@ -216,7 +216,7 @@ class DatabaseDetector(BaseAnalyzer): for file_path in ts_files: try: - content = file_path.read_text() + content = file_path.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): continue @@ -265,7 +265,7 @@ class DatabaseDetector(BaseAnalyzer): for file_path in schema_files: try: - content = file_path.read_text() + content = file_path.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): continue @@ -295,7 +295,7 @@ class DatabaseDetector(BaseAnalyzer): for file_path in model_files: try: - content = file_path.read_text() + content = file_path.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): continue diff --git a/apps/backend/analysis/analyzers/project_analyzer_module.py b/apps/backend/analysis/analyzers/project_analyzer_module.py index 948d487a..9e8ca7be 100644 --- a/apps/backend/analysis/analyzers/project_analyzer_module.py +++ b/apps/backend/analysis/analyzers/project_analyzer_module.py @@ -287,6 +287,6 @@ class ProjectAnalyzer: def _read_file(self, path: str) -> str: try: - return (self.project_dir / path).read_text() + return (self.project_dir / path).read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): return "" diff --git a/apps/backend/analysis/analyzers/route_detector.py b/apps/backend/analysis/analyzers/route_detector.py index 5442a538..0ff51e74 100644 --- a/apps/backend/analysis/analyzers/route_detector.py +++ b/apps/backend/analysis/analyzers/route_detector.py @@ -66,7 +66,7 @@ class RouteDetector(BaseAnalyzer): for file_path in files_to_check: try: - content = file_path.read_text() + content = file_path.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): continue @@ -130,7 +130,7 @@ class RouteDetector(BaseAnalyzer): for file_path in files_to_check: try: - content = file_path.read_text() + content = file_path.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): continue @@ -179,7 +179,7 @@ class RouteDetector(BaseAnalyzer): for file_path in url_files: try: - content = file_path.read_text() + content = file_path.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): continue @@ -218,7 +218,7 @@ class RouteDetector(BaseAnalyzer): files_to_check = js_files + ts_files for file_path in files_to_check: try: - content = file_path.read_text() + content = file_path.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): continue @@ -283,7 +283,7 @@ class RouteDetector(BaseAnalyzer): route_path = re.sub(r"\[([^\]]+)\]", r":\1", route_path) try: - content = route_file.read_text() + content = route_file.read_text(encoding="utf-8") # Detect exported methods: export async function GET(request) methods = re.findall( r"export\s+(?:async\s+)?function\s+(GET|POST|PUT|DELETE|PATCH)", @@ -348,7 +348,7 @@ class RouteDetector(BaseAnalyzer): for file_path in go_files: try: - content = file_path.read_text() + content = file_path.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): continue @@ -384,7 +384,7 @@ class RouteDetector(BaseAnalyzer): for file_path in rust_files: try: - content = file_path.read_text() + content = file_path.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): continue diff --git a/apps/backend/analysis/ci_discovery.py b/apps/backend/analysis/ci_discovery.py index 8aebd2e9..91025751 100644 --- a/apps/backend/analysis/ci_discovery.py +++ b/apps/backend/analysis/ci_discovery.py @@ -163,7 +163,7 @@ class CIDiscovery: ) try: - content = wf_file.read_text() + content = wf_file.read_text(encoding="utf-8") workflow_data = self._parse_yaml(content) if not workflow_data: @@ -242,7 +242,7 @@ class CIDiscovery: ) try: - content = config_file.read_text() + content = config_file.read_text(encoding="utf-8") data = self._parse_yaml(content) if not data: @@ -312,7 +312,7 @@ class CIDiscovery: ) try: - content = config_file.read_text() + content = config_file.read_text(encoding="utf-8") data = self._parse_yaml(content) if not data: @@ -370,7 +370,7 @@ class CIDiscovery: ) try: - content = jenkinsfile.read_text() + content = jenkinsfile.read_text(encoding="utf-8") # Extract sh commands using regex sh_pattern = re.compile(r'sh\s+[\'"]([^\'"]+)[\'"]') diff --git a/apps/backend/analysis/insight_extractor.py b/apps/backend/analysis/insight_extractor.py index 96b092a6..cd215c0f 100644 --- a/apps/backend/analysis/insight_extractor.py +++ b/apps/backend/analysis/insight_extractor.py @@ -237,7 +237,7 @@ def _get_subtask_description(spec_dir: Path, subtask_id: str) -> str: return f"Subtask: {subtask_id}" try: - with open(plan_file) as f: + with open(plan_file, encoding="utf-8") as f: plan = json.load(f) # Search through phases for the subtask @@ -280,7 +280,7 @@ def _build_extraction_prompt(inputs: dict) -> str: prompt_file = Path(__file__).parent / "prompts" / "insight_extractor.md" if prompt_file.exists(): - base_prompt = prompt_file.read_text() + base_prompt = prompt_file.read_text(encoding="utf-8") else: # Fallback if prompt file missing base_prompt = """Extract structured insights from this coding session. diff --git a/apps/backend/analysis/test_discovery.py b/apps/backend/analysis/test_discovery.py index a8ac6ca8..0ebafa55 100644 --- a/apps/backend/analysis/test_discovery.py +++ b/apps/backend/analysis/test_discovery.py @@ -302,7 +302,7 @@ class TestDiscovery: try: with open(package_json, encoding="utf-8") as f: pkg = json.load(f) - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): return deps = pkg.get("dependencies", {}) @@ -398,7 +398,7 @@ class TestDiscovery: # Check pyproject.toml pyproject = project_dir / "pyproject.toml" if pyproject.exists(): - content = pyproject.read_text() + content = pyproject.read_text(encoding="utf-8") # Check for pytest if "pytest" in content: @@ -418,7 +418,7 @@ class TestDiscovery: # Check requirements.txt requirements = project_dir / "requirements.txt" if requirements.exists(): - content = requirements.read_text().lower() + content = requirements.read_text(encoding="utf-8").lower() if "pytest" in content and not any( f.name == "pytest" for f in result.frameworks ): @@ -496,7 +496,7 @@ class TestDiscovery: if not gemfile.exists(): return - content = gemfile.read_text().lower() + content = gemfile.read_text(encoding="utf-8").lower() if "rspec" in content or (project_dir / ".rspec").exists(): result.frameworks.append( diff --git a/apps/backend/cli/batch_commands.py b/apps/backend/cli/batch_commands.py index 959df5ee..3c76f938 100644 --- a/apps/backend/cli/batch_commands.py +++ b/apps/backend/cli/batch_commands.py @@ -31,7 +31,7 @@ def handle_batch_create_command(batch_file: str, project_dir: str) -> bool: return False try: - with open(batch_path) as f: + with open(batch_path, encoding="utf-8") as f: batch_data = json.load(f) except json.JSONDecodeError as e: print_status(f"Invalid JSON in batch file: {e}", "error") @@ -81,7 +81,7 @@ def handle_batch_create_command(batch_file: str, project_dir: str) -> bool: } req_file = spec_dir / "requirements.json" - with open(req_file, "w") as f: + with open(req_file, "w", encoding="utf-8") as f: json.dump(requirements, f, indent=2, default=str) created_specs.append( @@ -145,7 +145,7 @@ def handle_batch_status_command(project_dir: str) -> bool: if req_file.exists(): try: - with open(req_file) as f: + with open(req_file, encoding="utf-8") as f: req = json.load(f) title = req.get("task_description", title) except json.JSONDecodeError: diff --git a/apps/backend/cli/build_commands.py b/apps/backend/cli/build_commands.py index 99fdb96f..39e2275e 100644 --- a/apps/backend/cli/build_commands.py +++ b/apps/backend/cli/build_commands.py @@ -421,7 +421,7 @@ def _handle_build_interrupt( if human_input: # Save to HUMAN_INPUT.md input_file = spec_dir / "HUMAN_INPUT.md" - input_file.write_text(human_input) + input_file.write_text(human_input, encoding="utf-8") content = [ success(f"{icon(Icons.SUCCESS)} INSTRUCTIONS SAVED"), diff --git a/apps/backend/cli/followup_commands.py b/apps/backend/cli/followup_commands.py index 89e399fb..5ce8d316 100644 --- a/apps/backend/cli/followup_commands.py +++ b/apps/backend/cli/followup_commands.py @@ -121,7 +121,7 @@ def collect_followup_task(spec_dir: Path, max_retries: int = 3) -> str | None: # Expand ~ and resolve path file_path = Path(file_path_str).expanduser().resolve() if file_path.exists(): - followup_task = file_path.read_text().strip() + followup_task = file_path.read_text(encoding="utf-8").strip() if followup_task: print_status( f"Loaded {len(followup_task)} characters from file", @@ -195,7 +195,7 @@ def collect_followup_task(spec_dir: Path, max_retries: int = 3) -> str | None: # Save to FOLLOWUP_REQUEST.md request_file = spec_dir / "FOLLOWUP_REQUEST.md" - request_file.write_text(followup_task) + request_file.write_text(followup_task, encoding="utf-8") # Show confirmation content = [ @@ -285,7 +285,7 @@ def handle_followup_command( # Check for prior follow-ups (for sequential follow-up context) prior_followup_count = 0 try: - with open(plan_file) as f: + with open(plan_file, encoding="utf-8") as f: plan_data = json.load(f) phases = plan_data.get("phases", []) # Count phases that look like follow-up phases (name contains "Follow" or high phase number) diff --git a/apps/backend/cli/input_handlers.py b/apps/backend/cli/input_handlers.py index 0c4c6e9d..6e564015 100644 --- a/apps/backend/cli/input_handlers.py +++ b/apps/backend/cli/input_handlers.py @@ -149,7 +149,7 @@ def read_from_file() -> str | None: # Expand ~ and resolve path file_path = Path(file_path_input).expanduser().resolve() if file_path.exists(): - content = file_path.read_text().strip() + content = file_path.read_text(encoding="utf-8").strip() if content: print_status( f"Loaded {len(content)} characters from file", diff --git a/apps/backend/cli/workspace_commands.py b/apps/backend/cli/workspace_commands.py index 85f9f732..5be776a8 100644 --- a/apps/backend/cli/workspace_commands.py +++ b/apps/backend/cli/workspace_commands.py @@ -182,7 +182,7 @@ def _detect_worktree_base_branch( config_path = worktree_path / ".auto-claude" / "worktree-config.json" if config_path.exists(): try: - config = json.loads(config_path.read_text()) + config = json.loads(config_path.read_text(encoding="utf-8")) if config.get("base_branch"): debug( MODULE, diff --git a/apps/backend/context/builder.py b/apps/backend/context/builder.py index e31aa431..aac2eebe 100644 --- a/apps/backend/context/builder.py +++ b/apps/backend/context/builder.py @@ -37,8 +37,12 @@ class ContextBuilder: """Load project index from file or create new one (.auto-claude is the installed instance).""" index_file = self.project_dir / ".auto-claude" / "project_index.json" if index_file.exists(): - with open(index_file) as f: - return json.load(f) + try: + with open(index_file, encoding="utf-8") as f: + return json.load(f) + except (OSError, json.JSONDecodeError, UnicodeDecodeError): + # Corrupted or legacy-encoded file, regenerate + pass # Try to create one from analyzer import analyze_project @@ -230,7 +234,9 @@ class ContextBuilder: if context_file.exists(): return { "source": "SERVICE_CONTEXT.md", - "content": context_file.read_text()[:2000], # First 2000 chars + "content": context_file.read_text(encoding="utf-8")[ + :2000 + ], # First 2000 chars } # Generate basic context from service info diff --git a/apps/backend/context/main.py b/apps/backend/context/main.py index 46d4b2fc..be9eeb32 100644 --- a/apps/backend/context/main.py +++ b/apps/backend/context/main.py @@ -72,7 +72,7 @@ def build_task_context( if output_file: output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as f: + with open(output_file, "w", encoding="utf-8") as f: json.dump(result, f, indent=2) print(f"Task context saved to: {output_file}") diff --git a/apps/backend/context/pattern_discovery.py b/apps/backend/context/pattern_discovery.py index 2a4f3c23..4983501a 100644 --- a/apps/backend/context/pattern_discovery.py +++ b/apps/backend/context/pattern_discovery.py @@ -38,7 +38,7 @@ class PatternDiscoverer: for match in reference_files[:max_files]: try: file_path = self.project_dir / match.path - content = file_path.read_text(errors="ignore") + content = file_path.read_text(encoding="utf-8", errors="ignore") # Look for common patterns for keyword in keywords: diff --git a/apps/backend/context/search.py b/apps/backend/context/search.py index e39c0a23..98011d4b 100644 --- a/apps/backend/context/search.py +++ b/apps/backend/context/search.py @@ -41,7 +41,7 @@ class CodeSearcher: for file_path in self._iter_code_files(service_path): try: - content = file_path.read_text(errors="ignore") + content = file_path.read_text(encoding="utf-8", errors="ignore") content_lower = content.lower() # Score this file diff --git a/apps/backend/context/serialization.py b/apps/backend/context/serialization.py index e712abbb..4a873b16 100644 --- a/apps/backend/context/serialization.py +++ b/apps/backend/context/serialization.py @@ -41,7 +41,7 @@ def save_context(context: TaskContext, output_file: Path) -> None: output_file: Path to output JSON file """ output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as f: + with open(output_file, "w", encoding="utf-8") as f: json.dump(serialize_context(context), f, indent=2) @@ -55,5 +55,5 @@ def load_context(input_file: Path) -> dict: Returns: Context dictionary """ - with open(input_file) as f: + with open(input_file, encoding="utf-8") as f: return json.load(f) diff --git a/apps/backend/core/client.py b/apps/backend/core/client.py index fc59a500..a8f4a318 100644 --- a/apps/backend/core/client.py +++ b/apps/backend/core/client.py @@ -875,7 +875,7 @@ def create_client( # Write settings to a file in the project directory settings_file = project_dir / ".claude_settings.json" - with open(settings_file, "w") as f: + with open(settings_file, "w", encoding="utf-8") as f: json.dump(security_settings, f, indent=2) print(f"Security settings: {settings_file}") diff --git a/apps/backend/core/debug.py b/apps/backend/core/debug.py index 9bef363d..df9ff4ed 100644 --- a/apps/backend/core/debug.py +++ b/apps/backend/core/debug.py @@ -110,7 +110,7 @@ def _write_log(message: str, to_file: bool = True) -> None: import re clean_message = re.sub(r"\033\[[0-9;]*m", "", message) - with open(log_file, "a") as f: + with open(log_file, "a", encoding="utf-8") as f: f.write(clean_message + "\n") except Exception: pass # Silently fail file logging diff --git a/apps/backend/core/progress.py b/apps/backend/core/progress.py index ce4f7f50..1bbe647a 100644 --- a/apps/backend/core/progress.py +++ b/apps/backend/core/progress.py @@ -43,7 +43,7 @@ def count_subtasks(spec_dir: Path) -> tuple[int, int]: return 0, 0 try: - with open(plan_file) as f: + with open(plan_file, encoding="utf-8") as f: plan = json.load(f) total = 0 @@ -56,7 +56,7 @@ def count_subtasks(spec_dir: Path) -> tuple[int, int]: completed += 1 return completed, total - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): return 0, 0 @@ -81,7 +81,7 @@ def count_subtasks_detailed(spec_dir: Path) -> dict: return result try: - with open(plan_file) as f: + with open(plan_file, encoding="utf-8") as f: plan = json.load(f) for phase in plan.get("phases", []): @@ -94,7 +94,7 @@ def count_subtasks_detailed(spec_dir: Path) -> dict: result["pending"] += 1 return result - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): return result @@ -182,7 +182,7 @@ def print_progress_summary(spec_dir: Path, show_next: bool = True) -> None: # Phase summary try: - with open(spec_dir / "implementation_plan.json") as f: + with open(spec_dir / "implementation_plan.json", encoding="utf-8") as f: plan = json.load(f) print("\nPhases:") @@ -230,8 +230,8 @@ def print_progress_summary(spec_dir: Path, show_next: bool = True) -> None: f" {icon(Icons.ARROW_RIGHT)} Next: {highlight(next_id)} - {next_desc}" ) - except (OSError, json.JSONDecodeError): - pass + except (OSError, json.JSONDecodeError, UnicodeDecodeError): + pass # Ignore corrupted/unreadable progress files else: print() print_status("No implementation subtasks yet - planner needs to run", "pending") @@ -302,7 +302,7 @@ def get_plan_summary(spec_dir: Path) -> dict: } try: - with open(plan_file) as f: + with open(plan_file, encoding="utf-8") as f: plan = json.load(f) summary = { @@ -355,7 +355,7 @@ def get_plan_summary(spec_dir: Path) -> dict: return summary - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): return { "workflow_type": None, "total_phases": 0, @@ -376,7 +376,7 @@ def get_current_phase(spec_dir: Path) -> dict | None: return None try: - with open(plan_file) as f: + with open(plan_file, encoding="utf-8") as f: plan = json.load(f) for phase in plan.get("phases", []): @@ -396,7 +396,7 @@ def get_current_phase(spec_dir: Path) -> dict | None: return None - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): return None @@ -470,7 +470,7 @@ def get_next_subtask(spec_dir: Path) -> dict | None: return None - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): return None diff --git a/apps/backend/core/sentry.py b/apps/backend/core/sentry.py index 50bab867..bd3414ad 100644 --- a/apps/backend/core/sentry.py +++ b/apps/backend/core/sentry.py @@ -53,7 +53,7 @@ def _get_version() -> str: if package_json.exists(): import json - with open(package_json) as f: + with open(package_json, encoding="utf-8") as f: data = json.load(f) return data.get("version", "0.0.0") except Exception as e: diff --git a/apps/backend/core/workspace/models.py b/apps/backend/core/workspace/models.py index 92d2178c..b22381f3 100644 --- a/apps/backend/core/workspace/models.py +++ b/apps/backend/core/workspace/models.py @@ -93,7 +93,7 @@ class MergeLock: os.close(fd) # Write our PID to the lock file - self.lock_file.write_text(str(os.getpid())) + self.lock_file.write_text(str(os.getpid()), encoding="utf-8") self.acquired = True return self @@ -101,7 +101,7 @@ class MergeLock: # Lock file exists - check if process is still running if self.lock_file.exists(): try: - pid = int(self.lock_file.read_text().strip()) + pid = int(self.lock_file.read_text(encoding="utf-8").strip()) # Import locally to avoid circular dependency import os as _os @@ -183,7 +183,7 @@ class SpecNumberLock: os.close(fd) # Write our PID to the lock file - self.lock_file.write_text(str(os.getpid())) + self.lock_file.write_text(str(os.getpid()), encoding="utf-8") self.acquired = True return self @@ -191,7 +191,7 @@ class SpecNumberLock: # Lock file exists - check if process is still running if self.lock_file.exists(): try: - pid = int(self.lock_file.read_text().strip()) + pid = int(self.lock_file.read_text(encoding="utf-8").strip()) import os as _os try: diff --git a/apps/backend/core/workspace/setup.py b/apps/backend/core/workspace/setup.py index 7529d508..b43907d9 100644 --- a/apps/backend/core/workspace/setup.py +++ b/apps/backend/core/workspace/setup.py @@ -412,10 +412,10 @@ def setup_workspace( if PROFILE_FILENAME in security_files_copied: profile_path = worktree_info.path / PROFILE_FILENAME try: - with open(profile_path) as f: + with open(profile_path, encoding="utf-8") as f: profile_data = json.load(f) profile_data["inherited_from"] = str(project_dir.resolve()) - with open(profile_path, "w") as f: + with open(profile_path, "w", encoding="utf-8") as f: json.dump(profile_data, f, indent=2) debug( MODULE, f"Marked security profile as inherited from {project_dir}" @@ -474,7 +474,7 @@ def ensure_timeline_hook_installed(project_dir: Path) -> None: # Handle worktrees (where .git is a file, not directory) if git_dir.is_file(): - content = git_dir.read_text().strip() + content = git_dir.read_text(encoding="utf-8").strip() if content.startswith("gitdir:"): git_dir = Path(content.split(":", 1)[1].strip()) else: @@ -484,7 +484,7 @@ def ensure_timeline_hook_installed(project_dir: Path) -> None: # Check if hook already installed if hook_path.exists(): - if "FileTimelineTracker" in hook_path.read_text(): + if "FileTimelineTracker" in hook_path.read_text(encoding="utf-8"): debug(MODULE, "FileTimelineTracker hook already installed") return @@ -522,7 +522,7 @@ def initialize_timeline_tracking( if source_spec_dir: plan_path = source_spec_dir / "implementation_plan.json" if plan_path.exists(): - with open(plan_path) as f: + with open(plan_path, encoding="utf-8") as f: plan = json.load(f) task_title = plan.get("title", spec_name) task_intent = plan.get("description", "") @@ -533,6 +533,7 @@ def initialize_timeline_tracking( files_to_modify.extend(subtask.get("files", [])) # Get the current branch point commit + # Note: run_git() already handles capture_output and encoding internally result = run_git( ["rev-parse", "HEAD"], cwd=project_dir, diff --git a/apps/backend/ideation/analyzer.py b/apps/backend/ideation/analyzer.py index f4012fea..6f89ea88 100644 --- a/apps/backend/ideation/analyzer.py +++ b/apps/backend/ideation/analyzer.py @@ -51,7 +51,7 @@ class ProjectAnalyzer: project_index_path = self.project_dir / ".auto-claude" / "project_index.json" if project_index_path.exists(): try: - with open(project_index_path) as f: + with open(project_index_path, encoding="utf-8") as f: index = json.load(f) # Extract tech stack from services for service_name, service_info in index.get("services", {}).items(): @@ -70,7 +70,7 @@ class ProjectAnalyzer: ) if roadmap_path.exists(): try: - with open(roadmap_path) as f: + with open(roadmap_path, encoding="utf-8") as f: roadmap = json.load(f) # Extract planned features for feature in roadmap.get("features", []): @@ -87,7 +87,7 @@ class ProjectAnalyzer: ) if discovery_path.exists() and not context["target_audience"]: try: - with open(discovery_path) as f: + with open(discovery_path, encoding="utf-8") as f: discovery = json.load(f) audience = discovery.get("target_audience", {}) context["target_audience"] = audience.get("primary_persona") @@ -109,7 +109,7 @@ class ProjectAnalyzer: spec_file = spec_dir / "spec.md" if spec_file.exists(): # Extract title from spec - content = spec_file.read_text() + content = spec_file.read_text(encoding="utf-8") lines = content.split("\n") for line in lines: if line.startswith("# "): diff --git a/apps/backend/ideation/formatter.py b/apps/backend/ideation/formatter.py index 7ae53e8c..6622bc83 100644 --- a/apps/backend/ideation/formatter.py +++ b/apps/backend/ideation/formatter.py @@ -39,7 +39,7 @@ class IdeationFormatter: existing_session = None if append and ideation_file.exists(): try: - with open(ideation_file) as f: + with open(ideation_file, encoding="utf-8") as f: existing_session = json.load(f) existing_ideas = existing_session.get("ideas", []) print_status( @@ -56,7 +56,7 @@ class IdeationFormatter: type_file = self.output_dir / f"{ideation_type}_ideas.json" if type_file.exists(): try: - with open(type_file) as f: + with open(type_file, encoding="utf-8") as f: data = json.load(f) ideas = data.get(ideation_type, []) new_ideas.extend(ideas) @@ -123,7 +123,7 @@ class IdeationFormatter: ideation_session["summary"]["by_status"].get(idea_status, 0) + 1 ) - with open(ideation_file, "w") as f: + with open(ideation_file, "w", encoding="utf-8") as f: json.dump(ideation_session, f, indent=2) action = "Updated" if append else "Created" @@ -139,7 +139,7 @@ class IdeationFormatter: context_data = {} if context_file.exists(): try: - with open(context_file) as f: + with open(context_file, encoding="utf-8") as f: context_data = json.load(f) except json.JSONDecodeError: pass diff --git a/apps/backend/ideation/generator.py b/apps/backend/ideation/generator.py index dcd34704..3068a68e 100644 --- a/apps/backend/ideation/generator.py +++ b/apps/backend/ideation/generator.py @@ -80,7 +80,7 @@ class IdeationGenerator: return False, f"Prompt not found: {prompt_path}" # Load prompt - prompt = prompt_path.read_text() + prompt = prompt_path.read_text(encoding="utf-8") # Add context prompt += f"\n\n---\n\n**Output Directory**: {self.output_dir}\n" diff --git a/apps/backend/ideation/phase_executor.py b/apps/backend/ideation/phase_executor.py index 991910bb..ec928782 100644 --- a/apps/backend/ideation/phase_executor.py +++ b/apps/backend/ideation/phase_executor.py @@ -85,7 +85,7 @@ class PhaseExecutor: if not is_graphiti_enabled(): print_status("Graphiti not enabled, skipping graph hints", "info") - with open(hints_file, "w") as f: + with open(hints_file, "w", encoding="utf-8") as f: json.dump( { "enabled": False, @@ -131,7 +131,7 @@ class PhaseExecutor: total_hints += len(result) # Save hints - with open(hints_file, "w") as f: + with open(hints_file, "w", encoding="utf-8") as f: json.dump( { "enabled": True, @@ -178,11 +178,11 @@ class PhaseExecutor: graph_hints = {} if hints_file.exists(): try: - with open(hints_file) as f: + with open(hints_file, encoding="utf-8") as f: hints_data = json.load(f) graph_hints = hints_data.get("hints_by_type", {}) - except (OSError, json.JSONDecodeError): - pass + except (OSError, json.JSONDecodeError, UnicodeDecodeError): + pass # Use empty hints if file is corrupted/unreadable # Write context file context_data = { @@ -200,7 +200,7 @@ class PhaseExecutor: "created_at": datetime.now().isoformat(), } - with open(context_file, "w") as f: + with open(context_file, "w", encoding="utf-8") as f: json.dump(context_data, f, indent=2) print_status("Created ideation_context.json", "success") @@ -252,7 +252,7 @@ class PhaseExecutor: if output_file.exists() and not self.refresh: # Load and validate existing ideas - only skip if we have valid ideas try: - with open(output_file) as f: + with open(output_file, encoding="utf-8") as f: data = json.load(f) count = len(data.get(ideation_type, [])) diff --git a/apps/backend/ideation/prioritizer.py b/apps/backend/ideation/prioritizer.py index 1dcad6e7..f1fdc5e5 100644 --- a/apps/backend/ideation/prioritizer.py +++ b/apps/backend/ideation/prioritizer.py @@ -48,7 +48,7 @@ class IdeaPrioritizer: } try: - content = output_file.read_text() + content = output_file.read_text(encoding="utf-8") data = json.loads(content) debug_verbose( "ideation_prioritizer", @@ -102,7 +102,7 @@ class IdeaPrioritizer: return { "success": False, "error": f"Invalid JSON: {e}", - "current_content": output_file.read_text() + "current_content": output_file.read_text(encoding="utf-8") if output_file.exists() else "", "count": 0, diff --git a/apps/backend/ideation/runner.py b/apps/backend/ideation/runner.py index c20d41f8..dfe1e81e 100644 --- a/apps/backend/ideation/runner.py +++ b/apps/backend/ideation/runner.py @@ -232,7 +232,7 @@ class IdeationOrchestrator: """Print summary of ideation generation results.""" ideation_file = self.output_dir / "ideation.json" if ideation_file.exists(): - with open(ideation_file) as f: + with open(ideation_file, encoding="utf-8") as f: ideation = json.load(f) ideas = ideation.get("ideas", []) diff --git a/apps/backend/init.py b/apps/backend/init.py index 19cfda82..b3ed46f9 100644 --- a/apps/backend/init.py +++ b/apps/backend/init.py @@ -57,7 +57,7 @@ def ensure_gitignore_entry(project_dir: Path, entry: str = ".auto-claude/") -> b # Check if .gitignore exists and if entry is already present if gitignore_path.exists(): - content = gitignore_path.read_text() + content = gitignore_path.read_text(encoding="utf-8") lines = content.splitlines() if _entry_exists_in_gitignore(lines, entry): @@ -72,14 +72,14 @@ def ensure_gitignore_entry(project_dir: Path, entry: str = ".auto-claude/") -> b content += "\n# Auto Claude data directory\n" content += entry + "\n" - gitignore_path.write_text(content) + gitignore_path.write_text(content, encoding="utf-8") return True else: # Create new .gitignore with the entry content = "# Auto Claude data directory\n" content += entry + "\n" - gitignore_path.write_text(content) + gitignore_path.write_text(content, encoding="utf-8") return True @@ -177,7 +177,7 @@ def ensure_all_gitignore_entries( # Read existing content or start fresh if gitignore_path.exists(): - content = gitignore_path.read_text() + content = gitignore_path.read_text(encoding="utf-8") lines = content.splitlines() else: content = "" @@ -203,7 +203,7 @@ def ensure_all_gitignore_entries( content += entry + "\n" added_entries.append(entry) - gitignore_path.write_text(content) + gitignore_path.write_text(content, encoding="utf-8") # Auto-commit if requested and entries were added if auto_commit and added_entries: diff --git a/apps/backend/integrations/graphiti/config.py b/apps/backend/integrations/graphiti/config.py index 71b538cd..45016f96 100644 --- a/apps/backend/integrations/graphiti/config.py +++ b/apps/backend/integrations/graphiti/config.py @@ -507,7 +507,7 @@ class GraphitiState: def save(self, spec_dir: Path) -> None: """Save state to the spec directory.""" marker_file = spec_dir / GRAPHITI_STATE_MARKER - with open(marker_file, "w") as f: + with open(marker_file, "w", encoding="utf-8") as f: json.dump(self.to_dict(), f, indent=2) @classmethod @@ -518,9 +518,9 @@ class GraphitiState: return None try: - with open(marker_file) as f: + with open(marker_file, encoding="utf-8") as f: return cls.from_dict(json.load(f)) - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): return None def record_error(self, error_msg: str) -> None: diff --git a/apps/backend/integrations/linear/config.py b/apps/backend/integrations/linear/config.py index 25bd149f..ae60b4a9 100644 --- a/apps/backend/integrations/linear/config.py +++ b/apps/backend/integrations/linear/config.py @@ -126,7 +126,7 @@ class LinearProjectState: def save(self, spec_dir: Path) -> None: """Save state to the spec directory.""" marker_file = spec_dir / LINEAR_PROJECT_MARKER - with open(marker_file, "w") as f: + with open(marker_file, "w", encoding="utf-8") as f: json.dump(self.to_dict(), f, indent=2) @classmethod @@ -137,9 +137,9 @@ class LinearProjectState: return None try: - with open(marker_file) as f: + with open(marker_file, encoding="utf-8") as f: return cls.from_dict(json.load(f)) - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): return None diff --git a/apps/backend/integrations/linear/integration.py b/apps/backend/integrations/linear/integration.py index a31b98f2..3559083d 100644 --- a/apps/backend/integrations/linear/integration.py +++ b/apps/backend/integrations/linear/integration.py @@ -158,9 +158,9 @@ class LinearManager: return None try: - with open(plan_file) as f: + with open(plan_file, encoding="utf-8") as f: return json.load(f) - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): return None def get_subtasks_for_sync(self) -> list[dict]: diff --git a/apps/backend/integrations/linear/updater.py b/apps/backend/integrations/linear/updater.py index 02d3880c..16431460 100644 --- a/apps/backend/integrations/linear/updater.py +++ b/apps/backend/integrations/linear/updater.py @@ -81,7 +81,7 @@ class LinearTaskState: def save(self, spec_dir: Path) -> None: """Save state to the spec directory.""" state_file = spec_dir / LINEAR_TASK_FILE - with open(state_file, "w") as f: + with open(state_file, "w", encoding="utf-8") as f: json.dump(self.to_dict(), f, indent=2) @classmethod @@ -92,9 +92,9 @@ class LinearTaskState: return None try: - with open(state_file) as f: + with open(state_file, encoding="utf-8") as f: return cls.from_dict(json.load(f)) - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): return None diff --git a/apps/backend/memory/codebase_map.py b/apps/backend/memory/codebase_map.py index 198a28a3..689d5203 100644 --- a/apps/backend/memory/codebase_map.py +++ b/apps/backend/memory/codebase_map.py @@ -38,9 +38,9 @@ def update_codebase_map(spec_dir: Path, discoveries: dict[str, str]) -> None: # Load existing map or create new if map_file.exists(): try: - with open(map_file) as f: + with open(map_file, encoding="utf-8") as f: codebase_map = json.load(f) - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): codebase_map = {} else: codebase_map = {} @@ -58,7 +58,7 @@ def update_codebase_map(spec_dir: Path, discoveries: dict[str, str]) -> None: ) # Write back - with open(map_file, "w") as f: + with open(map_file, "w", encoding="utf-8") as f: json.dump(codebase_map, f, indent=2, sort_keys=True) # Also save to Graphiti if enabled @@ -90,12 +90,12 @@ def load_codebase_map(spec_dir: Path) -> dict[str, str]: return {} try: - with open(map_file) as f: + with open(map_file, encoding="utf-8") as f: codebase_map = json.load(f) # Remove metadata before returning codebase_map.pop("_metadata", None) return codebase_map - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): return {} diff --git a/apps/backend/memory/patterns.py b/apps/backend/memory/patterns.py index e1d87267..208d63f4 100644 --- a/apps/backend/memory/patterns.py +++ b/apps/backend/memory/patterns.py @@ -36,7 +36,7 @@ def append_gotcha(spec_dir: Path, gotcha: str) -> None: # Load existing gotchas existing_gotchas = set() if gotchas_file.exists(): - content = gotchas_file.read_text() + content = gotchas_file.read_text(encoding="utf-8") # Extract bullet points for line in content.split("\n"): line = line.strip() @@ -47,7 +47,7 @@ def append_gotcha(spec_dir: Path, gotcha: str) -> None: gotcha_stripped = gotcha.strip() if gotcha_stripped and gotcha_stripped not in existing_gotchas: # Append to file - with open(gotchas_file, "a") as f: + with open(gotchas_file, "a", encoding="utf-8") as f: if gotchas_file.stat().st_size == 0: # First entry - add header f.write("# Gotchas and Pitfalls\n\n") @@ -80,7 +80,7 @@ def load_gotchas(spec_dir: Path) -> list[str]: if not gotchas_file.exists(): return [] - content = gotchas_file.read_text() + content = gotchas_file.read_text(encoding="utf-8") gotchas = [] for line in content.split("\n"): @@ -112,7 +112,7 @@ def append_pattern(spec_dir: Path, pattern: str) -> None: # Load existing patterns existing_patterns = set() if patterns_file.exists(): - content = patterns_file.read_text() + content = patterns_file.read_text(encoding="utf-8") # Extract bullet points for line in content.split("\n"): line = line.strip() @@ -123,7 +123,7 @@ def append_pattern(spec_dir: Path, pattern: str) -> None: pattern_stripped = pattern.strip() if pattern_stripped and pattern_stripped not in existing_patterns: # Append to file - with open(patterns_file, "a") as f: + with open(patterns_file, "a", encoding="utf-8") as f: if patterns_file.stat().st_size == 0: # First entry - add header f.write("# Code Patterns\n\n") @@ -156,7 +156,7 @@ def load_patterns(spec_dir: Path) -> list[str]: if not patterns_file.exists(): return [] - content = patterns_file.read_text() + content = patterns_file.read_text(encoding="utf-8") patterns = [] for line in content.split("\n"): diff --git a/apps/backend/memory/sessions.py b/apps/backend/memory/sessions.py index 3bd7fdb6..c72b5844 100644 --- a/apps/backend/memory/sessions.py +++ b/apps/backend/memory/sessions.py @@ -76,7 +76,7 @@ def save_session_insights( } # Write to file (always use file-based storage) - with open(session_file, "w") as f: + with open(session_file, "w", encoding="utf-8") as f: json.dump(session_data, f, indent=2) # Also save to Graphiti if enabled (non-blocking, errors logged but not raised) @@ -110,9 +110,9 @@ def load_all_insights(spec_dir: Path) -> list[dict[str, Any]]: insights = [] for session_file in session_files: try: - with open(session_file) as f: + with open(session_file, encoding="utf-8") as f: insights.append(json.load(f)) - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): # Skip corrupted files continue diff --git a/apps/backend/merge/file_evolution/storage.py b/apps/backend/merge/file_evolution/storage.py index a8998f6f..1ca28305 100644 --- a/apps/backend/merge/file_evolution/storage.py +++ b/apps/backend/merge/file_evolution/storage.py @@ -61,7 +61,7 @@ class EvolutionStorage: return {} try: - with open(self.evolution_file) as f: + with open(self.evolution_file, encoding="utf-8") as f: data = json.load(f) evolutions = {} @@ -88,7 +88,7 @@ class EvolutionStorage: for file_path, evolution in evolutions.items() } - with open(self.evolution_file, "w") as f: + with open(self.evolution_file, "w", encoding="utf-8") as f: json.dump(data, f, indent=2) logger.debug(f"Saved evolution data for {len(evolutions)} files") diff --git a/apps/backend/merge/file_merger.py b/apps/backend/merge/file_merger.py index 7fc3c35d..5bc7f358 100644 --- a/apps/backend/merge/file_merger.py +++ b/apps/backend/merge/file_merger.py @@ -83,10 +83,16 @@ def apply_single_task_changes( # Addition - need to determine where to add if change.change_type == ChangeType.ADD_IMPORT: # Add import at top + # Content is already normalized to LF, so only check for \n + has_trailing_newline = content.endswith("\n") lines = content.splitlines() import_end = find_import_end(lines, file_path) - lines.insert(import_end, change.content_after) + # Strip trailing newline from content_after to prevent double newlines + # (content_after may include newline from diff generation) + lines.insert(import_end, change.content_after.rstrip("\n\r")) content = line_ending.join(lines) + if has_trailing_newline: + content += line_ending elif change.change_type == ChangeType.ADD_FUNCTION: # Add function at end (before exports) content += f"{line_ending}{line_ending}{change.content_after}" @@ -149,13 +155,21 @@ def combine_non_conflicting_changes( # Add imports if imports: + # Content is already normalized to LF, so only check for \n + has_trailing_newline = content.endswith("\n") lines = content.splitlines() import_end = find_import_end(lines, file_path) for imp in imports: - if imp.content_after and imp.content_after not in content: - lines.insert(import_end, imp.content_after) + # Strip trailing newline from content_after to prevent double newlines + import_content = ( + imp.content_after.rstrip("\n\r") if imp.content_after else "" + ) + if import_content and import_content not in content: + lines.insert(import_end, import_content) import_end += 1 content = line_ending.join(lines) + if has_trailing_newline: + content += line_ending # Apply modifications for mod in modifications: diff --git a/apps/backend/merge/install_hook.py b/apps/backend/merge/install_hook.py index c9288d98..fd04eb6d 100644 --- a/apps/backend/merge/install_hook.py +++ b/apps/backend/merge/install_hook.py @@ -75,7 +75,7 @@ def install_hook(project_path: Path) -> bool: # Handle worktrees (where .git is a file, not directory) if git_dir.is_file(): # Read the gitdir from the file - content = git_dir.read_text().strip() + content = git_dir.read_text(encoding="utf-8").strip() if content.startswith("gitdir:"): git_dir = Path(content.split(":", 1)[1].strip()) else: @@ -93,7 +93,7 @@ def install_hook(project_path: Path) -> bool: # Check if hook already exists if hook_path.exists(): - existing = hook_path.read_text() + existing = hook_path.read_text(encoding="utf-8") if "FileTimelineTracker" in existing: print(f"Hook already installed at {hook_path}") return True @@ -104,13 +104,13 @@ def install_hook(project_path: Path) -> bool: print(f"Backed up existing hook to {backup_path}") # Append our hook to existing - with open(hook_path, "a") as f: + with open(hook_path, "a", encoding="utf-8") as f: f.write("\n\n# FileTimelineTracker integration\n") f.write(HOOK_SCRIPT.split("#!/bin/bash", 1)[1]) # Skip shebang print(f"Appended FileTimelineTracker hook to {hook_path}") else: # Write new hook - hook_path.write_text(HOOK_SCRIPT) + hook_path.write_text(HOOK_SCRIPT, encoding="utf-8") print(f"Created new hook at {hook_path}") # Make executable @@ -127,7 +127,7 @@ def uninstall_hook(project_path: Path) -> bool: git_dir = project_path / ".git" if git_dir.is_file(): - content = git_dir.read_text().strip() + content = git_dir.read_text(encoding="utf-8").strip() if content.startswith("gitdir:"): git_dir = Path(content.split(":", 1)[1].strip()) @@ -137,7 +137,7 @@ def uninstall_hook(project_path: Path) -> bool: print("No hook to uninstall") return True - content = hook_path.read_text() + content = hook_path.read_text(encoding="utf-8") if "FileTimelineTracker" not in content: print("Hook does not contain FileTimelineTracker integration") return True diff --git a/apps/backend/merge/models.py b/apps/backend/merge/models.py index 447dcc7a..6d9658f5 100644 --- a/apps/backend/merge/models.py +++ b/apps/backend/merge/models.py @@ -108,5 +108,5 @@ class MergeReport: def save(self, path: Path) -> None: """Save report to JSON file.""" - with open(path, "w") as f: + with open(path, "w", encoding="utf-8") as f: json.dump(self.to_dict(), f, indent=2) diff --git a/apps/backend/merge/timeline_persistence.py b/apps/backend/merge/timeline_persistence.py index afe29352..ceed5bd7 100644 --- a/apps/backend/merge/timeline_persistence.py +++ b/apps/backend/merge/timeline_persistence.py @@ -71,13 +71,13 @@ class TimelinePersistence: return timelines try: - with open(index_path) as f: + with open(index_path, encoding="utf-8") as f: index = json.load(f) for file_path in index.get("files", []): timeline_file = self._get_timeline_file_path(file_path) if timeline_file.exists(): - with open(timeline_file) as f: + with open(timeline_file, encoding="utf-8") as f: data = json.load(f) timelines[file_path] = FileTimeline.from_dict(data) @@ -101,7 +101,7 @@ class TimelinePersistence: timeline_file = self._get_timeline_file_path(file_path) timeline_file.parent.mkdir(parents=True, exist_ok=True) - with open(timeline_file, "w") as f: + with open(timeline_file, "w", encoding="utf-8") as f: json.dump(timeline.to_dict(), f, indent=2) except Exception as e: @@ -119,7 +119,7 @@ class TimelinePersistence: "files": file_paths, "last_updated": datetime.now().isoformat(), } - with open(index_path, "w") as f: + with open(index_path, "w", encoding="utf-8") as f: json.dump(index, f, indent=2) def _get_timeline_file_path(self, file_path: str) -> Path: diff --git a/apps/backend/phase_config.py b/apps/backend/phase_config.py index e196613e..41af2d81 100644 --- a/apps/backend/phase_config.py +++ b/apps/backend/phase_config.py @@ -164,7 +164,7 @@ def load_task_metadata(spec_dir: Path) -> TaskMetadataConfig | None: return None try: - with open(metadata_path) as f: + with open(metadata_path, encoding="utf-8") as f: return json.load(f) except (json.JSONDecodeError, OSError): return None diff --git a/apps/backend/planner_lib/context.py b/apps/backend/planner_lib/context.py index ef2cdb28..31e6fcd1 100644 --- a/apps/backend/planner_lib/context.py +++ b/apps/backend/planner_lib/context.py @@ -41,21 +41,29 @@ class ContextLoader: """Load all context files from spec directory.""" # Read spec.md spec_file = self.spec_dir / "spec.md" - spec_content = spec_file.read_text() if spec_file.exists() else "" + spec_content = ( + spec_file.read_text(encoding="utf-8") if spec_file.exists() else "" + ) # Read project_index.json index_file = self.spec_dir / "project_index.json" project_index = {} if index_file.exists(): - with open(index_file) as f: - project_index = json.load(f) + try: + with open(index_file, encoding="utf-8") as f: + project_index = json.load(f) + except (OSError, json.JSONDecodeError, UnicodeDecodeError): + pass # Use empty dict on error # Read context.json context_file = self.spec_dir / "context.json" task_context = {} if context_file.exists(): - with open(context_file) as f: - task_context = json.load(f) + try: + with open(context_file, encoding="utf-8") as f: + task_context = json.load(f) + except (OSError, json.JSONDecodeError, UnicodeDecodeError): + pass # Use empty dict on error # Determine services involved services = task_context.get("scoped_services", []) @@ -89,7 +97,7 @@ class ContextLoader: requirements_file = self.spec_dir / "requirements.json" if requirements_file.exists(): try: - with open(requirements_file) as f: + with open(requirements_file, encoding="utf-8") as f: requirements = json.load(f) declared_type = _normalize_workflow_type( requirements.get("workflow_type", "") @@ -103,7 +111,7 @@ class ContextLoader: assessment_file = self.spec_dir / "complexity_assessment.json" if assessment_file.exists(): try: - with open(assessment_file) as f: + with open(assessment_file, encoding="utf-8") as f: assessment = json.load(f) declared_type = _normalize_workflow_type( assessment.get("workflow_type", "") diff --git a/apps/backend/prediction/main.py b/apps/backend/prediction/main.py index f5a9d26a..674f3a74 100644 --- a/apps/backend/prediction/main.py +++ b/apps/backend/prediction/main.py @@ -52,7 +52,7 @@ def main(): print(f"Error: No implementation_plan.json found in {spec_dir}") sys.exit(1) - with open(plan_file) as f: + with open(plan_file, encoding="utf-8") as f: plan = json.load(f) # Find first pending subtask diff --git a/apps/backend/prediction/memory_loader.py b/apps/backend/prediction/memory_loader.py index a55b8204..6c0ff06d 100644 --- a/apps/backend/prediction/memory_loader.py +++ b/apps/backend/prediction/memory_loader.py @@ -33,7 +33,7 @@ class MemoryLoader: return [] gotchas = [] - content = self.gotchas_file.read_text() + content = self.gotchas_file.read_text(encoding="utf-8") # Parse markdown list items for line in content.split("\n"): @@ -56,7 +56,7 @@ class MemoryLoader: return [] patterns = [] - content = self.patterns_file.read_text() + content = self.patterns_file.read_text(encoding="utf-8") # Parse markdown sections current_pattern = None @@ -89,8 +89,8 @@ class MemoryLoader: return [] try: - with open(self.history_file) as f: + with open(self.history_file, encoding="utf-8") as f: history = json.load(f) return history.get("attempts", []) - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): return [] diff --git a/apps/backend/project/analyzer.py b/apps/backend/project/analyzer.py index 6054f8f5..89b19117 100644 --- a/apps/backend/project/analyzer.py +++ b/apps/backend/project/analyzer.py @@ -69,7 +69,7 @@ class ProjectAnalyzer: return None try: - with open(profile_path) as f: + with open(profile_path, encoding="utf-8") as f: data = json.load(f) return SecurityProfile.from_dict(data) except (OSError, json.JSONDecodeError, KeyError): @@ -80,7 +80,7 @@ class ProjectAnalyzer: profile_path = self.get_profile_path() profile_path.parent.mkdir(parents=True, exist_ok=True) - with open(profile_path, "w") as f: + with open(profile_path, "w", encoding="utf-8") as f: json.dump(profile.to_dict(), f, indent=2) def compute_project_hash(self) -> str: diff --git a/apps/backend/project/config_parser.py b/apps/backend/project/config_parser.py index ee457051..80234877 100644 --- a/apps/backend/project/config_parser.py +++ b/apps/backend/project/config_parser.py @@ -38,7 +38,7 @@ class ConfigParser: def read_json(self, filename: str) -> dict | None: """Read a JSON file from project root.""" try: - with open(self.project_dir / filename) as f: + with open(self.project_dir / filename, encoding="utf-8") as f: return json.load(f) except (FileNotFoundError, json.JSONDecodeError): return None @@ -59,7 +59,7 @@ class ConfigParser: def read_text(self, filename: str) -> str | None: """Read a text file from project root.""" try: - with open(self.project_dir / filename) as f: + with open(self.project_dir / filename, encoding="utf-8") as f: return f.read() except (OSError, FileNotFoundError): return None diff --git a/apps/backend/project/stack_detector.py b/apps/backend/project/stack_detector.py index 0fa67c29..71ac3847 100644 --- a/apps/backend/project/stack_detector.py +++ b/apps/backend/project/stack_detector.py @@ -248,7 +248,7 @@ class StackDetector: "**/*.yaml" ) + self.parser.glob_files("**/*.yml"): try: - with open(yaml_file) as f: + with open(yaml_file, encoding="utf-8") as f: content = f.read() if "apiVersion:" in content and "kind:" in content: self.stack.infrastructure.append("kubernetes") diff --git a/apps/backend/prompts/coder.md b/apps/backend/prompts/coder.md index 8b0acd9e..616a3aae 100644 --- a/apps/backend/prompts/coder.md +++ b/apps/backend/prompts/coder.md @@ -942,13 +942,13 @@ if insights["discoveries"]["patterns_found"]: # Load existing patterns existing_patterns = set() if patterns_file.exists(): - content = patterns_file.read_text() + content = patterns_file.read_text(encoding="utf-8") for line in content.split("\n"): if line.strip().startswith("- "): existing_patterns.add(line.strip()[2:]) # Add new patterns - with open(patterns_file, "a") as f: + with open(patterns_file, "a", encoding="utf-8") as f: if patterns_file.stat().st_size == 0: f.write("# Code Patterns\n\n") f.write("Established patterns to follow in this codebase:\n\n") @@ -965,13 +965,13 @@ if insights["discoveries"]["gotchas_encountered"]: # Load existing gotchas existing_gotchas = set() if gotchas_file.exists(): - content = gotchas_file.read_text() + content = gotchas_file.read_text(encoding="utf-8") for line in content.split("\n"): if line.strip().startswith("- "): existing_gotchas.add(line.strip()[2:]) # Add new gotchas - with open(gotchas_file, "a") as f: + with open(gotchas_file, "a", encoding="utf-8") as f: if gotchas_file.stat().st_size == 0: f.write("# Gotchas and Pitfalls\n\n") f.write("Things to watch out for in this codebase:\n\n") diff --git a/apps/backend/prompts_pkg/prompt_generator.py b/apps/backend/prompts_pkg/prompt_generator.py index 8f037299..cd632b2a 100644 --- a/apps/backend/prompts_pkg/prompt_generator.py +++ b/apps/backend/prompts_pkg/prompt_generator.py @@ -336,7 +336,7 @@ def load_subtask_context( full_path = project_dir / pattern_path if full_path.exists(): try: - lines = full_path.read_text().split("\n") + lines = full_path.read_text(encoding="utf-8").split("\n") if len(lines) > max_file_lines: content = "\n".join(lines[:max_file_lines]) content += ( @@ -353,7 +353,7 @@ def load_subtask_context( full_path = project_dir / file_path if full_path.exists(): try: - lines = full_path.read_text().split("\n") + lines = full_path.read_text(encoding="utf-8").split("\n") if len(lines) > max_file_lines: content = "\n".join(lines[:max_file_lines]) content += ( diff --git a/apps/backend/prompts_pkg/prompts.py b/apps/backend/prompts_pkg/prompts.py index dbccc74c..80ce5e43 100644 --- a/apps/backend/prompts_pkg/prompts.py +++ b/apps/backend/prompts_pkg/prompts.py @@ -173,7 +173,7 @@ def get_planner_prompt(spec_dir: Path) -> str: "Make sure the auto-claude/prompts/planner.md file exists." ) - prompt = prompt_file.read_text() + prompt = prompt_file.read_text(encoding="utf-8") # Inject spec directory information at the beginning spec_context = f"""## SPEC LOCATION @@ -216,7 +216,7 @@ def get_coding_prompt(spec_dir: Path) -> str: "Make sure the auto-claude/prompts/coder.md file exists." ) - prompt = prompt_file.read_text() + prompt = prompt_file.read_text(encoding="utf-8") spec_context = f"""## SPEC LOCATION @@ -240,7 +240,7 @@ The project root is the parent of auto-claude/. All code goes in the project roo # Check for human input file human_input_file = spec_dir / "HUMAN_INPUT.md" if human_input_file.exists(): - human_input = human_input_file.read_text().strip() + human_input = human_input_file.read_text(encoding="utf-8").strip() if human_input: spec_context += f"""## HUMAN INPUT (READ THIS FIRST!) @@ -275,7 +275,7 @@ def _get_recovery_context(spec_dir: Path) -> str: return "" try: - with open(attempt_history_file) as f: + with open(attempt_history_file, encoding="utf-8") as f: history = json.load(f) # Check for stuck subtasks @@ -321,7 +321,7 @@ Subtasks with previous attempts: return "" - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): return "" @@ -344,7 +344,7 @@ def get_followup_planner_prompt(spec_dir: Path) -> str: "Make sure the auto-claude/prompts/followup_planner.md file exists." ) - prompt = prompt_file.read_text() + prompt = prompt_file.read_text(encoding="utf-8") # Inject spec directory information at the beginning spec_context = f"""## SPEC LOCATION (FOLLOW-UP MODE) @@ -405,7 +405,7 @@ def is_first_run(spec_dir: Path) -> bool: # Check if any phase has subtasks total_subtasks = sum(len(phase.get("subtasks", [])) for phase in phases) return total_subtasks == 0 - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): # If we can't read the file, treat as first run return True @@ -426,7 +426,7 @@ def _load_prompt_file(filename: str) -> str: prompt_file = PROMPTS_DIR / filename if not prompt_file.exists(): raise FileNotFoundError(f"Prompt file not found: {prompt_file}") - return prompt_file.read_text() + return prompt_file.read_text(encoding="utf-8") def get_qa_reviewer_prompt(spec_dir: Path, project_dir: Path) -> str: diff --git a/apps/backend/qa/criteria.py b/apps/backend/qa/criteria.py index 1cada7f6..d58906d2 100644 --- a/apps/backend/qa/criteria.py +++ b/apps/backend/qa/criteria.py @@ -21,9 +21,9 @@ def load_implementation_plan(spec_dir: Path) -> dict | None: if not plan_file.exists(): return None try: - with open(plan_file) as f: + with open(plan_file, encoding="utf-8") as f: return json.load(f) - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): return None @@ -31,7 +31,7 @@ def save_implementation_plan(spec_dir: Path, plan: dict) -> bool: """Save the implementation plan JSON.""" plan_file = spec_dir / "implementation_plan.json" try: - with open(plan_file, "w") as f: + with open(plan_file, "w", encoding="utf-8") as f: json.dump(plan, f, indent=2) return True except OSError: diff --git a/apps/backend/qa/fixer.py b/apps/backend/qa/fixer.py index 163d27a4..f898add1 100644 --- a/apps/backend/qa/fixer.py +++ b/apps/backend/qa/fixer.py @@ -38,7 +38,7 @@ def load_qa_fixer_prompt() -> str: prompt_file = QA_PROMPTS_DIR / "qa_fixer.md" if not prompt_file.exists(): raise FileNotFoundError(f"QA fixer prompt not found: {prompt_file}") - return prompt_file.read_text() + return prompt_file.read_text(encoding="utf-8") # ============================================================================= diff --git a/apps/backend/qa/report.py b/apps/backend/qa/report.py index 6c841b46..f5d96652 100644 --- a/apps/backend/qa/report.py +++ b/apps/backend/qa/report.py @@ -324,7 +324,7 @@ These issues have appeared {RECURRING_ISSUE_THRESHOLD}+ times without being reso - `implementation_plan.json` - Full iteration history """ - escalation_file.write_text(content) + escalation_file.write_text(content, encoding="utf-8") print(f"\n📝 Escalation file created: {escalation_file}") @@ -345,7 +345,7 @@ def create_manual_test_plan(spec_dir: Path, spec_name: str) -> Path: spec_file = spec_dir / "spec.md" spec_content = "" if spec_file.exists(): - spec_content = spec_file.read_text() + spec_content = spec_file.read_text(encoding="utf-8") # Extract acceptance criteria from spec if present acceptance_criteria = [] @@ -439,7 +439,7 @@ _Add any observations or issues found during testing_ """ - manual_plan_file.write_text(content) + manual_plan_file.write_text(content, encoding="utf-8") return manual_plan_file @@ -460,9 +460,9 @@ def check_test_discovery(spec_dir: Path) -> dict[str, Any] | None: return None try: - with open(discovery_file) as f: + with open(discovery_file, encoding="utf-8") as f: return json.load(f) - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): return None diff --git a/apps/backend/review/formatters.py b/apps/backend/review/formatters.py index 5461c693..360b1316 100644 --- a/apps/backend/review/formatters.py +++ b/apps/backend/review/formatters.py @@ -159,7 +159,7 @@ def display_plan_summary(spec_dir: Path) -> None: return try: - with open(plan_file) as f: + with open(plan_file, encoding="utf-8") as f: plan = json.load(f) except (OSError, json.JSONDecodeError) as e: print_status(f"Could not read implementation_plan.json: {e}", "error") diff --git a/apps/backend/review/state.py b/apps/backend/review/state.py index cd536bc5..fa1b976d 100644 --- a/apps/backend/review/state.py +++ b/apps/backend/review/state.py @@ -85,7 +85,7 @@ class ReviewState: def save(self, spec_dir: Path) -> None: """Save state to the spec directory.""" state_file = Path(spec_dir) / REVIEW_STATE_FILE - with open(state_file, "w") as f: + with open(state_file, "w", encoding="utf-8") as f: json.dump(self.to_dict(), f, indent=2) @classmethod @@ -100,9 +100,9 @@ class ReviewState: return cls() try: - with open(state_file) as f: + with open(state_file, encoding="utf-8") as f: return cls.from_dict(json.load(f)) - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): return cls() def is_approved(self) -> bool: diff --git a/apps/backend/runners/ai_analyzer/cache_manager.py b/apps/backend/runners/ai_analyzer/cache_manager.py index b0be6a90..9ae74a6a 100644 --- a/apps/backend/runners/ai_analyzer/cache_manager.py +++ b/apps/backend/runners/ai_analyzer/cache_manager.py @@ -48,7 +48,7 @@ class CacheManager: return None print(f"✓ Using cached AI insights ({hours_old:.1f} hours old)") - return json.loads(self.cache_file.read_text()) + return json.loads(self.cache_file.read_text(encoding="utf-8")) def save_result(self, result: dict[str, Any]) -> None: """ @@ -57,5 +57,5 @@ class CacheManager: Args: result: Analysis result to cache """ - self.cache_file.write_text(json.dumps(result, indent=2)) + self.cache_file.write_text(json.dumps(result, indent=2), encoding="utf-8") print(f"\n✓ AI insights cached to: {self.cache_file}") diff --git a/apps/backend/runners/ai_analyzer/claude_client.py b/apps/backend/runners/ai_analyzer/claude_client.py index 5d3f0712..840f1101 100644 --- a/apps/backend/runners/ai_analyzer/claude_client.py +++ b/apps/backend/runners/ai_analyzer/claude_client.py @@ -87,7 +87,7 @@ class ClaudeAnalysisClient: } settings_file = self.project_dir / ".claude_ai_analyzer_settings.json" - with open(settings_file, "w") as f: + with open(settings_file, "w", encoding="utf-8") as f: json.dump(settings, f, indent=2) return settings_file diff --git a/apps/backend/runners/ai_analyzer_runner.py b/apps/backend/runners/ai_analyzer_runner.py index 62ea6c16..1a14f89a 100644 --- a/apps/backend/runners/ai_analyzer_runner.py +++ b/apps/backend/runners/ai_analyzer_runner.py @@ -56,7 +56,7 @@ def main() -> int: print(f"Run: python analyzer.py --project-dir {args.project_dir} --index") return 1 - project_index = json.loads(index_path.read_text()) + project_index = json.loads(index_path.read_text(encoding="utf-8")) # Import here to avoid import errors if dependencies are missing try: diff --git a/apps/backend/runners/github/audit.py b/apps/backend/runners/github/audit.py index 4f0172fa..9a482c89 100644 --- a/apps/backend/runners/github/audit.py +++ b/apps/backend/runners/github/audit.py @@ -369,7 +369,7 @@ class AuditLogger: try: log_file = self._get_log_file_path() - with open(log_file, "a") as f: + with open(log_file, "a", encoding="utf-8") as f: f.write(entry.to_json() + "\n") except Exception as e: logger.error(f"Failed to write audit log: {e}") @@ -585,7 +585,7 @@ class AuditLogger: for log_file in sorted(self.log_dir.glob("audit_*.jsonl"), reverse=True): try: - with open(log_file) as f: + with open(log_file, encoding="utf-8") as f: for line in f: if not line.strip(): continue diff --git a/apps/backend/runners/github/batch_issues.py b/apps/backend/runners/github/batch_issues.py index 59afed30..6429a60a 100644 --- a/apps/backend/runners/github/batch_issues.py +++ b/apps/backend/runners/github/batch_issues.py @@ -368,7 +368,7 @@ class IssueBatch: if not batch_file.exists(): return None - with open(batch_file) as f: + with open(batch_file, encoding="utf-8") as f: data = json.load(f) return cls.from_dict(data) @@ -448,7 +448,7 @@ class IssueBatcher: """Load batch index from disk.""" index_file = self.github_dir / "batches" / "index.json" if index_file.exists(): - with open(index_file) as f: + with open(index_file, encoding="utf-8") as f: data = json.load(f) self._batch_index = { int(k): v for k, v in data.get("issue_to_batch", {}).items() @@ -460,7 +460,7 @@ class IssueBatcher: batches_dir.mkdir(parents=True, exist_ok=True) index_file = batches_dir / "index.json" - with open(index_file, "w") as f: + with open(index_file, "w", encoding="utf-8") as f: json.dump( { "issue_to_batch": self._batch_index, @@ -1107,7 +1107,7 @@ class IssueBatcher: batches = [] for batch_file in batches_dir.glob("batch_*.json"): try: - with open(batch_file) as f: + with open(batch_file, encoding="utf-8") as f: data = json.load(f) batches.append(IssueBatch.from_dict(data)) except Exception as e: diff --git a/apps/backend/runners/github/batch_validator.py b/apps/backend/runners/github/batch_validator.py index 3bb2455d..39ccc329 100644 --- a/apps/backend/runners/github/batch_validator.py +++ b/apps/backend/runners/github/batch_validator.py @@ -230,7 +230,7 @@ class BatchValidator: } settings_file = self.project_dir / ".batch_validator_settings.json" - with open(settings_file, "w") as f: + with open(settings_file, "w", encoding="utf-8") as f: json.dump(settings, f) try: diff --git a/apps/backend/runners/github/bot_detection.py b/apps/backend/runners/github/bot_detection.py index 5d0146e5..46186821 100644 --- a/apps/backend/runners/github/bot_detection.py +++ b/apps/backend/runners/github/bot_detection.py @@ -85,7 +85,7 @@ class BotDetectionState: if not state_file.exists(): return cls() - with open(state_file) as f: + with open(state_file, encoding="utf-8") as f: return cls.from_dict(json.load(f)) diff --git a/apps/backend/runners/github/cleanup.py b/apps/backend/runners/github/cleanup.py index 0accd67b..27fddf57 100644 --- a/apps/backend/runners/github/cleanup.py +++ b/apps/backend/runners/github/cleanup.py @@ -248,9 +248,9 @@ class DataCleaner: ) -> bool: """Process a single file for cleanup.""" try: - with open(file_path) as f: + with open(file_path, encoding="utf-8") as f: data = json.load(f) - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): # Corrupted file, mark for deletion if not dry_run: file_size = file_path.stat().st_size @@ -327,7 +327,7 @@ class DataCleaner: data["_archived_at"] = datetime.now(timezone.utc).isoformat() data["_original_path"] = str(file_path) - with open(archive_path, "w") as f: + with open(archive_path, "w", encoding="utf-8") as f: json.dump(data, f, indent=2) # Remove original @@ -350,7 +350,7 @@ class DataCleaner: continue try: - with open(index_path) as f: + with open(index_path, encoding="utf-8") as f: index_data = json.load(f) if not isinstance(index_data, dict): @@ -375,12 +375,12 @@ class DataCleaner: for key in to_remove: del items[key] - with open(index_path, "w") as f: + with open(index_path, "w", encoding="utf-8") as f: json.dump(index_data, f, indent=2) result.pruned_index_entries += pruned - except (OSError, json.JSONDecodeError, KeyError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError, KeyError): result.errors.append(f"Error pruning index: {index_path}") async def _clean_audit_logs( diff --git a/apps/backend/runners/github/context_gatherer.py b/apps/backend/runners/github/context_gatherer.py index 31f5f015..f80e4d0c 100644 --- a/apps/backend/runners/github/context_gatherer.py +++ b/apps/backend/runners/github/context_gatherer.py @@ -782,7 +782,7 @@ class PRContextGatherer: # Check for package.json (Node.js) if (self.project_dir / "package.json").exists(): try: - with open(self.project_dir / "package.json") as f: + with open(self.project_dir / "package.json", encoding="utf-8") as f: pkg_data = json.load(f) if "workspaces" in pkg_data: structure_info.append( diff --git a/apps/backend/runners/github/duplicates.py b/apps/backend/runners/github/duplicates.py index 47d3dce4..577447d3 100644 --- a/apps/backend/runners/github/duplicates.py +++ b/apps/backend/runners/github/duplicates.py @@ -347,7 +347,7 @@ class DuplicateDetector: if not cache_file.exists(): return {} - with open(cache_file) as f: + with open(cache_file, encoding="utf-8") as f: data = json.load(f) cache = {} @@ -365,7 +365,7 @@ class DuplicateDetector: "embeddings": [e.to_dict() for e in cache.values()], "last_updated": datetime.now(timezone.utc).isoformat(), } - with open(cache_file, "w") as f: + with open(cache_file, "w", encoding="utf-8") as f: json.dump(data, f) async def get_embedding( diff --git a/apps/backend/runners/github/file_lock.py b/apps/backend/runners/github/file_lock.py index 065d2028..c70caa62 100644 --- a/apps/backend/runners/github/file_lock.py +++ b/apps/backend/runners/github/file_lock.py @@ -212,7 +212,7 @@ class FileLock: @contextmanager -def atomic_write(filepath: str | Path, mode: str = "w"): +def atomic_write(filepath: str | Path, mode: str = "w", encoding: str = "utf-8"): """ Atomic file write using temp file and rename. @@ -222,6 +222,7 @@ def atomic_write(filepath: str | Path, mode: str = "w"): Args: filepath: Target file path mode: File open mode (default: "w") + encoding: Text encoding (default: "utf-8") Example: with atomic_write("/path/to/file.json") as f: @@ -236,8 +237,9 @@ def atomic_write(filepath: str | Path, mode: str = "w"): ) try: - # Open temp file with requested mode - with os.fdopen(fd, mode) as f: + # Open temp file with requested mode and encoding + # Only use encoding for text modes (not binary modes) + with os.fdopen(fd, mode, encoding=encoding if "b" not in mode else None) as f: yield f # Atomic replace - succeeds or fails completely @@ -254,7 +256,10 @@ def atomic_write(filepath: str | Path, mode: str = "w"): @asynccontextmanager async def locked_write( - filepath: str | Path, timeout: float = 5.0, mode: str = "w" + filepath: str | Path, + timeout: float = 5.0, + mode: str = "w", + encoding: str = "utf-8", ) -> Any: """ Async context manager combining file locking and atomic writes. @@ -266,6 +271,7 @@ async def locked_write( filepath: Target file path timeout: Lock timeout in seconds (default: 5.0) mode: File open mode (default: "w") + encoding: Text encoding (default: "utf-8") Example: async with locked_write("/path/to/file.json", timeout=5.0) as f: @@ -291,7 +297,8 @@ async def locked_write( try: # Open temp file and yield to caller - f = os.fdopen(fd, mode) + # Only use encoding for text modes (not binary modes) + f = os.fdopen(fd, mode, encoding=encoding if "b" not in mode else None) try: yield f finally: @@ -348,7 +355,7 @@ async def locked_read(filepath: str | Path, timeout: float = 5.0) -> Any: try: # Open file for reading - with open(filepath) as f: + with open(filepath, encoding="utf-8") as f: yield f finally: # Release lock @@ -441,7 +448,7 @@ async def locked_json_update( # Read current data def _read_json(): if filepath.exists(): - with open(filepath) as f: + with open(filepath, encoding="utf-8") as f: return json.load(f) return None @@ -459,7 +466,7 @@ async def locked_json_update( ) try: - with os.fdopen(fd, "w") as f: + with os.fdopen(fd, "w", encoding="utf-8") as f: json.dump(updated_data, f, indent=indent) await asyncio.get_running_loop().run_in_executor( diff --git a/apps/backend/runners/github/learning.py b/apps/backend/runners/github/learning.py index f2389a97..d8993b0a 100644 --- a/apps/backend/runners/github/learning.py +++ b/apps/backend/runners/github/learning.py @@ -299,7 +299,7 @@ class LearningTracker: """Load all outcomes from disk.""" for file in self.learning_dir.glob("*_outcomes.json"): try: - with open(file) as f: + with open(file, encoding="utf-8") as f: data = json.load(f) for item in data.get("outcomes", []): outcome = ReviewOutcome.from_dict(item) diff --git a/apps/backend/runners/github/lifecycle.py b/apps/backend/runners/github/lifecycle.py index 38121fc5..d85297e7 100644 --- a/apps/backend/runners/github/lifecycle.py +++ b/apps/backend/runners/github/lifecycle.py @@ -409,7 +409,7 @@ class LifecycleManager: if not file.exists(): return None - with open(file) as f: + with open(file, encoding="utf-8") as f: data = json.load(f) return IssueLifecycle.from_dict(data) @@ -426,7 +426,7 @@ class LifecycleManager: def save(self, lifecycle: IssueLifecycle) -> None: """Save lifecycle state.""" file = self._get_file(lifecycle.repo, lifecycle.issue_number) - with open(file, "w") as f: + with open(file, "w", encoding="utf-8") as f: json.dump(lifecycle.to_dict(), f, indent=2) def transition( @@ -509,7 +509,7 @@ class LifecycleManager: safe_repo = repo.replace("/", "_") for file in self.lifecycle_dir.glob(f"{safe_repo}_*.json"): - with open(file) as f: + with open(file, encoding="utf-8") as f: data = json.load(f) lifecycle = IssueLifecycle.from_dict(data) if lifecycle.current_state == state: @@ -523,7 +523,7 @@ class LifecycleManager: safe_repo = repo.replace("/", "_") for file in self.lifecycle_dir.glob(f"{safe_repo}_*.json"): - with open(file) as f: + with open(file, encoding="utf-8") as f: data = json.load(f) state = data.get("current_state", "new") counts[state] = counts.get(state, 0) + 1 diff --git a/apps/backend/runners/github/memory_integration.py b/apps/backend/runners/github/memory_integration.py index e088c547..bff0d7f1 100644 --- a/apps/backend/runners/github/memory_integration.py +++ b/apps/backend/runners/github/memory_integration.py @@ -193,7 +193,7 @@ class GitHubMemoryIntegration: insights_file = self.memory_dir / f"{self.repo.replace('/', '_')}_insights.json" if insights_file.exists(): try: - with open(insights_file) as f: + with open(insights_file, encoding="utf-8") as f: self._local_insights = json.load(f).get("insights", []) except (json.JSONDecodeError, KeyError): self._local_insights = [] @@ -201,7 +201,7 @@ class GitHubMemoryIntegration: def _save_local_insights(self) -> None: """Save insights locally.""" insights_file = self.memory_dir / f"{self.repo.replace('/', '_')}_insights.json" - with open(insights_file, "w") as f: + with open(insights_file, "w", encoding="utf-8") as f: json.dump( { "repo": self.repo, diff --git a/apps/backend/runners/github/models.py b/apps/backend/runners/github/models.py index 21401d85..9a384525 100644 --- a/apps/backend/runners/github/models.py +++ b/apps/backend/runners/github/models.py @@ -550,7 +550,7 @@ class PRReviewResult: if not review_file.exists(): return None - with open(review_file) as f: + with open(review_file, encoding="utf-8") as f: return cls.from_dict(json.load(f)) @@ -663,7 +663,7 @@ class TriageResult: if not triage_file.exists(): return None - with open(triage_file) as f: + with open(triage_file, encoding="utf-8") as f: return cls.from_dict(json.load(f)) @@ -797,7 +797,7 @@ class AutoFixState: if not autofix_file.exists(): return None - with open(autofix_file) as f: + with open(autofix_file, encoding="utf-8") as f: return cls.from_dict(json.load(f)) @@ -879,7 +879,7 @@ class GitHubRunnerConfig: settings.pop("token", None) settings.pop("bot_token", None) - with open(config_file, "w") as f: + with open(config_file, "w", encoding="utf-8") as f: json.dump(settings, f, indent=2) @classmethod @@ -890,7 +890,7 @@ class GitHubRunnerConfig: config_file = github_dir / "config.json" if config_file.exists(): - with open(config_file) as f: + with open(config_file, encoding="utf-8") as f: settings = json.load(f) else: settings = {} diff --git a/apps/backend/runners/github/multi_repo.py b/apps/backend/runners/github/multi_repo.py index d0f531d4..314841fa 100644 --- a/apps/backend/runners/github/multi_repo.py +++ b/apps/backend/runners/github/multi_repo.py @@ -340,7 +340,7 @@ class MultiRepoConfig: "repos": [c.to_dict() for c in self.repos.values()], "last_updated": datetime.now(timezone.utc).isoformat(), } - with open(file_path, "w") as f: + with open(file_path, "w", encoding="utf-8") as f: json.dump(data, f, indent=2) @classmethod @@ -349,7 +349,7 @@ class MultiRepoConfig: if not config_file.exists(): return cls() - with open(config_file) as f: + with open(config_file, encoding="utf-8") as f: data = json.load(f) repos = [RepoConfig.from_dict(r) for r in data.get("repos", [])] diff --git a/apps/backend/runners/github/onboarding.py b/apps/backend/runners/github/onboarding.py index 59eb3442..da9d6f59 100644 --- a/apps/backend/runners/github/onboarding.py +++ b/apps/backend/runners/github/onboarding.py @@ -301,7 +301,7 @@ class OnboardingManager: if self.state_file.exists(): try: - with open(self.state_file) as f: + with open(self.state_file, encoding="utf-8") as f: data = json.load(f) self._state = OnboardingState.from_dict(data) except (json.JSONDecodeError, KeyError): @@ -315,7 +315,7 @@ class OnboardingManager: """Save onboarding state.""" state = self.get_state() self.state_file.parent.mkdir(parents=True, exist_ok=True) - with open(self.state_file, "w") as f: + with open(self.state_file, "w", encoding="utf-8") as f: json.dump(state.to_dict(), f, indent=2) async def run_setup( diff --git a/apps/backend/runners/github/override.py b/apps/backend/runners/github/override.py index fab53cb4..ac54c875 100644 --- a/apps/backend/runners/github/override.py +++ b/apps/backend/runners/github/override.py @@ -300,7 +300,7 @@ class OverrideManager: if not grace_file.exists(): return None - with open(grace_file) as f: + with open(grace_file, encoding="utf-8") as f: data = json.load(f) entry_data = data.get("entries", {}).get(str(issue_number)) @@ -778,7 +778,7 @@ class OverrideManager: if not history_file.exists(): return [] - with open(history_file) as f: + with open(history_file, encoding="utf-8") as f: data = json.load(f) records = [] @@ -809,7 +809,7 @@ class OverrideManager: if not history_file.exists(): return {"total": 0, "by_type": {}, "by_actor": {}} - with open(history_file) as f: + with open(history_file, encoding="utf-8") as f: data = json.load(f) stats = { diff --git a/apps/backend/runners/github/purge_strategy.py b/apps/backend/runners/github/purge_strategy.py index d9c20a01..001ee55d 100644 --- a/apps/backend/runners/github/purge_strategy.py +++ b/apps/backend/runners/github/purge_strategy.py @@ -224,7 +224,7 @@ class PurgeStrategy: result: PurgeResult to update """ try: - with open(file_path) as f: + with open(file_path, encoding="utf-8") as f: data = json.load(f) # Verify key matches value diff --git a/apps/backend/runners/github/runner.py b/apps/backend/runners/github/runner.py index e664bdda..ae1429c7 100644 --- a/apps/backend/runners/github/runner.py +++ b/apps/backend/runners/github/runner.py @@ -613,9 +613,9 @@ async def cmd_approve_batches(args) -> int: # Load approved batches from file try: - with open(args.batch_file) as f: + with open(args.batch_file, encoding="utf-8") as f: approved_batches = json.load(f) - except (json.JSONDecodeError, FileNotFoundError) as e: + except (json.JSONDecodeError, FileNotFoundError, UnicodeDecodeError) as e: safe_print(f"Error loading batch file: {e}") return 1 diff --git a/apps/backend/runners/github/test_context_gatherer.py b/apps/backend/runners/github/test_context_gatherer.py index ecd72894..19ed4498 100644 --- a/apps/backend/runners/github/test_context_gatherer.py +++ b/apps/backend/runners/github/test_context_gatherer.py @@ -75,11 +75,11 @@ def test_find_test_files(tmp_path): # Create source file source_file = src_dir / "utils.ts" - source_file.write_text("export const add = (a, b) => a + b;") + source_file.write_text("export const add = (a, b) => a + b;", encoding="utf-8") # Create test file test_file = src_dir / "utils.test.ts" - test_file.write_text("import { add } from './utils';") + test_file.write_text("import { add } from './utils';", encoding="utf-8") gatherer = PRContextGatherer(project_dir, 1) @@ -99,11 +99,11 @@ def test_resolve_import_path(tmp_path): # Create imported file utils_file = src_dir / "utils.ts" - utils_file.write_text("export const helper = () => {};") + utils_file.write_text("export const helper = () => {};", encoding="utf-8") # Create importing file app_file = src_dir / "app.ts" - app_file.write_text("import { helper } from './utils';") + app_file.write_text("import { helper } from './utils';", encoding="utf-8") gatherer = PRContextGatherer(project_dir, 1) @@ -128,7 +128,7 @@ def test_detect_repo_structure_monorepo(tmp_path): # Create package.json with workspaces package_json = project_dir / "package.json" - package_json.write_text('{"workspaces": ["apps/*"]}') + package_json.write_text('{"workspaces": ["apps/*"]}', encoding="utf-8") gatherer = PRContextGatherer(project_dir, 1) @@ -147,7 +147,7 @@ def test_detect_repo_structure_python(tmp_path): # Create pyproject.toml pyproject = project_dir / "pyproject.toml" - pyproject.write_text("[tool.poetry]\\nname = 'test'") + pyproject.write_text("[tool.poetry]\nname = 'test'", encoding="utf-8") gatherer = PRContextGatherer(project_dir, 1) @@ -163,8 +163,8 @@ def test_find_config_files(tmp_path): src_dir.mkdir(parents=True) # Create config files - (src_dir / "tsconfig.json").write_text("{}") - (src_dir / "package.json").write_text("{}") + (src_dir / "tsconfig.json").write_text("{}", encoding="utf-8") + (src_dir / "package.json").write_text("{}", encoding="utf-8") gatherer = PRContextGatherer(project_dir, 1) diff --git a/apps/backend/runners/github/test_file_lock.py b/apps/backend/runners/github/test_file_lock.py index eb755f7d..fe6c5ea8 100644 --- a/apps/backend/runners/github/test_file_lock.py +++ b/apps/backend/runners/github/test_file_lock.py @@ -28,7 +28,7 @@ async def test_basic_file_lock(): with tempfile.TemporaryDirectory() as tmpdir: test_file = Path(tmpdir) / "test.txt" - test_file.write_text("initial content") + test_file.write_text("initial content", encoding="utf-8") # Acquire lock and hold it async with FileLock(test_file, timeout=5.0): @@ -55,7 +55,7 @@ async def test_locked_write(): print(f"✓ Written to {test_file.name}") # Verify data was written correctly - with open(test_file) as f: + with open(test_file, encoding="utf-8") as f: loaded = json.load(f) assert loaded == data print(f"✓ Data verified: {loaded}") @@ -113,12 +113,12 @@ async def test_concurrent_updates_without_lock(): test_file = Path(tmpdir) / "unsafe.json" # Initialize counter - test_file.write_text(json.dumps({"count": 0})) + test_file.write_text(json.dumps({"count": 0}), encoding="utf-8") async def unsafe_increment(): """Increment without locking - RACE CONDITION!""" # Read - with open(test_file) as f: + with open(test_file, encoding="utf-8") as f: data = json.load(f) # Simulate some processing @@ -126,14 +126,14 @@ async def test_concurrent_updates_without_lock(): # Write data["count"] += 1 - with open(test_file, "w") as f: + with open(test_file, "w", encoding="utf-8") as f: json.dump(data, f) # Run 10 concurrent increments await asyncio.gather(*[unsafe_increment() for _ in range(10)]) # Check final count - with open(test_file) as f: + with open(test_file, encoding="utf-8") as f: final = json.load(f) print("✗ Expected count: 10") @@ -182,7 +182,7 @@ async def test_lock_timeout(): with tempfile.TemporaryDirectory() as tmpdir: test_file = Path(tmpdir) / "timeout.json" - test_file.write_text(json.dumps({"data": "test"})) + test_file.write_text(json.dumps({"data": "test"}), encoding="utf-8") # Acquire lock and hold it lock1 = FileLock(test_file, timeout=1.0) diff --git a/apps/backend/runners/github/trust.py b/apps/backend/runners/github/trust.py index 82fa1bc2..c5230d20 100644 --- a/apps/backend/runners/github/trust.py +++ b/apps/backend/runners/github/trust.py @@ -420,9 +420,13 @@ class TrustManager: state_file = self._get_state_file(repo) if state_file.exists(): - with open(state_file) as f: - data = json.load(f) - state = TrustState.from_dict(data) + try: + with open(state_file, encoding="utf-8") as f: + data = json.load(f) + state = TrustState.from_dict(data) + except (json.JSONDecodeError, UnicodeDecodeError): + # Return default state if file is corrupted + state = TrustState(repo=repo) else: state = TrustState(repo=repo) @@ -438,12 +442,9 @@ class TrustManager: # Write with restrictive permissions (0o600 = owner read/write only) fd = os.open(str(state_file), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) - try: - with os.fdopen(fd, "w") as f: - json.dump(state.to_dict(), f, indent=2) - except Exception: - os.close(fd) - raise + # os.fdopen takes ownership of fd and will close it when the with block exits + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(state.to_dict(), f, indent=2) def get_trust_level(self, repo: str) -> TrustLevel: """Get current trust level for a repository.""" @@ -514,9 +515,13 @@ class TrustManager: """Get trust states for all repos.""" states = [] for file in self.trust_dir.glob("*.json"): - with open(file) as f: - data = json.load(f) - states.append(TrustState.from_dict(data)) + try: + with open(file, encoding="utf-8") as f: + data = json.load(f) + states.append(TrustState.from_dict(data)) + except (json.JSONDecodeError, UnicodeDecodeError): + # Skip corrupted state files + continue return states def get_summary(self) -> dict[str, Any]: diff --git a/apps/backend/runners/gitlab/glab_client.py b/apps/backend/runners/gitlab/glab_client.py index c44d8d5e..4b2d47d1 100644 --- a/apps/backend/runners/gitlab/glab_client.py +++ b/apps/backend/runners/gitlab/glab_client.py @@ -253,7 +253,7 @@ def load_gitlab_config(project_dir: Path) -> GitLabConfig | None: return None try: - with open(config_path) as f: + with open(config_path, encoding="utf-8") as f: data = json.load(f) token = data.get("token") diff --git a/apps/backend/runners/gitlab/models.py b/apps/backend/runners/gitlab/models.py index 7fac1300..b0ccb4d6 100644 --- a/apps/backend/runners/gitlab/models.py +++ b/apps/backend/runners/gitlab/models.py @@ -184,7 +184,7 @@ class MRReviewResult: mr_dir.mkdir(parents=True, exist_ok=True) review_file = mr_dir / f"review_{self.mr_iid}.json" - with open(review_file, "w") as f: + with open(review_file, "w", encoding="utf-8") as f: json.dump(self.to_dict(), f, indent=2) @classmethod @@ -194,7 +194,7 @@ class MRReviewResult: if not review_file.exists(): return None - with open(review_file) as f: + with open(review_file, encoding="utf-8") as f: return cls.from_dict(json.load(f)) diff --git a/apps/backend/runners/gitlab/runner.py b/apps/backend/runners/gitlab/runner.py index 3b9df2bf..dad17680 100644 --- a/apps/backend/runners/gitlab/runner.py +++ b/apps/backend/runners/gitlab/runner.py @@ -91,7 +91,7 @@ def get_config(args) -> GitLabRunnerConfig: config_path = Path(args.project_dir) / ".auto-claude" / "gitlab" / "config.json" if config_path.exists(): try: - with open(config_path) as f: + with open(config_path, encoding="utf-8") as f: data = json.load(f) project = data.get("project", "") instance_url = data.get("instance_url", instance_url) diff --git a/apps/backend/runners/insights_runner.py b/apps/backend/runners/insights_runner.py index ee4ee449..7a1dc408 100644 --- a/apps/backend/runners/insights_runner.py +++ b/apps/backend/runners/insights_runner.py @@ -58,7 +58,7 @@ def load_project_context(project_dir: str) -> str: index_path = Path(project_dir) / ".auto-claude" / "project_index.json" if index_path.exists(): try: - with open(index_path) as f: + with open(index_path, encoding="utf-8") as f: index = json.load(f) # Summarize the index for context summary = { @@ -77,7 +77,7 @@ def load_project_context(project_dir: str) -> str: roadmap_path = Path(project_dir) / ".auto-claude" / "roadmap" / "roadmap.json" if roadmap_path.exists(): try: - with open(roadmap_path) as f: + with open(roadmap_path, encoding="utf-8") as f: roadmap = json.load(f) # Summarize roadmap features = roadmap.get("features", []) diff --git a/apps/backend/runners/roadmap/competitor_analyzer.py b/apps/backend/runners/roadmap/competitor_analyzer.py index 829558b8..01049b7d 100644 --- a/apps/backend/runners/roadmap/competitor_analyzer.py +++ b/apps/backend/runners/roadmap/competitor_analyzer.py @@ -123,7 +123,7 @@ Output your findings to competitor_analysis.json. Returns RoadmapPhaseResult if validation succeeds, None otherwise. """ try: - with open(self.analysis_file) as f: + with open(self.analysis_file, encoding="utf-8") as f: data = json.load(f) if "competitors" in data: @@ -146,7 +146,7 @@ Output your findings to competitor_analysis.json. def _create_disabled_analysis_file(self): """Create an analysis file indicating the feature is disabled.""" - with open(self.analysis_file, "w") as f: + with open(self.analysis_file, "w", encoding="utf-8") as f: json.dump( { "enabled": False, @@ -181,5 +181,5 @@ Output your findings to competitor_analysis.json. if errors: data["errors"] = errors - with open(self.analysis_file, "w") as f: + with open(self.analysis_file, "w", encoding="utf-8") as f: json.dump(data, f, indent=2) diff --git a/apps/backend/runners/roadmap/executor.py b/apps/backend/runners/roadmap/executor.py index 13509a18..d96ae81b 100644 --- a/apps/backend/runners/roadmap/executor.py +++ b/apps/backend/runners/roadmap/executor.py @@ -102,7 +102,7 @@ class AgentExecutor: return False, f"Prompt not found: {prompt_path}" # Load prompt - prompt = prompt_path.read_text() + prompt = prompt_path.read_text(encoding="utf-8") debug_detailed( "roadmap_executor", "Loaded prompt file", prompt_length=len(prompt) ) diff --git a/apps/backend/runners/roadmap/graph_integration.py b/apps/backend/runners/roadmap/graph_integration.py index 3a4dbac8..325dcb44 100644 --- a/apps/backend/runners/roadmap/graph_integration.py +++ b/apps/backend/runners/roadmap/graph_integration.py @@ -81,7 +81,7 @@ class GraphHintsProvider: def _create_disabled_hints_file(self): """Create a hints file indicating Graphiti is disabled.""" - with open(self.hints_file, "w") as f: + with open(self.hints_file, "w", encoding="utf-8") as f: json.dump( { "enabled": False, @@ -95,7 +95,7 @@ class GraphHintsProvider: def _save_hints(self, hints: list): """Save retrieved hints to file.""" - with open(self.hints_file, "w") as f: + with open(self.hints_file, "w", encoding="utf-8") as f: json.dump( { "enabled": True, @@ -109,7 +109,7 @@ class GraphHintsProvider: def _save_error_hints(self, error: str): """Save error information to hints file.""" - with open(self.hints_file, "w") as f: + with open(self.hints_file, "w", encoding="utf-8") as f: json.dump( { "enabled": True, diff --git a/apps/backend/runners/roadmap/orchestrator.py b/apps/backend/runners/roadmap/orchestrator.py index b49ca2c1..c2d3d335 100644 --- a/apps/backend/runners/roadmap/orchestrator.py +++ b/apps/backend/runners/roadmap/orchestrator.py @@ -198,7 +198,7 @@ class RoadmapOrchestrator: if not roadmap_file.exists(): return - with open(roadmap_file) as f: + with open(roadmap_file, encoding="utf-8") as f: roadmap = json.load(f) features = roadmap.get("features", []) diff --git a/apps/backend/runners/roadmap/phases.py b/apps/backend/runners/roadmap/phases.py index 79549692..00c6cb22 100644 --- a/apps/backend/runners/roadmap/phases.py +++ b/apps/backend/runners/roadmap/phases.py @@ -172,7 +172,7 @@ Do NOT ask questions. Make educated inferences and create the file. Returns RoadmapPhaseResult if validation succeeds, None otherwise. """ try: - with open(self.discovery_file) as f: + with open(self.discovery_file, encoding="utf-8") as f: data = json.load(f) required = ["project_name", "target_audience", "product_vision"] @@ -288,7 +288,7 @@ Output the complete roadmap to roadmap.json. Returns RoadmapPhaseResult if validation succeeds, None otherwise. """ try: - with open(self.roadmap_file) as f: + with open(self.roadmap_file, encoding="utf-8") as f: data = json.load(f) required = ["phases", "features", "vision", "target_audience"] diff --git a/apps/backend/runners/spec_runner.py b/apps/backend/runners/spec_runner.py index 210c7120..7cdc3b2f 100644 --- a/apps/backend/runners/spec_runner.py +++ b/apps/backend/runners/spec_runner.py @@ -232,7 +232,7 @@ Examples: if not args.task_file.exists(): print(f"Error: Task file not found: {args.task_file}") sys.exit(1) - task_description = args.task_file.read_text().strip() + task_description = args.task_file.read_text(encoding="utf-8").strip() if not task_description: print(f"Error: Task file is empty: {args.task_file}") sys.exit(1) diff --git a/apps/backend/security/scan_secrets.py b/apps/backend/security/scan_secrets.py index ab4c3138..c6ececc4 100644 --- a/apps/backend/security/scan_secrets.py +++ b/apps/backend/security/scan_secrets.py @@ -264,7 +264,7 @@ def load_secretsignore(project_dir: Path) -> list[str]: patterns = [] try: - content = ignore_file.read_text() + content = ignore_file.read_text(encoding="utf-8") for line in content.splitlines(): line = line.strip() # Skip comments and empty lines diff --git a/apps/backend/services/context.py b/apps/backend/services/context.py index b2a33812..5225544d 100644 --- a/apps/backend/services/context.py +++ b/apps/backend/services/context.py @@ -53,7 +53,7 @@ class ServiceContextGenerator: """Load project index from file (.auto-claude is the installed instance).""" index_file = self.project_dir / ".auto-claude" / "project_index.json" if index_file.exists(): - with open(index_file) as f: + with open(index_file, encoding="utf-8") as f: return json.load(f) return {"services": {}} @@ -132,7 +132,7 @@ class ServiceContextGenerator: requirements = service_path / "requirements.txt" if requirements.exists(): try: - content = requirements.read_text() + content = requirements.read_text(encoding="utf-8") for line in content.split("\n")[:20]: # Top 20 deps line = line.strip() if line and not line.startswith("#"): @@ -147,13 +147,13 @@ class ServiceContextGenerator: package_json = service_path / "package.json" if package_json.exists(): try: - with open(package_json) as f: + with open(package_json, encoding="utf-8") as f: pkg = json.load(f) deps = list(pkg.get("dependencies", {}).keys())[:15] context.dependencies.extend( [d for d in deps if d not in context.dependencies] ) - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): pass def _discover_api_patterns(self, service_path: Path, context: ServiceContext): @@ -170,7 +170,7 @@ class ServiceContextGenerator: for route_file in route_files[:5]: # Check first 5 try: - content = route_file.read_text() + content = route_file.read_text(encoding="utf-8") # Look for common route patterns if "@app.route" in content or "@router." in content: context.api_patterns.append( @@ -187,20 +187,20 @@ class ServiceContextGenerator: package_json = service_path / "package.json" if package_json.exists(): try: - with open(package_json) as f: + with open(package_json, encoding="utf-8") as f: pkg = json.load(f) scripts = pkg.get("scripts", {}) for name in ["dev", "start", "build", "test", "lint"]: if name in scripts: context.common_commands[name] = f"npm run {name}" - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): pass # From Makefile makefile = service_path / "Makefile" if makefile.exists(): try: - content = makefile.read_text() + content = makefile.read_text(encoding="utf-8") for line in content.split("\n"): if line and not line.startswith("\t") and ":" in line: target = line.split(":")[0].strip() @@ -236,7 +236,7 @@ class ServiceContextGenerator: env_path = service_path / env_file if env_path.exists(): try: - content = env_path.read_text() + content = env_path.read_text(encoding="utf-8") for line in content.split("\n"): line = line.strip() if line and not line.startswith("#") and "=" in line: @@ -381,7 +381,7 @@ class ServiceContextGenerator: output_path = service_path / "SERVICE_CONTEXT.md" output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(markdown) + output_path.write_text(markdown, encoding="utf-8") print(f"Generated SERVICE_CONTEXT.md for {service_name}: {output_path}") return output_path @@ -446,7 +446,7 @@ def main(): # Load project index if specified project_index = None if args.index and args.index.exists(): - with open(args.index) as f: + with open(args.index, encoding="utf-8") as f: project_index = json.load(f) if args.all: diff --git a/apps/backend/services/orchestrator.py b/apps/backend/services/orchestrator.py index ac6b4c5c..244ed8bd 100644 --- a/apps/backend/services/orchestrator.py +++ b/apps/backend/services/orchestrator.py @@ -147,7 +147,7 @@ class ServiceOrchestrator: if not HAS_YAML: # Basic parsing without yaml module - content = self._compose_file.read_text() + content = self._compose_file.read_text(encoding="utf-8") if "services:" in content: # Very basic service name extraction lines = content.split("\n") diff --git a/apps/backend/services/recovery.py b/apps/backend/services/recovery.py index af6fb66c..45126df7 100644 --- a/apps/backend/services/recovery.py +++ b/apps/backend/services/recovery.py @@ -86,7 +86,7 @@ class RecoveryManager: "last_updated": datetime.now().isoformat(), }, } - with open(self.attempt_history_file, "w") as f: + with open(self.attempt_history_file, "w", encoding="utf-8") as f: json.dump(initial_data, f, indent=2) def _init_build_commits(self) -> None: @@ -99,39 +99,39 @@ class RecoveryManager: "last_updated": datetime.now().isoformat(), }, } - with open(self.build_commits_file, "w") as f: + with open(self.build_commits_file, "w", encoding="utf-8") as f: json.dump(initial_data, f, indent=2) def _load_attempt_history(self) -> dict: """Load attempt history from JSON file.""" try: - with open(self.attempt_history_file) as f: + with open(self.attempt_history_file, encoding="utf-8") as f: return json.load(f) - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): self._init_attempt_history() - with open(self.attempt_history_file) as f: + with open(self.attempt_history_file, encoding="utf-8") as f: return json.load(f) def _save_attempt_history(self, data: dict) -> None: """Save attempt history to JSON file.""" data["metadata"]["last_updated"] = datetime.now().isoformat() - with open(self.attempt_history_file, "w") as f: + with open(self.attempt_history_file, "w", encoding="utf-8") as f: json.dump(data, f, indent=2) def _load_build_commits(self) -> dict: """Load build commits from JSON file.""" try: - with open(self.build_commits_file) as f: + with open(self.build_commits_file, encoding="utf-8") as f: return json.load(f) - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): self._init_build_commits() - with open(self.build_commits_file) as f: + with open(self.build_commits_file, encoding="utf-8") as f: return json.load(f) def _save_build_commits(self, data: dict) -> None: """Save build commits to JSON file.""" data["metadata"]["last_updated"] = datetime.now().isoformat() - with open(self.build_commits_file, "w") as f: + with open(self.build_commits_file, "w", encoding="utf-8") as f: json.dump(data, f, indent=2) def classify_failure(self, error: str, subtask_id: str) -> FailureType: diff --git a/apps/backend/spec/compaction.py b/apps/backend/spec/compaction.py index 9538585e..843b1408 100644 --- a/apps/backend/spec/compaction.py +++ b/apps/backend/spec/compaction.py @@ -144,7 +144,7 @@ def gather_phase_outputs(spec_dir: Path, phase_name: str) -> str: file_path = spec_dir / filename if file_path.exists(): try: - content = file_path.read_text() + content = file_path.read_text(encoding="utf-8") # Limit individual file size if len(content) > 10000: content = content[:10000] + "\n\n[... file truncated ...]" diff --git a/apps/backend/spec/complexity.py b/apps/backend/spec/complexity.py index 5be32544..9a1770a0 100644 --- a/apps/backend/spec/complexity.py +++ b/apps/backend/spec/complexity.py @@ -364,7 +364,7 @@ async def run_ai_complexity_assessment( # Load requirements if available requirements_file = spec_dir / "requirements.json" if requirements_file.exists(): - with open(requirements_file) as f: + with open(requirements_file, encoding="utf-8") as f: req = json.load(f) context += f""" ## Requirements (from user) @@ -397,7 +397,7 @@ async def run_ai_complexity_assessment( ) if success and assessment_file.exists(): - with open(assessment_file) as f: + with open(assessment_file, encoding="utf-8") as f: data = json.load(f) # Parse AI assessment into ComplexityAssessment @@ -440,7 +440,7 @@ def save_assessment(spec_dir: Path, assessment: ComplexityAssessment) -> Path: assessment_file = spec_dir / "complexity_assessment.json" phases = assessment.phases_to_run() - with open(assessment_file, "w") as f: + with open(assessment_file, "w", encoding="utf-8") as f: json.dump( { "complexity": assessment.complexity.value, diff --git a/apps/backend/spec/context.py b/apps/backend/spec/context.py index 7d31b032..61771999 100644 --- a/apps/backend/spec/context.py +++ b/apps/backend/spec/context.py @@ -62,7 +62,7 @@ def run_context_discovery( if result.returncode == 0 and context_file.exists(): # Validate and fix common schema issues try: - with open(context_file) as f: + with open(context_file, encoding="utf-8") as f: ctx = json.load(f) # Check for required field and fix common issues @@ -73,9 +73,9 @@ def run_context_discovery( else: ctx["task_description"] = task_description or "unknown task" - with open(context_file, "w") as f: + with open(context_file, "w", encoding="utf-8") as f: json.dump(ctx, f, indent=2) - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): context_file.unlink(missing_ok=True) return False, "Invalid context.json created" @@ -105,7 +105,7 @@ def create_minimal_context( "created_at": datetime.now().isoformat(), } - with open(context_file, "w") as f: + with open(context_file, "w", encoding="utf-8") as f: json.dump(minimal_context, f, indent=2) return context_file @@ -118,7 +118,7 @@ def get_context_stats(spec_dir: Path) -> dict: return {} try: - with open(context_file) as f: + with open(context_file, encoding="utf-8") as f: ctx = json.load(f) return { "files_to_modify": len(ctx.get("files_to_modify", [])), diff --git a/apps/backend/spec/discovery.py b/apps/backend/spec/discovery.py index 518627f1..e2254eb0 100644 --- a/apps/backend/spec/discovery.py +++ b/apps/backend/spec/discovery.py @@ -69,7 +69,7 @@ def get_project_index_stats(spec_dir: Path) -> dict: return {} try: - with open(spec_index) as f: + with open(spec_index, encoding="utf-8") as f: index_data = json.load(f) # Support both old and new analyzer formats diff --git a/apps/backend/spec/phases/requirements_phases.py b/apps/backend/spec/phases/requirements_phases.py index 05a72679..69d9a400 100644 --- a/apps/backend/spec/phases/requirements_phases.py +++ b/apps/backend/spec/phases/requirements_phases.py @@ -86,7 +86,7 @@ class RequirementsPhaseMixin: ) # Save hints to file - with open(hints_file, "w") as f: + with open(hints_file, "w", encoding="utf-8") as f: json.dump( { "enabled": True, diff --git a/apps/backend/spec/phases/spec_phases.py b/apps/backend/spec/phases/spec_phases.py index 29689c31..1c39daf2 100644 --- a/apps/backend/spec/phases/spec_phases.py +++ b/apps/backend/spec/phases/spec_phases.py @@ -123,7 +123,7 @@ Create: ) if critique_file.exists(): - with open(critique_file) as f: + with open(critique_file, encoding="utf-8") as f: critique = json.load(f) if critique.get("issues_fixed", False) or critique.get( "no_issues_found", False diff --git a/apps/backend/spec/pipeline/agent_runner.py b/apps/backend/spec/pipeline/agent_runner.py index fb657148..42c380d5 100644 --- a/apps/backend/spec/pipeline/agent_runner.py +++ b/apps/backend/spec/pipeline/agent_runner.py @@ -85,7 +85,7 @@ class AgentRunner: return False, f"Prompt not found: {prompt_path}" # Load prompt - prompt = prompt_path.read_text() + prompt = prompt_path.read_text(encoding="utf-8") debug_detailed( "agent_runner", "Loaded prompt file", diff --git a/apps/backend/spec/pipeline/models.py b/apps/backend/spec/pipeline/models.py index 6675c9c6..69614f88 100644 --- a/apps/backend/spec/pipeline/models.py +++ b/apps/backend/spec/pipeline/models.py @@ -218,7 +218,7 @@ def rename_spec_dir_from_requirements(spec_dir: Path) -> bool: return False try: - with open(requirements_file) as f: + with open(requirements_file, encoding="utf-8") as f: req = json.load(f) task_desc = req.get("task_description", "") diff --git a/apps/backend/spec/pipeline/orchestrator.py b/apps/backend/spec/pipeline/orchestrator.py index 3396f905..08d6d5bd 100644 --- a/apps/backend/spec/pipeline/orchestrator.py +++ b/apps/backend/spec/pipeline/orchestrator.py @@ -478,7 +478,7 @@ class SpecOrchestrator: if not requirements_file.exists(): return "" - with open(requirements_file) as f: + with open(requirements_file, encoding="utf-8") as f: req = json.load(f) self.task_description = req.get("task_description", self.task_description) return f""" @@ -581,7 +581,7 @@ class SpecOrchestrator: project_index = {} auto_build_index = self.project_dir / "auto-claude" / "project_index.json" if auto_build_index.exists(): - with open(auto_build_index) as f: + with open(auto_build_index, encoding="utf-8") as f: project_index = json.load(f) analyzer = complexity.ComplexityAnalyzer(project_index) diff --git a/apps/backend/spec/requirements.py b/apps/backend/spec/requirements.py index e6eac70d..7d49f143 100644 --- a/apps/backend/spec/requirements.py +++ b/apps/backend/spec/requirements.py @@ -19,7 +19,9 @@ def open_editor_for_input(field_name: str) -> str: editor = os.environ.get("EDITOR", os.environ.get("VISUAL", "nano")) # Create temp file with helpful instructions - with tempfile.NamedTemporaryFile(mode="w", suffix=".md", delete=False) as f: + with tempfile.NamedTemporaryFile( + mode="w", suffix=".md", delete=False, encoding="utf-8" + ) as f: f.write(f"# Enter your {field_name.replace('_', ' ')} below\n") f.write("# Lines starting with # will be ignored\n") f.write("# Save and close the editor when done\n\n") @@ -37,7 +39,7 @@ def open_editor_for_input(field_name: str) -> str: return "" # Read the content - with open(temp_path) as f: + with open(temp_path, encoding="utf-8") as f: lines = f.readlines() # Filter out comment lines and join @@ -167,7 +169,7 @@ def create_requirements_from_task(task_description: str) -> dict: def save_requirements(spec_dir: Path, requirements: dict) -> Path: """Save requirements to file.""" requirements_file = spec_dir / "requirements.json" - with open(requirements_file, "w") as f: + with open(requirements_file, "w", encoding="utf-8") as f: json.dump(requirements, f, indent=2) return requirements_file @@ -178,5 +180,5 @@ def load_requirements(spec_dir: Path) -> dict | None: if not requirements_file.exists(): return None - with open(requirements_file) as f: + with open(requirements_file, encoding="utf-8") as f: return json.load(f) diff --git a/apps/backend/spec/validate_pkg/auto_fix.py b/apps/backend/spec/validate_pkg/auto_fix.py index b47abdb9..81d2e0e1 100644 --- a/apps/backend/spec/validate_pkg/auto_fix.py +++ b/apps/backend/spec/validate_pkg/auto_fix.py @@ -143,7 +143,7 @@ def auto_fix_plan(spec_dir: Path) -> bool: with open(plan_file, encoding="utf-8") as f: content = f.read() plan = json.loads(content) - except json.JSONDecodeError: + except (json.JSONDecodeError, UnicodeDecodeError): # Attempt JSON syntax repair try: with open(plan_file, encoding="utf-8") as f: diff --git a/apps/backend/spec/validate_pkg/validators/context_validator.py b/apps/backend/spec/validate_pkg/validators/context_validator.py index 1f9668fd..2fb3ea15 100644 --- a/apps/backend/spec/validate_pkg/validators/context_validator.py +++ b/apps/backend/spec/validate_pkg/validators/context_validator.py @@ -43,7 +43,7 @@ class ContextValidator: return ValidationResult(False, "context", errors, warnings, fixes) try: - with open(context_file) as f: + with open(context_file, encoding="utf-8") as f: context = json.load(f) except json.JSONDecodeError as e: errors.append(f"context.json is invalid JSON: {e}") diff --git a/apps/backend/spec/validate_pkg/validators/implementation_plan_validator.py b/apps/backend/spec/validate_pkg/validators/implementation_plan_validator.py index f328add4..2b34157d 100644 --- a/apps/backend/spec/validate_pkg/validators/implementation_plan_validator.py +++ b/apps/backend/spec/validate_pkg/validators/implementation_plan_validator.py @@ -43,7 +43,7 @@ class ImplementationPlanValidator: return ValidationResult(False, "plan", errors, warnings, fixes) try: - with open(plan_file) as f: + with open(plan_file, encoding="utf-8") as f: plan = json.load(f) except json.JSONDecodeError as e: errors.append(f"implementation_plan.json is invalid JSON: {e}") diff --git a/apps/backend/spec/validate_pkg/validators/spec_document_validator.py b/apps/backend/spec/validate_pkg/validators/spec_document_validator.py index edf4a991..b29edb37 100644 --- a/apps/backend/spec/validate_pkg/validators/spec_document_validator.py +++ b/apps/backend/spec/validate_pkg/validators/spec_document_validator.py @@ -40,7 +40,7 @@ class SpecDocumentValidator: fixes.append("Create spec.md with required sections") return ValidationResult(False, "spec", errors, warnings, fixes) - content = spec_file.read_text() + content = spec_file.read_text(encoding="utf-8") # Check for required sections for section in SPEC_REQUIRED_SECTIONS: diff --git a/apps/backend/spec/validation_strategy.py b/apps/backend/spec/validation_strategy.py index cad467e2..687fa9ac 100644 --- a/apps/backend/spec/validation_strategy.py +++ b/apps/backend/spec/validation_strategy.py @@ -161,7 +161,7 @@ def detect_project_type(project_dir: Path) -> str: if "@angular/core" in all_deps: return "angular_spa" return "nodejs" - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): return "nodejs" # Check for Python projects @@ -171,9 +171,9 @@ def detect_project_type(project_dir: Path) -> str: # Try to detect API framework deps_text = "" if requirements.exists(): - deps_text = requirements.read_text().lower() + deps_text = requirements.read_text(encoding="utf-8").lower() if pyproject.exists(): - deps_text += pyproject.read_text().lower() + deps_text += pyproject.read_text(encoding="utf-8").lower() if "fastapi" in deps_text or "flask" in deps_text or "django" in deps_text: return "python_api" diff --git a/apps/backend/spec/validator.py b/apps/backend/spec/validator.py index 7f2d51d2..1cd69c1e 100644 --- a/apps/backend/spec/validator.py +++ b/apps/backend/spec/validator.py @@ -14,7 +14,7 @@ def create_minimal_research(spec_dir: Path, reason: str = "No research needed") """Create minimal research.json file.""" research_file = spec_dir / "research.json" - with open(research_file, "w") as f: + with open(research_file, "w", encoding="utf-8") as f: json.dump( { "integrations_researched": [], @@ -35,7 +35,7 @@ def create_minimal_critique( """Create minimal critique_report.json file.""" critique_file = spec_dir / "critique_report.json" - with open(critique_file, "w") as f: + with open(critique_file, "w", encoding="utf-8") as f: json.dump( { "issues_found": [], @@ -54,7 +54,7 @@ def create_empty_hints(spec_dir: Path, enabled: bool, reason: str) -> Path: """Create empty graph_hints.json file.""" hints_file = spec_dir / "graph_hints.json" - with open(hints_file, "w") as f: + with open(hints_file, "w", encoding="utf-8") as f: json.dump( { "enabled": enabled, diff --git a/apps/backend/spec/writer.py b/apps/backend/spec/writer.py index 87b57ab7..6f59934d 100644 --- a/apps/backend/spec/writer.py +++ b/apps/backend/spec/writer.py @@ -48,7 +48,7 @@ def create_minimal_plan(spec_dir: Path, task_description: str) -> Path: } plan_file = spec_dir / "implementation_plan.json" - with open(plan_file, "w") as f: + with open(plan_file, "w", encoding="utf-8") as f: json.dump(plan, f, indent=2) return plan_file @@ -61,7 +61,7 @@ def get_plan_stats(spec_dir: Path) -> dict: return {} try: - with open(plan_file) as f: + with open(plan_file, encoding="utf-8") as f: plan_data = json.load(f) total_subtasks = sum( len(p.get("subtasks", [])) for p in plan_data.get("phases", []) diff --git a/apps/backend/task_logger/storage.py b/apps/backend/task_logger/storage.py index 6e50e89d..be9d7380 100644 --- a/apps/backend/task_logger/storage.py +++ b/apps/backend/task_logger/storage.py @@ -34,7 +34,7 @@ class LogStorage: try: with open(self.log_file, encoding="utf-8") as f: return json.load(f) - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): pass return { @@ -176,7 +176,7 @@ def load_task_logs(spec_dir: Path) -> dict | None: try: with open(log_file, encoding="utf-8") as f: return json.load(f) - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): return None diff --git a/apps/backend/ui/status.py b/apps/backend/ui/status.py index d176280f..cc5c3595 100644 --- a/apps/backend/ui/status.py +++ b/apps/backend/ui/status.py @@ -122,11 +122,11 @@ class StatusManager: return BuildStatus() try: - with open(self.status_file) as f: + with open(self.status_file, encoding="utf-8") as f: data = json.load(f) self._status = BuildStatus.from_dict(data) return self._status - except (OSError, json.JSONDecodeError): + except (OSError, json.JSONDecodeError, UnicodeDecodeError): return BuildStatus() def _do_write(self) -> None: @@ -146,7 +146,7 @@ class StatusManager: status_dict = self._status.to_dict() try: - with open(self.status_file, "w") as f: + with open(self.status_file, "w", encoding="utf-8") as f: json.dump(status_dict, f, indent=2) if debug: diff --git a/guides/README.md b/guides/README.md index 78a0baad..6046349f 100644 --- a/guides/README.md +++ b/guides/README.md @@ -7,6 +7,8 @@ Detailed documentation for Auto Claude setup and usage. | Guide | Description | |-------|-------------| | **[CLI-USAGE.md](CLI-USAGE.md)** | Terminal-only usage for power users, headless servers, and CI/CD | +| **[windows-development.md](windows-development.md)** | Windows-specific development guide (file encoding, paths, line endings) | +| **[linux.md](linux.md)** | Linux-specific installation and build guide (Flatpak, AppImage) | ## Quick Links diff --git a/guides/windows-development.md b/guides/windows-development.md new file mode 100644 index 00000000..e054356b --- /dev/null +++ b/guides/windows-development.md @@ -0,0 +1,337 @@ +# Windows Development Guide + +This guide covers Windows-specific considerations when developing +Auto Claude. + +## File Encoding + +### Problem + +Windows Python defaults to the `cp1252` (Windows-1252) code page instead +of UTF-8. This causes encoding errors when reading/writing files with +non-ASCII characters. + +**Common Error:** + +```plaintext +UnicodeDecodeError: 'charmap' codec can't decode byte 0x8d in position 1234 +``` + +### Solution + +**Always specify `encoding="utf-8"` for all text file operations.** + +See [CONTRIBUTING.md - File Encoding](../CONTRIBUTING.md#file-encoding-python) +for detailed examples and patterns. + +### Testing on Windows + +To verify your code works on Windows: + +1. **Test with non-ASCII content:** + + ```python + # Include emoji, international chars in test data + test_data = {"message": "Test 🚀 with ñoño and 中文"} + ``` + +2. **Run pre-commit hooks:** + + ```bash + pre-commit run check-file-encoding --all-files + ``` + +3. **Run all tests:** + + ```bash + npm run test:backend + ``` + +### Common Pitfalls + +#### Pitfall 1: JSON files + +```python +# Wrong - no encoding +with open("config.json") as f: + data = json.load(f) + +# Correct +with open("config.json", encoding="utf-8") as f: + data = json.load(f) +``` + +#### Pitfall 2: Path methods + +```python +# Wrong +content = Path("README.md").read_text() + +# Correct +content = Path("README.md").read_text(encoding="utf-8") +``` + +#### Pitfall 3: Subprocess output + +```python +# Wrong +result = subprocess.run(cmd, capture_output=True, text=True) + +# Correct +result = subprocess.run(cmd, capture_output=True, encoding="utf-8") +``` + +## Line Endings + +### Problem + +Windows uses CRLF (`\r\n`) line endings while macOS/Linux use LF (`\n`). +This can cause git diffs to show every line as changed. + +### Solution + +1. **Configure git to handle line endings:** + + ```bash + git config --global core.autocrlf true + ``` + +2. **The project's `.gitattributes` handles this automatically:** + + ```plaintext + * text=auto + *.py text eol=lf + *.md text eol=lf + ``` + +3. **In code, normalize when processing:** + + ```python + # Normalize line endings to LF (idiomatic approach) + content = "\n".join(content.splitlines()) + ``` + +## Path Separators + +### Problem + +Windows uses backslash `\` for paths, while Unix uses `/`. +This can break path operations. + +### Solution + +1. **Always use `Path` from `pathlib`:** + + ```python + from pathlib import Path + + # Correct - works on all platforms + config_path = Path("config") / "settings.json" + + # Wrong - Unix only + config_path = "config/settings.json" + ``` + +2. **Use `os.path.join()` for strings:** + + ```python + import os + + # Correct + config_path = os.path.join("config", "settings.json") + ``` + +3. **Never hardcode separators:** + + ```python + # Wrong - Unix only + path = "apps/backend/core" + + # Correct + path = os.path.join("apps", "backend", "core") + # Or better + path = Path("apps") / "backend" / "core" + ``` + +## Shell Commands + +### Problem + +Windows doesn't have bash by default. Shell commands need to work across +platforms. + +### Solution + +1. **Use Python libraries instead of shell:** + + ```python + # Instead of shell commands + import shutil + shutil.copy("source.txt", "dest.txt") # Instead of cp + + import os + os.remove("file.txt") # Instead of rm + ``` + +2. **Use `shlex` for cross-platform commands:** + + ```python + import shlex + import subprocess + + cmd = shlex.split("git rev-parse HEAD") + result = subprocess.run(cmd, capture_output=True, encoding="utf-8") + ``` + +3. **Check platform when needed:** + + ```python + import sys + + if sys.platform == "win32": + # Windows-specific code + pass + else: + # Unix code + pass + ``` + +## Development Environment + +### Recommended Setup on Windows + +1. **Use WSL2 (Windows Subsystem for Linux)** - Recommended: + - Most consistent with production Linux environment + - Full bash support + - Better performance for file I/O + - Install from Microsoft Store or: `wsl --install` + +2. **Or use Git Bash:** + - Comes with Git for Windows + - Provides Unix-like shell + - Lighter than WSL + - Download from [gitforwindows.org](https://gitforwindows.org/) + +3. **Or use PowerShell with Python:** + - Native Windows environment + - Requires extra care with paths/encoding + - Built into Windows + +### Editor Configuration + +**VS Code settings for Windows (`settings.json`):** + +```json +{ + "files.encoding": "utf8", + "files.eol": "\n", + "python.analysis.typeCheckingMode": "basic", + "editor.formatOnSave": true +} +``` + +## Common Issues and Solutions + +### Issue: Permission errors when deleting files + +**Problem:** Windows file locking is stricter than Unix. + +**Solution:** Ensure files are properly closed using context managers: + +```python +# Use context managers +with open(path, encoding="utf-8") as f: + data = f.read() +# File is closed here - safe to delete +``` + +### Issue: Long path names + +**Problem:** Windows has a 260-character path limit (legacy). + +**Solution:** + +1. Enable long paths in Windows 10+ (Group Policy or Registry) +2. Or keep paths short +3. Or use WSL2 + +### Issue: Case-insensitive filesystem + +**Problem:** Windows filesystem is case-insensitive +(`File.txt` == `file.txt`). + +**Solution:** Be consistent with casing in filenames and imports: + +```python +# Consistent casing +from apps.backend.core import Client # File: client.py + +# Avoid mixing cases +from apps.backend.core import client # Could work on Windows but fail on Linux +``` + +## Testing Windows Compatibility + +### Before Submitting a PR + +1. **Run pre-commit hooks:** + + ```bash + pre-commit run --all-files + ``` + +2. **Run all tests:** + + ```bash + npm run test:backend + npm test # frontend tests + ``` + +3. **Test with special characters:** + + ```python + # Add test data with emoji, international chars + test_content = "Test 🚀 ñoño 中文 العربية" + ``` + +### Windows-Specific Test Cases + +Add tests for Windows compatibility when relevant: + +```python +import sys +import pytest + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows only") +def test_windows_encoding(): + """Test Windows encoding with special characters.""" + content = "Test 🚀 ñoño 中文" + Path("test.txt").write_text(content, encoding="utf-8") + loaded = Path("test.txt").read_text(encoding="utf-8") + assert loaded == content +``` + +## Getting Help + +If you encounter Windows-specific issues: + +1. Check this guide and [CONTRIBUTING.md](../CONTRIBUTING.md) +2. Search [existing issues](https://github.com/AndyMik90/Auto-Claude/issues) +3. Ask in [discussions](https://github.com/AndyMik90/Auto-Claude/discussions) +4. Create an issue with `[Windows]` tag + +## Resources + +- [Python on Windows](https://docs.python.org/3/using/windows.html) +- [pathlib Documentation](https://docs.python.org/3/library/pathlib.html) +- [Git for Windows](https://gitforwindows.org/) +- [WSL2 Documentation](https://docs.microsoft.com/en-us/windows/wsl/) + +## Related + +- [CONTRIBUTING.md](../CONTRIBUTING.md) - General contribution + guidelines +- [PR #782](https://github.com/AndyMik90/Auto-Claude/pull/782) - + Comprehensive UTF-8 encoding fix +- [PR #795](https://github.com/AndyMik90/Auto-Claude/pull/795) - + Pre-commit hooks for encoding enforcement diff --git a/scripts/check_encoding.py b/scripts/check_encoding.py new file mode 100644 index 00000000..f5b8195d --- /dev/null +++ b/scripts/check_encoding.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +""" +Check File Encoding +=================== + +Pre-commit hook to ensure all file operations specify UTF-8 encoding. + +This prevents Windows encoding issues where Python defaults to cp1252 instead of UTF-8. +""" + +import argparse +import re +import sys +from pathlib import Path + +# Fix Windows console encoding for emoji output +if sys.platform == "win32": + try: + sys.stdout.reconfigure(encoding='utf-8') + except AttributeError: + # Python < 3.7 + import codecs + sys.stdout = codecs.getwriter('utf-8')(sys.stdout.buffer, 'strict') + + +class EncodingChecker: + """Checks Python files for missing UTF-8 encoding parameters.""" + + def __init__(self): + self.issues = [] + + def check_file(self, filepath: Path) -> bool: + """ + Check a single Python file for encoding issues. + + Returns: + True if file passes checks, False if issues found + """ + try: + content = filepath.read_text(encoding="utf-8") + except UnicodeDecodeError: + self.issues.append(f"{filepath}: File is not UTF-8 encoded") + return False + except OSError as e: + self.issues.append(f"{filepath}: Cannot read file ({e})") + return False + + file_issues = [] + + # Check 1: open() without encoding + # Pattern: open(...) without encoding= parameter + # Use negative lookbehind to exclude os.open(), urlopen(), etc. + for match in re.finditer(r'(? 0: + if content[end_pos] == '(': + paren_depth += 1 + elif content[end_pos] == ')': + paren_depth -= 1 + end_pos += 1 + args = content[start_pos:end_pos - 1] if end_pos > start_pos else "" + + # Skip if it already has encoding + if re.search(r'\bencoding\s*=', args): + continue + + # Skip method calls on self/cls (custom methods, not Path) + if var_name in ('self', 'cls'): + continue + + # Skip if var_name is 'Path' (class name reference, not instance call) + if var_name == 'Path': + continue + + # Skip if it's a custom method call (e.g., self.parser.read_text) + # Check the characters immediately before the matched variable name + if var_name: + prefix_start = max(0, match.start() - 10) + prefix = content[prefix_start:match.start()] + if re.search(r'\bself\.$', prefix) or re.search(r'\bcls\.$', prefix): + continue + + line_num = content[:match.start()].count('\n') + 1 + file_issues.append( + f"{filepath}:{line_num} - .read_text() without encoding parameter" + ) + + # Check 3: Path.write_text() without encoding + # Match .write_text() calls - both variable.write_text() and Path(...).write_text() + for match in re.finditer(r'(?:(\w+)|(\))\s*)\.write_text\s*\(', content): + var_name = match.group(1) # Will be None if matched closing paren + start_pos = match.end() + + # Find the matching closing parenthesis (handle nesting) + paren_depth = 1 + end_pos = start_pos + while end_pos < len(content) and paren_depth > 0: + if content[end_pos] == '(': + paren_depth += 1 + elif content[end_pos] == ')': + paren_depth -= 1 + end_pos += 1 + args = content[start_pos:end_pos - 1] if end_pos > start_pos else "" + + # Skip if it already has encoding + if re.search(r'\bencoding\s*=', args): + continue + + # Skip method calls on self/cls (custom methods, not Path) + if var_name in ('self', 'cls'): + continue + + # Skip if var_name is 'Path' (class name reference, not instance call) + if var_name == 'Path': + continue + + # Skip if it's a custom method call (e.g., self.parser.write_text) + # Check the characters immediately before the matched variable name + if var_name: + prefix_start = max(0, match.start() - 10) + prefix = content[prefix_start:match.start()] + if re.search(r'\bself\.$', prefix) or re.search(r'\bcls\.$', prefix): + continue + + line_num = content[:match.start()].count('\n') + 1 + file_issues.append( + f"{filepath}:{line_num} - .write_text() without encoding parameter" + ) + + # Check 4: json.load() with open() without encoding + for match in re.finditer(r'json\.load\s*\(\s*open\s*\([^)]+\)', content): + call = match.group() + + # Skip if open() has encoding (use word boundary for robustness) + if re.search(r'\bencoding\s*=', call): + continue + + line_num = content[:match.start()].count('\n') + 1 + file_issues.append( + f"{filepath}:{line_num} - json.load(open()) without encoding in open()" + ) + + # Check 5: json.dump() with open() without encoding + for match in re.finditer(r'json\.dump\s*\([^,]+,\s*open\s*\([^)]+\)', content): + call = match.group() + + # Skip if open() has encoding (use word boundary for robustness) + if re.search(r'\bencoding\s*=', call): + continue + + line_num = content[:match.start()].count('\n') + 1 + file_issues.append( + f"{filepath}:{line_num} - json.dump(..., open()) without encoding in open()" + ) + + self.issues.extend(file_issues) + return len(file_issues) == 0 + + def check_files(self, filepaths: list[Path]) -> int: + """ + Check multiple files. + + Returns: + Number of files with issues + """ + for filepath in filepaths: + if not filepath.exists(): + continue + + if not filepath.suffix == '.py': + continue + + self.check_file(filepath) + + return len([f for f in self.issues if f]) + + +def main(): + """Main entry point for pre-commit hook.""" + parser = argparse.ArgumentParser( + description="Check Python files for missing UTF-8 encoding parameters" + ) + parser.add_argument( + 'filenames', + nargs='*', + help='Filenames to check' + ) + parser.add_argument( + '--verbose', + action='store_true', + help='Show all issues found' + ) + + args = parser.parse_args() + + # Convert filenames to Path objects + files = [Path(f) for f in args.filenames] + + # Run checks + checker = EncodingChecker() + checker.check_files(files) + + # Report results + if checker.issues: + print("❌ Encoding issues found:") + print() + for issue in checker.issues: + print(f" {issue}") + print() + print("💡 Fix: Add encoding=\"utf-8\" parameter to file operations") + print() + print("Examples:") + print(' open(path, encoding="utf-8")') + print(' Path(file).read_text(encoding="utf-8")') + print(' Path(file).write_text(content, encoding="utf-8")') + print() + return 1 + + if args.verbose: + print(f"✅ All {len(files)} files pass encoding checks") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/conftest.py b/tests/conftest.py index 9d43da7a..3fa4d432 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -496,7 +496,7 @@ Allow users to upload and manage their profile avatars. def spec_file(spec_dir: Path, sample_spec: str) -> Path: """Create a spec.md file in the spec directory.""" spec_file = spec_dir / "spec.md" - spec_file.write_text(sample_spec) + spec_file.write_text(sample_spec, encoding="utf-8") return spec_file diff --git a/tests/test_check_encoding.py b/tests/test_check_encoding.py new file mode 100644 index 00000000..add2330d --- /dev/null +++ b/tests/test_check_encoding.py @@ -0,0 +1,355 @@ +"""Tests for the encoding check script.""" + +import tempfile +from pathlib import Path + +# Import the checker +import sys +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) +from check_encoding import EncodingChecker + + +class TestEncodingChecker: + """Test the EncodingChecker class.""" + + def test_detects_open_without_encoding(self): + """Should detect open() calls without encoding parameter.""" + code = ''' +def read_file(path): + with open(path) as f: + return f.read() +''' + # Create temp file + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False, encoding="utf-8") as f: + f.write(code) + temp_path = Path(f.name) + + try: + checker = EncodingChecker() + result = checker.check_file(temp_path) + + assert result is False + assert len(checker.issues) == 1 + assert "open() without encoding" in checker.issues[0] + finally: + temp_path.unlink() + + def test_allows_open_with_encoding(self): + """Should allow open() calls with encoding parameter.""" + code = ''' +def read_file(path): + with open(path, encoding="utf-8") as f: + return f.read() +''' + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False, encoding="utf-8") as f: + f.write(code) + temp_path = Path(f.name) + + try: + checker = EncodingChecker() + result = checker.check_file(temp_path) + + assert result is True + assert len(checker.issues) == 0 + finally: + temp_path.unlink() + + def test_allows_binary_mode_without_encoding(self): + """Should allow binary mode without encoding (correct behavior).""" + code = ''' +def read_file(path): + with open(path, "rb") as f: + return f.read() +''' + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False, encoding="utf-8") as f: + f.write(code) + temp_path = Path(f.name) + + try: + checker = EncodingChecker() + result = checker.check_file(temp_path) + + assert result is True + assert len(checker.issues) == 0 + finally: + temp_path.unlink() + + def test_allows_write_binary_mode_without_encoding(self): + """Should allow write binary mode (wb) without encoding.""" + code = ''' +def write_file(path, data): + with open(path, "wb") as f: + f.write(data) +''' + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False, encoding="utf-8") as f: + f.write(code) + temp_path = Path(f.name) + + try: + checker = EncodingChecker() + result = checker.check_file(temp_path) + + assert result is True + assert len(checker.issues) == 0 + finally: + temp_path.unlink() + + def test_allows_append_binary_mode_without_encoding(self): + """Should allow append binary mode (ab) without encoding.""" + code = ''' +def append_file(path, data): + with open(path, "ab") as f: + f.write(data) +''' + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False, encoding="utf-8") as f: + f.write(code) + temp_path = Path(f.name) + + try: + checker = EncodingChecker() + result = checker.check_file(temp_path) + + assert result is True + assert len(checker.issues) == 0 + finally: + temp_path.unlink() + + def test_detects_text_write_mode_without_encoding(self): + """Should detect text write mode (w) without encoding.""" + code = ''' +def write_file(path, content): + with open(path, "w") as f: + f.write(content) +''' + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False, encoding="utf-8") as f: + f.write(code) + temp_path = Path(f.name) + + try: + checker = EncodingChecker() + result = checker.check_file(temp_path) + + assert result is False + assert len(checker.issues) == 1 + assert "open() without encoding" in checker.issues[0] + finally: + temp_path.unlink() + + def test_detects_path_read_text_without_encoding(self): + """Should detect Path.read_text() without encoding.""" + code = ''' +from pathlib import Path + +def read_file(path): + return Path(path).read_text() +''' + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False, encoding="utf-8") as f: + f.write(code) + temp_path = Path(f.name) + + try: + checker = EncodingChecker() + result = checker.check_file(temp_path) + + assert result is False + assert len(checker.issues) == 1 + assert "read_text() without encoding" in checker.issues[0] + finally: + temp_path.unlink() + + def test_detects_path_write_text_without_encoding(self): + """Should detect Path.write_text() without encoding.""" + code = ''' +from pathlib import Path + +def write_file(path, content): + Path(path).write_text(content) +''' + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False, encoding="utf-8") as f: + f.write(code) + temp_path = Path(f.name) + + try: + checker = EncodingChecker() + result = checker.check_file(temp_path) + + assert result is False + assert len(checker.issues) == 1 + assert "write_text() without encoding" in checker.issues[0] + finally: + temp_path.unlink() + + def test_detects_json_load_without_encoding(self): + """Should detect json.load(open()) without encoding in open().""" + code = ''' +import json + +def read_json(path): + with open(path) as f: + return json.load(f) +''' + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False, encoding="utf-8") as f: + f.write(code) + temp_path = Path(f.name) + + try: + checker = EncodingChecker() + result = checker.check_file(temp_path) + + assert result is False + assert len(checker.issues) == 1 + # Detects the open() call without encoding + finally: + temp_path.unlink() + + def test_allows_path_read_text_with_encoding(self): + """Should allow Path.read_text() with encoding parameter.""" + code = ''' +from pathlib import Path + +def read_file(path): + return Path(path).read_text(encoding="utf-8") +''' + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False, encoding="utf-8") as f: + f.write(code) + temp_path = Path(f.name) + + try: + checker = EncodingChecker() + result = checker.check_file(temp_path) + + assert result is True + assert len(checker.issues) == 0 + finally: + temp_path.unlink() + + def test_allows_path_write_text_with_encoding(self): + """Should allow Path.write_text() with encoding parameter.""" + code = ''' +from pathlib import Path + +def write_file(path, content): + Path(path).write_text(content, encoding="utf-8") +''' + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False, encoding="utf-8") as f: + f.write(code) + temp_path = Path(f.name) + + try: + checker = EncodingChecker() + result = checker.check_file(temp_path) + + assert result is True + assert len(checker.issues) == 0 + finally: + temp_path.unlink() + + def test_allows_json_dump_with_encoding(self): + """Should allow json.dump() with encoding in open().""" + code = ''' +import json + +def write_json(path, data): + with open(path, "w", encoding="utf-8") as f: + json.dump(data, f) +''' + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False, encoding="utf-8") as f: + f.write(code) + temp_path = Path(f.name) + + try: + checker = EncodingChecker() + result = checker.check_file(temp_path) + + assert result is True + assert len(checker.issues) == 0 + finally: + temp_path.unlink() + + def test_detects_json_dump_without_encoding(self): + """Should detect json.dump() with open() without encoding.""" + code = ''' +import json + +def write_json(path, data): + with open(path, "w") as f: + json.dump(data, f) +''' + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False, encoding="utf-8") as f: + f.write(code) + temp_path = Path(f.name) + + try: + checker = EncodingChecker() + result = checker.check_file(temp_path) + + assert result is False + assert len(checker.issues) == 1 + # Detects the open() call without encoding + finally: + temp_path.unlink() + + def test_multiple_issues_in_single_file(self): + """Should detect multiple encoding issues in a single file.""" + code = ''' +from pathlib import Path + +def process_files(input_path, output_path): + # Missing encoding in open() + with open(input_path) as f: + content = f.read() + + # Missing encoding in Path.write_text() + Path(output_path).write_text(content) + + return content +''' + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False, encoding="utf-8") as f: + f.write(code) + temp_path = Path(f.name) + + try: + checker = EncodingChecker() + result = checker.check_file(temp_path) + + assert result is False + assert len(checker.issues) == 2 + finally: + temp_path.unlink() + + def test_skips_non_python_files(self): + """Should skip files that are not Python files.""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False, encoding="utf-8") as f: + f.write("with open(path) as f: pass") + temp_path = Path(f.name) + + try: + checker = EncodingChecker() + failed_count = checker.check_files([temp_path]) + + assert failed_count == 0 + assert len(checker.issues) == 0 + finally: + temp_path.unlink() + + def test_detects_encoding_with_spaces(self): + """Should detect encoding parameter even with spaces around equals sign.""" + code = ''' +def read_file(path): + # This has spaces: encoding = "utf-8" + with open(path, encoding = "utf-8") as f: + return f.read() +''' + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False, encoding="utf-8") as f: + f.write(code) + temp_path = Path(f.name) + + try: + checker = EncodingChecker() + result = checker.check_file(temp_path) + + # Should pass because word boundary regex handles spaces + assert result is True + assert len(checker.issues) == 0 + finally: + temp_path.unlink()