From a01634d54809e9f4f1d6507d5bcbf55fdff68981 Mon Sep 17 00:00:00 2001
From: TamerineSky <168569051+TamerineSky@users.noreply.github.com>
Date: Wed, 7 Jan 2026 02:56:00 -0700
Subject: [PATCH] Fix Windows UTF-8 encoding errors in spec file reading (#744)

* Fix Windows UTF-8 encoding errors in spec file reading

Fixes UnicodeDecodeError on Windows when reading spec files with non-ASCII
characters. Python's read_text() defaults to cp1252 on Windows instead of
UTF-8, causing crashes when spec.md contains Unicode characters.

Changes:
- spec_document_validator.py: Add encoding='utf-8' to spec.md reading
- compaction.py: Add encoding='utf-8' to phase output file reading
- validation_strategy.py: Add encoding='utf-8' to requirements/pyproject reading
- agent_runner.py: Add encoding='utf-8' to prompt file reading

Impact:
- Fixes spec generation crashes on Windows with Unicode content
- No impact on Unix/Mac (already default to UTF-8)
- Prevents charmap codec errors during spec validation

Tested on Windows 10 with Auto-Claude v2.7.3

* Add error handling for file read operations

Implements Gemini Code Assist review suggestions to improve robustness:

1. spec_document_validator.py:
   - Wrap read_text() in try-except to handle corrupted UTF-8 files
   - Provide clear error message when spec.md is unreadable

2. validation_strategy.py:
   - Silently skip unreadable requirements.txt/pyproject.toml files
   - Prevents crashes during project type detection

This defensive programming prevents crashes from:
- Corrupted or invalid UTF-8 files
- Permission errors (OSError)
- Other file system issues

Addresses code review feedback from @gemini-code-assist

* Fix Windows UTF-8 encoding errors in project analyzer

- Add encoding='utf-8' to file reads in analyzer.py (load/save profile)
- Add encoding='utf-8' to file reads in config_parser.py (read_json, read_text)
- Add encoding='utf-8' to file reads in stack_detector.py (YAML files)

This fixes the 'charmap' codec error that was blocking roadmap generation on Windows:
  Warning: Could not load security profile: 'charmap' codec can't decode byte 0x8d

Same root cause as spec file reading - Windows defaults to cp1252 instead of UTF-8.

* Revert "Fix Windows UTF-8 encoding errors in project analyzer"

This reverts commit 41deed0aa388d8599371db16f822d9278464f09b.

---------

Co-authored-by: TamerineSky <TamerineSky@users.noreply.github.com>
---
 apps/backend/spec/compaction.py                        |  2 +-
 apps/backend/spec/pipeline/agent_runner.py             |  2 +-
 .../validate_pkg/validators/spec_document_validator.py |  7 ++++++-
 apps/backend/spec/validation_strategy.py               | 10 ++++++++--
 4 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/apps/backend/spec/compaction.py b/apps/backend/spec/compaction.py
index d74b377c..65365d34 100644
--- a/apps/backend/spec/compaction.py
+++ b/apps/backend/spec/compaction.py
@@ -141,7 +141,7 @@ def gather_phase_outputs(spec_dir: Path, phase_name: str) -> str:
         file_path = spec_dir / filename
         if file_path.exists():
             try:
-                content = file_path.read_text()
+                content = file_path.read_text(encoding="utf-8")
                 # Limit individual file size
                 if len(content) > 10000:
                     content = content[:10000] + "\n\n[... file truncated ...]"
diff --git a/apps/backend/spec/pipeline/agent_runner.py b/apps/backend/spec/pipeline/agent_runner.py
index d1ee2a78..b981a150 100644
--- a/apps/backend/spec/pipeline/agent_runner.py
+++ b/apps/backend/spec/pipeline/agent_runner.py
@@ -82,7 +82,7 @@ class AgentRunner:
             return False, f"Prompt not found: {prompt_path}"
 
         # Load prompt
-        prompt = prompt_path.read_text()
+        prompt = prompt_path.read_text(encoding="utf-8")
         debug_detailed(
             "agent_runner",
             "Loaded prompt file",
diff --git a/apps/backend/spec/validate_pkg/validators/spec_document_validator.py b/apps/backend/spec/validate_pkg/validators/spec_document_validator.py
index edf4a991..92a5f8ad 100644
--- a/apps/backend/spec/validate_pkg/validators/spec_document_validator.py
+++ b/apps/backend/spec/validate_pkg/validators/spec_document_validator.py
@@ -40,7 +40,12 @@ class SpecDocumentValidator:
             fixes.append("Create spec.md with required sections")
             return ValidationResult(False, "spec", errors, warnings, fixes)
 
-        content = spec_file.read_text()
+        try:
+            content = spec_file.read_text(encoding="utf-8")
+        except (OSError, UnicodeDecodeError) as e:
+            errors.append(f"Failed to read spec.md: {e}")
+            fixes.append("Ensure spec.md is a valid UTF-8 encoded file")
+            return ValidationResult(False, "spec", errors, warnings, fixes)
 
         # Check for required sections
         for section in SPEC_REQUIRED_SECTIONS:
diff --git a/apps/backend/spec/validation_strategy.py b/apps/backend/spec/validation_strategy.py
index cad467e2..fe71f8f1 100644
--- a/apps/backend/spec/validation_strategy.py
+++ b/apps/backend/spec/validation_strategy.py
@@ -171,9 +171,15 @@ def detect_project_type(project_dir: Path) -> str:
         # Try to detect API framework
         deps_text = ""
         if requirements.exists():
-            deps_text = requirements.read_text().lower()
+            try:
+                deps_text = requirements.read_text(encoding="utf-8").lower()
+            except (OSError, UnicodeDecodeError):
+                pass  # Skip if file is unreadable
         if pyproject.exists():
-            deps_text += pyproject.read_text().lower()
+            try:
+                deps_text += pyproject.read_text(encoding="utf-8").lower()
+            except (OSError, UnicodeDecodeError):
+                pass  # Skip if file is unreadable
 
         if "fastapi" in deps_text or "flask" in deps_text or "django" in deps_text:
             return "python_api"