From 94d941333bfc007d74e9cbd4a020f9e6ddc30d96 Mon Sep 17 00:00:00 2001
From: kaigler <bill@n8sw.com>
Date: Tue, 27 Jan 2026 02:51:55 -0600
Subject: [PATCH] fix: prevent planner from generating invalid verification
 types (#1388) (#1529)

The planner agent was generating invalid verification types like `code_review`
because the prompt didn't explicitly constrain allowed types or document all
valid options.

Changes:
- Add explicit warning in planner.md that ONLY 6 types are valid
- Document all 6 verification types (was missing `none`)
- Add guidance: use `manual` for code review, `command` for tests
- Add `manual` and `none` verification instructions to coder.md
- Add handlers for `e2e`, `manual`, `none` in checklist_generator.py
- Add missing schema fields: `command`, `expected`, `instructions`
- Mark `component` as legacy (kept for backward compatibility)

Fixes #1388

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
Co-authored-by: Andy <119136210+AndyMik90@users.noreply.github.com>
---
 apps/backend/prediction/checklist_generator.py | 18 +++++++++++++++++-
 apps/backend/prompts/coder.md                  | 13 +++++++++++++
 apps/backend/prompts/planner.md                |  9 +++++++--
 apps/backend/spec/validate_pkg/schemas.py      |  7 +++++--
 4 files changed, 42 insertions(+), 5 deletions(-)

diff --git a/apps/backend/prediction/checklist_generator.py b/apps/backend/prediction/checklist_generator.py
index 1ecabdf1..54e00bd2 100644
--- a/apps/backend/prediction/checklist_generator.py
+++ b/apps/backend/prediction/checklist_generator.py
@@ -146,6 +146,22 @@ class ChecklistGenerator:
                     f"Test in browser: {verification.get('scenario', 'Check functionality')}"
                 )
             elif ver_type == "command":
-                reminders.append(f"Run command: {verification.get('run', '')}")
+                reminders.append(
+                    f"Run command: {verification.get('run', verification.get('command', ''))}"
+                )
+            elif ver_type == "e2e":
+                steps = verification.get("steps", [])
+                if steps:
+                    reminders.append(
+                        f"E2E verification: {len(steps)} steps to complete"
+                    )
+                else:
+                    reminders.append("E2E verification required")
+            elif ver_type == "manual":
+                reminders.append(
+                    f"Manual check: {verification.get('instructions', 'Verify manually')}"
+                )
+            elif ver_type == "none":
+                pass  # No reminder needed
 
         return reminders
diff --git a/apps/backend/prompts/coder.md b/apps/backend/prompts/coder.md
index 223e4251..2c0e7b74 100644
--- a/apps/backend/prompts/coder.md
+++ b/apps/backend/prompts/coder.md
@@ -727,6 +727,19 @@ curl -X [method] [url] -H "Content-Type: application/json" -d '[body]'
 # Use combination of API calls and browser automation
 ```
 
+**Manual Verification:**
+```
+# For verification.type = "manual"
+# Read the instructions field and perform the described check
+# Mark subtask complete only after manual verification passes
+```
+
+**No Verification:**
+```
+# For verification.type = "none"
+# No verification required - mark subtask complete after implementation
+```
+
 ### FIX BUGS IMMEDIATELY
 
 **If verification fails: FIX IT NOW.**
diff --git a/apps/backend/prompts/planner.md b/apps/backend/prompts/planner.md
index 3209b521..ce811676 100644
--- a/apps/backend/prompts/planner.md
+++ b/apps/backend/prompts/planner.md
@@ -365,13 +365,18 @@ Use ONLY these values for the `type` field in phases:
 
 ### Verification Types
 
+**CRITICAL: ONLY these 6 verification types are valid. Any other type will cause validation failure.**
+
 | Type | When to Use | Format |
 |------|-------------|--------|
-| `command` | CLI verification | `{"type": "command", "command": "...", "expected": "..."}` |
+| `command` | CLI verification, running tests | `{"type": "command", "command": "...", "expected": "..."}` |
 | `api` | REST endpoint testing | `{"type": "api", "method": "GET/POST", "url": "...", "expected_status": 200}` |
 | `browser` | UI rendering checks | `{"type": "browser", "url": "...", "checks": [...]}` |
 | `e2e` | Full flow verification | `{"type": "e2e", "steps": [...]}` |
-| `manual` | Requires human judgment | `{"type": "manual", "instructions": "..."}` |
+| `manual` | Human judgment, code review | `{"type": "manual", "instructions": "..."}` |
+| `none` | No verification needed | `{"type": "none"}` |
+
+**DO NOT invent types like `code_review`, `component`, `test`, `lint`, `build`. Use `manual` for human review, `command` for running tests.**
 
 ### Special Subtask Types
 
diff --git a/apps/backend/spec/validate_pkg/schemas.py b/apps/backend/spec/validate_pkg/schemas.py
index f64db3d1..6683c101 100644
--- a/apps/backend/spec/validate_pkg/schemas.py
+++ b/apps/backend/spec/validate_pkg/schemas.py
@@ -72,21 +72,24 @@ IMPLEMENTATION_PLAN_SCHEMA = {
         "required_fields": ["type"],
         "optional_fields": [
             "run",
+            "command",
+            "expected",
             "url",
             "method",
             "expect_status",
             "expect_contains",
             "scenario",
             "steps",
+            "instructions",
         ],
         "verification_types": [
             "command",
             "api",
             "browser",
-            "component",
+            "component",  # Legacy - consider deprecating (use "command" with test)
+            "e2e",
             "manual",
             "none",
-            "e2e",
         ],
     },
 }