feat(tests): Mascarade integration + contract validation scripts
test_mascarade_integration.sh: - 4 tests: runtime health, dispatch mesh, langfuse, gateway - JSON structure validation with jq - cockpit-v1 contract compliance checks test_contracts.sh: - Validates all 10 schemas in specs/contracts/ - Validates all artifacts/ops/*/latest.json - Found: mascarade_agent_smoke missing contract_version Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
L'électron rare
Executable
+146
@@ -0,0 +1,146 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# Contract validation tests
|
||||
# Validates JSON schemas in specs/contracts/ and artifacts in artifacts/ops/
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
|
||||
CONTRACTS_DIR="$ROOT_DIR/specs/contracts"
|
||||
ARTIFACTS_DIR="$ROOT_DIR/artifacts/ops"
|
||||
|
||||
SCHEMA_VALID=0
|
||||
SCHEMA_INVALID=0
|
||||
SCHEMA_TOTAL=0
|
||||
ARTIFACT_VALID=0
|
||||
ARTIFACT_INVALID=0
|
||||
ARTIFACT_TOTAL=0
|
||||
|
||||
# Colors (disabled if not a terminal)
|
||||
if [[ -t 1 ]]; then
|
||||
GREEN=$'\033[0;32m'
|
||||
RED=$'\033[0;31m'
|
||||
BOLD=$'\033[1m'
|
||||
RESET=$'\033[0m'
|
||||
else
|
||||
GREEN="" RED="" BOLD="" RESET=""
|
||||
fi
|
||||
|
||||
# Require jq
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
echo "ERROR: jq is required but not found in PATH." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "${BOLD}Contract Validation Tests${RESET}"
|
||||
echo "========================="
|
||||
echo ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Part 1: Validate all .schema.json files parse as valid JSON
|
||||
# ---------------------------------------------------------------------------
|
||||
echo "${BOLD}1. JSON Schema Validation${RESET}"
|
||||
echo " Directory: $CONTRACTS_DIR"
|
||||
echo ""
|
||||
|
||||
if [[ ! -d "$CONTRACTS_DIR" ]]; then
|
||||
echo " WARNING: contracts directory not found at $CONTRACTS_DIR"
|
||||
else
|
||||
for schema_file in "$CONTRACTS_DIR"/*.schema.json; do
|
||||
[[ -f "$schema_file" ]] || continue
|
||||
SCHEMA_TOTAL=$((SCHEMA_TOTAL + 1))
|
||||
basename="$(basename "$schema_file")"
|
||||
|
||||
if jq empty "$schema_file" 2>/dev/null; then
|
||||
SCHEMA_VALID=$((SCHEMA_VALID + 1))
|
||||
printf " ${GREEN}VALID${RESET} %s\n" "$basename"
|
||||
else
|
||||
SCHEMA_INVALID=$((SCHEMA_INVALID + 1))
|
||||
printf " ${RED}INVALID${RESET} %s\n" "$basename"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$SCHEMA_TOTAL" -eq 0 ]]; then
|
||||
echo " No .schema.json files found."
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Part 2: Validate artifacts in artifacts/ops/*/latest.json
|
||||
# ---------------------------------------------------------------------------
|
||||
echo "${BOLD}2. Artifact Validation${RESET}"
|
||||
echo " Directory: $ARTIFACTS_DIR"
|
||||
echo ""
|
||||
|
||||
if [[ ! -d "$ARTIFACTS_DIR" ]]; then
|
||||
echo " WARNING: artifacts directory not found at $ARTIFACTS_DIR"
|
||||
else
|
||||
for latest_file in "$ARTIFACTS_DIR"/*/latest.json; do
|
||||
[[ -f "$latest_file" ]] || continue
|
||||
ARTIFACT_TOTAL=$((ARTIFACT_TOTAL + 1))
|
||||
|
||||
# Extract component directory name
|
||||
component_dir="$(basename "$(dirname "$latest_file")")"
|
||||
rel_path="$component_dir/latest.json"
|
||||
|
||||
# Check if the file is valid JSON
|
||||
if ! jq empty "$latest_file" 2>/dev/null; then
|
||||
ARTIFACT_INVALID=$((ARTIFACT_INVALID + 1))
|
||||
printf " ${RED}INVALID${RESET} %s — not valid JSON\n" "$rel_path"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Check for cockpit-v1 contract structure
|
||||
cv="$(jq -r '.contract_version // empty' "$latest_file" 2>/dev/null)"
|
||||
status="$(jq -r '.status // empty' "$latest_file" 2>/dev/null)"
|
||||
component="$(jq -r '.component // empty' "$latest_file" 2>/dev/null)"
|
||||
|
||||
issues=""
|
||||
if [[ -z "$cv" ]]; then
|
||||
issues="missing contract_version"
|
||||
fi
|
||||
if [[ -z "$status" ]]; then
|
||||
issues="${issues:+$issues, }missing status"
|
||||
fi
|
||||
|
||||
if [[ -n "$issues" ]]; then
|
||||
ARTIFACT_INVALID=$((ARTIFACT_INVALID + 1))
|
||||
printf " ${RED}INVALID${RESET} %s — %s\n" "$rel_path" "$issues"
|
||||
else
|
||||
ARTIFACT_VALID=$((ARTIFACT_VALID + 1))
|
||||
printf " ${GREEN}VALID${RESET} %s (component=%s, status=%s)\n" "$rel_path" "${component:-n/a}" "$status"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$ARTIFACT_TOTAL" -eq 0 ]]; then
|
||||
echo " No latest.json artifacts found."
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summary
|
||||
# ---------------------------------------------------------------------------
|
||||
echo "========================="
|
||||
echo "${BOLD}Summary${RESET}"
|
||||
printf " Schemas: %d valid / %d total" "$SCHEMA_VALID" "$SCHEMA_TOTAL"
|
||||
if [[ "$SCHEMA_INVALID" -gt 0 ]]; then
|
||||
printf " (${RED}%d invalid${RESET})" "$SCHEMA_INVALID"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
printf " Artifacts: %d valid / %d total" "$ARTIFACT_VALID" "$ARTIFACT_TOTAL"
|
||||
if [[ "$ARTIFACT_INVALID" -gt 0 ]]; then
|
||||
printf " (${RED}%d invalid${RESET})" "$ARTIFACT_INVALID"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
if [[ "$SCHEMA_INVALID" -gt 0 || "$ARTIFACT_INVALID" -gt 0 ]]; then
|
||||
printf "\n${RED}Validation failures detected.${RESET}\n"
|
||||
exit 1
|
||||
else
|
||||
printf "\n${GREEN}All validations passed.${RESET}\n"
|
||||
exit 0
|
||||
fi
|
||||
+189
@@ -0,0 +1,189 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# Integration tests for Mascarade cockpit scripts
|
||||
# Validates that each script produces well-formed cockpit-v1 JSON output.
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
|
||||
COCKPIT_DIR="$ROOT_DIR/tools/cockpit"
|
||||
|
||||
PASSED=0
|
||||
FAILED=0
|
||||
TOTAL=0
|
||||
|
||||
# Colors (disabled if not a terminal)
|
||||
if [[ -t 1 ]]; then
|
||||
GREEN=$'\033[0;32m'
|
||||
RED=$'\033[0;31m'
|
||||
BOLD=$'\033[1m'
|
||||
RESET=$'\033[0m'
|
||||
else
|
||||
GREEN="" RED="" BOLD="" RESET=""
|
||||
fi
|
||||
|
||||
pass() {
|
||||
PASSED=$((PASSED + 1))
|
||||
TOTAL=$((TOTAL + 1))
|
||||
printf " ${GREEN}PASS${RESET} %s\n" "$1"
|
||||
}
|
||||
|
||||
fail() {
|
||||
FAILED=$((FAILED + 1))
|
||||
TOTAL=$((TOTAL + 1))
|
||||
printf " ${RED}FAIL${RESET} %s — %s\n" "$1" "$2"
|
||||
}
|
||||
|
||||
# Require jq
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
echo "ERROR: jq is required but not found in PATH." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "${BOLD}Mascarade Integration Tests${RESET}"
|
||||
echo "=========================="
|
||||
echo ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 1: mascarade_runtime_health.sh --json
|
||||
# ---------------------------------------------------------------------------
|
||||
TEST_NAME="mascarade_runtime_health.sh --json"
|
||||
SCRIPT="$COCKPIT_DIR/mascarade_runtime_health.sh"
|
||||
|
||||
if [[ ! -x "$SCRIPT" ]]; then
|
||||
fail "$TEST_NAME" "script not found or not executable"
|
||||
else
|
||||
OUTPUT="$(bash "$SCRIPT" --json 2>/dev/null)" || true
|
||||
|
||||
if [[ -z "$OUTPUT" ]]; then
|
||||
fail "$TEST_NAME" "no output produced"
|
||||
elif ! echo "$OUTPUT" | jq empty 2>/dev/null; then
|
||||
fail "$TEST_NAME" "output is not valid JSON"
|
||||
else
|
||||
# Verify cockpit-v1 structure
|
||||
CV="$(echo "$OUTPUT" | jq -r '.contract_version // empty')"
|
||||
COMP="$(echo "$OUTPUT" | jq -r '.component // empty')"
|
||||
STAT="$(echo "$OUTPUT" | jq -r '.status // empty')"
|
||||
CHECKS="$(echo "$OUTPUT" | jq -r '.checks // empty')"
|
||||
|
||||
if [[ "$CV" != "cockpit-v1" ]]; then
|
||||
fail "$TEST_NAME" "contract_version is '$CV', expected 'cockpit-v1'"
|
||||
elif [[ -z "$COMP" ]]; then
|
||||
fail "$TEST_NAME" "missing .component field"
|
||||
elif [[ -z "$STAT" ]]; then
|
||||
fail "$TEST_NAME" "missing .status field"
|
||||
elif [[ "$CHECKS" == "" || "$CHECKS" == "null" ]]; then
|
||||
fail "$TEST_NAME" "missing .checks object"
|
||||
else
|
||||
pass "$TEST_NAME"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 2: mascarade_dispatch_mesh.sh --action route --profile heavy-code --json
|
||||
# ---------------------------------------------------------------------------
|
||||
TEST_NAME="mascarade_dispatch_mesh.sh --action route --profile heavy-code --json"
|
||||
SCRIPT="$COCKPIT_DIR/mascarade_dispatch_mesh.sh"
|
||||
|
||||
if [[ ! -x "$SCRIPT" ]]; then
|
||||
fail "$TEST_NAME" "script not found or not executable"
|
||||
else
|
||||
OUTPUT="$(bash "$SCRIPT" --action route --profile heavy-code --json 2>/dev/null)" || true
|
||||
|
||||
if [[ -z "$OUTPUT" ]]; then
|
||||
fail "$TEST_NAME" "no output produced"
|
||||
elif ! echo "$OUTPUT" | jq empty 2>/dev/null; then
|
||||
fail "$TEST_NAME" "output is not valid JSON"
|
||||
else
|
||||
CV="$(echo "$OUTPUT" | jq -r '.contract_version // empty')"
|
||||
STAT="$(echo "$OUTPUT" | jq -r '.status // empty')"
|
||||
|
||||
if [[ "$CV" != "cockpit-v1" ]]; then
|
||||
fail "$TEST_NAME" "contract_version is '$CV', expected 'cockpit-v1'"
|
||||
elif [[ -z "$STAT" ]]; then
|
||||
fail "$TEST_NAME" "missing .status field"
|
||||
else
|
||||
pass "$TEST_NAME"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 3: langfuse_health.sh --json
|
||||
# ---------------------------------------------------------------------------
|
||||
TEST_NAME="langfuse_health.sh --json"
|
||||
SCRIPT="$COCKPIT_DIR/langfuse_health.sh"
|
||||
|
||||
if [[ ! -x "$SCRIPT" ]]; then
|
||||
fail "$TEST_NAME" "script not found or not executable"
|
||||
else
|
||||
OUTPUT="$(bash "$SCRIPT" --json 2>/dev/null)" || true
|
||||
|
||||
if [[ -z "$OUTPUT" ]]; then
|
||||
fail "$TEST_NAME" "no output produced"
|
||||
elif ! echo "$OUTPUT" | jq empty 2>/dev/null; then
|
||||
fail "$TEST_NAME" "output is not valid JSON"
|
||||
else
|
||||
# Even if Langfuse is unreachable, JSON structure must be valid
|
||||
CV="$(echo "$OUTPUT" | jq -r '.contract_version // empty')"
|
||||
COMP="$(echo "$OUTPUT" | jq -r '.component // empty')"
|
||||
STAT="$(echo "$OUTPUT" | jq -r '.status // empty')"
|
||||
HAS_REASONS="$(echo "$OUTPUT" | jq 'has("degraded_reasons")')"
|
||||
|
||||
if [[ "$CV" != "cockpit-v1" ]]; then
|
||||
fail "$TEST_NAME" "contract_version is '$CV', expected 'cockpit-v1'"
|
||||
elif [[ -z "$COMP" ]]; then
|
||||
fail "$TEST_NAME" "missing .component field"
|
||||
elif [[ -z "$STAT" ]]; then
|
||||
fail "$TEST_NAME" "missing .status field"
|
||||
elif [[ "$HAS_REASONS" != "true" ]]; then
|
||||
fail "$TEST_NAME" "missing .degraded_reasons array"
|
||||
else
|
||||
pass "$TEST_NAME"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 4: runtime_ai_gateway.sh --action status --json
|
||||
# ---------------------------------------------------------------------------
|
||||
TEST_NAME="runtime_ai_gateway.sh --action status --json"
|
||||
SCRIPT="$COCKPIT_DIR/runtime_ai_gateway.sh"
|
||||
|
||||
if [[ ! -x "$SCRIPT" ]]; then
|
||||
fail "$TEST_NAME" "script not found or not executable"
|
||||
else
|
||||
OUTPUT="$(bash "$SCRIPT" --action status --json 2>/dev/null)" || true
|
||||
|
||||
if [[ -z "$OUTPUT" ]]; then
|
||||
fail "$TEST_NAME" "no output produced"
|
||||
elif ! echo "$OUTPUT" | jq empty 2>/dev/null; then
|
||||
fail "$TEST_NAME" "output is not valid JSON"
|
||||
else
|
||||
CV="$(echo "$OUTPUT" | jq -r '.contract_version // empty')"
|
||||
STAT="$(echo "$OUTPUT" | jq -r '.status // empty')"
|
||||
|
||||
if [[ "$CV" != "cockpit-v1" ]]; then
|
||||
fail "$TEST_NAME" "contract_version is '$CV', expected 'cockpit-v1'"
|
||||
elif [[ -z "$STAT" ]]; then
|
||||
fail "$TEST_NAME" "missing .status field"
|
||||
else
|
||||
pass "$TEST_NAME"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summary
|
||||
# ---------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "=========================="
|
||||
printf "${BOLD}Results: %d/%d tests passed${RESET}\n" "$PASSED" "$TOTAL"
|
||||
|
||||
if [[ "$FAILED" -gt 0 ]]; then
|
||||
printf "${RED}%d test(s) failed.${RESET}\n" "$FAILED"
|
||||
exit 1
|
||||
else
|
||||
printf "${GREEN}All tests passed.${RESET}\n"
|
||||
exit 0
|
||||
fi
|
||||
Reference in New Issue
Block a user