From 190d424aaa353efff23644936ec15cd9c26f6ff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20SAILLANT?= <108685187+electron-rare@users.noreply.github.com> Date: Sat, 21 Feb 2026 01:55:30 +0100 Subject: [PATCH] harden zeroclaw stack reuse and add optional prometheus wiring --- specs/zeroclaw_dual_hw_orchestration_spec.md | 21 +++ tools/ai/zeroclaw_stack_down.sh | 24 ++- tools/ai/zeroclaw_stack_up.sh | 149 ++++++++++++++++++- 3 files changed, 191 insertions(+), 3 deletions(-) diff --git a/specs/zeroclaw_dual_hw_orchestration_spec.md b/specs/zeroclaw_dual_hw_orchestration_spec.md index 90da0f2..b4676ce 100644 --- a/specs/zeroclaw_dual_hw_orchestration_spec.md +++ b/specs/zeroclaw_dual_hw_orchestration_spec.md @@ -68,12 +68,16 @@ This keeps `autonomy.workspace_only = true` effective on a per-repo boundary. - token sourcing from `gh auth token` at runtime only when `copilot` is selected. - `tools/ai/zeroclaw_stack_up.sh` - starts local gateway and local follow server, + - reuses existing listeners when ports are already bound (prevents duplicate-start failures), - generates live follow dashboard at `http://127.0.0.1:8788/`, - dashboard includes live polling panels for `/conversations.jsonl` and `/gateway.log` (1s polling), - preserves direct raw links: `/conversations.jsonl` and `/gateway.log`, + - writes `artifacts/zeroclaw/prometheus.yml` scrape config, + - supports local Prometheus startup via `ZEROCLAW_PROM_MODE` (`off`, `auto`, `binary`, `docker`), - stores pair token in `artifacts/zeroclaw/pair_token.txt`. - `tools/ai/zeroclaw_stack_down.sh` - stops local gateway/follow processes, + - stops local Prometheus process/container if managed by the stack, - confirms logs remain in `artifacts/zeroclaw/`. - `tools/ai/zeroclaw_webhook_send.sh` - requires `--allow-model-call` before any real webhook send, @@ -136,6 +140,7 @@ Raw URLs: - `http://127.0.0.1:8788/conversations.jsonl` - `http://127.0.0.1:8788/gateway.log` +- `http://127.0.0.1:8788/prometheus.yml` Conversation JSONL line schema (append-only): @@ -153,3 +158,19 @@ Compatibility rule: Credit-protection rule: - without `--allow-model-call`, `zeroclaw_webhook_send.sh` exits non-zero and does not send/write logs. + +## 9) Prometheus Integration + +Economical default: + +- `ZEROCLAW_PROM_MODE=auto` (start only if local `prometheus` binary exists) + +Optional modes: + +- `ZEROCLAW_PROM_MODE=off` disables Prometheus startup +- `ZEROCLAW_PROM_MODE=binary` requires local `prometheus` binary +- `ZEROCLAW_PROM_MODE=docker` runs `prom/prometheus:latest` locally + +Default local endpoint when running: + +- `http://127.0.0.1:9090/targets` diff --git a/tools/ai/zeroclaw_stack_down.sh b/tools/ai/zeroclaw_stack_down.sh index d8032f0..605ef0a 100755 --- a/tools/ai/zeroclaw_stack_down.sh +++ b/tools/ai/zeroclaw_stack_down.sh @@ -4,6 +4,11 @@ set -euo pipefail ART_DIR="${ZEROCLAW_ART_DIR:-/Users/cils/Documents/Lelectron_rare/Kill_LIFE/artifacts/zeroclaw}" GW_PID_FILE="$ART_DIR/gateway.pid" FW_PID_FILE="$ART_DIR/follow.pid" +PROM_PID_FILE="$ART_DIR/prometheus.pid" +PROM_CONTAINER="${ZEROCLAW_PROM_CONTAINER:-zeroclaw-prometheus}" +GW_MANAGED_FILE="$ART_DIR/gateway.managed" +FW_MANAGED_FILE="$ART_DIR/follow.managed" +PROM_MANAGED_FILE="$ART_DIR/prometheus.managed" stop_pid_file() { local pid_file="$1" @@ -20,6 +25,21 @@ stop_pid_file() { rm -f "$pid_file" } -stop_pid_file "$GW_PID_FILE" -stop_pid_file "$FW_PID_FILE" +if [[ -f "$GW_MANAGED_FILE" ]]; then + stop_pid_file "$GW_PID_FILE" +fi +if [[ -f "$FW_MANAGED_FILE" ]]; then + stop_pid_file "$FW_PID_FILE" +fi +if [[ -f "$PROM_MANAGED_FILE" ]]; then + prom_mode="$(cat "$PROM_MANAGED_FILE" 2>/dev/null || true)" + if [[ "$prom_mode" == "binary" ]]; then + stop_pid_file "$PROM_PID_FILE" + elif [[ "$prom_mode" == "docker" ]] && command -v docker >/dev/null 2>&1; then + if docker ps -a --filter "name=^/${PROM_CONTAINER}$" --format '{{.ID}}' | grep -q .; then + docker rm -f "$PROM_CONTAINER" >/dev/null 2>&1 || true + fi + fi +fi +rm -f "$GW_MANAGED_FILE" "$FW_MANAGED_FILE" "$PROM_MANAGED_FILE" echo "ZeroClaw local stack stopped. Logs preserved in artifacts/zeroclaw/." diff --git a/tools/ai/zeroclaw_stack_up.sh b/tools/ai/zeroclaw_stack_up.sh index 623851b..d681ad1 100755 --- a/tools/ai/zeroclaw_stack_up.sh +++ b/tools/ai/zeroclaw_stack_up.sh @@ -7,6 +7,12 @@ ZEROCLAW_BIN="${ZEROCLAW_BIN:-$ROOT_DIR/zeroclaw/target/release/zeroclaw}" GATEWAY_HOST="${ZEROCLAW_GATEWAY_HOST:-127.0.0.1}" GATEWAY_PORT="${ZEROCLAW_GATEWAY_PORT:-3000}" FOLLOW_PORT="${ZEROCLAW_FOLLOW_PORT:-8788}" +PROM_MODE="${ZEROCLAW_PROM_MODE:-auto}" +PROM_HOST="${ZEROCLAW_PROM_HOST:-127.0.0.1}" +PROM_PORT="${ZEROCLAW_PROM_PORT:-9090}" +PROM_RETENTION="${ZEROCLAW_PROM_RETENTION:-24h}" +PROM_SCRAPE_INTERVAL="${ZEROCLAW_PROM_SCRAPE_INTERVAL:-15s}" +PROM_CONTAINER="${ZEROCLAW_PROM_CONTAINER:-zeroclaw-prometheus}" GW_PID_FILE="$ART_DIR/gateway.pid" FW_PID_FILE="$ART_DIR/follow.pid" @@ -15,11 +21,23 @@ FW_LOG="$ART_DIR/follow.log" TOKEN_FILE="$ART_DIR/pair_token.txt" CONVO_FILE="$ART_DIR/conversations.jsonl" INDEX_FILE="$ART_DIR/index.html" +PROM_PID_FILE="$ART_DIR/prometheus.pid" +PROM_LOG="$ART_DIR/prometheus.log" +PROM_CONFIG_FILE="$ART_DIR/prometheus.yml" +PROM_DATA_DIR="$ART_DIR/prometheus-data" +PROM_STATUS="disabled" +GW_MANAGED_FILE="$ART_DIR/gateway.managed" +FW_MANAGED_FILE="$ART_DIR/follow.managed" +PROM_MANAGED_FILE="$ART_DIR/prometheus.managed" mkdir -p "$ART_DIR" touch "$CONVO_FILE" touch "$GW_LOG" +if [[ -f "$HOME/.zeroclaw/config.toml" ]]; then + chmod 600 "$HOME/.zeroclaw/config.toml" >/dev/null 2>&1 || true +fi + if [[ ! -x "$ZEROCLAW_BIN" ]]; then if command -v zeroclaw >/dev/null 2>&1; then ZEROCLAW_BIN="$(command -v zeroclaw)" @@ -38,18 +56,113 @@ is_running() { kill -0 "$pid" >/dev/null 2>&1 } +listener_pid_on_port() { + local port="$1" + lsof -nP -iTCP:"$port" -sTCP:LISTEN -t 2>/dev/null | head -n 1 +} + +write_prometheus_config() { + mkdir -p "$PROM_DATA_DIR" + cat >"$PROM_CONFIG_FILE" </dev/null 2>&1; then + return 1 + fi + nohup prometheus \ + --config.file="$PROM_CONFIG_FILE" \ + --storage.tsdb.path="$PROM_DATA_DIR" \ + --storage.tsdb.retention.time="$PROM_RETENTION" \ + --web.listen-address="$PROM_HOST:$PROM_PORT" \ + >"$PROM_LOG" 2>&1 & + echo "$!" >"$PROM_PID_FILE" + sleep 0.3 + if is_running "$PROM_PID_FILE"; then + printf '%s\n' "binary" >"$PROM_MANAGED_FILE" + PROM_STATUS="binary" + return 0 + fi + rm -f "$PROM_PID_FILE" + return 1 +} + +start_prometheus_docker() { + if ! command -v docker >/dev/null 2>&1; then + return 1 + fi + if ! docker info >/dev/null 2>&1; then + return 1 + fi + local running_id + running_id="$(docker ps --filter "name=^/${PROM_CONTAINER}$" --format '{{.ID}}' | head -n 1)" + if [[ -n "$running_id" ]]; then + printf '%s\n' "docker" >"$PROM_MANAGED_FILE" + PROM_STATUS="docker(existing:$PROM_CONTAINER)" + return 0 + fi + + if docker ps -a --filter "name=^/${PROM_CONTAINER}$" --format '{{.ID}}' | grep -q .; then + docker rm -f "$PROM_CONTAINER" >/dev/null 2>&1 || true + fi + + local container_id + container_id="$(docker run -d \ + --name "$PROM_CONTAINER" \ + -p "$PROM_HOST:$PROM_PORT:9090" \ + -v "$PROM_CONFIG_FILE:/etc/prometheus/prometheus.yml:ro" \ + -v "$PROM_DATA_DIR:/prometheus" \ + prom/prometheus:latest \ + --config.file=/etc/prometheus/prometheus.yml \ + --storage.tsdb.path=/prometheus \ + --storage.tsdb.retention.time="$PROM_RETENTION" \ + --web.listen-address=:9090 2>>"$PROM_LOG" || true)" + if [[ -n "$container_id" ]]; then + printf '%s\n' "docker" >"$PROM_MANAGED_FILE" + PROM_STATUS="docker($PROM_CONTAINER)" + return 0 + fi + return 1 +} + if is_running "$GW_PID_FILE"; then echo "[info] gateway already running (pid $(cat "$GW_PID_FILE"))." +elif [[ -n "$(listener_pid_on_port "$GATEWAY_PORT")" ]]; then + existing_gateway_pid="$(listener_pid_on_port "$GATEWAY_PORT")" + echo "[info] gateway port $GATEWAY_PORT already listening (pid $existing_gateway_pid). reusing." + printf '%s\n' "$existing_gateway_pid" >"$GW_PID_FILE" + rm -f "$GW_MANAGED_FILE" else nohup "$ZEROCLAW_BIN" gateway --port "$GATEWAY_PORT" --host "$GATEWAY_HOST" >"$GW_LOG" 2>&1 & echo "$!" >"$GW_PID_FILE" + printf '%s\n' "1" >"$GW_MANAGED_FILE" fi if is_running "$FW_PID_FILE"; then echo "[info] follow server already running (pid $(cat "$FW_PID_FILE"))." +elif [[ -n "$(listener_pid_on_port "$FOLLOW_PORT")" ]]; then + existing_follow_pid="$(listener_pid_on_port "$FOLLOW_PORT")" + echo "[info] follow port $FOLLOW_PORT already listening (pid $existing_follow_pid). reusing." + printf '%s\n' "$existing_follow_pid" >"$FW_PID_FILE" + rm -f "$FW_MANAGED_FILE" else nohup python3 -m http.server "$FOLLOW_PORT" --bind 127.0.0.1 --directory "$ART_DIR" >"$FW_LOG" 2>&1 & echo "$!" >"$FW_PID_FILE" + printf '%s\n' "1" >"$FW_MANAGED_FILE" fi for _ in $(seq 1 40); do @@ -75,6 +188,36 @@ print(obj.get("token",""), end="")')" fi fi +write_prometheus_config +case "$PROM_MODE" in + off) + rm -f "$PROM_MANAGED_FILE" + PROM_STATUS="disabled(mode=off)" + ;; + auto) + rm -f "$PROM_MANAGED_FILE" + if ! start_prometheus_binary; then + PROM_STATUS="disabled(prometheus missing; use ZEROCLAW_PROM_MODE=docker)" + fi + ;; + binary) + rm -f "$PROM_MANAGED_FILE" + if ! start_prometheus_binary; then + PROM_STATUS="disabled(prometheus binary unavailable)" + fi + ;; + docker) + rm -f "$PROM_MANAGED_FILE" + if ! start_prometheus_docker; then + PROM_STATUS="disabled(docker unavailable)" + fi + ;; + *) + rm -f "$PROM_MANAGED_FILE" + PROM_STATUS="disabled(invalid mode:$PROM_MODE)" + ;; +esac + cat >"$INDEX_FILE" < @@ -177,13 +320,17 @@ cat >"$INDEX_FILE" < Follow URL: http://127.0.0.1:$FOLLOW_PORT/ | Polling interval: 1s | - Display cap: 500 lines/panel + Display cap: 500 lines/panel | + Prometheus: $PROM_STATUS
/conversations.jsonl /gateway.log + /prometheus.yml /health /metrics + prometheus /targets + prometheus /graph