Files
hapi/scripts/verify-deploy.sh
T
c97180c54c chore(scripts): add post-deploy verification with a baseline
Checks hub health, both machine binaries/signers/runner exec paths, hub
metadata mtime, agent availability against the last passing baseline, and
Codex model loading. Prints a PASS/FAIL table and updates the baseline on
success.

via [HAPI](https://hapi.run)

Co-Authored-By: HAPI <noreply@hapi.run>
Co-Authored-By: Claude Code <noreply@anthropic.com>
2026-11-08 20:24:30 +08:00

295 lines
11 KiB
Bash
Executable File

#!/bin/bash
# Post-deploy functional verification for mini + k2lab.
#
# Proves the machines are actually USABLE, not just that a binary was installed:
# health / version / signer / mtime / runner process & executable
# hub metadata matches the installed file
# agent availability did not regress vs the last passing run
# Codex model loading still works where it worked before
#
# Usage:
# scripts/verify-deploy.sh # verify, update baseline on PASS
# scripts/verify-deploy.sh --init # only record current state as baseline
# scripts/verify-deploy.sh --json # machine-readable summary
# scripts/verify-deploy.sh --no-baseline # skip baseline comparison
#
# Exit: 0 = PASS, 1 = FAIL, 2 = setup error (hub unreachable / no JWT)
#
# Never prints the hub token.
set -uo pipefail
cd "$(dirname "$0")/.."
HUB=${HAPI_HUB_URL:-http://127.0.0.1:3006}
REMOTE_TARGET=${HAPI_REMOTE_TARGET:-k2lab}
LOCAL_NAME=${HAPI_LOCAL_NAME:-小吴家}
REMOTE_NAME=${HAPI_REMOTE_NAME:-小吴k2lab}
STABLE="$HOME/.hapi/bin/hapi"
BASELINE="$HOME/.hapi/deploy-baseline.json"
JSON=0
INIT=0
USE_BASELINE=1
for arg in "$@"; do
case "$arg" in
--json) JSON=1 ;;
--init) INIT=1 ;;
--no-baseline) USE_BASELINE=0 ;;
-h|--help) sed -n '2,20p' "$0"; exit 0 ;;
*) echo "unknown arg: $arg" >&2; exit 2 ;;
esac
done
fails=0
warns=0
rows=()
json_rows=()
record() { # status check detail
local status=$1 check=$2 detail=$3
rows+=("$status|$check|$detail")
json_rows+=("{\"status\":\"$status\",\"check\":\"$check\",\"detail\":\"$detail\"}")
case "$status" in
FAIL) fails=$((fails + 1)) ;;
WARN) warns=$((warns + 1)) ;;
esac
}
ok() { record OK "$1" "$2"; }
warn() { record WARN "$1" "$2"; }
fail() { record FAIL "$1" "$2"; }
get_jwt() {
grep -o 'token=eyJ[A-Za-z0-9._-]*' "$HOME/.hapi/logs/hub.log" 2>/dev/null \
| tail -1 | sed 's/^token=//'
}
api() { # path
curl -fsS --max-time 10 -H "Authorization: Bearer $JWT" "$HUB$1" 2>/dev/null
}
machine_id() { # display name
sqlite3 "$HOME/.hapi/hapi.db" \
"select id from machines where json_extract(metadata,'\$.displayName')='$1';" 2>/dev/null
}
# ---------- hub ----------
hub_ok=0
if curl -fsS --max-time 5 "$HUB/health" >/dev/null 2>&1; then
hub_ok=1
ok "hub /health" "200"
else
fail "hub /health" "unreachable at $HUB"
fi
JWT=$(get_jwt)
if [ -z "$JWT" ]; then
fail "hub token" "no token= in ~/.hapi/logs/hub.log"
[ "$hub_ok" = 1 ] && exit 2
fi
# ---------- local binary ----------
local_version=""
local_mtime=""
if [ -x "$STABLE" ]; then
local_version=$("$STABLE" --version 2>/dev/null | sed 's/^hapi version: //')
local_mtime=$(stat -f '%m' "$STABLE" 2>/dev/null)
[ -n "$local_version" ] && ok "mini binary" "version $local_version" \
|| fail "mini binary" "cannot run $STABLE"
ident=$(codesign -dv "$STABLE" 2>&1 | sed -n 's/^Identifier=//p')
team=$(codesign -dv "$STABLE" 2>&1 | sed -n 's/^TeamIdentifier=//p')
if [ "$ident" = "run.hapi.cli" ] && [ "$team" = "X75CUV458P" ]; then
ok "mini signer" "$ident / $team"
else
fail "mini signer" "got ${ident:-none} / ${team:-none}, expected run.hapi.cli / X75CUV458P"
fi
else
fail "mini binary" "missing $STABLE"
fi
# ---------- local runner ----------
local_pid=$(sed -n 's/.*"pid": *\([0-9][0-9]*\).*/\1/p' "$HOME/.hapi/runner.state.json" 2>/dev/null | head -n 1)
if [ -n "$local_pid" ] && kill -0 "$local_pid" 2>/dev/null; then
exe=$(lsof -p "$local_pid" 2>/dev/null | awk '$4=="txt"{print $NF}' | head -n 1)
[ "$exe" = "$STABLE" ] && ok "mini runner" "pid $local_pid -> $exe" \
|| fail "mini runner" "pid $local_pid runs ${exe:-unknown}, expected $STABLE"
else
fail "mini runner" "no live pid in runner.state.json"
fi
# ---------- remote ----------
remote_version=""
remote_mtime=""
remote_out=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$REMOTE_TARGET" '
v=$("$HOME/.hapi/bin/hapi" --version 2>/dev/null | sed "s/^hapi version: //")
m=$(stat -f %m "$HOME/.hapi/bin/hapi" 2>/dev/null)
i=$(codesign -dv "$HOME/.hapi/bin/hapi" 2>&1 | sed -n "s/^Identifier=//p")
t=$(codesign -dv "$HOME/.hapi/bin/hapi" 2>&1 | sed -n "s/^TeamIdentifier=//p")
p=$(sed -n "s/.*\"pid\": *\([0-9][0-9]*\).*/\1/p" "$HOME/.hapi/runner.state.json" 2>/dev/null | head -n 1)
if [ -n "$p" ] && kill -0 "$p" 2>/dev/null; then a=alive; else a=dead; fi
e=""
if [ "$a" = alive ]; then e=$(lsof -p "$p" 2>/dev/null | awk "\$4==\"txt\"{print \$NF}" | head -n 1); fi
printf "VERSION=%s\nMTIME=%s\nIDENT=%s\nTEAM=%s\nPID=%s\nALIVE=%s\nEXE=%s\n" \
"$v" "$m" "$i" "$t" "$p" "$a" "$e"
' 2>/dev/null)
if [ -z "$remote_out" ]; then
fail "k2lab ssh" "cannot reach $REMOTE_TARGET"
else
remote_version=$(printf '%s\n' "$remote_out" | sed -n 's/^VERSION=//p')
remote_mtime=$(printf '%s\n' "$remote_out" | sed -n 's/^MTIME=//p')
r_ident=$(printf '%s\n' "$remote_out" | sed -n 's/^IDENT=//p')
r_team=$(printf '%s\n' "$remote_out" | sed -n 's/^TEAM=//p')
r_pid=$(printf '%s\n' "$remote_out" | sed -n 's/^PID=//p')
r_alive=$(printf '%s\n' "$remote_out" | sed -n 's/^ALIVE=//p')
r_exe=$(printf '%s\n' "$remote_out" | sed -n 's/^EXE=//p')
[ -n "$remote_version" ] && ok "k2lab binary" "version $remote_version" \
|| fail "k2lab binary" "cannot run ~/.hapi/bin/hapi"
if [ "$r_ident" = "run.hapi.cli" ] && [ "$r_team" = "X75CUV458P" ]; then
ok "k2lab signer" "$r_ident / $r_team"
else
fail "k2lab signer" "got ${r_ident:-none} / ${r_team:-none}"
fi
if [ "$r_alive" = "alive" ]; then
case "$r_exe" in
*/".hapi/bin/hapi") ok "k2lab runner" "pid $r_pid -> $r_exe" ;;
*) fail "k2lab runner" "pid $r_pid runs ${r_exe:-unknown}" ;;
esac
else
fail "k2lab runner" "not running"
fi
fi
# ---------- version consistency ----------
if [ -n "$local_version" ] && [ -n "$remote_version" ]; then
[ "$local_version" = "$remote_version" ] \
&& ok "version match" "both $local_version" \
|| fail "version match" "mini=$local_version k2lab=$remote_version"
fi
# ---------- hub metadata ----------
check_meta() { # display name installed_mtime
local name=$1 mtime=$2 id row meta_ms meta_ver
id=$(machine_id "$name")
if [ -z "$id" ]; then
warn "hub $name" "machine not registered"
return
fi
local i
for i in $(seq 1 20); do
row=$(sqlite3 "$HOME/.hapi/hapi.db" \
"select coalesce(json_extract(metadata,'\$.happyCliVersion'),''), coalesce(json_extract(metadata,'\$.startedCliMtimeMs'),0) from machines where id='$id';" 2>/dev/null)
meta_ver=${row%%|*}
meta_ms=${row##*|}
meta_ms=${meta_ms%%.*} # sqlite/json may yield a float
case "$meta_ms" in ''|*[!0-9]*) meta_ms=0 ;; esac
[ -n "$meta_ver" ] && break
sleep 1
done
[ "$meta_ver" = "$local_version" ] \
&& ok "hub $name version" "$meta_ver" \
|| fail "hub $name version" "hub=$meta_ver binary=$local_version"
if [ -n "$mtime" ] && [ -n "$meta_ms" ] && [ "$meta_ms" != "0" ]; then
delta=$(( meta_ms / 1000 - mtime ))
[ "${delta#-}" -le 2 ] \
&& ok "hub $name mtime" "matches installed file" \
|| fail "hub $name mtime" "hub=${meta_ms}ms file=${mtime}s (delta ${delta}s)"
fi
}
[ "$hub_ok" = 1 ] && check_meta "$LOCAL_NAME" "$local_mtime"
[ "$hub_ok" = 1 ] && [ -n "$remote_mtime" ] && check_meta "$REMOTE_NAME" "$remote_mtime"
# ---------- agent availability + models ----------
baseline_get() { # name key
[ -f "$BASELINE" ] || return 1
jq -r --arg n "$1" --arg k "$2" '.[$n][$k] // empty | if type=="array" then join(",") else tostring end' "$BASELINE" 2>/dev/null
}
baseline_put() { # name json_fragment
mkdir -p "$(dirname "$BASELINE")"
[ -f "$BASELINE" ] || echo '{}' > "$BASELINE"
tmp=$(mktemp)
jq --arg n "$1" --argjson v "$2" '.[$n] = $v' "$BASELINE" > "$tmp" 2>/dev/null && mv "$tmp" "$BASELINE"
}
verify_machine_agents() { # display name
local name=$1 id resp now_agents base_agents
id=$(machine_id "$name")
[ -n "$id" ] || return
resp=$(api "/api/machines/$id/agent-availability")
if [ -z "$resp" ]; then
fail "agents $name" "agent-availability request failed"
return
fi
now_agents=$(printf '%s' "$resp" | jq -r '[.agents[] | select(.available) | .agent] | join(",")' 2>/dev/null)
[ -n "$now_agents" ] && ok "agents $name" "$(printf '%s' "$now_agents" | tr ',' ' ')" \
|| warn "agents $name" "none available"
if [ "$USE_BASELINE" = 1 ] && [ "$INIT" = 0 ]; then
base_agents=$(baseline_get "$name" agents)
if [ -n "$base_agents" ]; then
lost=$(comm -23 <(printf '%s' "$base_agents" | tr ',' '\n' | sort) \
<(printf '%s' "$now_agents" | tr ',' '\n' | sort) | paste -sd, -)
[ -z "$lost" ] && ok "agents $name baseline" "no regression" \
|| fail "agents $name baseline" "lost: $lost"
else
warn "agents $name baseline" "not recorded yet (run --init)"
fi
fi
resp=$(api "/api/machines/$id/codex-models")
if [ -n "$resp" ]; then
success=$(printf '%s' "$resp" | jq -r '.success // false' 2>/dev/null)
count=$(printf '%s' "$resp" | jq -r '(.models // []) | length' 2>/dev/null)
if [ "$success" = "true" ]; then
ok "codex $name" "$count models"
else
base=$(baseline_get "$name" codex_models)
[ "$base" = "true" ] && fail "codex $name" "was working, now success=false" \
|| warn "codex $name" "not available (also before)"
fi
fi
}
if [ "$hub_ok" = 1 ] && [ -n "$JWT" ]; then
verify_machine_agents "$LOCAL_NAME"
verify_machine_agents "$REMOTE_NAME"
fi
# ---------- baseline ----------
if [ "$INIT" = 1 ] || [ "$USE_BASELINE" = 1 ] && [ "$fails" = 0 ]; then
for name in "$LOCAL_NAME" "$REMOTE_NAME"; do
id=$(machine_id "$name")
[ -n "$id" ] || continue
agents=$(api "/api/machines/$id/agent-availability" | jq -c '[.agents[] | select(.available) | .agent]' 2>/dev/null)
codex=$(api "/api/machines/$id/codex-models" | jq -r '.success // false' 2>/dev/null)
[ -n "$agents" ] && baseline_put "$name" "{\"agents\":$agents,\"codex_models\":${codex:-false}}"
done
fi
# ---------- output ----------
if [ "$JSON" = 1 ]; then
printf '{"result":"%s","failed":%d,"warned":%d,"checks":[%s]}\n' \
"$([ "$fails" = 0 ] && echo PASS || echo FAIL)" "$fails" "$warns" \
"$(IFS=,; echo "${json_rows[*]}")"
else
printf '%-6s %-24s %s\n' "STATUS" "CHECK" "DETAIL"
printf '%-6s %-24s %s\n' "------" "------------------------" "----------------------------------------"
for r in "${rows[@]}"; do
IFS='|' read -r s c d <<< "$r"
printf '%-6s %-24s %s\n' "$s" "$c" "$d"
done
echo
if [ "$fails" = 0 ]; then
echo "RESULT: PASS (warn $warns)"
else
echo "RESULT: FAIL ($fails failed, $warns warned)"
echo "rollback: see SKILL.md / scripts print the exact command after each deploy"
fi
fi
[ "$fails" = 0 ] && exit 0 || exit 1