mirror of
https://github.com/wu736139669/hapi.git
synced 2026-11-18 03:27:08 +00:00
Checks hub health, both machine binaries/signers/runner exec paths, hub metadata mtime, agent availability against the last passing baseline, and Codex model loading. Prints a PASS/FAIL table and updates the baseline on success. via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> Co-Authored-By: Claude Code <noreply@anthropic.com>
295 lines
11 KiB
Bash
Executable File
295 lines
11 KiB
Bash
Executable File
#!/bin/bash
|
|
# Post-deploy functional verification for mini + k2lab.
|
|
#
|
|
# Proves the machines are actually USABLE, not just that a binary was installed:
|
|
# health / version / signer / mtime / runner process & executable
|
|
# hub metadata matches the installed file
|
|
# agent availability did not regress vs the last passing run
|
|
# Codex model loading still works where it worked before
|
|
#
|
|
# Usage:
|
|
# scripts/verify-deploy.sh # verify, update baseline on PASS
|
|
# scripts/verify-deploy.sh --init # only record current state as baseline
|
|
# scripts/verify-deploy.sh --json # machine-readable summary
|
|
# scripts/verify-deploy.sh --no-baseline # skip baseline comparison
|
|
#
|
|
# Exit: 0 = PASS, 1 = FAIL, 2 = setup error (hub unreachable / no JWT)
|
|
#
|
|
# Never prints the hub token.
|
|
set -uo pipefail
|
|
|
|
cd "$(dirname "$0")/.."
|
|
|
|
HUB=${HAPI_HUB_URL:-http://127.0.0.1:3006}
|
|
REMOTE_TARGET=${HAPI_REMOTE_TARGET:-k2lab}
|
|
LOCAL_NAME=${HAPI_LOCAL_NAME:-小吴家}
|
|
REMOTE_NAME=${HAPI_REMOTE_NAME:-小吴k2lab}
|
|
STABLE="$HOME/.hapi/bin/hapi"
|
|
BASELINE="$HOME/.hapi/deploy-baseline.json"
|
|
|
|
JSON=0
|
|
INIT=0
|
|
USE_BASELINE=1
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--json) JSON=1 ;;
|
|
--init) INIT=1 ;;
|
|
--no-baseline) USE_BASELINE=0 ;;
|
|
-h|--help) sed -n '2,20p' "$0"; exit 0 ;;
|
|
*) echo "unknown arg: $arg" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
fails=0
|
|
warns=0
|
|
rows=()
|
|
json_rows=()
|
|
|
|
record() { # status check detail
|
|
local status=$1 check=$2 detail=$3
|
|
rows+=("$status|$check|$detail")
|
|
json_rows+=("{\"status\":\"$status\",\"check\":\"$check\",\"detail\":\"$detail\"}")
|
|
case "$status" in
|
|
FAIL) fails=$((fails + 1)) ;;
|
|
WARN) warns=$((warns + 1)) ;;
|
|
esac
|
|
}
|
|
|
|
ok() { record OK "$1" "$2"; }
|
|
warn() { record WARN "$1" "$2"; }
|
|
fail() { record FAIL "$1" "$2"; }
|
|
|
|
get_jwt() {
|
|
grep -o 'token=eyJ[A-Za-z0-9._-]*' "$HOME/.hapi/logs/hub.log" 2>/dev/null \
|
|
| tail -1 | sed 's/^token=//'
|
|
}
|
|
|
|
api() { # path
|
|
curl -fsS --max-time 10 -H "Authorization: Bearer $JWT" "$HUB$1" 2>/dev/null
|
|
}
|
|
|
|
machine_id() { # display name
|
|
sqlite3 "$HOME/.hapi/hapi.db" \
|
|
"select id from machines where json_extract(metadata,'\$.displayName')='$1';" 2>/dev/null
|
|
}
|
|
|
|
# ---------- hub ----------
|
|
hub_ok=0
|
|
if curl -fsS --max-time 5 "$HUB/health" >/dev/null 2>&1; then
|
|
hub_ok=1
|
|
ok "hub /health" "200"
|
|
else
|
|
fail "hub /health" "unreachable at $HUB"
|
|
fi
|
|
|
|
JWT=$(get_jwt)
|
|
if [ -z "$JWT" ]; then
|
|
fail "hub token" "no token= in ~/.hapi/logs/hub.log"
|
|
[ "$hub_ok" = 1 ] && exit 2
|
|
fi
|
|
|
|
# ---------- local binary ----------
|
|
local_version=""
|
|
local_mtime=""
|
|
if [ -x "$STABLE" ]; then
|
|
local_version=$("$STABLE" --version 2>/dev/null | sed 's/^hapi version: //')
|
|
local_mtime=$(stat -f '%m' "$STABLE" 2>/dev/null)
|
|
[ -n "$local_version" ] && ok "mini binary" "version $local_version" \
|
|
|| fail "mini binary" "cannot run $STABLE"
|
|
ident=$(codesign -dv "$STABLE" 2>&1 | sed -n 's/^Identifier=//p')
|
|
team=$(codesign -dv "$STABLE" 2>&1 | sed -n 's/^TeamIdentifier=//p')
|
|
if [ "$ident" = "run.hapi.cli" ] && [ "$team" = "X75CUV458P" ]; then
|
|
ok "mini signer" "$ident / $team"
|
|
else
|
|
fail "mini signer" "got ${ident:-none} / ${team:-none}, expected run.hapi.cli / X75CUV458P"
|
|
fi
|
|
else
|
|
fail "mini binary" "missing $STABLE"
|
|
fi
|
|
|
|
# ---------- local runner ----------
|
|
local_pid=$(sed -n 's/.*"pid": *\([0-9][0-9]*\).*/\1/p' "$HOME/.hapi/runner.state.json" 2>/dev/null | head -n 1)
|
|
if [ -n "$local_pid" ] && kill -0 "$local_pid" 2>/dev/null; then
|
|
exe=$(lsof -p "$local_pid" 2>/dev/null | awk '$4=="txt"{print $NF}' | head -n 1)
|
|
[ "$exe" = "$STABLE" ] && ok "mini runner" "pid $local_pid -> $exe" \
|
|
|| fail "mini runner" "pid $local_pid runs ${exe:-unknown}, expected $STABLE"
|
|
else
|
|
fail "mini runner" "no live pid in runner.state.json"
|
|
fi
|
|
|
|
# ---------- remote ----------
|
|
remote_version=""
|
|
remote_mtime=""
|
|
remote_out=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$REMOTE_TARGET" '
|
|
v=$("$HOME/.hapi/bin/hapi" --version 2>/dev/null | sed "s/^hapi version: //")
|
|
m=$(stat -f %m "$HOME/.hapi/bin/hapi" 2>/dev/null)
|
|
i=$(codesign -dv "$HOME/.hapi/bin/hapi" 2>&1 | sed -n "s/^Identifier=//p")
|
|
t=$(codesign -dv "$HOME/.hapi/bin/hapi" 2>&1 | sed -n "s/^TeamIdentifier=//p")
|
|
p=$(sed -n "s/.*\"pid\": *\([0-9][0-9]*\).*/\1/p" "$HOME/.hapi/runner.state.json" 2>/dev/null | head -n 1)
|
|
if [ -n "$p" ] && kill -0 "$p" 2>/dev/null; then a=alive; else a=dead; fi
|
|
e=""
|
|
if [ "$a" = alive ]; then e=$(lsof -p "$p" 2>/dev/null | awk "\$4==\"txt\"{print \$NF}" | head -n 1); fi
|
|
printf "VERSION=%s\nMTIME=%s\nIDENT=%s\nTEAM=%s\nPID=%s\nALIVE=%s\nEXE=%s\n" \
|
|
"$v" "$m" "$i" "$t" "$p" "$a" "$e"
|
|
' 2>/dev/null)
|
|
|
|
if [ -z "$remote_out" ]; then
|
|
fail "k2lab ssh" "cannot reach $REMOTE_TARGET"
|
|
else
|
|
remote_version=$(printf '%s\n' "$remote_out" | sed -n 's/^VERSION=//p')
|
|
remote_mtime=$(printf '%s\n' "$remote_out" | sed -n 's/^MTIME=//p')
|
|
r_ident=$(printf '%s\n' "$remote_out" | sed -n 's/^IDENT=//p')
|
|
r_team=$(printf '%s\n' "$remote_out" | sed -n 's/^TEAM=//p')
|
|
r_pid=$(printf '%s\n' "$remote_out" | sed -n 's/^PID=//p')
|
|
r_alive=$(printf '%s\n' "$remote_out" | sed -n 's/^ALIVE=//p')
|
|
r_exe=$(printf '%s\n' "$remote_out" | sed -n 's/^EXE=//p')
|
|
|
|
[ -n "$remote_version" ] && ok "k2lab binary" "version $remote_version" \
|
|
|| fail "k2lab binary" "cannot run ~/.hapi/bin/hapi"
|
|
if [ "$r_ident" = "run.hapi.cli" ] && [ "$r_team" = "X75CUV458P" ]; then
|
|
ok "k2lab signer" "$r_ident / $r_team"
|
|
else
|
|
fail "k2lab signer" "got ${r_ident:-none} / ${r_team:-none}"
|
|
fi
|
|
if [ "$r_alive" = "alive" ]; then
|
|
case "$r_exe" in
|
|
*/".hapi/bin/hapi") ok "k2lab runner" "pid $r_pid -> $r_exe" ;;
|
|
*) fail "k2lab runner" "pid $r_pid runs ${r_exe:-unknown}" ;;
|
|
esac
|
|
else
|
|
fail "k2lab runner" "not running"
|
|
fi
|
|
fi
|
|
|
|
# ---------- version consistency ----------
|
|
if [ -n "$local_version" ] && [ -n "$remote_version" ]; then
|
|
[ "$local_version" = "$remote_version" ] \
|
|
&& ok "version match" "both $local_version" \
|
|
|| fail "version match" "mini=$local_version k2lab=$remote_version"
|
|
fi
|
|
|
|
# ---------- hub metadata ----------
|
|
check_meta() { # display name installed_mtime
|
|
local name=$1 mtime=$2 id row meta_ms meta_ver
|
|
id=$(machine_id "$name")
|
|
if [ -z "$id" ]; then
|
|
warn "hub $name" "machine not registered"
|
|
return
|
|
fi
|
|
local i
|
|
for i in $(seq 1 20); do
|
|
row=$(sqlite3 "$HOME/.hapi/hapi.db" \
|
|
"select coalesce(json_extract(metadata,'\$.happyCliVersion'),''), coalesce(json_extract(metadata,'\$.startedCliMtimeMs'),0) from machines where id='$id';" 2>/dev/null)
|
|
meta_ver=${row%%|*}
|
|
meta_ms=${row##*|}
|
|
meta_ms=${meta_ms%%.*} # sqlite/json may yield a float
|
|
case "$meta_ms" in ''|*[!0-9]*) meta_ms=0 ;; esac
|
|
[ -n "$meta_ver" ] && break
|
|
sleep 1
|
|
done
|
|
[ "$meta_ver" = "$local_version" ] \
|
|
&& ok "hub $name version" "$meta_ver" \
|
|
|| fail "hub $name version" "hub=$meta_ver binary=$local_version"
|
|
if [ -n "$mtime" ] && [ -n "$meta_ms" ] && [ "$meta_ms" != "0" ]; then
|
|
delta=$(( meta_ms / 1000 - mtime ))
|
|
[ "${delta#-}" -le 2 ] \
|
|
&& ok "hub $name mtime" "matches installed file" \
|
|
|| fail "hub $name mtime" "hub=${meta_ms}ms file=${mtime}s (delta ${delta}s)"
|
|
fi
|
|
}
|
|
|
|
[ "$hub_ok" = 1 ] && check_meta "$LOCAL_NAME" "$local_mtime"
|
|
[ "$hub_ok" = 1 ] && [ -n "$remote_mtime" ] && check_meta "$REMOTE_NAME" "$remote_mtime"
|
|
|
|
# ---------- agent availability + models ----------
|
|
baseline_get() { # name key
|
|
[ -f "$BASELINE" ] || return 1
|
|
jq -r --arg n "$1" --arg k "$2" '.[$n][$k] // empty | if type=="array" then join(",") else tostring end' "$BASELINE" 2>/dev/null
|
|
}
|
|
|
|
baseline_put() { # name json_fragment
|
|
mkdir -p "$(dirname "$BASELINE")"
|
|
[ -f "$BASELINE" ] || echo '{}' > "$BASELINE"
|
|
tmp=$(mktemp)
|
|
jq --arg n "$1" --argjson v "$2" '.[$n] = $v' "$BASELINE" > "$tmp" 2>/dev/null && mv "$tmp" "$BASELINE"
|
|
}
|
|
|
|
verify_machine_agents() { # display name
|
|
local name=$1 id resp now_agents base_agents
|
|
id=$(machine_id "$name")
|
|
[ -n "$id" ] || return
|
|
|
|
resp=$(api "/api/machines/$id/agent-availability")
|
|
if [ -z "$resp" ]; then
|
|
fail "agents $name" "agent-availability request failed"
|
|
return
|
|
fi
|
|
now_agents=$(printf '%s' "$resp" | jq -r '[.agents[] | select(.available) | .agent] | join(",")' 2>/dev/null)
|
|
[ -n "$now_agents" ] && ok "agents $name" "$(printf '%s' "$now_agents" | tr ',' ' ')" \
|
|
|| warn "agents $name" "none available"
|
|
|
|
if [ "$USE_BASELINE" = 1 ] && [ "$INIT" = 0 ]; then
|
|
base_agents=$(baseline_get "$name" agents)
|
|
if [ -n "$base_agents" ]; then
|
|
lost=$(comm -23 <(printf '%s' "$base_agents" | tr ',' '\n' | sort) \
|
|
<(printf '%s' "$now_agents" | tr ',' '\n' | sort) | paste -sd, -)
|
|
[ -z "$lost" ] && ok "agents $name baseline" "no regression" \
|
|
|| fail "agents $name baseline" "lost: $lost"
|
|
else
|
|
warn "agents $name baseline" "not recorded yet (run --init)"
|
|
fi
|
|
fi
|
|
|
|
resp=$(api "/api/machines/$id/codex-models")
|
|
if [ -n "$resp" ]; then
|
|
success=$(printf '%s' "$resp" | jq -r '.success // false' 2>/dev/null)
|
|
count=$(printf '%s' "$resp" | jq -r '(.models // []) | length' 2>/dev/null)
|
|
if [ "$success" = "true" ]; then
|
|
ok "codex $name" "$count models"
|
|
else
|
|
base=$(baseline_get "$name" codex_models)
|
|
[ "$base" = "true" ] && fail "codex $name" "was working, now success=false" \
|
|
|| warn "codex $name" "not available (also before)"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
if [ "$hub_ok" = 1 ] && [ -n "$JWT" ]; then
|
|
verify_machine_agents "$LOCAL_NAME"
|
|
verify_machine_agents "$REMOTE_NAME"
|
|
fi
|
|
|
|
# ---------- baseline ----------
|
|
if [ "$INIT" = 1 ] || [ "$USE_BASELINE" = 1 ] && [ "$fails" = 0 ]; then
|
|
for name in "$LOCAL_NAME" "$REMOTE_NAME"; do
|
|
id=$(machine_id "$name")
|
|
[ -n "$id" ] || continue
|
|
agents=$(api "/api/machines/$id/agent-availability" | jq -c '[.agents[] | select(.available) | .agent]' 2>/dev/null)
|
|
codex=$(api "/api/machines/$id/codex-models" | jq -r '.success // false' 2>/dev/null)
|
|
[ -n "$agents" ] && baseline_put "$name" "{\"agents\":$agents,\"codex_models\":${codex:-false}}"
|
|
done
|
|
fi
|
|
|
|
# ---------- output ----------
|
|
if [ "$JSON" = 1 ]; then
|
|
printf '{"result":"%s","failed":%d,"warned":%d,"checks":[%s]}\n' \
|
|
"$([ "$fails" = 0 ] && echo PASS || echo FAIL)" "$fails" "$warns" \
|
|
"$(IFS=,; echo "${json_rows[*]}")"
|
|
else
|
|
printf '%-6s %-24s %s\n' "STATUS" "CHECK" "DETAIL"
|
|
printf '%-6s %-24s %s\n' "------" "------------------------" "----------------------------------------"
|
|
for r in "${rows[@]}"; do
|
|
IFS='|' read -r s c d <<< "$r"
|
|
printf '%-6s %-24s %s\n' "$s" "$c" "$d"
|
|
done
|
|
echo
|
|
if [ "$fails" = 0 ]; then
|
|
echo "RESULT: PASS (warn $warns)"
|
|
else
|
|
echo "RESULT: FAIL ($fails failed, $warns warned)"
|
|
echo "rollback: see SKILL.md / scripts print the exact command after each deploy"
|
|
fi
|
|
fi
|
|
|
|
[ "$fails" = 0 ] && exit 0 || exit 1
|