140 lines
6.6 KiB
Bash
140 lines
6.6 KiB
Bash
#!/usr/bin/env bash
|
|
set -eo pipefail
|
|
|
|
# The watchdog used to report what it was about to do. ai_heal_report() was
|
|
# called before the repair, its fourth parameter was dead, and the recovery
|
|
# facts the controller published (PROXY_UP / DNS_UP) had no subscriber — so a
|
|
# rate-limited restart that never ran still announced itself as a repair, and a
|
|
# restart that ran and did not help was indistinguishable from one that did.
|
|
#
|
|
# This pins the outcome path: act, then report what actually happened.
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
TACHYON_LIB="$ROOT_DIR/tachyon/files/usr/lib"
|
|
WATCHDOG_UC="$TACHYON_LIB/service/watchdog.uc"
|
|
CONTROLLER_UC="$TACHYON_LIB/service/event_controller.uc"
|
|
|
|
fail() {
|
|
printf 'FAIL: %s\n' "$1" >&2
|
|
exit 1
|
|
}
|
|
|
|
# --- the outcome parameter is alive ---
|
|
# It was declared as `status_code` and never read; all call sites passed a
|
|
# literal into nothing.
|
|
grep -q 'function ai_heal_report(event_type, description, resolution, outcome)' "$WATCHDOG_UC" \
|
|
|| fail "ai_heal_report no longer takes an outcome parameter"
|
|
grep -q 'let status_code = as_string(outcome || "fixed");' "$WATCHDOG_UC" \
|
|
|| fail "ai_heal_report ignores its outcome argument again"
|
|
grep -q 'outcome: status_code,' "$WATCHDOG_UC" \
|
|
|| fail "the incident record carries no outcome field"
|
|
|
|
# The two fields telegram.uc:1823 reads must stay in the incident record.
|
|
for field in 'description: description,' 'resolution: resolution,'; do
|
|
grep -q "$field" "$WATCHDOG_UC" \
|
|
|| fail "last_incident lost '$field'; telegram.uc reads it"
|
|
done
|
|
|
|
# --- a repair in flight does not notify ---
|
|
# Otherwise every asynchronous repair sends two messages: one on the attempt and
|
|
# one on the outcome.
|
|
grep -q 'if (status_code != "pending" &&' "$WATCHDOG_UC" \
|
|
|| fail "a pending outcome would send a Telegram notification of its own"
|
|
|
|
# --- the recovery watch exists and is settled from both sides ---
|
|
grep -q 'function watch_recovery(key, incident, reason)' "$WATCHDOG_UC" \
|
|
|| fail "watch_recovery() is gone; asynchronous repairs cannot report an outcome"
|
|
# The recovery_event parameter was removed: settle_recovery() settles on whatever
|
|
# fact caused it, not on a specific EV.* value stored at registration time. The
|
|
# reason field (used to escalate or reset the ladder) must still be stored.
|
|
grep -q 'reason: reason,' "$WATCHDOG_UC" \
|
|
|| fail "watch_recovery() no longer records the reason behind the repair"
|
|
grep -q 'function settle_recovery(key, outcome)' "$WATCHDOG_UC" \
|
|
|| fail "settle_recovery() is gone"
|
|
grep -q 'function settle_expired_recoveries()' "$WATCHDOG_UC" \
|
|
|| fail "nothing expires an unclosed watch; a failed repair would report nothing"
|
|
grep -q 'settle_recovery(key, "failed");' "$WATCHDOG_UC" \
|
|
|| fail "an expired watch no longer settles as failed"
|
|
grep -q 'settle_recovery("proxy", "fixed");' "$WATCHDOG_UC" \
|
|
|| fail "PROXY_UP no longer settles the proxy watch as fixed"
|
|
grep -q 'settle_recovery("dns", "fixed");' "$WATCHDOG_UC" \
|
|
|| fail "DNS_UP no longer settles the dns watch as fixed"
|
|
|
|
# --- the recovery facts have subscribers ---
|
|
# They were published from the very first version of event_controller.uc and
|
|
# subscribed by nobody, which is the whole reason outcomes were unmeasurable.
|
|
for pair in 'PROXY_UP:note_proxy_recovered' 'DNS_UP:note_dns_recovered'; do
|
|
ev="${pair%%:*}"; handler="${pair##*:}"
|
|
grep -q "bus.emit(EV.$ev" "$CONTROLLER_UC" \
|
|
|| fail "event_controller no longer publishes EV.$ev"
|
|
grep -qE "subscribe\(EV\.$ev, $handler," "$WATCHDOG_UC" \
|
|
|| fail "EV.$ev has no subscriber; a successful repair would expire as failed"
|
|
done
|
|
|
|
# --- expiry is swept on a tier that actually runs ---
|
|
# A watch nobody sweeps stays open forever and reports neither outcome.
|
|
grep -q 'safe_call(settle_expired_recoveries, "settle_expired_recoveries");' "$WATCHDOG_UC" \
|
|
|| fail "settle_expired_recoveries() is never called from a tick"
|
|
grep -A6 'function perform_fast_checks()' "$WATCHDOG_UC" \
|
|
| grep -q 'settle_expired_recoveries' \
|
|
|| fail "the fast tier does not sweep expired watches"
|
|
# Order matters: probing first lets a recovery observed on this tick close its
|
|
# watch before the deadline is tested against it.
|
|
grep -A6 'function perform_fast_checks()' "$WATCHDOG_UC" \
|
|
| grep -nE 'probe_fast|settle_expired_recoveries' \
|
|
| awk -F: '/probe_fast/{p=$1} /settle_expired/{s=$1} END{exit !(p && s && p < s)}' \
|
|
|| fail "expiries are swept before the probes run; a same-tick recovery would report failed"
|
|
|
|
# --- no repair reports before it acts ---
|
|
# Every healer that goes through safe_proxy_restart() must branch on its return
|
|
# value: a rate-limited or locked attempt did not restart anything.
|
|
for healer in heal_dns_stall heal_proxy_connectivity heal_proxy_health; do
|
|
body="$(awk -v fn="function $healer(ev) {" '
|
|
index($0, fn) { inside=1 }
|
|
inside { print }
|
|
inside && /^}$/ { exit }
|
|
' "$WATCHDOG_UC")"
|
|
[ -n "$body" ] || fail "$healer not found"
|
|
|
|
grep -q 'if (!safe_proxy_restart(' <<< "$body" \
|
|
|| fail "$healer does not check the safe_proxy_restart() return value; a skipped restart would report as a repair"
|
|
grep -q '"skipped");' <<< "$body" \
|
|
|| fail "$healer does not report a skipped outcome when the restart never ran"
|
|
grep -q 'watch_recovery(' <<< "$body" \
|
|
|| fail "$healer restarts without registering a recovery watch; its outcome is unknowable"
|
|
|
|
# The literal "fixed" is what the old code passed unconditionally. An
|
|
# asynchronous healer cannot know that at call time.
|
|
if grep -q '"fixed"' <<< "$body"; then
|
|
fail "$healer still claims a fixed outcome for an asynchronous restart"
|
|
fi
|
|
done
|
|
|
|
# --- synchronous repairs report their real exit status ---
|
|
# These do not need a watch: the command returns before the healer does.
|
|
for healer in heal_dns_continuous heal_rpcd; do
|
|
body="$(awk -v fn="function $healer(ev) {" '
|
|
index($0, fn) { inside=1 }
|
|
inside { print }
|
|
inside && /^}$/ { exit }
|
|
' "$WATCHDOG_UC")"
|
|
[ -n "$body" ] || fail "$healer not found"
|
|
grep -q 'rc == 0 ? "fixed" : "failed"' <<< "$body" \
|
|
|| fail "$healer reports a fixed outcome without checking the command's exit status"
|
|
done
|
|
|
|
# --- the CLI contract is unchanged ---
|
|
# diagnostics/status.uc:1178 and telegram.uc:1823 read this file; ai-status must
|
|
# keep answering with the same shape whether or not an incident exists.
|
|
status_json="$(ucode -L "$TACHYON_LIB" "$WATCHDOG_UC" ai-status)"
|
|
grep -q '"ai_active"' <<< "$status_json" \
|
|
|| fail "ai-status no longer reports ai_active: $status_json"
|
|
|
|
full_json="$(ucode -L "$TACHYON_LIB" "$WATCHDOG_UC" ai-status-full)"
|
|
for key in '"status"' '"ai_active"' '"last_incident"' '"incidents_total"'; do
|
|
grep -q "$key" <<< "$full_json" \
|
|
|| fail "ai-status-full lost $key from its contract"
|
|
done
|
|
|
|
printf 'heal outcome checks passed\n'
|