145 lines
6.5 KiB
Bash
145 lines
6.5 KiB
Bash
#!/usr/bin/env bash
|
|
set -eo pipefail
|
|
|
|
# Pins the wiring between the two halves of the split: event_controller.uc
|
|
# observes and emits, watchdog.uc subscribes and repairs. A fact that nobody
|
|
# subscribes to is a repair that silently stopped happening — exactly the
|
|
# regression this refactor could introduce and that no other test would catch.
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
TACHYON_LIB="$ROOT_DIR/tachyon/files/usr/lib"
|
|
CONTROLLER_UC="$TACHYON_LIB/service/event_controller.uc"
|
|
WATCHDOG_UC="$TACHYON_LIB/service/watchdog.uc"
|
|
|
|
fail() {
|
|
printf 'FAIL: %s\n' "$1" >&2
|
|
exit 1
|
|
}
|
|
|
|
# Facts that are deliberately published without a repair attached.
|
|
# PROXY_UP / DNS_UP are the recovery side of a fault pair, kept so a future
|
|
# subscriber can act on recovery; FIREWALL_RELOADED and TICK are informational.
|
|
is_informational() {
|
|
case "$1" in
|
|
PROXY_UP|DNS_UP|FIREWALL_RELOADED|TICK) return 0 ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
emitted="$(grep -oE 'bus\.emit(_once)?\(EV\.[A-Z_]+' "$CONTROLLER_UC" \
|
|
| grep -oE 'EV\.[A-Z_]+' | sed 's/^EV\.//' | sort -u)"
|
|
[ -n "$emitted" ] || fail "no emit sites found in event_controller.uc"
|
|
|
|
# Registrations go through subscribe(), a wrapper that turns a bus.on() rejection
|
|
# into a startup failure instead of a silently missing repair.
|
|
subscribed="$(grep -oE 'subscribe\(EV\.[A-Z_]+' "$WATCHDOG_UC" \
|
|
| grep -oE 'EV\.[A-Z_]+' | sed 's/^EV\.//' | sort -u)"
|
|
[ -n "$subscribed" ] || fail "no subscribe() registrations found in watchdog.uc"
|
|
|
|
# --- every emitted fault has a subscriber ---
|
|
for ev in $emitted; do
|
|
is_informational "$ev" && continue
|
|
grep -qxF "$ev" <<< "$subscribed" \
|
|
|| fail "event_controller emits EV.$ev but watchdog subscribes to nothing for it"
|
|
done
|
|
|
|
# --- every subscription has a matching emit site ---
|
|
# A dead subscription means a repair that can never run.
|
|
for ev in $subscribed; do
|
|
grep -qxF "$ev" <<< "$emitted" \
|
|
|| fail "watchdog subscribes to EV.$ev but event_controller never emits it"
|
|
done
|
|
|
|
# --- every referenced EV name exists in the vocabulary ---
|
|
vocabulary="$(ucode -L "$TACHYON_LIB" "$CONTROLLER_UC" event-types \
|
|
| sed 's/=.*//' | sort -u)"
|
|
for ev in $emitted $subscribed; do
|
|
grep -qxF "$ev" <<< "$vocabulary" \
|
|
|| fail "EV.$ev is used but missing from the exported event vocabulary"
|
|
done
|
|
|
|
# --- the faults that used to be repaired on a timer are still covered ---
|
|
# Each name here was an ai_heal_* function called from a worker() tier before the
|
|
# refactor. If one loses its subscription, the router stops self-healing it.
|
|
for required in \
|
|
SINGBOX_STOPPED WAN_DOWN CONFIG_CORRUPT NFT_MISSING QOS_MISSING TPROXY_DOWN \
|
|
SUBNETS_EMPTY PROXY_DOWN DNS_DOWN SECTIONS_EMPTY SUBSCRIPTION_UNREACHABLE \
|
|
OOM_DETECTED OOM_RECOVERABLE MEMORY_LOW RPCD_FD_LEAK ANOMALY_RECONNECTS \
|
|
SMARTDETECT_CANDIDATE URLTEST_SWITCHED HONEYPOT_HIT
|
|
do
|
|
grep -qxF "$required" <<< "$subscribed" \
|
|
|| fail "no subscriber for EV.$required (was repaired on a timer before the refactor)"
|
|
done
|
|
|
|
# --- subscriptions are registered exactly once ---
|
|
# register_subscribers() is called from both worker() and the ai-heal CLI mode;
|
|
# without the idempotence guard every repair would run twice per fact.
|
|
grep -q 'if (subscribers_registered) return;' "$WATCHDOG_UC" \
|
|
|| fail "register_subscribers() lost its idempotence guard"
|
|
|
|
# --- a rejected registration is fatal, not silent ---
|
|
# bus.on() returns false for a non-function handler rather than raising, and
|
|
# ucode does not hoist function declarations: moving a heal_* below
|
|
# register_subscribers() would register nothing, emit nothing and throw nothing.
|
|
# subscribe() exists to make that loud. Registering directly through bus.on()
|
|
# would reopen the hole.
|
|
grep -q 'function subscribe(event_type, handler, opts)' "$WATCHDOG_UC" \
|
|
|| fail "the subscribe() wrapper is gone; a lost handler would fail silently"
|
|
grep -q 'if (!bus.on(event_type, handler, opts))' "$WATCHDOG_UC" \
|
|
|| fail "subscribe() no longer checks the bus.on() return value"
|
|
if grep -qE '^\s+bus\.on\(EV\.' "$WATCHDOG_UC"; then
|
|
fail "a subscriber registers via bus.on() directly, bypassing the subscribe() guard"
|
|
fi
|
|
|
|
# --- cooldowns that replaced the old last_*_time globals are still declared ---
|
|
# These four were the only per-repair timers in the old code; the bus owns them
|
|
# now, so the values have to survive here.
|
|
check_cooldown() {
|
|
local handler="$1" seconds="$2"
|
|
grep -A1 -E "subscribe\(EV\.[A-Z_]+, $handler," "$WATCHDOG_UC" \
|
|
| grep -q "cooldown: $seconds" \
|
|
|| fail "$handler lost its cooldown of ${seconds}s"
|
|
}
|
|
check_cooldown heal_wan_and_gateway 300
|
|
check_cooldown heal_community_subnet_sets 300
|
|
check_cooldown heal_empty_sections 120
|
|
check_cooldown heal_anomaly_reconnects 300
|
|
|
|
# --- the two-tier repairs keep their original pacing ---
|
|
# Before the split, proxy and DNS were measured twice at two different rates:
|
|
# ai_heal_proxy_health / ai_heal_dns_continuous on the fast tier (15s), and
|
|
# ai_heal_proxy_connectivity / ai_heal_dns inside the normal-tier audit (120s+).
|
|
# One probe now serves both, at the fast rate. Without these bounds a single
|
|
# stall would restart sing-box eight times as often as it used to — the
|
|
# aggressive-restart regression fixed in c8052ee6.
|
|
check_cooldown heal_proxy_connectivity 120
|
|
check_cooldown heal_dns_stall 120
|
|
|
|
# The DNS stall threshold counts a streak of 3. That streak has to be stepped at
|
|
# the old normal-tier rate, or "3 failures" would mean 45 seconds of trouble
|
|
# instead of ~6 minutes.
|
|
grep -q 'const DNS_STREAK_INTERVAL = 120;' "$CONTROLLER_UC" \
|
|
|| fail "DNS_STREAK_INTERVAL is missing or no longer 120s"
|
|
grep -q 'now - state.dns_streak_stamp >= DNS_STREAK_INTERVAL' "$CONTROLLER_UC" \
|
|
|| fail "the DNS stall streak is no longer paced by DNS_STREAK_INTERVAL"
|
|
|
|
# The fast-tier counters must NOT be paced — they are the ones the old fast
|
|
# healers incremented every 15s, and ai-status-full reports them.
|
|
grep -q 'state.dns_consecutive_fails++;' "$CONTROLLER_UC" \
|
|
|| fail "dns_consecutive_fails is no longer incremented every probe"
|
|
grep -q 'state.proxy_consecutive_fails++;' "$CONTROLLER_UC" \
|
|
|| fail "proxy_consecutive_fails is no longer incremented every probe"
|
|
|
|
# --- the observation layer does not repair ---
|
|
# The whole point of the split: no restart, reload or nft mutation may leak back
|
|
# into event_controller.uc.
|
|
for forbidden in 'safe_proxy_restart' 'safe_reload_firewall' 'ai_heal_report' \
|
|
'/etc/init.d/tachyon' 'service sing-box' 'nft add' 'nft flush'
|
|
do
|
|
if grep -q -- "$forbidden" "$CONTROLLER_UC"; then
|
|
fail "event_controller.uc performs a repair action ($forbidden); observation must not heal"
|
|
fi
|
|
done
|
|
|
|
printf 'watchdog subscription checks passed\n'
|