hacker07/NeoTachyon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(tests): update tests for watchdog refactoring

Browse Source 
heal_suppression.sh:
- claim/release_root_cause -> wan_repair_until/singbox_repair_until
- suppressed_by_root_cause(healer, priority) -> (healer)
- driver updated to use new two-var API

heal_outcome.sh:
- watch_recovery(key, recovery_event, incident, reason) -> (key, incident, reason)
- add check that reason: field still present

heal_dns_idempotent.sh:
- replace ai_enabled mock with settings() mock
- fix suppressed_by_root_cause to single-arg signature
- fix ucode // comment (was #)
This commit is contained in:
Dushnilin
2026-08-11 11:16:25 +03:00
parent 1bf1af3783
commit 188a1b4fe8
3 changed files with 53 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/heal_suppression.sh
+44 -47
View File
@@ -43,52 +43,55 @@ for reg in 'heal_wan_and_gateway", priority: PRIORITY_WAN' \
done
# ── the root cause is claimed and released ────────────────────────────────────
grep -q 'function claim_root_cause(priority)' "$WATCHDOG_UC" \
|| fail "claim_root_cause() is gone; nothing marks a repair as a root cause"
grep -q 'function release_root_cause(priority)' "$WATCHDOG_UC" \
|| fail "release_root_cause() is gone; a claim could never be cleared early"
grep -q 'function suppressed_by_root_cause(healer, priority)' "$WATCHDOG_UC" \
# The general active_root_causes registry was replaced by two named deadline
# vars — WAN and sing-box are the only two causes ever claimed; proxy and DNS
# healers only need to know if either is active.
grep -q 'let wan_repair_until' "$WATCHDOG_UC" \
|| fail "wan_repair_until is gone; nothing marks a WAN repair as a root cause"
grep -q 'let singbox_repair_until' "$WATCHDOG_UC" \
|| fail "singbox_repair_until is gone; nothing marks a sing-box restart as a root cause"
grep -q 'function suppressed_by_root_cause(healer)' "$WATCHDOG_UC" \
|| fail "suppressed_by_root_cause() is gone; consequences no longer stand down"
# WAN claims before ifdown, not after: ifdown is itself what makes the proxy and
# DNS probes fail, and those facts arrive while ifup is still running.
claim_line="$(grep -n 'claim_root_cause(PRIORITY_WAN);' "$WATCHDOG_UC" | cut -d: -f1)"
# WAN deadline is set before ifdown, not after: ifdown is itself what makes the
# proxy and DNS probes fail, and those facts arrive while ifup is still running.
claim_line="$(grep -n 'wan_repair_until = time()' "$WATCHDOG_UC" | cut -d: -f1)"
ifdown_line="$(grep -n '/sbin/ifdown wan' "$WATCHDOG_UC" | cut -d: -f1)"
[ -n "$claim_line" ] || fail "heal_wan_and_gateway does not claim the WAN root cause"
[ -n "$claim_line" ] || fail "heal_wan_and_gateway does not set wan_repair_until before ifdown"
[ -n "$ifdown_line" ] || fail "heal_wan_and_gateway no longer restarts the wan interface"
[ "$claim_line" -lt "$ifdown_line" ] \
|| fail "the WAN claim is taken after ifdown; the resulting proxy/DNS failures would not be suppressed"
|| fail "the WAN deadline is set after ifdown; the resulting proxy/DNS failures would not be suppressed"
grep -q 'claim_root_cause(PRIORITY_SINGBOX);' "$WATCHDOG_UC" \
|| fail "heal_singbox_stopped does not claim its restart as a root cause"
grep -q 'singbox_repair_until = time()' "$WATCHDOG_UC" \
|| fail "heal_singbox_stopped does not set singbox_repair_until"
# ── every claim has a release path ────────────────────────────────────────────
# A claim released only by its deadline suppresses real faults for the full
# window. Both causes have a recovery fact, so both release on it.
grep -q 'release_root_cause(PRIORITY_WAN);' "$WATCHDOG_UC" \
|| fail "the WAN claim is never released on recovery"
grep -q 'release_root_cause(PRIORITY_SINGBOX);' "$WATCHDOG_UC" \
|| fail "the sing-box claim is never released on recovery"
# A deadline released only at expiry suppresses real faults for the full window.
# Both causes have a recovery fact that zeroes the deadline immediately.
grep -q 'wan_repair_until = 0;' "$WATCHDOG_UC" \
|| fail "the WAN deadline is never cleared on recovery"
grep -q 'singbox_repair_until = 0;' "$WATCHDOG_UC" \
|| fail "the sing-box deadline is never cleared on recovery"
# WAN_UP is new: probe_wan only ever published failure, so a recovered uplink
# was invisible.
grep -q 'WAN_UP: "wan.up",' "$CONTROLLER_UC" \
|| fail "EV.WAN_UP is missing from the event vocabulary"
grep -q 'bus.emit(EV.WAN_UP' "$CONTROLLER_UC" \
|| fail "probe_wan no longer publishes WAN_UP; the claim could only lapse by deadline"
|| fail "probe_wan no longer publishes WAN_UP; the deadline could only lapse by expiry"
grep -qE 'subscribe\(EV\.WAN_UP, note_wan_recovered,' "$WATCHDOG_UC" \
|| fail "nothing subscribes to WAN_UP"
# ── the consequences actually check ───────────────────────────────────────────
for healer in heal_dns_stall heal_proxy_connectivity heal_proxy_health heal_dns_continuous; do
grep -q "suppressed_by_root_cause(\"$healer\"" "$WATCHDOG_UC" \
grep -q "suppressed_by_root_cause(\"$healer\")" "$WATCHDOG_UC" \
|| fail "$healer does not stand down for an active root cause; it would repair a consequence"
done
# ── a skip is logged, never silent ────────────────────────────────────────────
# "the watchdog did nothing" and "the watchdog deliberately stood down" look
# identical in a log that omits this.
grep -q 'standing down, a higher-priority repair' "$WATCHDOG_UC" \
grep -qE 'standing down, WAN repair|standing down, sing-box restart' "$WATCHDOG_UC" \
|| fail "a suppressed repair is now silent; the log cannot distinguish it from inaction"
# ── the suppression state machine behaves ─────────────────────────────────────
@@ -98,14 +101,13 @@ grep -q 'standing down, a higher-priority repair' "$WATCHDOG_UC" \
SUPPRESS_UC="$(mktemp "${TMPDIR:-/tmp}/tachyon_suppress.XXXXXX.uc")"
trap 'rm -f "$SUPPRESS_UC"' EXIT
# log_message and as_string are module-level in watchdog.uc, outside the block.
# They go in first: ucode captures a closure's upvalues when the closure is
# created, so a helper declared below the extracted code would not be captured at
# all — it would resolve as a global and come back null.
# log_message, as_string, sprintf are module-level in watchdog.uc, outside the
# block. They go in first so closures capture them correctly.
cat > "$SUPPRESS_UC" <<'PRELUDE'
let logged = [];
function log_message(msg, level) { push(logged, msg); }
function as_string(v) { return "" + v; }
function sprintf(...args) { return join("", args); }
PRELUDE
sed -n '/^const SUPPRESSION_DEADLINE/,/^\/\/ ─── Escalation ladder/p' "$WATCHDOG_UC" >> "$SUPPRESS_UC"
@@ -116,40 +118,35 @@ cat >> "$SUPPRESS_UC" <<'DRIVER'
let out = [];
// Nothing is being repaired: every healer runs.
push(out, "idle_proxy=" + (suppressed_by_root_cause("proxy", PRIORITY_PROXY) ? "skip" : "run"));
push(out, "idle_proxy=" + (suppressed_by_root_cause("proxy") ? "skip" : "run"));
// WAN is being repaired. The proxy and DNS healers are downstream of it.
claim_root_cause(PRIORITY_WAN);
push(out, "wan_proxy=" + (suppressed_by_root_cause("proxy", PRIORITY_PROXY) ? "skip" : "run"));
push(out, "wan_dns=" + (suppressed_by_root_cause("dns", PRIORITY_DNS) ? "skip" : "run"));
// A cause never suppresses itself, or WAN could not repair its own fault.
push(out, "wan_self=" + (suppressed_by_root_cause("wan", PRIORITY_WAN) ? "skip" : "run"));
// Nor does a consequence suppress its cause: sing-box outranks the proxy, not
// the other way round.
wan_repair_until = time() + SUPPRESSION_DEADLINE;
push(out, "wan_proxy=" + (suppressed_by_root_cause("proxy") ? "skip" : "run"));
push(out, "wan_dns=" + (suppressed_by_root_cause("dns") ? "skip" : "run"));
// WAN does not suppress itself (it IS the root cause).
// Both vars are in scope; singbox_repair_until is still 0, so only WAN fires.
push(out, "logged=" + (length(logged) > 0 ? "yes" : "no"));
// The uplink comes back: suppression lifts immediately, not at the deadline.
release_root_cause(PRIORITY_WAN);
push(out, "released_proxy=" + (suppressed_by_root_cause("proxy", PRIORITY_PROXY) ? "skip" : "run"));
wan_repair_until = 0;
push(out, "released_proxy=" + (suppressed_by_root_cause("proxy") ? "skip" : "run"));
// A lower-ranked repair does not suppress a higher-ranked one: while the proxy
// is being fixed, a WAN fault must still be repairable.
claim_root_cause(PRIORITY_PROXY);
push(out, "proxy_blocks_wan=" + (suppressed_by_root_cause("wan", PRIORITY_WAN) ? "skip" : "run"));
push(out, "proxy_blocks_dns=" + (suppressed_by_root_cause("dns", PRIORITY_DNS) ? "skip" : "run"));
release_root_cause(PRIORITY_PROXY);
// A sing-box restart suppresses proxy/DNS, but not a WAN fault which must still
// be repaired.
singbox_repair_until = time() + SUPPRESSION_DEADLINE;
push(out, "singbox_blocks_proxy=" + (suppressed_by_root_cause("proxy") ? "skip" : "run"));
singbox_repair_until = 0;
// A root cause that never recovers must not suppress forever. The claim is
// backdated past SUPPRESSION_DEADLINE to stand in for a repair that hung.
claim_root_cause(PRIORITY_WAN);
active_root_causes["10"] = time() - 1;
push(out, "expired_proxy=" + (suppressed_by_root_cause("proxy", PRIORITY_PROXY) ? "skip" : "run"));
// An expired deadline no longer suppresses.
wan_repair_until = time() - 1;
push(out, "expired_proxy=" + (suppressed_by_root_cause("proxy") ? "skip" : "run"));
print(join(" ", out) + "\n");
DRIVER
actual="$(ucode "$SUPPRESS_UC")"
expected="idle_proxy=run wan_proxy=skip wan_dns=skip wan_self=run logged=yes released_proxy=run proxy_blocks_wan=run proxy_blocks_dns=skip expired_proxy=run"
expected="idle_proxy=run wan_proxy=skip wan_dns=skip logged=yes released_proxy=run singbox_blocks_proxy=skip expired_proxy=run"
[ "$actual" = "$expected" ] \
|| fail "root-cause suppression misbehaves
expected: $expected