fix(watchdog): не перезапускать WAN по одиночному сбою пробы (issue #31)

This commit is contained in:
Dushnilin
2026-08-13 21:24:43 +03:00
parent 305e14d44e
commit ac3fbc90fc
5 changed files with 75 additions and 9 deletions
@@ -1377,7 +1377,7 @@ function renderDiagnosticAvailableActionsWidget() {
}, },
doctor: { doctor: {
loading: diagnosticsActions.doctor.loading, loading: diagnosticsActions.doctor.loading,
visible: false, visible: true,
onClick: handleRunDoctor, onClick: handleRunDoctor,
disabled: utilityActionsDisabled, disabled: utilityActionsDisabled,
}, },
@@ -12523,7 +12523,7 @@ function renderDiagnosticAvailableActionsWidget() {
}, },
doctor: { doctor: {
loading: diagnosticsActions.doctor.loading, loading: diagnosticsActions.doctor.loading,
visible: false, visible: true,
onClick: handleRunDoctor, onClick: handleRunDoctor,
disabled: utilityActionsDisabled disabled: utilityActionsDisabled
}, },
@@ -29,6 +29,13 @@ const CONFIG_NAME = getenv("TACHYON_CONFIG_NAME") || "tachyon";
// ai_heal_dns() used to step that counter before probing moved to the fast tier. // ai_heal_dns() used to step that counter before probing moved to the fast tier.
const DNS_STREAK_INTERVAL = 120; const DNS_STREAK_INTERVAL = 120;
// Consecutive WAN probe misses required before WAN_DOWN is published. The old
// check fired on a single observation: one transient gap in `ip route` (a
// DHCP renew re-applying the route, a reload_firewall window) made the healer
// bounce the whole interface, which then really dropped the address and the
// route — a self-inflicted WAN flap every ~5 minutes (issue #31).
const WAN_FAIL_THRESHOLD = 3;
let as_string = common.as_string; let as_string = common.as_string;
let shell_quote = common.shell_quote; let shell_quote = common.shell_quote;
let command_from_args = common.command_from_args; let command_from_args = common.command_from_args;
@@ -360,7 +367,8 @@ function controller(bus, opts) {
logread_pipe_fd: -1, logread_pipe_fd: -1,
last_oom_time: 0, last_oom_time: 0,
healthy_streak: 0, healthy_streak: 0,
subscription_fail_streak: 0 subscription_fail_streak: 0,
wan_fail_streak: 0
}; };
let self = { EV: EV, state: state }; let self = { EV: EV, state: state };
@@ -707,8 +715,17 @@ function controller(bus, opts) {
} }
// ── Probe: WAN address and default route ────────────────────────────────── // ── Probe: WAN address and default route ──────────────────────────────────
// WAN_DOWN is published only after WAN_FAIL_THRESHOLD consecutive misses,
// mirroring the streak logic of the other probes. A reload restarts the
// firewall, which can briefly drop the default route; failures seen while
// that is in progress are not counted, and a healthy pass resets the
// streak so one transient gap cannot accumulate into a false trigger.
function probe_wan() { function probe_wan() {
if (is_reload_in_progress()) return; if (is_reload_in_progress()) {
state.wan_fail_streak = 0;
return;
}
if (setting("recovery_bypass", "0") == "1") return;
let proto = uci_core.get("network", "wan", "proto") || "pppoe"; let proto = uci_core.get("network", "wan", "proto") || "pppoe";
let device = trim(uci_core.get("network", "wan", "device") || "eth0"); let device = trim(uci_core.get("network", "wan", "device") || "eth0");
@@ -720,10 +737,23 @@ function controller(bus, opts) {
let route_out = command_capture("ip route 2>/dev/null").output; let route_out = command_capture("ip route 2>/dev/null").output;
let no_gateway = index(route_out, "default") < 0; let no_gateway = index(route_out, "default") < 0;
if (no_address || no_gateway) if (!no_address && !no_gateway) {
bus.emit(EV.WAN_DOWN, { iface: iface, no_address: no_address, no_gateway: no_gateway }); state.wan_fail_streak = 0;
else
bus.emit(EV.WAN_UP, { iface: iface }); bus.emit(EV.WAN_UP, { iface: iface });
return;
}
state.wan_fail_streak++;
log("Watchdog: WAN check failed (" + as_string(state.wan_fail_streak) + "/" +
as_string(WAN_FAIL_THRESHOLD) + " misses)", "warn");
if (state.wan_fail_streak < WAN_FAIL_THRESHOLD) return;
bus.emit(EV.WAN_DOWN, {
iface: iface,
no_address: no_address,
no_gateway: no_gateway,
streak: state.wan_fail_streak
});
} }
// ── Probe: community subnet nft sets ────────────────────────────────────── // ── Probe: community subnet nft sets ──────────────────────────────────────
+14 -1
View File
@@ -858,6 +858,7 @@ function heal_tproxy_port(ev) {
} }
function heal_wan_and_gateway(ev) { function heal_wan_and_gateway(ev) {
if (settings().recovery_bypass == "1") return;
let tcfg = common.object_or_empty(uci_core.get_all(CONFIG_NAME, "telegram")); let tcfg = common.object_or_empty(uci_core.get_all(CONFIG_NAME, "telegram"));
if (tcfg.notify_crash != "0") { if (tcfg.notify_crash != "0") {
send_telegram_notification("⚠️ *Watchdog:* WAN/Gateway проблема. Перезапуск wan..."); send_telegram_notification("⚠️ *Watchdog:* WAN/Gateway проблема. Перезапуск wan...");
@@ -866,7 +867,19 @@ function heal_wan_and_gateway(ev) {
// makes the proxy and DNS probes fail, and those facts arrive while ifup is // makes the proxy and DNS probes fail, and those facts arrive while ifup is
// still running. // still running.
wan_repair_until = time() + SUPPRESSION_DEADLINE; wan_repair_until = time() + SUPPRESSION_DEADLINE;
bg_system("/sbin/ifdown wan >/dev/null 2>&1 && /sbin/ifup wan >/dev/null 2>&1"); // The address is still there but the default route vanished: a graceful
// renew re-applies the routes without dropping the interface (issue #31).
// Only when the address itself is gone does the interface get recycled,
// and even then only if the renew did not restore the route within 5s.
let cmd;
if (ev.payload.no_address) {
cmd = "/sbin/ifdown wan >/dev/null 2>&1 && /sbin/ifup wan >/dev/null 2>&1";
} else {
cmd = "/sbin/ubus call network.interface.wan renew >/dev/null 2>&1; sleep 5; " +
"ip route 2>/dev/null | grep -q default || " +
"(/sbin/ifdown wan >/dev/null 2>&1 && /sbin/ifup wan >/dev/null 2>&1)";
}
bg_system(cmd);
} }
// The uplink answering again is what ends the WAN repair. probe_wan publishes // The uplink answering again is what ends the WAN repair. probe_wan publishes
+24 -1
View File
@@ -56,7 +56,7 @@ grep -q 'function suppressed_by_root_cause(healer)' "$WATCHDOG_UC" \
# WAN deadline is set before ifdown, not after: ifdown is itself what makes the # WAN deadline is set before ifdown, not after: ifdown is itself what makes the
# proxy and DNS probes fail, and those facts arrive while ifup is still running. # proxy and DNS probes fail, and those facts arrive while ifup is still running.
claim_line="$(grep -n 'wan_repair_until = time()' "$WATCHDOG_UC" | cut -d: -f1)" claim_line="$(grep -n 'wan_repair_until = time()' "$WATCHDOG_UC" | cut -d: -f1)"
ifdown_line="$(grep -n '/sbin/ifdown wan' "$WATCHDOG_UC" | cut -d: -f1)" ifdown_line="$(grep -n -m1 '/sbin/ifdown wan' "$WATCHDOG_UC" | cut -d: -f1)"
[ -n "$claim_line" ] || fail "heal_wan_and_gateway does not set wan_repair_until before ifdown" [ -n "$claim_line" ] || fail "heal_wan_and_gateway does not set wan_repair_until before ifdown"
[ -n "$ifdown_line" ] || fail "heal_wan_and_gateway no longer restarts the wan interface" [ -n "$ifdown_line" ] || fail "heal_wan_and_gateway no longer restarts the wan interface"
[ "$claim_line" -lt "$ifdown_line" ] \ [ "$claim_line" -lt "$ifdown_line" ] \
@@ -82,6 +82,29 @@ grep -q 'bus.emit(EV.WAN_UP' "$CONTROLLER_UC" \
grep -qE 'subscribe\(EV\.WAN_UP, note_wan_recovered,' "$WATCHDOG_UC" \ grep -qE 'subscribe\(EV\.WAN_UP, note_wan_recovered,' "$WATCHDOG_UC" \
|| fail "nothing subscribes to WAN_UP" || fail "nothing subscribes to WAN_UP"
# ── a transient WAN glitch must not bounce the interface ──────────────────────
# probe_wan used to publish WAN_DOWN on a single miss; the healer's ifdown/ifup
# then really dropped the interface and its route, repeating every cooldown
# window — a self-inflicted flap every ~5 minutes (issue #31). The probe now
# counts consecutive misses, the heal renews gracefully first, and both honour
# recovery_bypass like every other probe/healer.
grep -q 'const WAN_FAIL_THRESHOLD' "$CONTROLLER_UC" \
|| fail "probe_wan has no debounce threshold"
wan_probe_body="$(sed -n '/function probe_wan()/,/^ }$/p' "$CONTROLLER_UC")"
echo "$wan_probe_body" | grep -q 'state.wan_fail_streak = 0;' \
|| fail "probe_wan no longer resets its miss streak on a healthy pass"
echo "$wan_probe_body" | grep -q 'state.wan_fail_streak++;' \
|| fail "probe_wan no longer counts consecutive misses"
echo "$wan_probe_body" | grep -q 'if (state.wan_fail_streak < WAN_FAIL_THRESHOLD) return;' \
|| fail "probe_wan publishes WAN_DOWN before the debounce threshold"
echo "$wan_probe_body" | grep -q 'if (setting("recovery_bypass", "0") == "1") return;' \
|| fail "probe_wan ignores recovery_bypass"
wan_heal_body="$(sed -n '/function heal_wan_and_gateway/,/^}/p' "$WATCHDOG_UC")"
echo "$wan_heal_body" | grep -q 'if (settings().recovery_bypass == "1") return;' \
|| fail "heal_wan_and_gateway ignores recovery_bypass; it was the only healer that could not be disabled"
echo "$wan_heal_body" | grep -q '/sbin/ubus call network.interface.wan renew' \
|| fail "heal_wan_and_gateway no longer tries a graceful renew before ifdown/ifup"
# ── the consequences actually check ─────────────────────────────────────────── # ── the consequences actually check ───────────────────────────────────────────
for healer in heal_dns_stall heal_proxy_connectivity heal_proxy_health heal_dns_continuous; do for healer in heal_dns_stall heal_proxy_connectivity heal_proxy_health heal_dns_continuous; do
grep -q "suppressed_by_root_cause(\"$healer\")" "$WATCHDOG_UC" \ grep -q "suppressed_by_root_cause(\"$healer\")" "$WATCHDOG_UC" \