fix(watchdog): не перезапускать WAN по одиночному сбою пробы (issue #31)
This commit is contained in:
@@ -1377,7 +1377,7 @@ function renderDiagnosticAvailableActionsWidget() {
|
|||||||
},
|
},
|
||||||
doctor: {
|
doctor: {
|
||||||
loading: diagnosticsActions.doctor.loading,
|
loading: diagnosticsActions.doctor.loading,
|
||||||
visible: false,
|
visible: true,
|
||||||
onClick: handleRunDoctor,
|
onClick: handleRunDoctor,
|
||||||
disabled: utilityActionsDisabled,
|
disabled: utilityActionsDisabled,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -12523,7 +12523,7 @@ function renderDiagnosticAvailableActionsWidget() {
|
|||||||
},
|
},
|
||||||
doctor: {
|
doctor: {
|
||||||
loading: diagnosticsActions.doctor.loading,
|
loading: diagnosticsActions.doctor.loading,
|
||||||
visible: false,
|
visible: true,
|
||||||
onClick: handleRunDoctor,
|
onClick: handleRunDoctor,
|
||||||
disabled: utilityActionsDisabled
|
disabled: utilityActionsDisabled
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -29,6 +29,13 @@ const CONFIG_NAME = getenv("TACHYON_CONFIG_NAME") || "tachyon";
|
|||||||
// ai_heal_dns() used to step that counter before probing moved to the fast tier.
|
// ai_heal_dns() used to step that counter before probing moved to the fast tier.
|
||||||
const DNS_STREAK_INTERVAL = 120;
|
const DNS_STREAK_INTERVAL = 120;
|
||||||
|
|
||||||
|
// Consecutive WAN probe misses required before WAN_DOWN is published. The old
|
||||||
|
// check fired on a single observation: one transient gap in `ip route` (a
|
||||||
|
// DHCP renew re-applying the route, a reload_firewall window) made the healer
|
||||||
|
// bounce the whole interface, which then really dropped the address and the
|
||||||
|
// route — a self-inflicted WAN flap every ~5 minutes (issue #31).
|
||||||
|
const WAN_FAIL_THRESHOLD = 3;
|
||||||
|
|
||||||
let as_string = common.as_string;
|
let as_string = common.as_string;
|
||||||
let shell_quote = common.shell_quote;
|
let shell_quote = common.shell_quote;
|
||||||
let command_from_args = common.command_from_args;
|
let command_from_args = common.command_from_args;
|
||||||
@@ -360,7 +367,8 @@ function controller(bus, opts) {
|
|||||||
logread_pipe_fd: -1,
|
logread_pipe_fd: -1,
|
||||||
last_oom_time: 0,
|
last_oom_time: 0,
|
||||||
healthy_streak: 0,
|
healthy_streak: 0,
|
||||||
subscription_fail_streak: 0
|
subscription_fail_streak: 0,
|
||||||
|
wan_fail_streak: 0
|
||||||
};
|
};
|
||||||
|
|
||||||
let self = { EV: EV, state: state };
|
let self = { EV: EV, state: state };
|
||||||
@@ -707,8 +715,17 @@ function controller(bus, opts) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ── Probe: WAN address and default route ──────────────────────────────────
|
// ── Probe: WAN address and default route ──────────────────────────────────
|
||||||
|
// WAN_DOWN is published only after WAN_FAIL_THRESHOLD consecutive misses,
|
||||||
|
// mirroring the streak logic of the other probes. A reload restarts the
|
||||||
|
// firewall, which can briefly drop the default route; failures seen while
|
||||||
|
// that is in progress are not counted, and a healthy pass resets the
|
||||||
|
// streak so one transient gap cannot accumulate into a false trigger.
|
||||||
function probe_wan() {
|
function probe_wan() {
|
||||||
if (is_reload_in_progress()) return;
|
if (is_reload_in_progress()) {
|
||||||
|
state.wan_fail_streak = 0;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (setting("recovery_bypass", "0") == "1") return;
|
||||||
|
|
||||||
let proto = uci_core.get("network", "wan", "proto") || "pppoe";
|
let proto = uci_core.get("network", "wan", "proto") || "pppoe";
|
||||||
let device = trim(uci_core.get("network", "wan", "device") || "eth0");
|
let device = trim(uci_core.get("network", "wan", "device") || "eth0");
|
||||||
@@ -720,10 +737,23 @@ function controller(bus, opts) {
|
|||||||
let route_out = command_capture("ip route 2>/dev/null").output;
|
let route_out = command_capture("ip route 2>/dev/null").output;
|
||||||
let no_gateway = index(route_out, "default") < 0;
|
let no_gateway = index(route_out, "default") < 0;
|
||||||
|
|
||||||
if (no_address || no_gateway)
|
if (!no_address && !no_gateway) {
|
||||||
bus.emit(EV.WAN_DOWN, { iface: iface, no_address: no_address, no_gateway: no_gateway });
|
state.wan_fail_streak = 0;
|
||||||
else
|
|
||||||
bus.emit(EV.WAN_UP, { iface: iface });
|
bus.emit(EV.WAN_UP, { iface: iface });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
state.wan_fail_streak++;
|
||||||
|
log("Watchdog: WAN check failed (" + as_string(state.wan_fail_streak) + "/" +
|
||||||
|
as_string(WAN_FAIL_THRESHOLD) + " misses)", "warn");
|
||||||
|
if (state.wan_fail_streak < WAN_FAIL_THRESHOLD) return;
|
||||||
|
|
||||||
|
bus.emit(EV.WAN_DOWN, {
|
||||||
|
iface: iface,
|
||||||
|
no_address: no_address,
|
||||||
|
no_gateway: no_gateway,
|
||||||
|
streak: state.wan_fail_streak
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Probe: community subnet nft sets ──────────────────────────────────────
|
// ── Probe: community subnet nft sets ──────────────────────────────────────
|
||||||
|
|||||||
@@ -858,6 +858,7 @@ function heal_tproxy_port(ev) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function heal_wan_and_gateway(ev) {
|
function heal_wan_and_gateway(ev) {
|
||||||
|
if (settings().recovery_bypass == "1") return;
|
||||||
let tcfg = common.object_or_empty(uci_core.get_all(CONFIG_NAME, "telegram"));
|
let tcfg = common.object_or_empty(uci_core.get_all(CONFIG_NAME, "telegram"));
|
||||||
if (tcfg.notify_crash != "0") {
|
if (tcfg.notify_crash != "0") {
|
||||||
send_telegram_notification("⚠️ *Watchdog:* WAN/Gateway проблема. Перезапуск wan...");
|
send_telegram_notification("⚠️ *Watchdog:* WAN/Gateway проблема. Перезапуск wan...");
|
||||||
@@ -866,7 +867,19 @@ function heal_wan_and_gateway(ev) {
|
|||||||
// makes the proxy and DNS probes fail, and those facts arrive while ifup is
|
// makes the proxy and DNS probes fail, and those facts arrive while ifup is
|
||||||
// still running.
|
// still running.
|
||||||
wan_repair_until = time() + SUPPRESSION_DEADLINE;
|
wan_repair_until = time() + SUPPRESSION_DEADLINE;
|
||||||
bg_system("/sbin/ifdown wan >/dev/null 2>&1 && /sbin/ifup wan >/dev/null 2>&1");
|
// The address is still there but the default route vanished: a graceful
|
||||||
|
// renew re-applies the routes without dropping the interface (issue #31).
|
||||||
|
// Only when the address itself is gone does the interface get recycled,
|
||||||
|
// and even then only if the renew did not restore the route within 5s.
|
||||||
|
let cmd;
|
||||||
|
if (ev.payload.no_address) {
|
||||||
|
cmd = "/sbin/ifdown wan >/dev/null 2>&1 && /sbin/ifup wan >/dev/null 2>&1";
|
||||||
|
} else {
|
||||||
|
cmd = "/sbin/ubus call network.interface.wan renew >/dev/null 2>&1; sleep 5; " +
|
||||||
|
"ip route 2>/dev/null | grep -q default || " +
|
||||||
|
"(/sbin/ifdown wan >/dev/null 2>&1 && /sbin/ifup wan >/dev/null 2>&1)";
|
||||||
|
}
|
||||||
|
bg_system(cmd);
|
||||||
}
|
}
|
||||||
|
|
||||||
// The uplink answering again is what ends the WAN repair. probe_wan publishes
|
// The uplink answering again is what ends the WAN repair. probe_wan publishes
|
||||||
|
|||||||
@@ -56,7 +56,7 @@ grep -q 'function suppressed_by_root_cause(healer)' "$WATCHDOG_UC" \
|
|||||||
# WAN deadline is set before ifdown, not after: ifdown is itself what makes the
|
# WAN deadline is set before ifdown, not after: ifdown is itself what makes the
|
||||||
# proxy and DNS probes fail, and those facts arrive while ifup is still running.
|
# proxy and DNS probes fail, and those facts arrive while ifup is still running.
|
||||||
claim_line="$(grep -n 'wan_repair_until = time()' "$WATCHDOG_UC" | cut -d: -f1)"
|
claim_line="$(grep -n 'wan_repair_until = time()' "$WATCHDOG_UC" | cut -d: -f1)"
|
||||||
ifdown_line="$(grep -n '/sbin/ifdown wan' "$WATCHDOG_UC" | cut -d: -f1)"
|
ifdown_line="$(grep -n -m1 '/sbin/ifdown wan' "$WATCHDOG_UC" | cut -d: -f1)"
|
||||||
[ -n "$claim_line" ] || fail "heal_wan_and_gateway does not set wan_repair_until before ifdown"
|
[ -n "$claim_line" ] || fail "heal_wan_and_gateway does not set wan_repair_until before ifdown"
|
||||||
[ -n "$ifdown_line" ] || fail "heal_wan_and_gateway no longer restarts the wan interface"
|
[ -n "$ifdown_line" ] || fail "heal_wan_and_gateway no longer restarts the wan interface"
|
||||||
[ "$claim_line" -lt "$ifdown_line" ] \
|
[ "$claim_line" -lt "$ifdown_line" ] \
|
||||||
@@ -82,6 +82,29 @@ grep -q 'bus.emit(EV.WAN_UP' "$CONTROLLER_UC" \
|
|||||||
grep -qE 'subscribe\(EV\.WAN_UP, note_wan_recovered,' "$WATCHDOG_UC" \
|
grep -qE 'subscribe\(EV\.WAN_UP, note_wan_recovered,' "$WATCHDOG_UC" \
|
||||||
|| fail "nothing subscribes to WAN_UP"
|
|| fail "nothing subscribes to WAN_UP"
|
||||||
|
|
||||||
|
# ── a transient WAN glitch must not bounce the interface ──────────────────────
|
||||||
|
# probe_wan used to publish WAN_DOWN on a single miss; the healer's ifdown/ifup
|
||||||
|
# then really dropped the interface and its route, repeating every cooldown
|
||||||
|
# window — a self-inflicted flap every ~5 minutes (issue #31). The probe now
|
||||||
|
# counts consecutive misses, the heal renews gracefully first, and both honour
|
||||||
|
# recovery_bypass like every other probe/healer.
|
||||||
|
grep -q 'const WAN_FAIL_THRESHOLD' "$CONTROLLER_UC" \
|
||||||
|
|| fail "probe_wan has no debounce threshold"
|
||||||
|
wan_probe_body="$(sed -n '/function probe_wan()/,/^ }$/p' "$CONTROLLER_UC")"
|
||||||
|
echo "$wan_probe_body" | grep -q 'state.wan_fail_streak = 0;' \
|
||||||
|
|| fail "probe_wan no longer resets its miss streak on a healthy pass"
|
||||||
|
echo "$wan_probe_body" | grep -q 'state.wan_fail_streak++;' \
|
||||||
|
|| fail "probe_wan no longer counts consecutive misses"
|
||||||
|
echo "$wan_probe_body" | grep -q 'if (state.wan_fail_streak < WAN_FAIL_THRESHOLD) return;' \
|
||||||
|
|| fail "probe_wan publishes WAN_DOWN before the debounce threshold"
|
||||||
|
echo "$wan_probe_body" | grep -q 'if (setting("recovery_bypass", "0") == "1") return;' \
|
||||||
|
|| fail "probe_wan ignores recovery_bypass"
|
||||||
|
wan_heal_body="$(sed -n '/function heal_wan_and_gateway/,/^}/p' "$WATCHDOG_UC")"
|
||||||
|
echo "$wan_heal_body" | grep -q 'if (settings().recovery_bypass == "1") return;' \
|
||||||
|
|| fail "heal_wan_and_gateway ignores recovery_bypass; it was the only healer that could not be disabled"
|
||||||
|
echo "$wan_heal_body" | grep -q '/sbin/ubus call network.interface.wan renew' \
|
||||||
|
|| fail "heal_wan_and_gateway no longer tries a graceful renew before ifdown/ifup"
|
||||||
|
|
||||||
# ── the consequences actually check ───────────────────────────────────────────
|
# ── the consequences actually check ───────────────────────────────────────────
|
||||||
for healer in heal_dns_stall heal_proxy_connectivity heal_proxy_health heal_dns_continuous; do
|
for healer in heal_dns_stall heal_proxy_connectivity heal_proxy_health heal_dns_continuous; do
|
||||||
grep -q "suppressed_by_root_cause(\"$healer\")" "$WATCHDOG_UC" \
|
grep -q "suppressed_by_root_cause(\"$healer\")" "$WATCHDOG_UC" \
|
||||||
|
|||||||
Reference in New Issue
Block a user