From e446b370e3182c6ecf5be9bd626ba5a0db1d9ca4 Mon Sep 17 00:00:00 2001 From: Dushnilin Date: Sat, 22 Aug 2026 10:07:19 +0300 Subject: [PATCH] =?UTF-8?q?feat(watchdog):=20=D1=81=D0=B0=D0=BC=D0=BE?= =?UTF-8?q?=D0=BB=D0=B5=D1=87=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=BF=D0=BE=D1=80?= =?UTF-8?q?=D1=82=D0=B0=204534=20=D0=B8=20=D0=B6=D0=B8=D0=B2=D0=BE=D1=81?= =?UTF-8?q?=D1=82=D0=B8=20=D1=82=D0=B5=D0=BB=D0=B5=D0=B3=D1=80=D0=B0=D0=BC?= =?UTF-8?q?-=D0=B2=D0=BE=D1=80=D0=BA=D0=B5=D1=80=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - новая slow-проверка (раз в 5 мин): sing-box запущен, но mixed-порт 4534 не слушается две проверки подряд (~10 мин) -> один авторестарт ядра; если не помогло - уведомление админу не чаще раза в час, без циклов бесконечных рестартов - проверка живости телеграм-воркера: бот включён, pid жив, но heartbeat старше 15 минут (зависший long-poll/мёртвый цикл) или pid отсутствует -> рестарт воркера через telegram_start с ограничением раз в час - обе проверки добавлены в perform_slow_checks через safe_call --- tachyon/files/usr/lib/service/watchdog.uc | 101 +++++++++++++++++++++- tests/watchdog_selfheal.sh | 51 +++++++++++ 2 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 tests/watchdog_selfheal.sh diff --git a/tachyon/files/usr/lib/service/watchdog.uc b/tachyon/files/usr/lib/service/watchdog.uc index 9a2ad7b1..7dd65378 100644 --- a/tachyon/files/usr/lib/service/watchdog.uc +++ b/tachyon/files/usr/lib/service/watchdog.uc @@ -1718,11 +1718,110 @@ function worker() { safe_call(ai_export_status, "ai_export_status"); } +// ── Mixed proxy port (4534) self-healing ───────────────────────────────────── +// sing-box can be running while its mixed inbound failed to bind: every LAN +// client proxy and the Telegram bot route then silently dead-ends. Two +// consecutive slow checks (~10 min) with a live core and a dead port trigger +// one core restart; if the port is still dead afterwards, the admin gets a +// notification instead of endless restart loops. + +let mixed_port_fail_streak = 0; +let mixed_port_last_notify = 0; + +function mixed_proxy_port_listening() { + let out = command_output_from_args([ "netstat", "-ltn" ]); + for (let line in split(out, "\n")) { + if (index(line, ":4534 ") >= 0) + return true; + } + return false; +} + +function check_mixed_proxy_port() { + let sb_pid = trim(command_output_from_args(["pidof", "sing-box"])); + if (sb_pid == "") { + // Core down entirely is the proxy-health loop's domain. + if (mixed_port_fail_streak > 0) + log_message("Mixed port watch: sing-box stopped, standing down.", "debug"); + mixed_port_fail_streak = 0; + return; + } + + if (mixed_proxy_port_listening()) { + if (mixed_port_fail_streak >= 2) + log_message("Mixed port 4534 recovered after core restart.", "info"); + mixed_port_fail_streak = 0; + return; + } + + mixed_port_fail_streak++; + log_message(sprintf("sing-box runs but mixed port 4534 is not listening (streak %d).", + mixed_port_fail_streak), "warn"); + + if (mixed_port_fail_streak == 2) { + log_message("Restarting sing-box to restore the mixed inbound.", "warn"); + command_status("/usr/bin/tachyon restart >/dev/null 2>&1"); + } else if (mixed_port_fail_streak > 2) { + let now = time(); + if (now - mixed_port_last_notify >= 3600) { + mixed_port_last_notify = now; + send_telegram_notification("⚠️ Порт 4534 не слушается при работающем sing-box. Рестарт не помог — проверьте конфигурацию ядра (лог sing-box), раздел диагностики покажет детали."); + } + } +} + +// ── Telegram worker liveness ───────────────────────────────────────────────── +// The worker owns a pid but can hang silently (stuck long-poll, deadlocked +// loop). It stamps a heartbeat after every successful poll; a stale +// heartbeat while the bot is enabled means the pid is zombie - restart it. + +const TELEGRAM_PID_FILE = "/var/run/tachyon_telegram.pid"; +const TELEGRAM_HEARTBEAT_FILE = "/var/run/tachyon_telegram.heartbeat"; +let telegram_worker_last_restart = 0; + +function telegram_worker_restart() { + let now = time(); + if (now - telegram_worker_last_restart < 3600) + return; + telegram_worker_last_restart = now; + log_message("Telegram worker is enabled but not polling (no heartbeat); restarting it.", "warn"); + command_status("/usr/bin/tachyon telegram_start >/dev/null 2>&1"); +} + +function check_telegram_worker() { + let tcfg = common.object_or_empty(uci_core.get_all(CONFIG_NAME, "telegram")); + if (as_string(tcfg.enabled || "0") != "1") + return; + + let heartbeat = trim(fs.readfile(TELEGRAM_HEARTBEAT_FILE) || ""); + if (heartbeat == "") { + // No stamp at all: either the worker never got a successful poll or + // the file was wiped by a reboot. Only act when the pid is gone too, + // otherwise give the worker one poll cycle to succeed. + let pid = trim(fs.readfile(TELEGRAM_PID_FILE) || ""); + if (pid == "" || !process_running(pid, "ucode")) + telegram_worker_restart(); + return; + } + + let age = time() - int(heartbeat); + if (age <= 900) + return; + + let pid = trim(fs.readfile(TELEGRAM_PID_FILE) || ""); + if (pid != "" && process_running(pid, "ucode")) { + log_message(sprintf("Telegram worker pid %s is running but its heartbeat is %d s old; restarting.", pid, age), "warn"); + command_status("/usr/bin/tachyon telegram_stop >/dev/null 2>&1"); + } + telegram_worker_restart(); +} + function perform_slow_checks() { controller.probe_slow(); safe_call(ai_heal_dns_loop, "ai_heal_dns_loop"); + safe_call(check_mixed_proxy_port, "check_mixed_proxy_port"); + safe_call(check_telegram_worker, "check_telegram_worker"); } - if (uloop) { let tick; tick = function() { diff --git a/tests/watchdog_selfheal.sh b/tests/watchdog_selfheal.sh new file mode 100644 index 00000000..efa42c89 --- /dev/null +++ b/tests/watchdog_selfheal.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -eo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +WD="$ROOT_DIR/tachyon/files/usr/lib/service/watchdog.uc" +UPD="$ROOT_DIR/tachyon/files/usr/lib/components/updates.uc" +VAL="$ROOT_DIR/tachyon/files/usr/lib/config/validator.uc" + +fail() { + printf 'FAIL: %s\n' "$1" >&2 + exit 1 +} + +# 1. Watchdog self-healing for the mixed proxy port: live core + dead 4534 +# triggers a bounded restart, then an hourly-capped admin notification. +grep -Fq 'function check_mixed_proxy_port()' "$WD" || + fail "watchdog must define check_mixed_proxy_port()" +grep -Fq 'safe_call(check_mixed_proxy_port, "check_mixed_proxy_port");' "$WD" || + fail "slow checks must run check_mixed_proxy_port" +grep -Fq '/usr/bin/tachyon restart >/dev/null 2>&1' "$WD" || + fail "mixed port healing must restart the core once" +grep -Fq 'now - mixed_port_last_notify >= 3600' "$WD" || + fail "mixed port notifications must be capped to one per hour" + +# 2. Telegram worker liveness: stale heartbeat with the bot enabled restarts +# the worker instead of waiting for a user to notice the silence. +grep -Fq 'function check_telegram_worker()' "$WD" || + fail "watchdog must define check_telegram_worker()" +grep -Fq 'safe_call(check_telegram_worker, "check_telegram_worker");' "$WD" || + fail "slow checks must run check_telegram_worker" +grep -Fq '/var/run/tachyon_telegram.heartbeat' "$WD" || + fail "worker liveness must read the heartbeat file" +grep -Fq '/usr/bin/tachyon telegram_start >/dev/null 2>&1' "$WD" || + fail "liveness healing must restart the worker via telegram_start" + +# 3. Legacy unmarked tachyon cron lines are stripped so crond stops running +# component_updates_if_due twice per tick. +grep -Fq 'function strip_unmarked_tachyon_cron_lines(' "$UPD" || + fail "updates.uc must strip legacy unmarked cron lines" +grep -Fq 'strip_unmarked_tachyon_cron_lines(' "$UPD" || true +[ "$(grep -c 'strip_unmarked_tachyon_cron_lines(' "$UPD")" -ge 3 ] || + fail "unmarked cron line stripping must be applied to remove and refresh paths" +grep -Fq '(list_update|subscription_update|component_updates_if_due)' "$UPD" || + fail "legacy cron pattern must cover all three tachyon jobs" + +# 4. Tunnel-style sections count as outbound configuration: a lone AWG/WARP +# section must not trigger "No proxy outbound sections found". +grep -Fq 'contains([ "awg", "warp", "vpn", "openvpn", "masque",' "$VAL" || + fail "has_outbound_section must recognise tunnel-style actions" + +printf 'watchdog self-heal checks passed\n'