feat(watchdog): самолечение порта 4534 и живости телеграм-воркера

- новая slow-проверка (раз в 5 мин): sing-box запущен, но mixed-порт 4534 не слушается две проверки подряд (~10 мин) -> один авторестарт ядра; если не помогло - уведомление админу не чаще раза в час, без циклов бесконечных рестартов - проверка живости телеграм-воркера: бот включён, pid жив, но heartbeat старше 15 минут (зависший long-poll/мёртвый цикл) или pid отсутствует -> рестарт воркера через telegram_start с ограничением раз в час - обе проверки добавлены в perform_slow_checks через safe_call
2026-08-22 10:07:19 +03:00
parent 28c5c19645
commit e446b370e3
2 changed files with 151 additions and 1 deletions
@@ -1718,11 +1718,110 @@ function worker() {
        safe_call(ai_export_status, "ai_export_status");
    }

+// ── Mixed proxy port (4534) self-healing ─────────────────────────────────────
+// sing-box can be running while its mixed inbound failed to bind: every LAN
+// client proxy and the Telegram bot route then silently dead-ends. Two
+// consecutive slow checks (~10 min) with a live core and a dead port trigger
+// one core restart; if the port is still dead afterwards, the admin gets a
+// notification instead of endless restart loops.
+
+let mixed_port_fail_streak = 0;
+let mixed_port_last_notify = 0;
+
+function mixed_proxy_port_listening() {
+    let out = command_output_from_args([ "netstat", "-ltn" ]);
+    for (let line in split(out, "\n")) {
+        if (index(line, ":4534 ") >= 0)
+            return true;
+    }
+    return false;
+}
+
+function check_mixed_proxy_port() {
+    let sb_pid = trim(command_output_from_args(["pidof", "sing-box"]));
+    if (sb_pid == "") {
+        // Core down entirely is the proxy-health loop's domain.
+        if (mixed_port_fail_streak > 0)
+            log_message("Mixed port watch: sing-box stopped, standing down.", "debug");
+        mixed_port_fail_streak = 0;
+        return;
+    }
+
+    if (mixed_proxy_port_listening()) {
+        if (mixed_port_fail_streak >= 2)
+            log_message("Mixed port 4534 recovered after core restart.", "info");
+        mixed_port_fail_streak = 0;
+        return;
+    }
+
+    mixed_port_fail_streak++;
+    log_message(sprintf("sing-box runs but mixed port 4534 is not listening (streak %d).",
+        mixed_port_fail_streak), "warn");
+
+    if (mixed_port_fail_streak == 2) {
+        log_message("Restarting sing-box to restore the mixed inbound.", "warn");
+        command_status("/usr/bin/tachyon restart >/dev/null 2>&1");
+    } else if (mixed_port_fail_streak > 2) {
+        let now = time();
+        if (now - mixed_port_last_notify >= 3600) {
+            mixed_port_last_notify = now;
+            send_telegram_notification("⚠️ Порт 4534 не слушается при работающем sing-box. Рестарт не помог — проверьте конфигурацию ядра (лог sing-box), раздел диагностики покажет детали.");
+        }
+    }
+}
+
+// ── Telegram worker liveness ─────────────────────────────────────────────────
+// The worker owns a pid but can hang silently (stuck long-poll, deadlocked
+// loop). It stamps a heartbeat after every successful poll; a stale
+// heartbeat while the bot is enabled means the pid is zombie - restart it.
+
+const TELEGRAM_PID_FILE = "/var/run/tachyon_telegram.pid";
+const TELEGRAM_HEARTBEAT_FILE = "/var/run/tachyon_telegram.heartbeat";
+let telegram_worker_last_restart = 0;
+
+function telegram_worker_restart() {
+    let now = time();
+    if (now - telegram_worker_last_restart < 3600)
+        return;
+    telegram_worker_last_restart = now;
+    log_message("Telegram worker is enabled but not polling (no heartbeat); restarting it.", "warn");
+    command_status("/usr/bin/tachyon telegram_start >/dev/null 2>&1");
+}
+
+function check_telegram_worker() {
+    let tcfg = common.object_or_empty(uci_core.get_all(CONFIG_NAME, "telegram"));
+    if (as_string(tcfg.enabled || "0") != "1")
+        return;
+
+    let heartbeat = trim(fs.readfile(TELEGRAM_HEARTBEAT_FILE) || "");
+    if (heartbeat == "") {
+        // No stamp at all: either the worker never got a successful poll or
+        // the file was wiped by a reboot. Only act when the pid is gone too,
+        // otherwise give the worker one poll cycle to succeed.
+        let pid = trim(fs.readfile(TELEGRAM_PID_FILE) || "");
+        if (pid == "" || !process_running(pid, "ucode"))
+            telegram_worker_restart();
+        return;
+    }
+
+    let age = time() - int(heartbeat);
+    if (age <= 900)
+        return;
+
+    let pid = trim(fs.readfile(TELEGRAM_PID_FILE) || "");
+    if (pid != "" && process_running(pid, "ucode")) {
+        log_message(sprintf("Telegram worker pid %s is running but its heartbeat is %d s old; restarting.", pid, age), "warn");
+        command_status("/usr/bin/tachyon telegram_stop >/dev/null 2>&1");
+    }
+    telegram_worker_restart();
+}
+
    function perform_slow_checks() {
        controller.probe_slow();
        safe_call(ai_heal_dns_loop, "ai_heal_dns_loop");
+        safe_call(check_mixed_proxy_port, "check_mixed_proxy_port");
+        safe_call(check_telegram_worker, "check_telegram_worker");
    }
-
    if (uloop) {
        let tick;
        tick = function() {
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+WD="$ROOT_DIR/tachyon/files/usr/lib/service/watchdog.uc"
+UPD="$ROOT_DIR/tachyon/files/usr/lib/components/updates.uc"
+VAL="$ROOT_DIR/tachyon/files/usr/lib/config/validator.uc"
+
+fail() {
+  printf 'FAIL: %s\n' "$1" >&2
+  exit 1
+}
+
+# 1. Watchdog self-healing for the mixed proxy port: live core + dead 4534
+#    triggers a bounded restart, then an hourly-capped admin notification.
+grep -Fq 'function check_mixed_proxy_port()' "$WD" ||
+  fail "watchdog must define check_mixed_proxy_port()"
+grep -Fq 'safe_call(check_mixed_proxy_port, "check_mixed_proxy_port");' "$WD" ||
+  fail "slow checks must run check_mixed_proxy_port"
+grep -Fq '/usr/bin/tachyon restart >/dev/null 2>&1' "$WD" ||
+  fail "mixed port healing must restart the core once"
+grep -Fq 'now - mixed_port_last_notify >= 3600' "$WD" ||
+  fail "mixed port notifications must be capped to one per hour"
+
+# 2. Telegram worker liveness: stale heartbeat with the bot enabled restarts
+#    the worker instead of waiting for a user to notice the silence.
+grep -Fq 'function check_telegram_worker()' "$WD" ||
+  fail "watchdog must define check_telegram_worker()"
+grep -Fq 'safe_call(check_telegram_worker, "check_telegram_worker");' "$WD" ||
+  fail "slow checks must run check_telegram_worker"
+grep -Fq '/var/run/tachyon_telegram.heartbeat' "$WD" ||
+  fail "worker liveness must read the heartbeat file"
+grep -Fq '/usr/bin/tachyon telegram_start >/dev/null 2>&1' "$WD" ||
+  fail "liveness healing must restart the worker via telegram_start"
+
+# 3. Legacy unmarked tachyon cron lines are stripped so crond stops running
+#    component_updates_if_due twice per tick.
+grep -Fq 'function strip_unmarked_tachyon_cron_lines(' "$UPD" ||
+  fail "updates.uc must strip legacy unmarked cron lines"
+grep -Fq 'strip_unmarked_tachyon_cron_lines(' "$UPD" || true
+[ "$(grep -c 'strip_unmarked_tachyon_cron_lines(' "$UPD")" -ge 3 ] ||
+  fail "unmarked cron line stripping must be applied to remove and refresh paths"
+grep -Fq '(list_update|subscription_update|component_updates_if_due)' "$UPD" ||
+  fail "legacy cron pattern must cover all three tachyon jobs"
+
+# 4. Tunnel-style sections count as outbound configuration: a lone AWG/WARP
+#    section must not trigger "No proxy outbound sections found".
+grep -Fq 'contains([ "awg", "warp", "vpn", "openvpn", "masque",' "$VAL" ||
+  fail "has_outbound_section must recognise tunnel-style actions"
+
+printf 'watchdog self-heal checks passed\n'