feat(watchdog): самолечение порта 4534 и живости телеграм-воркера

- новая slow-проверка (раз в 5 мин): sing-box запущен, но mixed-порт 4534
  не слушается две проверки подряд (~10 мин) -> один авторестарт ядра;
  если не помогло - уведомление админу не чаще раза в час, без циклов
  бесконечных рестартов
- проверка живости телеграм-воркера: бот включён, pid жив, но heartbeat
  старше 15 минут (зависший long-poll/мёртвый цикл) или pid отсутствует ->
  рестарт воркера через telegram_start с ограничением раз в час
- обе проверки добавлены в perform_slow_checks через safe_call
This commit is contained in:
Dushnilin
2026-08-22 10:07:19 +03:00
parent 28c5c19645
commit e446b370e3
2 changed files with 151 additions and 1 deletions
+51
View File
@@ -0,0 +1,51 @@
#!/usr/bin/env bash
set -eo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
WD="$ROOT_DIR/tachyon/files/usr/lib/service/watchdog.uc"
UPD="$ROOT_DIR/tachyon/files/usr/lib/components/updates.uc"
VAL="$ROOT_DIR/tachyon/files/usr/lib/config/validator.uc"
fail() {
printf 'FAIL: %s\n' "$1" >&2
exit 1
}
# 1. Watchdog self-healing for the mixed proxy port: live core + dead 4534
# triggers a bounded restart, then an hourly-capped admin notification.
grep -Fq 'function check_mixed_proxy_port()' "$WD" ||
fail "watchdog must define check_mixed_proxy_port()"
grep -Fq 'safe_call(check_mixed_proxy_port, "check_mixed_proxy_port");' "$WD" ||
fail "slow checks must run check_mixed_proxy_port"
grep -Fq '/usr/bin/tachyon restart >/dev/null 2>&1' "$WD" ||
fail "mixed port healing must restart the core once"
grep -Fq 'now - mixed_port_last_notify >= 3600' "$WD" ||
fail "mixed port notifications must be capped to one per hour"
# 2. Telegram worker liveness: stale heartbeat with the bot enabled restarts
# the worker instead of waiting for a user to notice the silence.
grep -Fq 'function check_telegram_worker()' "$WD" ||
fail "watchdog must define check_telegram_worker()"
grep -Fq 'safe_call(check_telegram_worker, "check_telegram_worker");' "$WD" ||
fail "slow checks must run check_telegram_worker"
grep -Fq '/var/run/tachyon_telegram.heartbeat' "$WD" ||
fail "worker liveness must read the heartbeat file"
grep -Fq '/usr/bin/tachyon telegram_start >/dev/null 2>&1' "$WD" ||
fail "liveness healing must restart the worker via telegram_start"
# 3. Legacy unmarked tachyon cron lines are stripped so crond stops running
# component_updates_if_due twice per tick.
grep -Fq 'function strip_unmarked_tachyon_cron_lines(' "$UPD" ||
fail "updates.uc must strip legacy unmarked cron lines"
grep -Fq 'strip_unmarked_tachyon_cron_lines(' "$UPD" || true
[ "$(grep -c 'strip_unmarked_tachyon_cron_lines(' "$UPD")" -ge 3 ] ||
fail "unmarked cron line stripping must be applied to remove and refresh paths"
grep -Fq '(list_update|subscription_update|component_updates_if_due)' "$UPD" ||
fail "legacy cron pattern must cover all three tachyon jobs"
# 4. Tunnel-style sections count as outbound configuration: a lone AWG/WARP
# section must not trigger "No proxy outbound sections found".
grep -Fq 'contains([ "awg", "warp", "vpn", "openvpn", "masque",' "$VAL" ||
fail "has_outbound_section must recognise tunnel-style actions"
printf 'watchdog self-heal checks passed\n'