refactor(watchdog): разделение наблюдения и восстановления через шину событий

Ранее вочдог опрашивал состояние системы и устранял проблемы в едином смешанном цикле. Добавление новой проверки или действия по восстановлению требовало изменения монолитной функции, где мониторинг и ремонт были переплетены, что затрудняло аудит порядка выполнения и изолированное тестирование.

Вводится `usr/lib/service/event_controller.uc` — шина событий pub/sub для вочдога. Теперь зонды публикуют факты (`EV.PROXY_DOWN`, `EV.DNS_DOWN`, `EV.SINGBOX_STOPPED`, `EV.OOM_DETECTED`, `EV.WAN_DOWN` и т.д.), не зная, кто их обрабатывает. Восстановители (healers) подписываются на эти факты и независимо выполняют ремонт.

Ключевые изменения:
- `event_controller.uc` управляет словарём событий, регистрацией подписчиков, сортировкой по приоритетам и синхронной отправкой.
- Зонды (`probe_proxy`, `probe_dns`, `probe_singbox`, `probe_oom`, `probe_wan`) публикуют события вместо прямого вызова функций ремонта.
- Восстановители регистрируются как подписчики с явными приоритетами, чтобы первопричины устранялись раньше вторичных сбоев.
- `tests/event_bus.sh` добавляет интеграционные тесты для отправки событий, сортировки приоритетов и обработки подписчиков.

tests/watchdog_subscriptions.sh

@@ -0,0 +1,144 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# Pins the wiring between the two halves of the split: event_controller.uc
+# observes and emits, watchdog.uc subscribes and repairs. A fact that nobody
+# subscribes to is a repair that silently stopped happening — exactly the
+# regression this refactor could introduce and that no other test would catch.
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+TACHYON_LIB="$ROOT_DIR/tachyon/files/usr/lib"
+CONTROLLER_UC="$TACHYON_LIB/service/event_controller.uc"
+WATCHDOG_UC="$TACHYON_LIB/service/watchdog.uc"
+
+fail() {
+  printf 'FAIL: %s\n' "$1" >&2
+  exit 1
+}
+
+# Facts that are deliberately published without a repair attached.
+# PROXY_UP / DNS_UP are the recovery side of a fault pair, kept so a future
+# subscriber can act on recovery; FIREWALL_RELOADED and TICK are informational.
+is_informational() {
+  case "$1" in
+    PROXY_UP|DNS_UP|FIREWALL_RELOADED|TICK) return 0 ;;
+    *) return 1 ;;
+  esac
+}
+
+emitted="$(grep -oE 'bus\.emit(_once)?\(EV\.[A-Z_]+' "$CONTROLLER_UC" \
+  | grep -oE 'EV\.[A-Z_]+' | sed 's/^EV\.//' | sort -u)"
+[ -n "$emitted" ] || fail "no emit sites found in event_controller.uc"
+
+# Registrations go through subscribe(), a wrapper that turns a bus.on() rejection
+# into a startup failure instead of a silently missing repair.
+subscribed="$(grep -oE 'subscribe\(EV\.[A-Z_]+' "$WATCHDOG_UC" \
+  | grep -oE 'EV\.[A-Z_]+' | sed 's/^EV\.//' | sort -u)"
+[ -n "$subscribed" ] || fail "no subscribe() registrations found in watchdog.uc"
+
+# --- every emitted fault has a subscriber ---
+for ev in $emitted; do
+  is_informational "$ev" && continue
+  printf '%s\n' "$subscribed" | grep -qx "$ev" \
+    || fail "event_controller emits EV.$ev but watchdog subscribes to nothing for it"
+done
+
+# --- every subscription has a matching emit site ---
+# A dead subscription means a repair that can never run.
+for ev in $subscribed; do
+  printf '%s\n' "$emitted" | grep -qx "$ev" \
+    || fail "watchdog subscribes to EV.$ev but event_controller never emits it"
+done
+
+# --- every referenced EV name exists in the vocabulary ---
+vocabulary="$(ucode -L "$TACHYON_LIB" "$CONTROLLER_UC" event-types \
+  | sed 's/=.*//' | sort -u)"
+for ev in $emitted $subscribed; do
+  printf '%s\n' "$vocabulary" | grep -qx "$ev" \
+    || fail "EV.$ev is used but missing from the exported event vocabulary"
+done
+
+# --- the faults that used to be repaired on a timer are still covered ---
+# Each name here was an ai_heal_* function called from a worker() tier before the
+# refactor. If one loses its subscription, the router stops self-healing it.
+for required in \
+  SINGBOX_STOPPED WAN_DOWN CONFIG_CORRUPT NFT_MISSING QOS_MISSING TPROXY_DOWN \
+  SUBNETS_EMPTY PROXY_DOWN DNS_DOWN SECTIONS_EMPTY SUBSCRIPTION_UNREACHABLE \
+  OOM_DETECTED OOM_RECOVERABLE MEMORY_LOW RPCD_FD_LEAK ANOMALY_RECONNECTS \
+  SMARTDETECT_CANDIDATE URLTEST_SWITCHED HONEYPOT_HIT
+do
+  printf '%s\n' "$subscribed" | grep -qx "$required" \
+    || fail "no subscriber for EV.$required (was repaired on a timer before the refactor)"
+done
+
+# --- subscriptions are registered exactly once ---
+# register_subscribers() is called from both worker() and the ai-heal CLI mode;
+# without the idempotence guard every repair would run twice per fact.
+grep -q 'if (subscribers_registered) return;' "$WATCHDOG_UC" \
+  || fail "register_subscribers() lost its idempotence guard"
+
+# --- a rejected registration is fatal, not silent ---
+# bus.on() returns false for a non-function handler rather than raising, and
+# ucode does not hoist function declarations: moving a heal_* below
+# register_subscribers() would register nothing, emit nothing and throw nothing.
+# subscribe() exists to make that loud. Registering directly through bus.on()
+# would reopen the hole.
+grep -q 'function subscribe(event_type, handler, opts)' "$WATCHDOG_UC" \
+  || fail "the subscribe() wrapper is gone; a lost handler would fail silently"
+grep -q 'if (!bus.on(event_type, handler, opts))' "$WATCHDOG_UC" \
+  || fail "subscribe() no longer checks the bus.on() return value"
+if grep -qE '^\s+bus\.on\(EV\.' "$WATCHDOG_UC"; then
+  fail "a subscriber registers via bus.on() directly, bypassing the subscribe() guard"
+fi
+
+# --- cooldowns that replaced the old last_*_time globals are still declared ---
+# These four were the only per-repair timers in the old code; the bus owns them
+# now, so the values have to survive here.
+check_cooldown() {
+  local handler="$1" seconds="$2"
+  grep -A1 -E "subscribe\(EV\.[A-Z_]+, $handler," "$WATCHDOG_UC" \
+    | grep -q "cooldown: $seconds" \
+    || fail "$handler lost its cooldown of ${seconds}s"
+}
+check_cooldown heal_wan_and_gateway 300
+check_cooldown heal_community_subnet_sets 300
+check_cooldown heal_empty_sections 120
+check_cooldown heal_anomaly_reconnects 300
+
+# --- the two-tier repairs keep their original pacing ---
+# Before the split, proxy and DNS were measured twice at two different rates:
+# ai_heal_proxy_health / ai_heal_dns_continuous on the fast tier (15s), and
+# ai_heal_proxy_connectivity / ai_heal_dns inside the normal-tier audit (120s+).
+# One probe now serves both, at the fast rate. Without these bounds a single
+# stall would restart sing-box eight times as often as it used to — the
+# aggressive-restart regression fixed in c8052ee6.
+check_cooldown heal_proxy_connectivity 120
+check_cooldown heal_dns_stall 120
+
+# The DNS stall threshold counts a streak of 3. That streak has to be stepped at
+# the old normal-tier rate, or "3 failures" would mean 45 seconds of trouble
+# instead of ~6 minutes.
+grep -q 'const DNS_STREAK_INTERVAL = 120;' "$CONTROLLER_UC" \
+  || fail "DNS_STREAK_INTERVAL is missing or no longer 120s"
+grep -q 'now - state.dns_streak_stamp >= DNS_STREAK_INTERVAL' "$CONTROLLER_UC" \
+  || fail "the DNS stall streak is no longer paced by DNS_STREAK_INTERVAL"
+
+# The fast-tier counters must NOT be paced — they are the ones the old fast
+# healers incremented every 15s, and ai-status-full reports them.
+grep -q 'state.dns_consecutive_fails++;' "$CONTROLLER_UC" \
+  || fail "dns_consecutive_fails is no longer incremented every probe"
+grep -q 'state.proxy_consecutive_fails++;' "$CONTROLLER_UC" \
+  || fail "proxy_consecutive_fails is no longer incremented every probe"
+
+# --- the observation layer does not repair ---
+# The whole point of the split: no restart, reload or nft mutation may leak back
+# into event_controller.uc.
+for forbidden in 'safe_proxy_restart' 'safe_reload_firewall' 'ai_heal_report' \
+  '/etc/init.d/tachyon' 'service sing-box' 'nft add' 'nft flush'
+do
+  if grep -q -- "$forbidden" "$CONTROLLER_UC"; then
+    fail "event_controller.uc performs a repair action ($forbidden); observation must not heal"
+  fi
+done
+
+printf 'watchdog subscription checks passed\n'