feat(failover): автоматическое переключение прокси-секций

- генератор: при section_failover_enabled catch-all указывает на селектор tachyon-failover со всеми секциями-кандидатами - выбор по умолчанию переживает рестарты через /etc/tachyon/failover_section, устаревшее имя откатывается на первую секцию - service/failover.uc: пробер каждой секции через Clash API delay, порог N подряд — переключение на живую, TG-уведомление - watchdog: вызов проверки в slow-тире (модуль сам отключён при выключенной опции) - настройки: флаг + порог во вкладке Advanced, RU-локализация - тесты: генерация вкл/выкл/персист/устаревшее состояние + логика переключения со стабом Clash API
2026-08-26 14:15:23 +03:00
parent b12f2fbdee
commit 9bc6fe28fd
8 changed files with 480 additions and 5 deletions
@@ -836,6 +836,18 @@ msgstr "Дашборд сейчас недоступен"
 msgid "Days of Week"
 msgstr "Дни недели"

+msgid "Auto-failover of proxy sections"
+msgstr "Авто-failover прокси-секций"
+
+msgid "Watchdog probes every Connection section; when the active one fails several checks in a row, traffic switches to a healthy backup and Telegram is notified. Requires a config reload to take effect."
+msgstr "Watchdog проверяет каждую Connection-секцию; если активная падает несколько проверок подряд, трафик переключается на живую резервную, а Telegram получает уведомление. Нужен reload конфигурации."
+
+msgid "Failover failure threshold"
+msgstr "Порог срабатывания failover"
+
+msgid "Number of consecutive failed probes before switching sections"
+msgstr "Сколько неудачных проверок подряд до переключения секции"
+
 msgid "Daily time quota (minutes)"
 msgstr "Дневной лимит времени (минуты)"

@@ -1507,6 +1507,31 @@ function createSettingsContent(section, capabilities) {
  o.default = "0";
  o.rmempty = false;

+  o = section.taboption(
+    "advanced",
+    form.Flag,
+    "section_failover_enabled",
+    _("Auto-failover of proxy sections"),
+    _(
+      "Watchdog probes every Connection section; when the active one fails several checks in a row, traffic switches to a healthy backup and Telegram is notified. Requires a config reload to take effect.",
+    ),
+  );
+  o.default = "0";
+  o.rmempty = false;
+
+  o = section.taboption(
+    "advanced",
+    form.Value,
+    "section_failover_threshold",
+    _("Failover failure threshold"),
+    _("Number of consecutive failed probes before switching sections"),
+  );
+  o.depends("section_failover_enabled", "1");
+  o.default = "3";
+  o.placeholder = "3";
+  o.rmempty = false;
+  o.datatype = "range(1,30)";
+
  o = section.taboption(
    "advanced",
    form.ListValue,
@@ -836,6 +836,18 @@ msgstr "Дашборд сейчас недоступен"
 msgid "Days of Week"
 msgstr "Дни недели"

+msgid "Auto-failover of proxy sections"
+msgstr "Авто-failover прокси-секций"
+
+msgid "Watchdog probes every Connection section; when the active one fails several checks in a row, traffic switches to a healthy backup and Telegram is notified. Requires a config reload to take effect."
+msgstr "Watchdog проверяет каждую Connection-секцию; если активная падает несколько проверок подряд, трафик переключается на живую резервную, а Telegram получает уведомление. Нужен reload конфигурации."
+
+msgid "Failover failure threshold"
+msgstr "Порог срабатывания failover"
+
+msgid "Number of consecutive failed probes before switching sections"
+msgstr "Сколько неудачных проверок подряд до переключения секции"
+
 msgid "Daily time quota (minutes)"
 msgstr "Дневной лимит времени (минуты)"

@@ -212,6 +212,7 @@ function command_spec(command) {
        get_tailscale_peers: [ "diagnostics/runtime.uc", "get-tailscale-peers", 0 ],
        tailscale_restart: [ "providers/tailscale/runtime.uc", "start-runtime", 0 ],
        parental_quota_tick: [ "service/parental_quota.uc", "tick", 0 ],
+        failover_check: [ "service/failover.uc", "check", 0 ],
        get_system_info: [ "diagnostics/runtime.uc", "get-system-info", 0 ],
        get_server_capabilities: [ "diagnostics/runtime.uc", "get-server-capabilities", 0 ],
        get_ui_capabilities: [ "service/ui.uc", "get-ui-capabilities", 0 ],
@@ -0,0 +1,196 @@
+#!/usr/bin/env ucode
+
+// Cross-section proxy failover: probe every candidate section through the
+// Clash API delay endpoint; when the active section misses the failure
+// threshold in a row, switch the tachyon-failover selector to a healthy one,
+// persist the choice and notify Telegram.
+//
+// The runtime switch only works when sing-box was generated with
+// section_failover_enabled=1 (the catch-all route then points at the
+// tachyon-failover selector group).
+//
+// NOTE: ucode does not hoist function declarations - keep callees above
+// their first caller.
+
+let fs = require("fs");
+let common = require("core.common");
+let uci_core = require("core.uci");
+
+let as_string = common.as_string;
+let object_or_empty = common.object_or_empty;
+let option = common.option;
+let int_option = common.int_option;
+let bool_option = common.bool_option;
+let shell_quote = common.shell_quote;
+let command_from_args = common.command_from_args;
+let command_status = common.command_status;
+let command_success_from_args = common.command_success_from_args;
+let command_output_from_args = common.command_output_from_args;
+
+const CONFIG_NAME = getenv("TACHYON_CONFIG_NAME") || "tachyon";
+const TACHYON_BIN = getenv("TACHYON_BIN") || "/usr/bin/tachyon";
+const STATE_FILE = getenv("TACHYON_FAILOVER_STATE_FILE") || "/etc/tachyon/failover_section";
+const STREAK_FILE = getenv("TACHYON_FAILOVER_STREAK_FILE") || "/var/run/tachyon/failover_streak";
+const PROBE_TIMEOUT_MS = getenv("TACHYON_FAILOVER_PROBE_TIMEOUT") || "3000";
+
+const FAILOVER_GROUP_TAG = "tachyon-failover";
+
+function log_message(message, level) {
+    level = as_string(level || "info");
+    command_success_from_args([ "logger", "-t", "tachyon", "[failover] [" + level + "] " + as_string(message) ]);
+}
+
+function settings() {
+    return object_or_empty(uci_core.get_all(CONFIG_NAME, "settings"));
+}
+
+function enabled() {
+    return bool_option(settings(), "section_failover_enabled", false);
+}
+
+function threshold() {
+    let value = int_option(settings(), "section_failover_threshold", 3);
+    return value > 0 ? value : 3;
+}
+
+// Same ordering semantics as the generator: UCI order of enabled sections.
+function candidates() {
+    let result = [];
+    for (let section in uci_core.section_objects(CONFIG_NAME, "section")) {
+        if (!bool_option(section, "enabled", true))
+            continue;
+        let action = option(section, "action", "");
+        if (action == "connection" || action == "proxy" || action == "outbound" || action == "vpn" ||
+            action == "awg" || action == "warp" || action == "byedpi" || action == "zapret" || action == "zapret2" ||
+            action == "anytls" || action == "snell" || action == "mieru" || action == "sudoku" ||
+            action == "masque" || action == "openvpn")
+            push(result, as_string(section[".name"]));
+    }
+    return result;
+}
+
+function read_active(candidates_list) {
+    let raw = "";
+    try {
+        raw = trim(as_string(fs.readfile(STATE_FILE)) || "");
+    } catch (e) {
+        raw = "";
+    }
+    if (raw != "") {
+        for (let name in candidates_list)
+            if (as_string(name) == raw)
+                return raw;
+    }
+    return length(candidates_list) > 0 ? as_string(candidates_list[0]) : "";
+}
+
+function write_active(name) {
+    fs.unlink(STATE_FILE);
+    if (!fs.writefile(STATE_FILE, as_string(name) + "\n"))
+        log_message("failed to persist failover state", "warn");
+}
+
+function read_streak() {
+    let raw = trim(as_string(fs.readfile(STREAK_FILE)) || "");
+    let value = int(raw);
+    return value > 0 ? value : 0;
+}
+
+function write_streak(value) {
+    fs.unlink(STREAK_FILE);
+    fs.writefile(STREAK_FILE, as_string(value) + "\n");
+}
+
+function clash_delay_ok(tag) {
+    let out = trim(command_output_from_args([ TACHYON_BIN, "clash_api", "get_proxy_latency", tag, PROBE_TIMEOUT_MS ]) || "");
+    if (out == "")
+        return false;
+    let parsed = null;
+    try {
+        parsed = json(out);
+    } catch (e) {
+        return false;
+    }
+    return object_or_empty(parsed)["delay"] != null;
+}
+
+function switch_group(section_name, outbound_name) {
+    let out = trim(command_output_from_args([ TACHYON_BIN, "clash_api", "set_group_proxy", FAILOVER_GROUP_TAG, outbound_name ]) || "");
+    // set_group_proxy prints a JSON error object on failure.
+    if (index(out, '"error"') >= 0) {
+        log_message("runtime switch failed: " + out, "warn");
+        return false;
+    }
+    return true;
+}
+
+function send_notification(message) {
+    let tcfg = object_or_empty(uci_core.get_all(CONFIG_NAME, "telegram"));
+    if (as_string(tcfg.enabled || "0") != "1" || !tcfg.bot_token || !tcfg.admin_ids)
+        return;
+    system(common.background_command(TACHYON_BIN + " telegram send " + shell_quote(message)));
+}
+
+function check() {
+    if (!enabled())
+        return 0;
+    let list = candidates();
+    if (length(list) < 2)
+        return 0;
+
+    let active = read_active(list);
+    if (active == "")
+        return 0;
+
+    if (clash_delay_ok(active + "-out")) {
+        write_streak(0);
+        return 0;
+    }
+
+    let streak = read_streak() + 1;
+    if (streak < threshold()) {
+        write_streak(streak);
+        log_message(active + " failed " + as_string(streak) + "/" + as_string(threshold()), "debug");
+        return 0;
+    }
+    write_streak(0);
+
+    for (let name in list) {
+        name = as_string(name);
+        if (name == active)
+            continue;
+        if (!clash_delay_ok(name + "-out"))
+            continue;
+        write_active(name);
+        if (!switch_group(name, name + "-out"))
+            log_message("switched state to " + name + ", but live group update failed; will apply on next reload", "warn");
+        log_message("active section switched: " + active + " -> " + name, "info");
+        send_notification("🔀 *Failover*: секция `" + active + "` недоступна, переключаюсь на `" + name + "`.");
+        return 0;
+    }
+
+    log_message("active section failed and no healthy backup found", "warn");
+    return 0;
+}
+
+function status_json() {
+    let list = candidates();
+    write_json({
+        enabled: enabled(),
+        threshold: threshold(),
+        candidates: list,
+        active: read_active(list),
+        streak: read_streak()
+    });
+}
+
+let mode = ARGV[0] || "";
+
+if (mode == "check")
+    exit(check());
+else if (mode == "status")
+    status_json();
+else {
+    warn("Usage: service/failover.uc <check|status>\n");
+    exit(1);
+}
@@ -1907,12 +1907,19 @@ function check_tailscale_worker() {
    }
 }

+// Cross-section failover probe; the module self-gates on
+// settings.section_failover_enabled so this is a no-op by default.
+function check_section_failover() {
+    command_status("/usr/bin/tachyon failover_check >/dev/null 2>&1");
+}
+
    function perform_slow_checks() {
        controller.probe_slow();
        safe_call(ai_heal_dns_loop, "ai_heal_dns_loop");
        safe_call(check_mixed_proxy_port, "check_mixed_proxy_port");
        safe_call(check_telegram_worker, "check_telegram_worker");
        safe_call(check_tailscale_worker, "check_tailscale_worker");
+        safe_call(check_section_failover, "check_section_failover");
    }
    if (uloop) {
        let tick;
@@ -1437,18 +1437,48 @@ function add_route_for_section(config, section) {
        add_combined_route_for_section(config, section);
 }

-function add_service_route_rules(config, sections) {
-    let first = null;
+function failover_candidate(sections) {
+    let result = [];
    for (let section in sections) {
        let action = option(section, "action", "");
        if (connections.is_connections_action(action) ||
            action == "awg" || action == "warp" || action == "byedpi" || action == "zapret" || action == "zapret2" ||
            action == "anytls" || action == "snell" || action == "mieru" || action == "sudoku" ||
            action == "masque" || action == "openvpn") {
-            first = section;
-            break;
+            push(result, section);
        }
    }
+    return result;
+}
+
+function failover_state_file() {
+    return getenv("TACHYON_FAILOVER_STATE_FILE") || "/etc/tachyon/failover_section";
+}
+
+// Persisted user/agent choice survives restarts; ignored when the section
+// disappeared from the candidate list.
+function failover_default_name(candidates) {
+    let raw = "";
+    try {
+        raw = trim(as_string(fs.readfile(failover_state_file())) || "");
+    } catch (e) {
+        raw = "";
+    }
+    if (raw != "") {
+        for (let section in candidates)
+            if (as_string(section[".name"]) == raw)
+                return raw;
+    }
+    return as_string(candidates[0][".name"]);
+}
+
+const FAILOVER_GROUP_TAG = "tachyon-failover";
+
+function add_service_route_rules(config, sections) {
+    let candidates = failover_candidate(sections);
+    let first = length(candidates) > 0 ? candidates[0] : null;
+    let failover_active = bool_option(object_or_empty(ctx.uci_cursor().get_all(CONFIG_NAME, "settings")), "section_failover_enabled", false) &&
+        length(candidates) > 1;
    if (first != null) {
        push(config.route.rules, {
            action: "resolve",
@@ -1456,10 +1486,24 @@ function add_service_route_rules(config, sections) {
            server: runtime_constants.DNS_SERVER_TAG,
            domain: runtime_constants.CHECK_PROXY_IP_DOMAIN
        });
+        let catchall_target = outbound_tag(first[".name"]);
+        if (failover_active) {
+            let member_tags = [];
+            for (let section in candidates)
+                push(member_tags, outbound_tag(section[".name"]));
+            push(config.outbounds, {
+                type: "selector",
+                tag: FAILOVER_GROUP_TAG,
+                outbounds: member_tags,
+                default: outbound_tag(failover_default_name(candidates)),
+                interrupt_exist_connections: true
+            });
+            catchall_target = FAILOVER_GROUP_TAG;
+        }
        push(config.route.rules, {
            action: "route",
            inbound: tproxy_inbound_matcher(),
-            outbound: outbound_tag(first[".name"]),
+            outbound: catchall_target,
            domain: runtime_constants.CHECK_PROXY_IP_DOMAIN
        });
    }
@@ -0,0 +1,178 @@
+#!/usr/bin/env bash
+# Auto-failover: generator emits the tachyon-failover selector group only when
+# enabled; the failover module switches to a healthy section after N failures
+# (Clash API stubbed).
+set -eo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+TACHYON_LIB="$ROOT_DIR/tachyon/files/usr/lib"
+GENERATOR_UC="$TACHYON_LIB/singbox/generator.uc"
+FAILOVER_UC="$TACHYON_LIB/service/failover.uc"
+WORK_DIR="$(mktemp -d)"
+
+cleanup() {
+  rm -rf "$WORK_DIR"
+}
+trap cleanup EXIT
+
+fail() {
+  printf 'FAIL: %s\n' "$1" >&2
+  exit 1
+}
+
+generate_config() {
+  local fixture="$1" output="$2"
+  mkdir -p "${output}.section-cache"
+  ucode -L "$TACHYON_LIB" "$GENERATOR_UC" generate-config-fixture \
+    "$fixture" "$output" "127.0.0.1"
+}
+
+make_fixture() {
+  local enabled_flag="$1" out="$2"
+  cat >"$out" <<JSON
+{
+  "settings": {
+    ".name": "settings",
+    ".type": "settings",
+    "dns_server": [ "77.88.8.8" ],
+    "bootstrap_dns_server": [ "77.88.8.8" ],
+    "section_failover_enabled": "$enabled_flag",
+    "section_failover_threshold": "2"
+  },
+  "section": [
+    {
+      ".name": "main_sec", ".type": "section", "enabled": "1", "action": "connection",
+      "outbound_jsons": [ "{\"type\":\"vless\",\"tag\":\"A\",\"server\":\"a.example\",\"server_port\":443,\"uuid\":\"00000000-0000-4000-8000-000000000001\",\"tls\":{\"enabled\":true}}" ]
+    },
+    {
+      ".name": "backup_sec", ".type": "section", "enabled": "1", "action": "connection",
+      "outbound_jsons": [ "{\"type\":\"vless\",\"tag\":\"B\",\"server\":\"b.example\",\"server_port\":443,\"uuid\":\"00000000-0000-4000-8000-000000000002\",\"tls\":{\"enabled\":true}}" ]
+    }
+  ]
+}
+JSON
+}
+
+# --- generator: disabled (default) -> no failover group ---------------------
+make_fixture 0 "$WORK_DIR/off.json"
+generate_config "$WORK_DIR/off.json" "$WORK_DIR/off.out"
+grep -Fq 'tachyon-failover' "$WORK_DIR/off.out" &&
+  fail "disabled failover must not emit the group"
+
+# catch-all still targets the first section outbound
+grep -Fq '"outbound": "main_sec-out"' "$WORK_DIR/off.out" ||
+  fail "disabled failover must keep the direct catch-all route"
+
+# --- generator: enabled -> group with both members, persisted default -------
+make_fixture 1 "$WORK_DIR/on.json"
+generate_config "$WORK_DIR/on.json" "$WORK_DIR/on.out"
+grep -Fq '"tag": "tachyon-failover"' "$WORK_DIR/on.out" ||
+  fail "enabled failover must emit the selector group"
+grep -Fq '"main_sec-out"' "$WORK_DIR/on.out" && grep -Fq '"backup_sec-out"' "$WORK_DIR/on.out" ||
+  fail "group must contain both section outbounds"
+grep -Fq '"default": "main_sec-out"' "$WORK_DIR/on.out" ||
+  fail "group default must be the first candidate"
+
+printf '%s' 'backup_sec' >"$WORK_DIR/state"
+TACHYON_FAILOVER_STATE_FILE="$WORK_DIR/state" \
+  generate_config "$WORK_DIR/on.json" "$WORK_DIR/on_persisted.out"
+grep -Fq '"default": "backup_sec-out"' "$WORK_DIR/on_persisted.out" ||
+  fail "persisted choice must become the group default"
+
+# stale persisted name falls back to the first candidate
+printf '%s' 'gone_sec' >"$WORK_DIR/stale"
+TACHYON_FAILOVER_STATE_FILE="$WORK_DIR/stale" \
+  generate_config "$WORK_DIR/on.json" "$WORK_DIR/on_stale.out"
+grep -Fq '"default": "main_sec-out"' "$WORK_DIR/on_stale.out" ||
+  fail "stale persisted section must fall back to first candidate"
+
+# --- switcher logic with stubbed Clash API ---------------------------------
+STUB_BIN="$WORK_DIR/bin"
+mkdir -p "$STUB_BIN"
+FAIL_LOG="$WORK_DIR/clash.log"
+
+cat >"$STUB_BIN/tachyon" <<EOF
+#!/usr/bin/env bash
+printf '%s\n' "\$*" >> '$FAIL_LOG'
+case "\$*" in
+  *"get_proxy_latency main_sec-out"*)
+    if [ "\$MAIN_DEAD" = "1" ]; then echo '{}'; else echo '{"delay":120}'; fi ;;
+  *"get_proxy_latency backup_sec-out"*) echo '{"delay":200}' ;;
+  *"get_proxy_latency gone_sec-out"*) echo '{}' ;;
+  *) exit 0 ;;
+esac
+EOF
+chmod +x "$STUB_BIN/tachyon"
+
+export PATH="$STUB_BIN:$PATH"
+export TACHYON_CONFIG_NAME="tachyon"
+export TACHYON_BIN="$STUB_BIN/tachyon"
+export TACHYON_FAILOVER_STATE_FILE="$WORK_DIR/state"
+export TACHYON_FAILOVER_STREAK_FILE="$WORK_DIR/streak"
+
+# --- switcher logic with stubbed Clash API ---------------------------------
+STUB_BIN="$WORK_DIR/bin"
+mkdir -p "$STUB_BIN"
+FAIL_LOG="$WORK_DIR/clash.log"
+
+cat >"$STUB_BIN/tachyon" <<EOF
+#!/usr/bin/env bash
+printf '%s\n' "\$*" >> '$FAIL_LOG'
+case "\$*" in
+  *"get_proxy_latency main_sec-out"*)
+    if [ "\${MAIN_DEAD:-0}" = "1" ]; then echo '{}'; else echo '{"delay":120}'; fi ;;
+  *"get_proxy_latency backup_sec-out"*) echo '{"delay":200}' ;;
+  *) exit 0 ;;
+esac
+EOF
+chmod +x "$STUB_BIN/tachyon"
+
+export PATH="$STUB_BIN:$PATH"
+export TACHYON_CONFIG_NAME="tachyon"
+export TACHYON_BIN="$STUB_BIN/tachyon"
+export TACHYON_FAILOVER_STATE_FILE="$WORK_DIR/state"
+export TACHYON_FAILOVER_STREAK_FILE="$WORK_DIR/streak"
+
+make_state() {
+  printf '%s\n' \
+    "tachyon.settings=settings" \
+    "tachyon.settings.section_failover_enabled=$1" \
+    "tachyon.settings.section_failover_threshold=2" \
+    "tachyon.main_sec=section" "tachyon.main_sec.enabled=1" "tachyon.main_sec.action=connection" \
+    "tachyon.backup_sec=section" "tachyon.backup_sec.enabled=1" "tachyon.backup_sec.action=connection" \
+    "tachyon.telegram=telegram" "tachyon.telegram.enabled=1" "tachyon.telegram.bot_token=x" "tachyon.telegram.admin_ids=1" \
+    >"$WORK_DIR/uci.state"
+  printf '%s' 'main_sec' >"$WORK_DIR/state"
+  rm -f "$WORK_DIR/streak"
+}
+
+run_check() {
+  UCI_STATE="$WORK_DIR/uci.state" ucode -L "$TACHYON_LIB" "$FAILOVER_UC" check
+}
+
+# disabled -> no-op even with dead primary
+make_state 0
+MAIN_DEAD=1 run_check || fail "check must not fail when disabled"
+[ ! -s "$WORK_DIR/streak" ] || fail "disabled check must not track streaks"
+
+# enabled, healthy primary -> no switch
+make_state 1
+run_check || fail "healthy check failed"
+[ "$(cat "$WORK_DIR/state")" = "main_sec" ] || fail "healthy primary must stay active"
+
+# primary dies twice (threshold=2) -> switch to backup + persist
+export MAIN_DEAD=1
+run_check; run_check || fail "switch path errored"
+[ "$(cat "$WORK_DIR/state")" = "backup_sec" ] ||
+  fail "must switch to backup after threshold, state=$(cat "$WORK_DIR/state")"
+grep -Fq "set_group_proxy tachyon-failover backup_sec-out" "$FAIL_LOG" ||
+  fail "runtime switch via Clash API missing"
+grep -Fq "telegram send" "$FAIL_LOG" ||
+  fail "telegram notification missing"
+
+# recovery of streak file on success
+unset MAIN_DEAD
+run_check
+[ "$(cat "$WORK_DIR/streak")" = "0" ] || fail "success must reset the streak counter"
+
+printf 'auto-failover checks passed\n'