исправление: утечки ФД и временных файлов, переполнение ОЗУ
Проблема с FD-каскадом (зависание sing-box):
watchdog.uc открывал logread -f через popen() без O_CLOEXEC. Каждый
фоновый system('... &') порождал дочерний процесс, наследующий read-end
logread pipe. После kill watchdog — дочерний процесс держал pipe, logread
не получал SIGPIPE и оставался живым. За N перезапусков накапливалось
N orphan-процессов logread → исчерпание лимита 1024 FD → генератор
конфига sing-box не мог открыть .uc файлы → sing-box не стартовал.
Исправления FD-утечки:
- watchdog.uc: добавлена переменная logread_pipe_fd + помощник bg_system(),
который закрывает FD logread перед запуском фоновых процессов
- watchdog.uc: killall logread перед открытием нового logread -f в
setup_syslog_listener() для очистки orphan-процессов от предыдущих экземпляров
- watchdog.uc: все 7 вызовов system('... &') переведены на bg_system():
restart, reload_firewall (x3), WAN ifdown/ifup, subscription_update_async (x3)
- initd.uc: killall logread в stop_service() после остановки tachyon
- nft/apply.uc: добавлены </dev/null >/dev/null 2>&1 к голому reload_firewall &
- updates.uc: добавлены редиректы к tachyon reload &
- telegram.uc: добавлен </dev/null к reload_firewall &
Проблема с утечкой временных файлов (500MB в /tmp):
При каждом tachyon restart (SIGKILL) процессы-генераторы умирали на
полуслове, оставляя /tmp/tmp.* файлы (sing-box config JSON ~3.4MB,
reload state). За 8 дней накопилось 227 файлов = 506MB в tmpfs,
что занимало 25% всего ОЗУ (519MB из 2007MB shared).
Исправления утечки временных файлов:
- temp_path() в runtime.uc и updates.uc теперь создаёт /tmp/tachyon-XXXXXX
вместо /tmp/tmp.XXXXXX (безопасный для очистки префикс)
- nft/apply.uc и lifecycle.uc: аналогичное изменение mktemp
- lifecycle.uc start_impl(): автоматическая очистка /tmp/tachyon-* файлов
старше 30 минут при каждом старте сервиса
Результат на роутере 192.168.2.1:
logread после stop = 0 (было 1+)
logread после start = 1 (было 2+ в каскаде)
sing-box = запущен
/tmp shared = 27MB (было 519MB, освобождено ~490MB)
ОЗУ available = 1691MB (было 1195MB)
This commit is contained in:
@@ -332,7 +332,7 @@ function ensure_parent_dir(path) {
|
||||
}
|
||||
|
||||
function temp_path() {
|
||||
return trim(command_output_from_args([ "mktemp" ]));
|
||||
return trim(command_output_from_args([ "mktemp", "/tmp/tachyon-XXXXXX" ]));
|
||||
}
|
||||
|
||||
function remove_files(paths) {
|
||||
@@ -2309,7 +2309,7 @@ function import_builtin_subnets_from_rule(section, settings) {
|
||||
write_file(cached_file, join("\n", combined_lines) + "\n");
|
||||
ensure_dir("/etc/tachyon/rulesets");
|
||||
write_file(persistent_file, join("\n", combined_lines) + "\n");
|
||||
system("/etc/init.d/tachyon reload &");
|
||||
system("/etc/init.d/tachyon reload </dev/null >/dev/null 2>&1 &");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1574,7 +1574,7 @@ function signature_add_value(body, key, value) {
|
||||
}
|
||||
|
||||
function signature_hash(body) {
|
||||
let path = trim(command_output_from_args([ "mktemp" ]));
|
||||
let path = trim(command_output_from_args([ "mktemp", "/tmp/tachyon-XXXXXX" ]));
|
||||
if (path == "")
|
||||
return "";
|
||||
|
||||
@@ -1907,7 +1907,7 @@ function nft_populate_runtime_sets_from_uci(populate_enabled, deferred_section_n
|
||||
|
||||
if (!nft_table_present(table)) {
|
||||
log_warn("nft_populate_runtime_sets_from_uci: Table " + table + " does not exist. Rebuilding firewall rules dynamically.");
|
||||
system("/usr/bin/tachyon reload_firewall &");
|
||||
system("/usr/bin/tachyon reload_firewall </dev/null >/dev/null 2>&1 &");
|
||||
}
|
||||
|
||||
return nft_populate_runtime_sets_from_sections(uci_sections("section"), populate_enabled, deferred_section_names, table, common_set, port_set, ip_port_set, interface_set, localv4_set, mark, common6_set, ip_port6_set, localv6_set);
|
||||
|
||||
@@ -573,6 +573,10 @@ function stop_service(owner_pid) {
|
||||
}
|
||||
|
||||
let status = command_status_from_args([ BIN_PATH, "stop" ]);
|
||||
// Clean up orphaned logread -f processes left by watchdog instances.
|
||||
// These survive tachyon stop because background subprocesses (reload_firewall,
|
||||
// restart) inherit the logread pipe read-end, preventing SIGPIPE delivery.
|
||||
system("killall logread 2>/dev/null; true");
|
||||
return stop_finish(job_id, status);
|
||||
}
|
||||
|
||||
|
||||
@@ -794,6 +794,11 @@ function start_main() {
|
||||
}
|
||||
|
||||
function start_impl() {
|
||||
// Clean up orphaned /tmp/tachyon-* temp files from processes killed mid-operation
|
||||
// (e.g. config generator killed during restart). 30-min threshold is safe since
|
||||
// all tachyon temp files are created and deleted within seconds under normal operation.
|
||||
command_status("find /tmp -maxdepth 1 -name 'tachyon-*' -mmin +30 -delete 2>/dev/null; true");
|
||||
|
||||
// Start Watchdog & Telegram Bot early so they remain active even if proxy configuration fails
|
||||
let wd_status = module_status(WATCHDOG_UC, [ "start-runtime" ]);
|
||||
if (wd_status != 0)
|
||||
@@ -1180,7 +1185,7 @@ function reload(reason) {
|
||||
if (status != 0)
|
||||
return status;
|
||||
|
||||
let current_reload_state_file = trim(command_output_from_args([ "mktemp" ]));
|
||||
let current_reload_state_file = trim(command_output_from_args([ "mktemp", "/tmp/tachyon-XXXXXX" ]));
|
||||
if (current_reload_state_file == "")
|
||||
return abort_reload(1, false);
|
||||
|
||||
|
||||
@@ -1689,7 +1689,7 @@ function handle_qos_toggle(token, chat_id, msg_id) {
|
||||
let new_val = (cfg.qos_priority_engine == "0") ? "1" : "0";
|
||||
c.set(CONFIG_NAME, "settings", "qos_priority_engine", new_val);
|
||||
c.commit(CONFIG_NAME);
|
||||
system("/usr/bin/tachyon reload_firewall >/dev/null 2>&1 &");
|
||||
system("/usr/bin/tachyon reload_firewall </dev/null >/dev/null 2>&1 &");
|
||||
view_qos(token, chat_id, msg_id);
|
||||
}
|
||||
|
||||
|
||||
@@ -22,6 +22,8 @@ let proxy_restart_window_start = time();
|
||||
const PROXY_RESTART_LOCK = "/var/run/tachyon_proxy_restart.lock";
|
||||
let telegram_msg_count = 0;
|
||||
let telegram_msg_window = time();
|
||||
// FD-cascade prevention: track logread pipe FD to close it in background spawns
|
||||
let logread_pipe_fd = -1;
|
||||
|
||||
let command_from_args = common.command_from_args;
|
||||
let command_status = common.command_status;
|
||||
@@ -47,6 +49,17 @@ function command_output_from_args(args) {
|
||||
return command_output(command_from_args(args) + " 2>/dev/null");
|
||||
}
|
||||
|
||||
// Run a command in background, explicitly closing the logread pipe FD to
|
||||
// prevent FD-cascade: each restart/reload inherits read-end of logread pipe,
|
||||
// keeping orphaned logread -f processes alive across watchdog generations.
|
||||
function bg_system(cmd) {
|
||||
if (logread_pipe_fd >= 0) {
|
||||
system(sprintf("%d<&- ", logread_pipe_fd) + cmd);
|
||||
} else {
|
||||
system(cmd);
|
||||
}
|
||||
}
|
||||
|
||||
function settings() {
|
||||
return common.object_or_empty(uci_core.get_all(CONFIG_NAME, "settings"));
|
||||
}
|
||||
@@ -456,7 +469,7 @@ function safe_proxy_restart(reason) {
|
||||
}
|
||||
try { fs.writefile(PROXY_RESTART_LOCK, as_string(now)); } catch(e) {}
|
||||
proxy_restart_count++;
|
||||
system("/etc/init.d/tachyon restart >/dev/null 2>&1 &");
|
||||
bg_system("/etc/init.d/tachyon restart </dev/null >/dev/null 2>&1 &");
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -664,7 +677,7 @@ function ai_heal_community_subnet_sets() {
|
||||
"Восстановлены nftables sets из persistent кеша (/etc/tachyon/rulesets/)",
|
||||
"fixed"
|
||||
);
|
||||
system("/usr/bin/tachyon reload_firewall </dev/null >/dev/null 2>&1 &");
|
||||
bg_system("/usr/bin/tachyon reload_firewall </dev/null >/dev/null 2>&1 &");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
@@ -758,7 +771,7 @@ function ai_heal_wan_and_gateway() {
|
||||
if (tcfg.notify_crash != "0") {
|
||||
send_telegram_notification("⚠️ *Watchdog:* WAN/Gateway проблема. Перезапуск wan...");
|
||||
}
|
||||
system("/sbin/ifdown wan >/dev/null 2>&1 && /sbin/ifup wan >/dev/null 2>&1 &");
|
||||
bg_system("/sbin/ifdown wan >/dev/null 2>&1 && /sbin/ifup wan >/dev/null 2>&1 &");
|
||||
}
|
||||
|
||||
function ai_heal_subscriptions() {
|
||||
@@ -818,7 +831,7 @@ function safe_reload_firewall() {
|
||||
let min_interval = 120;
|
||||
if (ai_enabled("ai_reload_dedup_enabled", "1") && elapsed < min_interval) return;
|
||||
last_reload_time = now;
|
||||
system("/usr/bin/tachyon reload_firewall >/dev/null 2>&1 &");
|
||||
bg_system("/usr/bin/tachyon reload_firewall </dev/null >/dev/null 2>&1 &");
|
||||
}
|
||||
|
||||
// ─── Config validation: sing-box check ────────────────────────────────────────
|
||||
@@ -965,7 +978,7 @@ function ai_heal_empty_sections() {
|
||||
|
||||
// Section with subscriptions but 0 usable outbounds — trigger async reload
|
||||
log_message("Empty proxy section '" + name + "' — triggering subscription update", "warn");
|
||||
system("/usr/bin/tachyon subscription_update_async " + common.shell_quote(name) + " >/dev/null 2>&1 1000<&- &");
|
||||
bg_system("/usr/bin/tachyon subscription_update_async " + common.shell_quote(name) + " </dev/null >/dev/null 2>&1 1000<&- &");
|
||||
push(recovered, name);
|
||||
}
|
||||
|
||||
@@ -1046,7 +1059,7 @@ function ai_heal_dns_loop() {
|
||||
if (recovery.phase == "detour_disabled") {
|
||||
// Already in recovery — just wait and retry
|
||||
log_message("DNS loop recovery: DNS still dead, retrying subscription update", "info");
|
||||
system("/usr/bin/tachyon subscription_update_async " + common.shell_quote(detour_section) + " >/dev/null 2>&1 1000<&- &");
|
||||
bg_system("/usr/bin/tachyon subscription_update_async " + common.shell_quote(detour_section) + " </dev/null >/dev/null 2>&1 1000<&- &");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1070,7 +1083,7 @@ function ai_heal_dns_loop() {
|
||||
write_dns_recovery_state({ phase: "detour_disabled", section: detour_section, ts: now });
|
||||
|
||||
// Trigger subscription update for the empty section
|
||||
system("/usr/bin/tachyon subscription_update_async " + common.shell_quote(detour_section) + " >/dev/null 2>&1 1000<&- &");
|
||||
bg_system("/usr/bin/tachyon subscription_update_async " + common.shell_quote(detour_section) + " </dev/null >/dev/null 2>&1 1000<&- &");
|
||||
|
||||
ai_heal_report(
|
||||
"dns_loop_detected",
|
||||
@@ -1434,8 +1447,15 @@ function setup_honeypot_listener() {
|
||||
|
||||
function setup_syslog_listener() {
|
||||
if (!uloop) return null;
|
||||
// Kill orphaned logread -f processes from previous watchdog instances.
|
||||
// Without this, every restart cascades: new watchdog inherits old watchdog's
|
||||
// logread pipe read-end, keeping old logread alive. Over N restarts,
|
||||
// watchdog accumulates N inherited FDs → hits 1024 limit → config generator fails.
|
||||
system("killall logread 2>/dev/null; true");
|
||||
let log_pipe = fs.popen("logread -f 2>/dev/null", "r");
|
||||
if (!log_pipe) return null;
|
||||
// Track the FD so bg_system() can close it before spawning background processes
|
||||
logread_pipe_fd = log_pipe.fileno();
|
||||
|
||||
try {
|
||||
uloop.handle(log_pipe.fileno(), function(events) {
|
||||
|
||||
@@ -183,7 +183,7 @@ function ensure_parent_dir(path) {
|
||||
}
|
||||
|
||||
function temp_path() {
|
||||
return trim(command_output_from_args([ "mktemp" ]));
|
||||
return trim(command_output_from_args([ "mktemp", "/tmp/tachyon-XXXXXX" ]));
|
||||
}
|
||||
|
||||
function md5_file(path) {
|
||||
|
||||
Reference in New Issue
Block a user