feat: custom hosts lists — remote URL download, dnsmasq addn-hosts, sing-box DNS rules, LuCI UI

luci-app-tachyon/htdocs/luci-static/resources/view/tachyon/settings.js

@@ -1222,6 +1222,44 @@ function createSettingsContent(section, capabilities) {
     ]);
   };
 
+  // ─── Remote Hosts Lists ──────────────────────────────────────────────────
+
+  const hostsListUrlsOpt = section.option(
+    form.DynamicList,
+    "hosts_list_urls",
+    _("Remote Hosts Lists"),
+    _("URLs of remote hosts files to download and apply. Supports standard hosts format: <code>IP domain</code>. Entries are resolved via both dnsmasq and sing-box DNS."),
+  );
+  hostsListUrlsOpt.placeholder = "https://raw.githubusercontent.com/.../hosts";
+
+  o = section.option(
+    form.Flag,
+    "hosts_list_auto_update",
+    _("Auto-update Hosts Lists"),
+    _("Automatically update remote hosts lists on schedule"),
+  );
+  o.default = "0";
+  o.rmempty = false;
+
+  o = section.option(
+    form.Value,
+    "hosts_list_update_interval",
+    _("Hosts Lists Update Interval"),
+    _("Use sing-box duration format like 1d, 12h or 6h"),
+  );
+  o.depends("hosts_list_auto_update", "1");
+  o.placeholder = "24h";
+  o.default = "24h";
+  o.rmempty = false;
+  o.validate = function (_section_id, value) {
+    if (!value) return true;
+    return /^(?=.*[1-9])([0-9]+(?:\.[0-9]+)?(?:ns|us|ms|s|m|h|d))+$/.test(value)
+      ? true
+      : _("Use sing-box duration format like 1d, 12h or 6h");
+  };
+
+  // ─── DNS Strategy ────────────────────────────────────────────────────────
+
   o = section.option(form.ListValue, "dns_strategy", _("DNS Strategy"));
   o.value("prefer_ipv4", _("Prefer IPv4"));
   o.value("ipv4_only", _("IPv4 only"));

tachyon/files/etc/config/tachyon

@@ -67,6 +67,11 @@ config settings 'settings'
         option ai_anomaly_detection_enabled '1'
         option ai_anomaly_reconnect_threshold '10'
 
+        # ─── Hosts Lists ────────────────────────────────────────────────────────
+        # list hosts_list_urls 'https://raw.githubusercontent.com/.../hosts'
+        option hosts_list_auto_update '0'
+        option hosts_list_update_interval '24h'
+
 
 #config section 'subscription_example'
 #        option label 'Subscription'

tachyon/files/usr/bin/tachyon

@@ -119,6 +119,8 @@ function show_help() {
     print("    component_action_status Get background component action status\n");
     print("    component_updates_if_due Check installed components for updates when due\n");
     print("    component_update_check_cache Get cached component update check results\n");
+    print("    hosts_list_update [url] Update custom hosts lists (all or specific URL)\n");
+    print("    hosts_list_status       Show hosts lists status\n");
     print("    check_dns_available     Check DNS server availability\n");
     print("    global_check            Run global system check\n");
     print("    doctor                  Run doctor diagnostics checks\n");
@@ -201,6 +203,8 @@ function command_spec(command) {
         component_action_status: [ "components/updates.uc", "component-action-status", 1 ],
         component_updates_if_due: [ "components/updates.uc", "component-updates-if-due", 0 ],
         component_update_check_cache: [ "components/updates.uc", "component-update-check-cache", 0 ],
+        hosts_list_update: [ "components/hosts.uc", "list-update", 1 ],
+        hosts_list_status: [ "components/hosts.uc", "list-status", 0 ],
         extract_ruleset: [ "diagnostics/runtime.uc", "extract-ruleset", 1 ],
         check_dns_available: [ "diagnostics/runtime.uc", "check-dns-available", 0 ],
         global_check: [ "diagnostics/runtime.uc", "global-check", 2 ],

tachyon/files/usr/lib/components/hosts.uc

@@ -0,0 +1,223 @@
+#!/usr/bin/env ucode
+
+let fs = require("fs");
+let uci = require("core.uci");
+
+const CONFIG_NAME = getenv("TACHYON_CONFIG_NAME") || "tachyon";
+const HOSTS_CACHE_DIR = getenv("TACHYON_HOSTS_CACHE_DIR") || "/etc/tachyon/hosts-lists";
+const HOSTS_CACHE_FILE = HOSTS_CACHE_DIR + "/combined.txt";
+const HOSTS_TMP_DIR = getenv("TACHYON_HOSTS_TMP_DIR") || "/tmp/tachyon/hosts-lists";
+const CONNECT_TIMEOUT = "30";
+
+function as_string(value) {
+    return value == null ? "" : "" + value;
+}
+
+function shell_quote(value) {
+    return "'" + replace(as_string(value), /'/g, "'\\''") + "'";
+}
+
+function run(command) {
+    return system(command) == 0;
+}
+
+function log(message, level) {
+    level = as_string(level || "info");
+    run("logger -t " + shell_quote("tachyon-hosts") + " " + shell_quote("[" + level + "] " + as_string(message)));
+}
+
+function mkdir_p(path) {
+    run("mkdir -p " + shell_quote(path));
+}
+
+function file_nonempty(path) {
+    let st = fs.stat(path);
+    return st != null && int(st.size) > 0;
+}
+
+function remove_file(path) {
+    fs.remove(path);
+}
+
+function http_get_to_file(url, output_path) {
+    mkdir_p(dirname(output_path));
+    let cmd = "wget -q -O " + shell_quote(output_path) + " --timeout=" + CONNECT_TIMEOUT + " " + shell_quote(url);
+    return run(cmd);
+}
+
+function download_with_retry(url, output_path, label) {
+    for (let attempt = 1; attempt <= 3; attempt++) {
+        let current_url = url;
+        if (attempt == 2 && index(current_url, "https://github.com/") == 0) {
+            log("Retrying " + as_string(label) + " via gh-proxy.com mirror", "warn");
+            current_url = "https://gh-proxy.com/" + url;
+        } else if (attempt == 3 && index(current_url, "https://github.com/") == 0) {
+            log("Retrying " + as_string(label) + " via ghproxy.net mirror", "warn");
+            current_url = "https://ghproxy.net/" + url;
+        } else if (attempt > 1) {
+            log("Retrying " + as_string(label), "warn");
+        }
+
+        log("Downloading " + as_string(label) + " (attempt " + attempt + "/3)");
+        if (http_get_to_file(current_url, output_path) && file_nonempty(output_path))
+            return output_path;
+        remove_file(output_path);
+    }
+    return null;
+}
+
+function parse_hosts_file(path) {
+    let entries = [];
+    let fh = fs.open(path, "r");
+    if (fh == null)
+        return entries;
+
+    let line;
+    while ((line = fh.read("line")) != null) {
+        line = trim(line);
+        if (line == "" || substr(line, 0, 1) == "#")
+            continue;
+
+        let parts = split(line, /[ \t]+/);
+        if (length(parts) < 2)
+            continue;
+
+        let p1 = parts[0];
+        let p2 = parts[1];
+
+        if (core_ip.valid_ip(p1)) {
+            let ip = p1;
+            for (let i = 1; i < length(parts); i++) {
+                let domain = parts[i];
+                if (domain != "" && substr(domain, 0, 1) != "#" && index(domain, ":") == -1)
+                    push(entries, { ip: ip, domain: domain });
+            }
+        } else if (core_ip.valid_ip(p2)) {
+            let domain = p1;
+            let ip = p2;
+            if (index(domain, ":") == -1)
+                push(entries, { ip: ip, domain: domain });
+        }
+    }
+    fh.close();
+    return entries;
+}
+
+function write_hosts_cache(entries, source_urls) {
+    mkdir_p(HOSTS_CACHE_DIR);
+
+    let fh = fs.open(HOSTS_CACHE_FILE, "w");
+    if (fh == null) {
+        log("Failed to open " + HOSTS_CACHE_FILE + " for writing", "error");
+        return false;
+    }
+
+    fh.write("# Tachyon hosts lists — combined cache\n");
+    fh.write("# Generated: " + strftime("%Y-%m-%d %H:%M:%S", localtime()) + "\n");
+    if (source_urls != null) {
+        for (let url in source_urls)
+            fh.write("# Source: " + url + "\n");
+    }
+    fh.write("# Entries: " + length(entries) + "\n");
+    fh.write("#\n");
+
+    for (let entry in entries)
+        fh.write(entry.ip + " " + entry.domain + "\n");
+
+    fh.close();
+    log("Wrote " + length(entries) + " hosts entries to " + HOSTS_CACHE_FILE);
+    return true;
+}
+
+function get_hosts_urls() {
+    let urls = [];
+    let uci_urls = uci.get(CONFIG_NAME + ".settings.hosts_list_urls");
+    if (uci_urls == "")
+        return urls;
+
+    if (type(uci_urls) == "array") {
+        for (let url in uci_urls) {
+            let s = trim(url);
+            if (s != "")
+                push(urls, s);
+        }
+    } else {
+        let s = trim(uci_urls);
+        if (s != "")
+            push(urls, s);
+    }
+    return urls;
+}
+
+function hosts_list_update(target_url) {
+    let urls = target_url != null && target_url != "" ? [target_url] : get_hosts_urls();
+    if (length(urls) == 0) {
+        log("No hosts list URLs configured", "warn");
+        print('{"success":false,"error":"no_urls","entries":0}');
+        return false;
+    }
+
+    mkdir_p(HOSTS_TMP_DIR);
+
+    let all_entries = [];
+    let updated_urls = [];
+
+    for (let url in urls) {
+        let label = "hosts list from " + url;
+        let safe_name = replace(url, /[^a-zA-Z0-9]/, "_");
+        let tmp_file = HOSTS_TMP_DIR + "/list-" + safe_name + ".txt";
+
+        if (download_with_retry(url, tmp_file, label) != null) {
+            let entries = parse_hosts_file(tmp_file);
+            log("Parsed " + length(entries) + " entries from " + url);
+            for (let entry in entries)
+                push(all_entries, entry);
+            push(updated_urls, url);
+        } else {
+            log("Failed to download hosts list from " + url, "error");
+        }
+        remove_file(tmp_file);
+    }
+
+    if (length(all_entries) == 0) {
+        log("No valid hosts entries found in any list", "error");
+        print('{"success":false,"error":"empty","entries":0}');
+        return false;
+    }
+
+    if (!write_hosts_cache(all_entries, updated_urls)) {
+        print('{"success":false,"error":"write_failed","entries":0}');
+        return false;
+    }
+
+    log("Hosts lists updated: " + length(all_entries) + " entries from " + length(updated_urls) + " sources");
+
+    print('{"success":true,"entries":' + length(all_entries) + ',"sources":' + length(updated_urls) + '}');
+    return true;
+}
+
+function hosts_list_status() {
+    let urls = get_hosts_urls();
+    let cache_exists = fs.stat(HOSTS_CACHE_FILE) != null;
+    let entry_count = 0;
+
+    if (cache_exists) {
+        let st = fs.stat(HOSTS_CACHE_FILE);
+        entry_count = int(st.size);
+    }
+
+    print('{"urls":' + length(urls) + ',"cache_exists":' + (cache_exists ? 'true' : 'false') + ',"cache_size":' + entry_count + '}');
+}
+
+let command = ARGV[0] || "";
+
+if (command == "list-update") {
+    let target_url = ARGV[1] || "";
+    exit(hosts_list_update(target_url) ? 0 : 1);
+} else if (command == "list-status") {
+    hosts_list_status();
+    exit(0);
+}
+
+warn("Usage: components/hosts.uc <list-update [url]|list-status>\n");
+exit(1);

tachyon/files/usr/lib/dns/apply.uc

@@ -6,6 +6,7 @@ let uci = require("core.uci");
 const CONFIG_NAME = getenv("TACHYON_CONFIG_NAME") || "tachyon";
 const SB_DNS_INBOUND_ADDRESS = getenv("SB_DNS_INBOUND_ADDRESS") || "127.0.0.42";
 const DNSMASQ_INIT = getenv("DNSMASQ_INIT") || "/etc/init.d/dnsmasq";
+const HOSTS_CACHE_FILE = getenv("TACHYON_HOSTS_CACHE_FILE") || "/etc/tachyon/hosts-lists/combined.txt";
 
 function as_string(value) {
     return value == null ? "" : "" + value;
@@ -188,10 +189,16 @@ function dnsmasq_configure_default_instance() {
         backup_dnsmasq_config_option("cachesize", "tachyon_cachesize");
     }
 
+    backup_dnsmasq_config_option("addn_hosts", "tachyon_addn_hosts");
+
     uci_delete("dhcp.@dnsmasq[0].server");
     uci_add_list("dhcp.@dnsmasq[0].server", SB_DNS_INBOUND_ADDRESS);
     uci_set("dhcp.@dnsmasq[0].noresolv", "1");
     uci_set("dhcp.@dnsmasq[0].cachesize", "0");
+
+    if (fs.stat(HOSTS_CACHE_FILE) != null) {
+        uci_set("dhcp.@dnsmasq[0].addn_hosts", HOSTS_CACHE_FILE);
+    }
 }
 
 function dnsmasq_restore_default_instance() {
@@ -224,6 +231,8 @@ function dnsmasq_restore_default_instance() {
         restore_dnsmasq_config_option("cachesize", "tachyon_cachesize", "");
     else if (managed_global_dns)
         uci_set("dhcp.@dnsmasq[0].cachesize", "150");
+
+    restore_dnsmasq_config_option("addn_hosts", "tachyon_addn_hosts", "");
 }
 
 function dnsmasq_configure(force) {

tachyon/files/usr/lib/singbox/generator.uc

@@ -22,6 +22,7 @@ let fixture_uci_data = null;
 let runtime_settings_cache = null;
 let runtime_ruleset_folder = runtime_constants.TMP_RULESET_FOLDER;
 let runtime_supports_xhttp = true;
+let HOSTS_CACHE_FILE = "/etc/tachyon/hosts-lists/combined.txt";
 
 let as_string = common.as_string;
 let read_json_file = common.read_json_file;
@@ -448,6 +449,51 @@ function base_config(settings, service_address, runtime_context) {
         }
     }
 
+    // Read remote hosts lists cache file
+    let hosts_cache_st = fs.stat(HOSTS_CACHE_FILE);
+    if (hosts_cache_st != null && int(hosts_cache_st.size) > 0) {
+        let fh = fs.open(HOSTS_CACHE_FILE, "r");
+        if (fh != null) {
+            let line;
+            while ((line = fh.read("line")) != null) {
+                line = trim(line);
+                if (line == "" || substr(line, 0, 1) == "#")
+                    continue;
+                let parts = split(line, /[ \t]+/);
+                if (length(parts) >= 2) {
+                    let p1 = parts[0];
+                    let p2 = parts[1];
+                    if (core_ip.valid_ip(p1)) {
+                        let ip = p1;
+                        let rr_type = index(ip, ":") != -1 ? "AAAA" : "A";
+                        for (let i = 1; i < length(parts); i++) {
+                            let d = parts[i];
+                            if (d != "" && substr(d, 0, 1) != "#" && index(d, ":") == -1) {
+                                push(dns_rules, {
+                                    action: "predefined",
+                                    domain: [d],
+                                    answer: [d + ". 60 IN " + rr_type + " " + ip]
+                                });
+                            }
+                        }
+                    } else if (core_ip.valid_ip(p2)) {
+                        let domain = p1;
+                        let ip = p2;
+                        if (index(domain, ":") == -1) {
+                            let rr_type = index(ip, ":") != -1 ? "AAAA" : "A";
+                            push(dns_rules, {
+                                action: "predefined",
+                                domain: [domain],
+                                answer: [domain + ". 60 IN " + rr_type + " " + ip]
+                            });
+                        }
+                    }
+                }
+            }
+            fh.close();
+        }
+    }
+    
     for (let rule in dns_config.rules)
         push(dns_rules, rule);
     for (let rule in [

tests/hosts_list.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+HOSTS_UC="$ROOT_DIR/tachyon/files/usr/lib/components/hosts.uc"
+TACHYON_LIB="$ROOT_DIR/tachyon/files/usr/lib"
+TMP_DIR="$ROOT_DIR/tmp_hosts_list_test"
+PASS=0
+FAIL=0
+
+rm -rf "$TMP_DIR"
+mkdir -p "$TMP_DIR"
+
+ucode() {
+  command ucode -L "$TACHYON_LIB" "$@"
+}
+
+fail() {
+  printf 'FAIL: %s\n' "$1" >&2
+  FAIL=$((FAIL + 1))
+}
+
+pass() {
+  PASS=$((PASS + 1))
+}
+
+assert_eq() {
+  local expected="$1"
+  local actual="$2"
+  local label="$3"
+  if [ "$actual" = "$expected" ]; then
+    pass
+  else
+    fail "$label: expected '$expected', got '$actual'"
+  fi
+}
+
+# --- Test 1: Parse hosts file with mixed formats ---
+cat > "$TMP_DIR/test_hosts.txt" <<'EOF'
+# This is a comment
+127.0.0.1 localhost
+localhost 127.0.0.1
+45.155.204.190 chatgpt.com
+45.155.204.190 ab.chatgpt.com
+0.0.0.0 ad.doubleclick.net
+0.0.0.0 tracker.example.com
+
+# IPv6
+::1 localhost
+fe80::1 host.local
+EOF
+
+# --- Test 2: Parse file with only comments ---
+cat > "$TMP_DIR/comments_only.txt" <<'EOF'
+# Just comments
+# Nothing else
+EOF
+
+# --- Test 3: Parse empty file ---
+: > "$TMP_DIR/empty.txt"
+
+# --- Test 4: Parse file with invalid lines ---
+cat > "$TMP_DIR/invalid.txt" <<'EOF'
+not a valid line
+also not valid
+45.155.204.190 chatgpt.com
+valid.com 1.2.3.4
+EOF
+
+# Run the hosts list status command (should not fail with no URLs)
+echo "=== Test: hosts_list_status with no URLs ==="
+output=$(ucode "$HOSTS_UC" list-status 2>/dev/null || true)
+assert_eq "false" "$(echo "$output" | grep -o '"cache_exists":[a-z]*' | cut -d: -f2)" "cache_exists should be false initially"
+
+echo ""
+echo "=== Results: $PASS passed, $FAIL failed ==="
+[ "$FAIL" -eq 0 ] || exit 1