feat(rulesets): add Twitch preset and resilient CDN fallback

fe-app-tachyon/src/constants.ts

@@ -36,6 +36,7 @@ export const DOMAIN_LIST_OPTIONS = {
   ads_hagezi_pro: 'Ads (Hagezi Pro)',
   supercell: 'Supercell',
   github: 'GitHub',
+  twitch: 'Twitch',
   hetzner: 'Hetzner ASN',
   ovh: 'OVH ASN',
   digitalocean: 'Digital Ocean ASN',

luci-app-tachyon/htdocs/luci-static/resources/view/tachyon/main.js

@@ -1255,6 +1255,7 @@ var DOMAIN_LIST_OPTIONS = {
   ads_hagezi_pro: "Ads (Hagezi Pro)",
   supercell: "Supercell",
   github: "GitHub",
+  twitch: "Twitch",
   hetzner: "Hetzner ASN",
   ovh: "OVH ASN",
   digitalocean: "Digital Ocean ASN",

luci-app-tachyon/htdocs/luci-static/resources/view/tachyon/section.js

@@ -7509,6 +7509,7 @@ var DOMAIN_LIST_HINTS = {
   ads_hagezi_pro: "Блокировка рекламы (Hagezi Pro список)",
   supercell: "supercell.com, clashofclans.com, ...",
   github: "github.com, raw.githubusercontent.com, ...",
+  twitch: "twitch.tv, ttvnw.net, jtvnw.net, ...",
   hetzner: "IP-адреса Hetzner (ASN)",
   ovh: "IP-адреса OVH (ASN)",
   digitalocean: "IP-адреса DigitalOcean (ASN)",

tachyon/files/usr/lib/components/updates.uc

@@ -2469,17 +2469,35 @@ function import_community_srs_file(service, settings) {
     if (tmpfile == "")
         return false;
 
-    let ok = true;
+    let core_url_mod = core_url_module_or_null();
-    if (download_to_file(url, tmpfile, service_proxy_address(settings, "lists")) && file_nonempty(tmpfile) && helpers.file_is_usable(tmpfile, 100)) {
+    let candidates = core_url_mod && type(core_url_mod.download_candidates) == "function" ?
-        ensure_dir(TMP_RULESET_FOLDER);
+        core_url_mod.download_candidates(url) : [ url ];
-        copy_file(tmpfile, cached_file);
+    let downloaded = false;
-        ensure_dir("/etc/tachyon/rulesets");
+
-        copy_file(tmpfile, persistent_file);
+    for (let candidate in candidates) {
-        log_message("Successfully cached preset ruleset " + service, "info");
+        if (download_to_file(candidate, tmpfile, service_proxy_address(settings, "lists")) &&
+            file_nonempty(tmpfile) &&
+            singbox_rulesets_module().is_valid_srs_file(tmpfile) &&
+            helpers.file_is_usable(tmpfile, 100)) {
+            ensure_dir(TMP_RULESET_FOLDER);
+            copy_file(tmpfile, cached_file);
+            ensure_dir("/etc/tachyon/rulesets");
+            copy_file(tmpfile, persistent_file);
+            log_message("Successfully cached preset ruleset " + service + (candidate != url ? " via mirror " + candidate : ""), "info");
+            downloaded = true;
+            break;
+        }
+        remove_file(tmpfile);
     }
-    else if (helpers.file_is_usable(persistent_file, 100)) {
+
+    let ok = true;
+    if (downloaded) {
+        // Cached successfully
+    }
+    else if (singbox_rulesets_module().is_valid_srs_file(persistent_file) && helpers.file_is_usable(persistent_file, 100)) {
         ensure_dir(TMP_RULESET_FOLDER);
         copy_file(persistent_file, cached_file);
+        log_message("Using persistent cached ruleset for " + service + " after download failure", "warn");
     }
     else {
         remove_file(persistent_file);

tachyon/files/usr/lib/core/constants.uc

@@ -151,6 +151,8 @@ function constants_map() {
     c.SRS_MAIN_URL = env("SRS_MAIN_URL", "https://github.com/itdoginfo/allow-domains/releases/latest/download");
     c.SRS_ADS_HAGEZI_PRO_URL = env("SRS_ADS_HAGEZI_PRO_URL", "https://github.com/zxc-rv/ad-filter/releases/latest/download/adlist.srs");
     c.SRS_SUPERCELL_URL = env("SRS_SUPERCELL_URL", "https://raw.githubusercontent.com/ushan0v/sing-box-supercell-ruleset/main/supercell.srs");
+    c.SRS_GITHUB_URL = env("SRS_GITHUB_URL", "https://raw.githubusercontent.com/MetaCubeX/meta-rules-dat/sing/geo/geosite/github.srs");
+    c.SRS_TWITCH_URL = env("SRS_TWITCH_URL", "https://raw.githubusercontent.com/MetaCubeX/meta-rules-dat/sing/geo/geosite/twitch.srs");
     c.SUBNETS_TWITTER = env("SUBNETS_TWITTER", c.GITHUB_RAW_URL + "/Subnets/IPv4/twitter.lst");
     c.SUBNETS_META = env("SUBNETS_META", c.GITHUB_RAW_URL + "/Subnets/IPv4/meta.lst");
     c.SUBNETS_DISCORD = env("SUBNETS_DISCORD", c.GITHUB_RAW_URL + "/Subnets/IPv4/discord.lst");
@@ -161,7 +163,7 @@ function constants_map() {
     c.SUBNETS_OVH = env("SUBNETS_OVH", c.GITHUB_RAW_URL + "/Subnets/IPv4/ovh.lst");
     c.SUBNETS_DIGITALOCEAN = env("SUBNETS_DIGITALOCEAN", c.GITHUB_RAW_URL + "/Subnets/IPv4/digitalocean.lst");
     c.SUBNETS_CLOUDFRONT = env("SUBNETS_CLOUDFRONT", c.GITHUB_RAW_URL + "/Subnets/IPv4/cloudfront.lst");
-    c.COMMUNITY_SERVICES = env("COMMUNITY_SERVICES", "russia_inside russia_outside ukraine_inside geoblock block porn news anime youtube hdrezka tiktok google_ai google_play hodca discord meta twitter cloudflare cloudfront digitalocean hetzner ovh telegram roblox ads_hagezi_pro supercell github");
+    c.COMMUNITY_SERVICES = env("COMMUNITY_SERVICES", "russia_inside russia_outside ukraine_inside geoblock block porn news anime youtube hdrezka tiktok google_ai google_play hodca discord meta twitter cloudflare cloudfront digitalocean hetzner ovh telegram roblox ads_hagezi_pro supercell github twitch");
 
     c.ZAPRET_PROVIDER_BASE_DIR = env("ZAPRET_PROVIDER_BASE_DIR", "/opt/zapret");
     let default_nfqws_bin = c.ZAPRET_PROVIDER_BASE_DIR + "/nfq/nfqws";

tachyon/files/usr/lib/singbox/rulesets.uc

@@ -10,6 +10,7 @@ const SRS_MAIN_URL = "https://github.com/itdoginfo/allow-domains/releases/latest
 const SRS_ADS_HAGEZI_PRO_URL = "https://github.com/zxc-rv/ad-filter/releases/latest/download/adlist.srs";
 const SRS_SUPERCELL_URL = "https://raw.githubusercontent.com/ushan0v/sing-box-supercell-ruleset/main/supercell.srs";
 const SRS_GITHUB_URL = "https://raw.githubusercontent.com/MetaCubeX/meta-rules-dat/sing/geo/geosite/github.srs";
+const SRS_TWITCH_URL = "https://raw.githubusercontent.com/MetaCubeX/meta-rules-dat/sing/geo/geosite/twitch.srs";
 const SRS_GEOIP_RU_URL = "https://raw.githubusercontent.com/MetaCubeX/meta-rules-dat/sing/geo/geoip/ru.srs";
 const SRS_GEOSITE_RU_URL = "https://raw.githubusercontent.com/MetaCubeX/meta-rules-dat/sing/geo/geosite/category-ru.srs";
 const SRS_GEOIP_US_URL = "https://raw.githubusercontent.com/MetaCubeX/meta-rules-dat/sing/geo/geoip/us.srs";
@@ -43,6 +44,7 @@ const COMMUNITY_SERVICES = {
     ads_hagezi_pro: true,
     supercell: true,
     github: true,
+    twitch: true,
     geoip_ru: true,
     geosite_ru: true,
     geoip_us: true,
@@ -68,6 +70,8 @@ function community_url(name) {
         return SRS_SUPERCELL_URL;
     if (name == "github")
         return SRS_GITHUB_URL;
+    if (name == "twitch")
+        return SRS_TWITCH_URL;
     if (name == "geosite_ru")
         return SRS_GEOSITE_RU_URL;
 

tests/core_url.sh

@@ -46,6 +46,13 @@ assert(url.github_to_jsdelivr("https://raw.githubusercontent.com/owner/repo/main
 let candidates = url.download_candidates("https://github.com/owner/repo/blob/main/hosts");
 assert(candidates[0] == "https://raw.githubusercontent.com/owner/repo/main/hosts", "download_candidates first entry normalized");
 
+let twitch_candidates = url.download_candidates("https://raw.githubusercontent.com/MetaCubeX/meta-rules-dat/sing/geo/geosite/twitch.srs");
+assert(twitch_candidates[0] == "https://raw.githubusercontent.com/MetaCubeX/meta-rules-dat/sing/geo/geosite/twitch.srs", "twitch candidate 0 is raw github");
+assert(twitch_candidates[1] == "https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@sing/geo/geosite/twitch.srs", "twitch candidate 1 is jsdelivr cdn");
+for (let c in twitch_candidates) {
+    assert(index(c, "forkop") < 0, "no forkop mirrors allowed");
+}
+
 // authority, host, port, userinfo with @ in query parameters
 let complex = "vless://user-uuid@example.com:8443?authority=name@channel&mode=gun#my-fragment";
 assert(url.authority(complex) == "example.com:8443", "authority strips userinfo and query with @");

tests/singbox_rulesets.sh

@@ -47,6 +47,10 @@ assert_eq binary \
 
 ucode -L "$TACHYON_LIB" -e 'let rulesets = require("singbox.rulesets"); if (rulesets.kind_from_reference_hint("geoip") != "subnets") exit(1);'
 ucode -L "$TACHYON_LIB" -e 'let rulesets = require("singbox.rulesets"); if (type(rulesets.COMMUNITY_SERVICES) != "object" || !rulesets.COMMUNITY_SERVICES.discord) exit(1);'
+ucode -L "$TACHYON_LIB" -e 'let rulesets = require("singbox.rulesets"); if (!rulesets.COMMUNITY_SERVICES.twitch) exit(1);'
+assert_eq "https://raw.githubusercontent.com/MetaCubeX/meta-rules-dat/sing/geo/geosite/twitch.srs" \
+  "$(ucode -L "$TACHYON_LIB" -e 'let rulesets = require("singbox.rulesets"); print(rulesets.community_url("twitch"));')" \
+  "twitch community ruleset url"
 
 printf 'singbox rulesets checks passed\n'