diff --git a/luci-app-tachyon/po/ru/tachyon.ru.lmo b/luci-app-tachyon/po/ru/tachyon.ru.lmo index a414d35b..ac3d4fdb 100644 Binary files a/luci-app-tachyon/po/ru/tachyon.ru.lmo and b/luci-app-tachyon/po/ru/tachyon.ru.lmo differ diff --git a/tools/debug_lmo.js b/tools/debug_lmo.js new file mode 100644 index 00000000..095b5c23 --- /dev/null +++ b/tools/debug_lmo.js @@ -0,0 +1,93 @@ +const fs = require('fs'); + +// LuCI uses this specific SuperFastHash from OpenWrt +// Let me try to match it against the reference lmo +// adblock.bg.lmo entry 1: keyId=0x1545b51f +// We need to know what string produces this hash + +// Try the OpenWrt luci sfh_hash +function luci_sfh_hash(data) { + let hash = 0; + const len = data.length; + let i; + for (i = 0; i < len; i++) { + hash += data.charCodeAt(i); + hash = (hash << 10) + hash; + hash ^= hash >>> 6; + } + hash += hash << 3; + hash ^= hash >>> 11; + hash += hash << 15; + return hash >>> 0; +} + +console.log('luci_sfh empty:', luci_sfh_hash('').toString(16)); +console.log('luci_sfh Plural-Forms:', luci_sfh_hash('Plural-Forms').toString(16)); + +// Now check my po2lmo sfh_hash +function po2lmo_sfh(data) { + let hash = 0; + let i = data.length; + let k; + while (i >= 4) { + k = (data.charCodeAt(i - 4) & 0xff) | + ((data.charCodeAt(i - 3) & 0xff) << 8) | + ((data.charCodeAt(i - 2) & 0xff) << 16) | + ((data.charCodeAt(i - 1) & 0xff) << 24); + k = Math.imul(k, 0x5bd1e995); + k = k >>> 0; + k ^= k >>> 24; + k = Math.imul(k, 0x5bd1e995); + k = k >>> 0; + hash = Math.imul(hash, 0x5bd1e995) + k; + hash = hash >>> 0; + i -= 4; + } + switch (i) { + case 3: hash ^= (data.charCodeAt(0) & 0xff) << 16; + case 2: hash ^= (data.charCodeAt(i - 2) & 0xff) << 8; + case 1: hash ^= (data.charCodeAt(i - 1) & 0xff); + hash = Math.imul(hash, 0x5bd1e995); + hash = hash >>> 0; + } + hash ^= hash >>> 13; + hash = Math.imul(hash, 0x5bd1e995); + hash = hash >>> 0; + hash ^= hash >>> 15; + return hash >>> 0; +} + +console.log('po2lmo_sfh empty:', po2lmo_sfh('').toString(16)); +console.log('po2lmo_sfh Plural-Forms:', po2lmo_sfh('Plural-Forms').toString(16)); + +// Read the reference LMO +const buf = fs.readFileSync('tools/test_ref.lmo'); +const indexOffset = buf.readUInt32BE(buf.length - 4); +const entryCount = (buf.length - 4 - indexOffset) / 16; +console.log('\nReference LMO entries:'); +for (let i = 0; i < Math.min(10, entryCount); i++) { + const off = indexOffset + i * 16; + const keyId = buf.readUInt32BE(off); + console.log(' keyId:', keyId.toString(16)); +} + +// Read payload to get actual strings +for (let i = 0; i < Math.min(10, entryCount); i++) { + const off = indexOffset + i * 16; + const keyId = buf.readUInt32BE(off); + const valId = buf.readUInt32BE(off + 4); + const offset = buf.readUInt32BE(off + 8); + const length = buf.readUInt32BE(off + 12); + + // Read the payload at offset + const valBuf = buf.slice(offset, offset + length); + const valStr = valBuf.toString('utf8').replace(/\0/g, ''); + + console.log(` [${i}] keyId:${keyId.toString(16).padStart(8,'0')} off:${offset} len:${length} val:"${valStr}"`); + + // If this is entry 1+, try to compute hash of the key + if (i > 0 && valStr.length > 0) { + console.log(` luci_sfh("${valStr}") = ${luci_sfh_hash(valStr).toString(16)}`); + console.log(` po2lmo_sfh("${valStr}") = ${po2lmo_sfh(valStr).toString(16)}`); + } +} diff --git a/tools/hash_test.js b/tools/hash_test.js new file mode 100644 index 00000000..993df92d --- /dev/null +++ b/tools/hash_test.js @@ -0,0 +1,26 @@ +// OpenWrt luci-core uses this hash (from lmo.c): +function lmo_hash(s) { + let h = 0; + for (let i = 0; i < s.length; i++) { + h += s.charCodeAt(i); + h = (h + (h << 10)) >>> 0; + h ^= h >>> 6; + } + h = (h + (h << 3)) >>> 0; + h ^= h >>> 11; + h = (h + (h << 15)) >>> 0; + return h >>> 0; +} +// Reference: entry 1 keyId=0x1545b51f +console.log('lmo_hash(Client):', lmo_hash('Client').toString(16), '(target: 1545b51f)'); +console.log('lmo_hash(Download):', lmo_hash('Download').toString(16)); +console.log('lmo_hash(Save):', lmo_hash('Save').toString(16)); +console.log('lmo_hash(Action):', lmo_hash('Action').toString(16)); +console.log('lmo_hash(Enabled):', lmo_hash('Enabled').toString(16)); +console.log('lmo_hash(General):', lmo_hash('General').toString(16)); +console.log('lmo_hash(Email):', lmo_hash('Email').toString(16)); +console.log('lmo_hash(Subject):', lmo_hash('Subject').toString(16)); +console.log('lmo_hash(Enable detailed logs):', lmo_hash('Enable detailed logs').toString(16)); +console.log('lmo_hash(Enable):', lmo_hash('Enable').toString(16)); +console.log('lmo_hash(Close):', lmo_hash('Close').toString(16)); +console.log('lmo_hash(Save & Apply):', lmo_hash('Save & Apply').toString(16)); diff --git a/tools/po2lmo.js b/tools/po2lmo.js index 4e702422..1bae795c 100644 --- a/tools/po2lmo.js +++ b/tools/po2lmo.js @@ -3,31 +3,58 @@ const fs = require('fs'); -// Paul Hsieh's SuperFastHash -function sfh_hash(data) { - let hash = 0; - let i = data.length; - let k; - while (i >= 4) { - k = (data.charCodeAt(i - 4) & 0xff) | - ((data.charCodeAt(i - 3) & 0xff) << 8) | - ((data.charCodeAt(i - 2) & 0xff) << 16) | - ((data.charCodeAt(i - 1) & 0xff) << 24); - k = (k * 0x5bd1e995) >>> 0; - k ^= k >>> 24; - k = (k * 0x5bd1e995) >>> 0; - hash = (hash * 0x5bd1e995 + k) >>> 0; - i -= 4; +// OpenWrt LuCI SuperFastHash (from LMO.md / template_lmo.c) +// sfh_get16 reads uint16 in little-endian +function sfh_get16(d, off) { + return (d[off] & 0xff) + ((d[off + 1] & 0xff) << 8); +} + +function sfh_hash(str) { + const data = Buffer.from(str, 'utf8'); + const len = data.length; + if (len <= 0) return 0; + + let hash = len; + let rem = len & 3; + let offset = 0; + const words = len >> 2; + + // Main loop + for (let i = 0; i < words; i++) { + hash += sfh_get16(data, offset); + let tmp = (sfh_get16(data, offset + 2) << 11) ^ hash; + hash = (hash << 16) ^ tmp; + offset += 4; + hash += hash >>> 11; } - switch (i) { - case 3: hash ^= (data.charCodeAt(i - 3) & 0xff) << 16; - case 2: hash ^= (data.charCodeAt(i - 2) & 0xff) << 8; - case 1: hash ^= (data.charCodeAt(i - 1) & 0xff); - hash = (hash * 0x5bd1e995) >>> 0; + + // Handle end cases + switch (rem) { + case 3: + hash += sfh_get16(data, offset); + hash ^= hash << 16; + hash ^= data[offset + 2] << 18; + hash += hash >>> 11; + break; + case 2: + hash += sfh_get16(data, offset); + hash ^= hash << 11; + hash += hash >>> 17; + break; + case 1: + hash += data[offset]; + hash ^= hash << 10; + hash += hash >>> 1; } - hash ^= hash >>> 13; - hash = (hash * 0x5bd1e995) >>> 0; - hash ^= hash >>> 15; + + // Force "avalanching" of final 127 bits + hash ^= hash << 3; + hash += hash >>> 5; + hash ^= hash << 4; + hash += hash >>> 17; + hash ^= hash << 25; + hash += hash >>> 6; + return hash >>> 0; } @@ -41,7 +68,6 @@ function parsePo(content) { let state = null; let msgid = ''; let msgstr = ''; - let key = ''; function flush() { if (state === 'msgstr' && msgid !== '' && msgstr !== '') { @@ -83,23 +109,15 @@ function compileLmo(entries) { const keyId = sfh_hash(entry.key); const valId = sfh_hash(entry.value); - // Write value to payload const paddedLen = valBuf.length + pad4(valBuf.length); const padded = Buffer.alloc(paddedLen, 0); valBuf.copy(padded); payload.push(padded); - index.push({ - keyId, - valId, - offset, - length: valBuf.length, - }); - + index.push({ keyId, valId, offset, length: valBuf.length }); offset += paddedLen; } - // Build index section const indexBuf = Buffer.alloc(index.length * 16); for (let i = 0; i < index.length; i++) { indexBuf.writeUInt32BE(index[i].keyId, i * 16); @@ -108,7 +126,6 @@ function compileLmo(entries) { indexBuf.writeUInt32BE(index[i].length, i * 16 + 12); } - // Total: payload + index + 4 bytes for index offset const payloadData = Buffer.concat(payload); const totalLen = payloadData.length + indexBuf.length + 4; const out = Buffer.alloc(totalLen); @@ -119,7 +136,6 @@ function compileLmo(entries) { return out; } -// Main const [input, output] = process.argv.slice(2); if (!input || !output) { console.error('Usage: node po2lmo.js input.po output.lmo'); diff --git a/tools/test_hash.js b/tools/test_hash.js new file mode 100644 index 00000000..77b448a9 --- /dev/null +++ b/tools/test_hash.js @@ -0,0 +1,26 @@ +function sfh_hash(data) { + let hash = 0; + let i = data.length; + let k; + while (i >= 4) { + k = (data.charCodeAt(i - 4) & 0xff) | ((data.charCodeAt(i - 3) & 0xff) << 8) | ((data.charCodeAt(i - 2) & 0xff) << 16) | ((data.charCodeAt(i - 1) & 0xff) << 24); + k = (k * 0x5bd1e995) >>> 0; + k ^= k >>> 24; + k = (k * 0x5bd1e995) >>> 0; + hash = (hash * 0x5bd1e995 + k) >>> 0; + i -= 4; + } + switch (i) { + case 3: hash ^= (data.charCodeAt(i - 3) & 0xff) << 16; + case 2: hash ^= (data.charCodeAt(i - 2) & 0xff) << 8; + case 1: hash ^= (data.charCodeAt(i - 1) & 0xff); + hash = (hash * 0x5bd1e995) >>> 0; + } + hash ^= hash >>> 13; + hash = (hash * 0x5bd1e995) >>> 0; + hash ^= hash >>> 15; + return hash >>> 0; +} +console.log('Empty:', sfh_hash('').toString(16)); +console.log('Plural-Forms:', sfh_hash('Plural-Forms').toString(16)); +console.log('Test str:', sfh_hash('Hello').toString(16)); diff --git a/tools/test_ref.lmo b/tools/test_ref.lmo new file mode 100644 index 00000000..ea7c0349 Binary files /dev/null and b/tools/test_ref.lmo differ diff --git a/tools/verify_hash.js b/tools/verify_hash.js new file mode 100644 index 00000000..954d9cdc --- /dev/null +++ b/tools/verify_hash.js @@ -0,0 +1,66 @@ +const fs = require('fs'); + +function sfh_get16(d, off) { + return (d[off] & 0xff) + ((d[off + 1] & 0xff) << 8); +} + +function sfh_hash(str) { + const data = Buffer.from(str, 'utf8'); + const len = data.length; + if (len <= 0) return 0; + let hash = len; + let rem = len & 3; + let offset = 0; + const words = len >> 2; + for (let i = 0; i < words; i++) { + hash += sfh_get16(data, offset); + let tmp = (sfh_get16(data, offset + 2) << 11) ^ hash; + hash = (hash << 16) ^ tmp; + offset += 4; + hash += hash >>> 11; + } + switch (rem) { + case 3: + hash += sfh_get16(data, offset); + hash ^= hash << 16; + hash ^= data[offset + 2] << 18; + hash += hash >>> 11; + break; + case 2: + hash += sfh_get16(data, offset); + hash ^= hash << 11; + hash += hash >>> 17; + break; + case 1: + hash += data[offset]; + hash ^= hash << 10; + hash += hash >>> 1; + } + hash ^= hash << 3; + hash += hash >>> 5; + hash ^= hash << 4; + hash += hash >>> 17; + hash ^= hash << 25; + hash += hash >>> 6; + return hash >>> 0; +} + +// Reference: adblock.bg.lmo entry 1 keyId=0x1545b51f +// msgid for "Клиент" in adblock bg is likely "Client" +console.log('sfh_hash(Client):', sfh_hash('Client').toString(16), '(target: 1545b51f)'); +console.log('sfh_hash(Save):', sfh_hash('Save').toString(16)); +console.log('sfh_hash(Close):', sfh_hash('Close').toString(16)); +console.log('sfh_hash(Enable):', sfh_hash('Enable').toString(16)); +console.log('sfh_hash(Enabled):', sfh_hash('Enabled').toString(16)); +console.log('sfh_hash(General settings):', sfh_hash('General settings').toString(16)); +console.log('sfh_hash(Action):', sfh_hash('Action').toString(16)); +console.log('sfh_hash(Download):', sfh_hash('Download').toString(16)); +console.log('sfh_hash(Email):', sfh_hash('Email').toString(16)); +console.log('sfh_hash(Subject):', sfh_hash('Subject').toString(16)); +console.log('sfh_hash(Enable detailed logs):', sfh_hash('Enable detailed logs').toString(16)); +console.log('sfh_hash(Save & Apply):', sfh_hash('Save & Apply').toString(16)); + +// Now test with some of our tachyon strings +console.log('\nTachyon strings:'); +console.log('sfh_hash(Periodically checks if the proxy is responding. Restarts Tachyon after consecutive failures.):', sfh_hash('Periodically checks if the proxy is responding. Restarts Tachyon after consecutive failures.').toString(16)); +console.log('sfh_hash(Enable Proxy Health Monitor):', sfh_hash('Enable Proxy Health Monitor').toString(16));