fix: bound package hooks so apk cannot roll the upgrade back

The router upgrade died at the installer's 120s ceiling while apk was
running tachyon-1.2.66.pre-upgrade, so apk rolled the transaction back
and the old build stayed installed.

prerm reached INIT_PATH status through rc.common, which waits on
flock -w 1000. A stuck start or retry_start_on_wan_up holding that lock
stalls the hook for up to a quarter of an hour. Every init call in the
hooks is now bounded, the status probe moved after the flock waiters are
killed, and the Makefile hook bounds the old version's cleanup from the
outside since that code cannot be trusted to bound itself.

The installer also called a rollback a success: a rebuilt tag keeps its
version, so both the presence and version checks still passed. It now
compares the commit SHA baked into the installed constants module and
fails loudly when it did not move. Package installs get 300s, since a
kill mid-transaction is what caused the rollback.

tests/installer_rollback_detection.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+# A rebuilt release keeps its version string, so "the package is installed and the
+# version matches" stays true even when the package manager rolled the upgrade
+# back — which is how a failed install used to end with a green success banner.
+# The only thing that moves is the commit SHA baked into the installed constants
+# module, so these checks pin how the installer reads it.
+set -eo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+INSTALLER="$ROOT_DIR/install.sh"
+
+WORK_DIR="$(mktemp -d)"
+trap 'rm -rf "$WORK_DIR"' EXIT
+
+fail() {
+  printf 'FAIL: %s\n' "$1" >&2
+  exit 1
+}
+
+assert_eq() {
+  local expected="$1" actual="$2" label="$3"
+  [ "$actual" = "$expected" ] || fail "$label: expected '$expected', got '$actual'"
+}
+
+[ -r "$INSTALLER" ] || fail "install.sh must exist"
+
+# The function is lifted out of the installer rather than sourced: install.sh runs
+# main() at the bottom, so sourcing it would attempt a real installation. Only the
+# hardcoded path is redirected, so the extraction logic under test is the shipped
+# one, character for character.
+sed -n '/^installed_backend_sha() {/,/^}/p' "$INSTALLER" >"$WORK_DIR/sha.sh"
+[ -s "$WORK_DIR/sha.sh" ] ||
+  fail "install.sh must define installed_backend_sha for rollback detection"
+grep -Fq '/usr/lib/tachyon/core/constants.uc' "$WORK_DIR/sha.sh" ||
+  fail "installed_backend_sha must read the installed constants module"
+sed 's#/usr/lib/tachyon/core/constants.uc#'"$WORK_DIR"'/constants.uc#' \
+  "$WORK_DIR/sha.sh" >"$WORK_DIR/sha-under-test.sh"
+
+read_sha() {
+  local constants="$1"
+  rm -f "$WORK_DIR/constants.uc"
+  [ "$constants" = "" ] || cp "$constants" "$WORK_DIR/constants.uc"
+  (
+    # shellcheck disable=SC1090
+    . "$WORK_DIR/sha-under-test.sh"
+    installed_backend_sha
+  )
+}
+
+printf '    c.TACHYON_COMMIT_SHA = env("TACHYON_COMMIT_SHA", "ba69e3d2");\n' \
+  >"$WORK_DIR/stamped.uc"
+assert_eq "ba69e3d2" "$(read_sha "$WORK_DIR/stamped.uc")" "a stamped build must report its SHA"
+
+# An unprocessed tree keeps the literal placeholder; treating it as a SHA would
+# make every install look like a rollback.
+printf '    c.TACHYON_COMMIT_SHA = env("TACHYON_COMMIT_SHA", "__COMPILED_COMMIT_SHA__");\n' \
+  >"$WORK_DIR/placeholder.uc"
+assert_eq "" "$(read_sha "$WORK_DIR/placeholder.uc")" "the build placeholder must not read as a SHA"
+
+# The Makefile substitutes "unknown" when git is unavailable at build time. It
+# never changes between builds, so it must not be compared as a SHA either.
+printf '    c.TACHYON_COMMIT_SHA = env("TACHYON_COMMIT_SHA", "unknown");\n' \
+  >"$WORK_DIR/unknown.uc"
+assert_eq "" "$(read_sha "$WORK_DIR/unknown.uc")" "an unstamped build must not read as a SHA"
+
+assert_eq "" "$(read_sha "")" "a missing constants module must yield no SHA"
+
+# A rolled-back rebuild is only detectable by comparing SHAs, so the backend
+# install must refuse to accept a mere presence check when the SHA did not move.
+grep -Fq 'rolled the upgrade back' "$INSTALLER" ||
+  fail "install.sh must fail loudly when the backend SHA did not move after install"
+
+# 120s was below the time the package hooks legitimately need to stop and restart
+# the service; being killed there makes apk roll the transaction back.
+for pm in apk opkg; do
+  if grep -nE "(apk_with_lock_retry|run_logged_timeout) \"$pm\" 120 $pm (add|install)" "$INSTALLER" >/dev/null; then
+    fail "$pm package install must not keep the 120s ceiling that rolled upgrades back"
+  fi
+done
+
+printf 'PASS: installer rollback detection\n'

tests/package_lifecycle.sh

@@ -47,6 +47,21 @@ if grep -n -E 'grep -q "105 tachyon"|sed -i "/105 tachyon|tachyon_dont_touch_dhc
 fi
 grep -Fq '#!/usr/bin/ucode' "$TACHYON_MAKEFILE" ||
   fail "tachyon Makefile package hooks must use ucode entrypoints"
+
+# The prerm hook lives inside a Makefile define, so ucode_syntax_lint.sh never
+# sees it. A syntax error there only surfaces on a router mid-upgrade.
+sed -n '/define Package\/tachyon\/prerm/,/^endef/p' "$TACHYON_MAKEFILE" |
+  sed '1d;$d' >"$WORK_DIR/prerm-hook.uc"
+[ -s "$WORK_DIR/prerm-hook.uc" ] ||
+  fail "tachyon Makefile must define a prerm hook"
+ucode -c -o /dev/null "$WORK_DIR/prerm-hook.uc" 2>"$WORK_DIR/prerm-hook.err" ||
+  fail "tachyon Makefile prerm hook must compile: $(cat "$WORK_DIR/prerm-hook.err")"
+
+# apk runs prerm as pre-upgrade and rolls the whole transaction back if it does
+# not finish, so the cleanup call — which executes the OLD installed build — must
+# be bounded from the outside.
+grep -Fq 'timeout' "$WORK_DIR/prerm-hook.uc" ||
+  fail "tachyon Makefile prerm must bound package_prerm so a stuck cleanup cannot roll the upgrade back"
 grep -Fq '/usr/bin/tachyon package_prerm' "$TACHYON_MAKEFILE" ||
   fail "tachyon Makefile prerm must delegate cleanup to package_prerm"
 grep -Fq '/usr/bin/tachyon package_postinst' "$TACHYON_MAKEFILE" ||
@@ -190,4 +205,40 @@ TACHYON_RT_TABLES="$WORK_DIR/rt_tables_upgrade" \
 [ ! -e "$TACHYON_PACKAGE_UPGRADE_STATE" ] ||
   fail "package pre-upgrade must not mark an already stopped service"
 
+# Every init.d call goes through rc.common's `flock -w 1000`, so a wedged start
+# or retry_start_on_wan_up holding that lock used to stall pre-upgrade for a
+# quarter of an hour — long enough for apk (or the installer) to kill the hook and
+# roll the new build back. The hook must give up on its own instead.
+cat >"$WORK_DIR/hanging-init" <<'SH'
+#!/usr/bin/env bash
+sleep 600
+SH
+chmod 0755 "$WORK_DIR/hanging-init"
+: >"$WORK_DIR/rt_tables_hang"
+hang_start="$(date +%s)"
+TACHYON_HOOK_COMMAND_TIMEOUT=2 \
+TACHYON_INIT="$WORK_DIR/hanging-init" \
+TACHYON_BIN="$WORK_DIR/hanging-init" \
+TACHYON_DNS_APPLY_UC="$WORK_DIR/missing-dns-apply.uc" \
+TACHYON_SING_BOX_INIT="$WORK_DIR/missing-sing-box-init" \
+TACHYON_UCI_STATE_FILE="$WORK_DIR/dont-touch.state" \
+TACHYON_RT_TABLES="$WORK_DIR/rt_tables_hang" \
+  ucode -L "$TACHYON_LIB" "$PACKAGE_UC" prerm upgrade
+hang_elapsed="$(( $(date +%s) - hang_start ))"
+[ "$hang_elapsed" -lt 30 ] ||
+  fail "package pre-upgrade must abandon a blocked init call, took ${hang_elapsed}s"
+
+# Same for the restart in postinst: a start that never returns must not hold the
+# package transaction open.
+printf '1\n' >"$TACHYON_PACKAGE_UPGRADE_STATE"
+hang_start="$(date +%s)"
+TACHYON_HOOK_START_TIMEOUT=2 \
+TACHYON_INIT="$WORK_DIR/hanging-init" \
+  ucode -L "$TACHYON_LIB" "$PACKAGE_UC" postinst
+hang_elapsed="$(( $(date +%s) - hang_start ))"
+[ "$hang_elapsed" -lt 30 ] ||
+  fail "package postinst must abandon a blocked service start, took ${hang_elapsed}s"
+[ ! -e "$TACHYON_PACKAGE_UPGRADE_STATE" ] ||
+  fail "package postinst must clear the upgrade state even when the start times out"
+
 printf 'package lifecycle checks passed\n'