144 lines
5.6 KiB
Bash
144 lines
5.6 KiB
Bash
#!/usr/bin/env bash
|
|
set -eo pipefail
|
|
|
|
# A Tachyon self-update restarts the service mid-worker. If the worker dies
|
|
# before writing its result, the job state stays "running" with a dead pid.
|
|
# The stale detector used a bare `kill -0` liveness check: the recycled pid of
|
|
# the next process would keep the job running forever, and LuCI's progress
|
|
# modal hung for minutes over a finished install (issue #31). The pid must
|
|
# still belong to the component-action worker (checked via /proc cmdline).
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
TACHYON_LIB="$ROOT_DIR/tachyon/files/usr/lib"
|
|
UPDATES_UC="$TACHYON_LIB/components/updates.uc"
|
|
WORK_DIR="$(mktemp -d /tmp/tachyon-stale-pid-test.XXXXXX)"
|
|
|
|
cleanup() {
|
|
rm -rf "$WORK_DIR"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
fail() {
|
|
printf 'FAIL: %s\n' "$1" >&2
|
|
ls -la "$WORK_DIR" >&2 || true
|
|
exit 1
|
|
}
|
|
|
|
assert_stale_uc_ready() {
|
|
[ -f "$STALE_UC" ] || fail "stale.uc is missing right before ucode runs"
|
|
[ -s "$STALE_UC" ] || fail "stale.uc is empty right before ucode runs"
|
|
}
|
|
|
|
# ── the identity check exists and is used ────────────────────────────────────
|
|
grep -q 'function pid_is_component_worker' "$UPDATES_UC" \
|
|
|| fail "pid_is_component_worker() is gone; a recycled pid can keep a dead job running forever"
|
|
grep -q 'pid_is_component_worker(pid)' "$UPDATES_UC" \
|
|
|| fail "refresh_component_running_job_state no longer verifies the worker identity"
|
|
|
|
# ── behavior: extract the stale-detection block verbatim and drive it ────────
|
|
STALE_UC="$WORK_DIR/stale.uc"
|
|
|
|
build_stale_uc() {
|
|
cat > "$STALE_UC" <<'PRELUDE'
|
|
let fs = require("fs");
|
|
function now_seconds() { return int(time()); }
|
|
function as_string(v) { return v == null ? "" : "" + v; }
|
|
function arg_number(v) { return int(v); }
|
|
function arg_bool(v) { return v == true || v == "1" || v == 1; }
|
|
function command_success_from_args(args) {
|
|
let parts = [];
|
|
for (let a in args)
|
|
push(parts, "'" + replace(as_string(a), /'/g, "'\\''") + "'");
|
|
return system(join(" ", parts) + " >/dev/null 2>&1") == 0;
|
|
}
|
|
function read_json_file(path) {
|
|
let raw = fs.readfile(path);
|
|
if (raw == null) return null;
|
|
let value = json(raw);
|
|
return type(value) == "object" ? value : null;
|
|
}
|
|
let stale_calls = [];
|
|
function write_component_stale_job_state(path) { push(stale_calls, path); }
|
|
PRELUDE
|
|
|
|
sed -n '/^const COMPONENT_JOB_STALE_GRACE_SECONDS/,/^$/p' "$UPDATES_UC" >> "$STALE_UC"
|
|
sed -n '/^function job_started_at_within_grace/,/^}$/p' "$UPDATES_UC" >> "$STALE_UC"
|
|
sed -n '/^function job_pid_valid/,/^}$/p' "$UPDATES_UC" >> "$STALE_UC"
|
|
sed -n '/^function pid_running/,/^}$/p' "$UPDATES_UC" >> "$STALE_UC"
|
|
sed -n '/^function pid_is_component_worker/,/^}$/p' "$UPDATES_UC" >> "$STALE_UC"
|
|
sed -n '/^function refresh_component_running_job_state/,/^}$/p' "$UPDATES_UC" >> "$STALE_UC"
|
|
|
|
cat >> "$STALE_UC" <<'DRIVER'
|
|
function write_state(path, pid, started_at) {
|
|
let tmp = path + ".tmp";
|
|
let payload = sprintf('{ "running": true, "pid": "%s", "started_at": %d }', pid, started_at);
|
|
if (!fs.writefile(tmp, payload))
|
|
return false;
|
|
return fs.rename(tmp, path);
|
|
}
|
|
let state_path = ARGV[0];
|
|
let pid = ARGV[1];
|
|
let started_at = int(ARGV[2] || (time() - 60));
|
|
write_state(state_path, pid, started_at);
|
|
refresh_component_running_job_state(state_path);
|
|
print((length(stale_calls) > 0 ? "stale" : "running") + "\n");
|
|
DRIVER
|
|
assert_stale_uc_ready
|
|
}
|
|
|
|
run_stale_check() {
|
|
mkdir -p "$WORK_DIR"
|
|
if [ ! -f "$STALE_UC" ] || [ ! -s "$STALE_UC" ]; then
|
|
printf 'WARN: stale.uc vanished (parallel /tmp race); rebuilding before ucode\n' >&2
|
|
build_stale_uc
|
|
fi
|
|
local err="$WORK_DIR/ucode.err"
|
|
local out
|
|
if ! out="$(ucode -L "$TACHYON_LIB" "$STALE_UC" "$@" 2>"$err")"; then
|
|
cat "$err" >&2 || true
|
|
fail "ucode failed: ucode -L \"$TACHYON_LIB\" \"$STALE_UC\" $*"
|
|
fi
|
|
printf '%s' "$out"
|
|
}
|
|
|
|
build_stale_uc
|
|
|
|
# A live worker is not stale.
|
|
mkdir -p "$WORK_DIR"
|
|
bash -c 'while true; do sleep 1; done' component-action-worker &
|
|
worker_pid=$!
|
|
actual="$(run_stale_check "$WORK_DIR/job.json" "$worker_pid")"
|
|
[ "$actual" = "running" ] || fail "a live component-action worker was marked stale (pid $worker_pid): '$actual'"
|
|
kill "$worker_pid" 2>/dev/null || true
|
|
|
|
# The regression: a recycled pid — a live process that is NOT our worker —
|
|
# must be marked stale, not kept "running" forever.
|
|
sleep 30 &
|
|
alien_pid=$!
|
|
actual="$(run_stale_check "$WORK_DIR/job.json" "$alien_pid")"
|
|
[ "$actual" = "stale" ] || fail "a dead job whose pid was recycled is never marked stale (alien pid $alien_pid): '$actual'"
|
|
kill "$alien_pid" 2>/dev/null || true
|
|
|
|
# A dead pid is stale too.
|
|
sleep 30 &
|
|
dead_pid=$!
|
|
kill "$dead_pid" 2>/dev/null || true
|
|
wait "$dead_pid" 2>/dev/null || true
|
|
actual="$(run_stale_check "$WORK_DIR/job.json" "$dead_pid")"
|
|
[ "$actual" = "stale" ] || fail "a dead worker pid is not marked stale (pid $dead_pid): '$actual'"
|
|
|
|
# Grace still protects a fresh job: an alien pid inside the grace window is
|
|
# left alone, so a worker that just started is not raced by the detector.
|
|
sleep 30 &
|
|
fresh_pid=$!
|
|
fresh_start="$(($(date +%s) - 5))"
|
|
actual="$(run_stale_check "$WORK_DIR/job.json" "$fresh_pid" "$fresh_start")"
|
|
[ "$actual" = "running" ] || fail "a fresh job was marked stale inside the grace window (pid $fresh_pid): '$actual'"
|
|
kill "$fresh_pid" 2>/dev/null || true
|
|
|
|
# ── ui.uc's running-for must not report a recycled pid as a running action ──
|
|
UI_UC="$TACHYON_LIB/service/ui.uc"
|
|
grep -q 'component-action-worker' "$UI_UC" \
|
|
|| fail "ui.uc component_action_running_for no longer verifies the worker identity"
|
|
|
|
printf 'component job stale-pid checks passed\n' |