#!/usr/bin/env bash
# Stranded-local-cohort self-healing: with a remote host configured, the gate
# caps the local queue wait (injected BUILD_SLOT_TIMEOUT), retries remote on
# an admission timeout, and re-queues locally while remote stays down. A
# child's own 75 and a caller's own BUILD_SLOT_TIMEOUT pass through.
set -uo pipefail

GATE="$HOME/.claude/bin/local-gate"
TMP=$(mktemp -d "$HOME/.cache/lg-requeue-XXXXXX")
HOLDERS=()
cleanup() {
  local pid
  for pid in ${HOLDERS[@]+"${HOLDERS[@]}"}; do kill "$pid" 2>/dev/null; done
  rm -rf "$TMP"
}
trap cleanup EXIT
PASS=0; FAIL=0

check() {
  local name="$1" want="$2" got="$3" extra="${4:-0}"
  if [ "$want" = "$got" ] && [ "$extra" = "0" ]; then
    echo "PASS $name"; PASS=$((PASS+1))
  else
    echo "FAIL $name (want=$want got=$got extra=$extra)"; FAIL=$((FAIL+1))
  fi
}

# Unreachable remote (closed port on localhost) with fast probes; local runs
# route through a stub cpu-guard so only the gate's own loop is under test.
cat > "$TMP/build-remote.json" <<EOF
{ "enabled": true, "port": 1, "ssh_user": "u",
  "remote_root": "$TMP/rb", "health_ttl_sec": 1, "connect_timeout_sec": 2,
  "ssh_probe_timeout_sec": 2, "local_fallback": true, "local_requeue_sec": 2 }
EOF
cat > "$TMP/local-gate.json" <<EOF
{ "min_slots": 4, "max_slots": 4, "load_per_slot": 2, "min_mem_available_gb": 0,
  "max_swap_used_gb": 4096, "state_dir": "$TMP/state" }
EOF
mkdir "$TMP/bin"
cat > "$TMP/bin/ssh" <<'EOF'
#!/usr/bin/env bash
for file in "$BUILD_SLOT_TEST_STATE"/.buildslot-status-*/status; do
  [ -f "$file" ] || continue
  cat "$file" >> "$BUILD_SLOT_TEST_OBSERVED"
done
exit 255
EOF
chmod +x "$TMP/bin/ssh"
export PATH="$TMP/bin:$PATH"
export BUILD_REMOTE_CONFIG="$TMP/build-remote.json"
export LOCAL_GATE_CONFIG="$TMP/local-gate.json"
export BUILD_SLOT_TEST_STATE="$TMP/state"
export BUILD_SLOT_TEST_OBSERVED="$TMP/observed-status"
unset BUILD_SLOT_TIMEOUT BUILD_SLOT_HELD

# child sees the injected cap
out=$("$GATE" --key rq-inject -- bash -c 'echo "T=$BUILD_SLOT_TIMEOUT"' 2>/dev/null | grep '^T=')
check "requeue-timeout-injected" "T=2" "$out"

# admitted child 75 is the child's result, not an admission timeout
marker="$TMP/rq-own-75-count"
"$GATE" --key rq-own-75 -- bash -c '
  count=$(cat "$1" 2>/dev/null || echo 0); count=$((count + 1)); printf "%s\n" "$count" > "$1"
  if (( count > 1 )); then exit 0; fi
  printf "admitted\n" > "$BUILD_SLOT_STATUS"
  exit 75
' _ "$marker" >/dev/null 2>&1
rc=$?
runs=$(cat "$marker")
check "child-exit-75-after-admission-passes-through" 75 "$rc" "$((runs != 1))"

# real buildslot admission timeout -> remote retried -> local re-queued.
# The holder releases slot-0 on the gate's own admission-timeout log line, not on a
# wall clock: gate startup spends an unbounded probe window on the unreachable host
# before the child is even spawned, so a fixed hold races that window and the child
# gets admitted immediately instead of timing out. The cap keeps a regression a
# failure rather than a hang.
GATE_LOG="$HOME/.claude/local-gate.log"
TIMEOUT_LINE='local queue timeout key=rq-timeout retrying-remote'
log_mark=$(wc -l < "$GATE_LOG" 2>/dev/null || echo 0)
# The gate log is machine-global and append-only; every assertion below reads only
# the lines this run appended, so a previous run's line cannot stand in for this one.
log_since() { tail -n +"$((log_mark + 1))" "$GATE_LOG" 2>/dev/null; }
slot_dir="$TMP/slot-timeout"
mkdir "$slot_dir"
bash -c '
  exec {fd}>>"$1/slot-0.lock"; flock -x "$fd"; : >"$1/held"
  deadline=$((SECONDS + 60))
  while (( SECONDS < deadline )); do
    tail -n +"$2" "$3" 2>/dev/null | grep -q "$4" && break
    sleep 0.1
  done
' _ "$slot_dir" "$((log_mark + 1))" "$GATE_LOG" "$TIMEOUT_LINE" &
HOLDERS+=("$!")
for _ in $(seq 1 100); do [ -f "$slot_dir/held" ] && break; sleep 0.05; done
out=$(BUILD_SLOT_DIR="$slot_dir" BUILD_SLOTS=1 BUILD_SLOT_FIXED=1 \
  "$GATE" --key rq-timeout -- "$HOME/.claude/lib/buildslot.sh" \
  bash -c 'printf "S=%s\n" "$(cat "$BUILD_SLOT_STATUS" 2>/dev/null || echo missing)"' 2>/dev/null)
rc=$?
requeued=1
log_since | grep -q "$TIMEOUT_LINE" \
  && log_since | grep -q 'remote requeue skip key=rq-timeout' && requeued=0
timed_out=1
grep -qx 'timeout' "$BUILD_SLOT_TEST_OBSERVED" 2>/dev/null && timed_out=0
check "admission-timeout-retries-remote-then-requeues-local" "0:S=admitted" "$rc:$out" "$((requeued || timed_out))"

# absent status file -> one warning and legacy exit-75 requeue behavior
marker="$TMP/rq-missing-count"
before=$(grep -c 'status-file-missing.*key=rq-missing' "$HOME/.claude/local-gate.log" 2>/dev/null || true)
"$GATE" --key rq-missing -- bash -c '
  count=$(cat "$1" 2>/dev/null || echo 0); count=$((count + 1)); printf "%s\n" "$count" > "$1"
  (( count == 1 )) && exit 75
  exit 0
' _ "$marker" >/dev/null 2>&1
rc=$?
after=$(grep -c 'status-file-missing.*key=rq-missing' "$HOME/.claude/local-gate.log" 2>/dev/null || true)
runs=$(cat "$marker")
check "status-file-missing-warns-once-and-uses-legacy-requeue" 0 "$rc" "$((runs != 2 || after - before != 1))"

# caller-owned timeout: no injection, 75 passes through untouched
out=$(BUILD_SLOT_TIMEOUT=7 "$GATE" --key rq-owned-env -- bash -c 'echo "T=$BUILD_SLOT_TIMEOUT"' 2>/dev/null | grep '^T=')
check "caller-timeout-not-overridden" "T=7" "$out"
BUILD_SLOT_TIMEOUT=7 "$GATE" --key rq-owned-75 -- bash -c 'exit 75' >/dev/null 2>&1
check "caller-timeout-75-passes-through" 75 "$?"

# remote disabled: no injection at all
cat > "$TMP/build-remote.json" <<EOF
{ "enabled": false }
EOF
out=$("$GATE" --key rq-disabled -- bash -c 'echo "T=${BUILD_SLOT_TIMEOUT:-none}"' 2>/dev/null | grep '^T=')
check "disabled-remote-no-injection" "T=none" "$out"

echo
echo "passed=$PASS failed=$FAIL"
[ "$FAIL" -eq 0 ]