#!/usr/bin/env bash
# buildslot.sh — machine-global admission gate for heavy builds/tests.
# At most N jobs run concurrently; excess invocations BLOCK until a slot frees.
# Admission is strict FIFO via an on-disk ticket queue: only the queue head may
# probe slots, so a waiter can never lose a freed slot to a later arrival
# (non-FIFO polling starved long waiters indefinitely under sustained load).
# Each ticket owns a flock held for its waiting lifetime. Acquirable ownership
# locks are pruned, so crashed or cross-PID-namespace waiters cannot wedge it.
# N adapts to real-time interactive demand measured from /proc/stat: non-nice
# CPU time (user+system+irq+softirq+steal) is "demand"; nice time is NOT —
# queued builds run at nice 19, so a box saturated only by its own builds still
# reads as idle and admits up to BUILD_SLOTS. A box busy with interactive work
# shrinks to BUILD_SLOT_FLOOR. Holders are never preempted; adaptation only
# gates NEW admissions.
# Slot lock = flock on an fd held open across exec: the kernel releases it when
# the command exits or crashes — no PID files, no daemon.
# Priority admission: a priority job enqueues at the HEAD of the ticket queue
# (still no preemption of running holders). Priority is granted by
# BUILD_SLOT_PRIORITY=1 or by a `--key <value>` argument whose value matches a
# prefix line in ~/.claude/build-priority.conf (one prefix per line, `#`
# comments). Priority tickets are LIFO among themselves.
#
# Tunables:
#   BUILD_SLOTS=4          max concurrent slots (adaptive ceiling)
#   BUILD_SLOT_FLOOR=1     slots still admitted under full interactive load
#   BUILD_SLOT_FIXED=1     disable adaptation; always exactly BUILD_SLOTS
#   BUILD_SLOT_TIMEOUT=    seconds to wait before giving up (default: forever)
#   BUILD_SLOT_PRIORITY=1  enqueue this job at the head of the queue
#   BUILD_SLOT_DIR=        lock directory override (tests)
#   BUILD_SLOT_STAT=       /proc/stat override (tests)
#   BUILD_SLOT_ISOLATED=1  admission is private to this invocation: an inherited
#                          BUILD_SLOT_HELD does not satisfy it, and neither the
#                          slot it takes nor the tunables above are handed to the
#                          admitted command
set -euo pipefail

write_status() {
  [[ -n "${BUILD_SLOT_STATUS:-}" ]] || return 0
  printf '%s\n' "$1" > "$BUILD_SLOT_STATUS"
}

exec_admitted() {
  if [[ "${BUILD_SUPERVISOR_AFTER_ADMISSION:-}" != 1 || "${COMMAND_SUPERVISOR_ACTIVE:-}" == 1 ]]; then
    exec "$@"
  fi
  local supervisor="${COMMAND_SUPERVISOR_BIN:-$HOME/.claude/lib/command-supervisor.mjs}"
  local supervisor_node="${COMMAND_SUPERVISOR_NODE:-node}"
  [[ -r "$supervisor" ]] || { printf '[buildslot] command supervisor missing: %s\n' "$supervisor" >&2; exit 254; }
  local job_root="${COMMAND_SUPERVISOR_JOB_ROOT:-$HOME/.claude/run/command-supervisor}"
  local job_dir="${COMMAND_SUPERVISOR_JOB_DIR:-}"
  if [[ -z "$job_dir" ]]; then
    mkdir -p "$job_root" || { printf '[buildslot] unable to create supervisor job root: %s\n' "$job_root" >&2; exit 254; }
    job_dir=$(mktemp -d "$job_root/job.XXXXXX") || { printf '[buildslot] unable to create supervisor job directory\n' >&2; exit 254; }
  else
    mkdir -p "$job_dir" || { printf '[buildslot] unable to create supervisor job directory: %s\n' "$job_dir" >&2; exit 254; }
  fi
  export COMMAND_SUPERVISOR_ACTIVE=1
  local rc
  if "$supervisor_node" "$supervisor" \
      --mode "${SUPERVISOR_MODE:-opaque}" \
      --job-dir "$job_dir" \
      --lease-ms "${SUPERVISOR_LEASE_MS:-60000}" \
      --terminal-grace-ms "${SUPERVISOR_TERMINAL_GRACE_MS:-5000}" \
      -- "$@"; then
    rc=$?
  else
    rc=$?
  fi
  [[ -n "${COMMAND_SUPERVISOR_JOB_DIR:-}" ]] || rm -rf "$job_dir"
  exit "$rc"
}

if (( $# == 0 )); then
  printf 'usage: %s COMMAND [ARG...]\n' "$0" >&2
  exit 64
fi

isolated=0
if [[ "${BUILD_SLOT_ISOLATED:-}" == 1 ]]; then
  isolated=1
fi

# An isolated admission gates a different pool, so its slot must not silence the
# build gate for the command it admits, and its pool selection must not follow the
# command into descendant builds.
mark_held() {
  if (( isolated )); then
    unset BUILD_SLOT_HELD BUILD_SLOT_ISOLATED BUILD_SLOT_DIR BUILD_SLOTS \
      BUILD_SLOT_FIXED BUILD_SLOT_FLOOR BUILD_SLOT_TIMEOUT
  else
    export BUILD_SLOT_HELD=1
  fi
}

# Reentrancy: a nested invocation inside a slot-holding tree inherits the slot.
if [[ -n "${BUILD_SLOT_HELD:-}" ]] && (( ! isolated )); then
  write_status admitted
  exec_admitted "$@"
fi

slots_max=${BUILD_SLOTS:-1}
if ! [[ "$slots_max" =~ ^[1-9][0-9]*$ ]]; then
  printf '[buildslot] BUILD_SLOTS must be a positive integer: %s\n' "$slots_max" >&2
  exit 64
fi

floor=${BUILD_SLOT_FLOOR:-1}
if ! [[ "$floor" =~ ^[1-9][0-9]*$ ]] || (( floor > slots_max )); then
  printf '[buildslot] BUILD_SLOT_FLOOR must be an integer in [1, BUILD_SLOTS]: %s\n' "$floor" >&2
  exit 64
fi

timeout=${BUILD_SLOT_TIMEOUT:-}
if [[ -n "$timeout" ]] && ! [[ "$timeout" =~ ^[0-9]+$ ]]; then
  printf '[buildslot] BUILD_SLOT_TIMEOUT must be a non-negative integer: %s\n' "$timeout" >&2
  exit 64
fi

if [[ -n "${BUILD_SLOT_DIR:-}" ]]; then
  lock_dir=$BUILD_SLOT_DIR
elif [[ -d /run/dev-build-slot && -w /run/dev-build-slot && -x /run/dev-build-slot ]]; then
  lock_dir=/run/dev-build-slot
else
  lock_dir=/run/user/$(id -u)/buildslot
fi
if ! mkdir -p "$lock_dir" 2>/dev/null || [[ ! -d "$lock_dir" || ! -w "$lock_dir" || ! -x "$lock_dir" ]]; then
  printf '[buildslot] unable to use lock directory: %s\n' "$lock_dir" >&2
  exit 73
fi

# Field 22 of /proc/PID/stat, indexed past the parenthesised comm.
proc_start() {
  local raw rest fields
  [[ -r "/proc/$1/stat" ]] || return 1
  raw=$(<"/proc/$1/stat")
  rest=${raw##*") "}
  read -r -a fields <<< "$rest"
  printf '%s\n' "${fields[19]}"
}

slot_locked() {
  local probe_fd
  exec {probe_fd}>>"$1" || return 1
  if flock -n "$probe_fd"; then
    exec {probe_fd}>&-
    return 1
  fi
  exec {probe_fd}>&-
  return 0
}

# The stamp a holder writes at acquire time (pid, starttime, command) survives
# exec unchanged, so pid+starttime match = the acquiring process is alive.
stamp_pid_alive() {
  local pid start _cmd cur
  IFS=$'\t' read -r pid start _cmd < "$1" 2>/dev/null || return 1
  [[ "$pid" =~ ^[0-9]+$ && "$start" =~ ^[0-9]+$ ]] || return 1
  cur=$(proc_start "$pid") || return 1
  [[ "$cur" == "$start" ]]
}

# Reentrancy fallback: env-filtering runners (turbo passthrough, env -i) strip
# BUILD_SLOT_HELD from descendants of a slot holder. The holder's stamp pid is
# stable across exec and every process in the holder's tree has it as an
# ancestor — if any of OUR ancestors matches a held slot's stamp, this tree
# already owns one. Without this, holder -> env-scrubbed child -> child queues
# behind its own ancestor = machine-wide deadlock. (lsof-free: scanning every
# process's fd table burned a core per probe on busy boxes.)
ancestor_holds_slot() {
  local file pid start
  local -a held_pids=() held_starts=()
  for file in "$lock_dir"/slot-*.lock; do
    [[ -e "$file" ]] || continue
    slot_locked "$file" || continue
    IFS=$'\t' read -r pid start _ < "$file" 2>/dev/null || continue
    [[ "$pid" =~ ^[0-9]+$ && "$start" =~ ^[0-9]+$ ]] || continue
    held_pids+=("$pid")
    held_starts+=("$start")
  done
  (( ${#held_pids[@]} )) || return 1
  local self=$$ stat rest ppid i cur
  while [[ "$self" =~ ^[0-9]+$ ]] && (( self > 1 )); do
    for i in "${!held_pids[@]}"; do
      if [[ "${held_pids[i]}" == "$self" ]]; then
        cur=$(proc_start "$self") || continue
        [[ "$cur" == "${held_starts[i]}" ]] && return 0
      fi
    done
    [[ -r "/proc/$self/stat" ]] || return 1
    stat=$(<"/proc/$self/stat")
    rest=${stat##*) }
    read -r _ ppid _ <<< "$rest"
    self=$ppid
  done
  return 1
}

if ancestor_holds_slot; then
  mark_held
  write_status admitted
  exec_admitted "$@"
fi

stat_file=${BUILD_SLOT_STAT:-/proc/stat}
adaptive=1
if [[ "${BUILD_SLOT_FIXED:-}" == 1 ]] || [[ ! -r "$stat_file" ]]; then
  adaptive=0
fi

# Prints "busy total" from the aggregate cpu line. busy excludes the nice field
# so CPU consumed by nice>0 tasks (the queued builds themselves) never counts
# as demand — otherwise a build-saturated idle box would choke its own queue.
read_stat() {
  local _c user nice system idle iowait irq softirq steal _rest
  read -r _c user nice system idle iowait irq softirq steal _rest < "$stat_file" || return 1
  printf '%s %s\n' "$((user + system + irq + softirq + steal))" \
    "$((user + nice + system + idle + iowait + irq + softirq + steal))"
}

# Map interactive-demand percent over the last sample window to a slot count.
effective_slots() {
  local prev_busy=$1 prev_total=$2 cur_busy=$3 cur_total=$4
  local db=$((cur_busy - prev_busy)) dt=$((cur_total - prev_total))
  if (( dt <= 0 )); then
    printf '%s\n' "$floor"
    return
  fi
  local pct=$((100 * db / dt))
  if (( pct < 30 )); then
    printf '%s\n' "$slots_max"
  elif (( pct < 60 )); then
    local half=$(((slots_max + 1) / 2))
    (( half < floor )) && half=$floor
    printf '%s\n' "$half"
  else
    printf '%s\n' "$floor"
  fi
}

queue_file="$lock_dir/queue"
queue_lock="$lock_dir/queue.lock"
exec {queue_fd}>"$queue_lock"

if ! ticket_file=$(mktemp "$lock_dir/ticket.XXXXXX.lock"); then
  printf '[buildslot] unable to use lock directory: %s\n' "$lock_dir" >&2
  exit 73
fi
ticket_id=${ticket_file##*/}
if ! exec {ticket_fd}>"$ticket_file" || ! flock -n "$ticket_fd"; then
  rm -f "$ticket_file"
  printf '[buildslot] unable to use lock directory: %s\n' "$lock_dir" >&2
  exit 73
fi

# Priority: env wins; otherwise match this job's `--key <value>` argument
# against prefix lines in the conf. No conf / no key / no match => normal FIFO.
priority=0
if [[ "${BUILD_SLOT_PRIORITY:-}" == 1 ]]; then
  priority=1
else
  priority_conf="$HOME/.claude/build-priority.conf"
  if [[ -r "$priority_conf" ]]; then
    job_key=""
    job_args=("$@")
    for (( arg_i = 0; arg_i < ${#job_args[@]} - 1; arg_i++ )); do
      if [[ "${job_args[arg_i]}" == "--key" ]]; then
        job_key=${job_args[arg_i + 1]}
        break
      fi
    done
    if [[ -n "$job_key" ]]; then
      while IFS= read -r prefix; do
        [[ -n "$prefix" && "${prefix:0:1}" != "#" ]] || continue
        if [[ "$job_key" == "$prefix"* ]]; then
          priority=1
          break
        fi
      done <"$priority_conf"
    fi
  fi
fi

enqueue() {
  flock "$queue_fd"
  if (( priority )) && [[ -s "$queue_file" ]]; then
    { printf '%s\n' "$ticket_id"; cat "$queue_file"; } >"$queue_file.new"
    mv "$queue_file.new" "$queue_file"
  else
    printf '%s\n' "$ticket_id" >>"$queue_file"
  fi
  flock -u "$queue_fd"
}

dequeue_self() {
  flock "$queue_fd"
  if [[ -f "$queue_file" ]]; then
    grep -Fxv -- "$ticket_id" "$queue_file" >"$queue_file.new" || true
    mv "$queue_file.new" "$queue_file"
  fi
  flock -u "$queue_fd"
}

release_ticket() {
  dequeue_self
  exec {ticket_fd}>&-
  rm -f "$ticket_file"
}

# Head = first ticket whose ownership lock is held. Numeric records from the
# PID-based queue format have no owner lock and are therefore pruned.
at_head() {
  local head="" ticket owner_file owner_fd orphan_file orphan_id referenced live=()
  flock "$queue_fd"
  if [[ -f "$queue_file" ]]; then
    while read -r ticket; do
      if [[ "$ticket" =~ ^ticket\.[[:alnum:]]+\.lock$ ]]; then
        owner_file="$lock_dir/$ticket"
      elif [[ "$ticket" =~ ^[0-9]+$ ]]; then
        owner_file="$lock_dir/legacy-ticket.$ticket.lock"
      else
        continue
      fi
      if ! exec {owner_fd}>"$owner_file"; then
        flock -u "$queue_fd"
        printf '[buildslot] unable to use lock directory: %s\n' "$lock_dir" >&2
        exit 73
      fi
      if flock -n "$owner_fd"; then
        exec {owner_fd}>&-
        rm -f "$owner_file"
      else
        live+=("$ticket")
        exec {owner_fd}>&-
      fi
    done <"$queue_file"
    if (( ${#live[@]} )); then
      printf '%s\n' "${live[@]}" >"$queue_file.new"
      mv "$queue_file.new" "$queue_file"
      head=${live[0]}
    else
      : >"$queue_file"
    fi
  fi
  referenced=0
  for ticket in "${live[@]}"; do
    if [[ "$ticket" == "$ticket_id" ]]; then
      referenced=1
      break
    fi
  done
  if (( ! referenced )); then
    if (( priority )); then
      live=("$ticket_id" "${live[@]}")
      head=$ticket_id
    else
      live+=("$ticket_id")
      [[ -n "$head" ]] || head=$ticket_id
    fi
    printf '%s\n' "${live[@]}" >"$queue_file.new"
    mv "$queue_file.new" "$queue_file"
  fi
  for orphan_file in "$lock_dir"/ticket.*.lock; do
    [[ -e "$orphan_file" ]] || continue
    orphan_id=${orphan_file##*/}
    referenced=0
    for ticket in "${live[@]}"; do
      if [[ "$ticket" == "$orphan_id" ]]; then
        referenced=1
        break
      fi
    done
    (( referenced )) && continue
    if ! exec {owner_fd}>"$orphan_file"; then
      flock -u "$queue_fd"
      printf '[buildslot] unable to use lock directory: %s\n' "$lock_dir" >&2
      exit 73
    fi
    if flock -n "$owner_fd"; then
      rm -f "$orphan_file"
    fi
    exec {owner_fd}>&-
  done
  flock -u "$queue_fd"
  [[ "$head" == "$ticket_id" ]]
}

slot_available() {
  local index probe_fd probe_file
  for (( index = 0; index < slots_max; index++ )); do
    probe_file="$lock_dir/slot-${index}.lock"
    exec {probe_fd}>>"$probe_file" || return 1
    if flock -n "$probe_fd"; then
      exec {probe_fd}>&-
      return 0
    fi
    exec {probe_fd}>&-
  done
  return 1
}

# A ghost: the flock is held while its stamped holder is dead, so nothing can
# ever release it. /proc/locks is no help — it names the process that CREATED
# the lock, which for `exec {fd}>f; flock -x $fd` is the flock helper that
# exited immediately, so a dead owner there is the normal state of every
# healthy slot. The stamp is the primary liveness check: cheap, and immune to
# other waiters' momentary probe fds (concurrent lsof-based scans read each
# other's probes as holders and can wedge admission for hours). lsof runs only
# as the final confirm so a live fd-inheriting descendant of a dead holder is
# never rotated out from under; the resample covers the window between flock
# acquire and stamp write.
slot_is_ghost() {
  local file=$1 sample
  for sample in 1 2; do
    slot_locked "$file" || return 1
    stamp_pid_alive "$file" && return 1
    if (( sample < 2 )); then
      sleep 0.5
    fi
  done
  command -v lsof >/dev/null 2>&1 || return 1
  [[ -z "$(lsof -t -- "$file" 2>/dev/null)" ]]
}

# Unlink, so the next opener creates a fresh inode with a fresh lock domain.
# Stale fds on the old inode are caught by fd_holds_path at acquire time.
rotate_ghost_slot() {
  local file=$1 rotated=1
  flock "$queue_fd"
  if slot_locked "$file" && [[ -z "$(lsof -t -- "$file" 2>/dev/null)" ]]; then
    rm -f "$file"
    printf '[buildslot] cleared ghost lock on %s (held, no fd holder)\n' "$file" >&2
    rotated=0
  fi
  flock -u "$queue_fd"
  return $rotated
}

ghost_scan() {
  local index file
  for (( index = 0; index < slots_max; index++ )); do
    file="$lock_dir/slot-${index}.lock"
    [[ -e "$file" ]] || continue
    if slot_is_ghost "$file" && rotate_ghost_slot "$file"; then
      return 0
    fi
  done
  return 1
}

fd_holds_path() {
  local held path
  held=$(stat -Lc %i "/proc/self/fd/$1" 2>/dev/null) || return 1
  path=$(stat -c %i "$2" 2>/dev/null) || return 1
  [[ "$held" == "$path" ]]
}

# pid + start time + command of the holder: a slot that outlives its holder is
# then a named ghost instead of an anonymous kernel record.
stamp_slot() {
  local file=$1 start
  shift
  start=$(proc_start $$) || start="?"
  printf '%s\t%s\t%s\n' "$$" "$start" "$*" >"$file"
}

trap 'release_ticket' EXIT
enqueue

prev_busy="" prev_total=""
if (( adaptive )); then
  if sample="$(read_stat)"; then
    read -r prev_busy prev_total <<< "$sample"
    sleep 0.1
  else
    printf '[buildslot] cannot read %s; falling back to fixed %s slots\n' "$stat_file" "$slots_max" >&2
    adaptive=0
  fi
fi

started=$SECONDS
announced=0
all_free_since=""
ghost_next=0

while :; do
  may_probe=0
  is_head=0
  if at_head; then
    may_probe=1
    is_head=1
    all_free_since=""
  elif slot_available; then
    [[ -n "$all_free_since" ]] || all_free_since=$SECONDS
    if (( SECONDS - all_free_since >= 1 )); then
      may_probe=1
    fi
  else
    all_free_since=""
  fi
  if (( may_probe )); then
    eff=$slots_max
    if (( adaptive )); then
      if sample="$(read_stat)"; then
        read -r cur_busy cur_total <<< "$sample"
        eff="$(effective_slots "$prev_busy" "$prev_total" "$cur_busy" "$cur_total")"
        prev_busy=$cur_busy prev_total=$cur_total
      else
        printf '[buildslot] cannot read %s; falling back to fixed %s slots\n' "$stat_file" "$slots_max" >&2
        adaptive=0
      fi
    fi
    # Admission counts held slots across every index rather than probing only
    # indexes below eff: with a busy low index the old walk both blocked on a
    # slot it was not entitled to and, when eff sat above a held index, granted
    # a slot past the ceiling.
    held=0
    free_slots=()
    for (( index = 0; index < slots_max; index++ )); do
      probe_file="$lock_dir/slot-${index}.lock"
      if ! exec {probe_fd}>>"$probe_file"; then
        printf '[buildslot] unable to use lock directory: %s\n' "$lock_dir" >&2
        exit 73
      fi
      if flock -n "$probe_fd"; then
        free_slots+=("$index")
      else
        held=$((held + 1))
      fi
      exec {probe_fd}>&-
    done
    # Head-only: concurrent scans from several waiters poison each other's
    # lsof confirms with their own probe fds.
    if (( is_head && held >= eff && SECONDS >= ghost_next )); then
      ghost_next=$((SECONDS + 2))
      ghost_scan || true
    fi
    if (( held < eff )); then
      for index in "${free_slots[@]}"; do
        lock_file="$lock_dir/slot-${index}.lock"
        if ! exec {lock_fd}>>"$lock_file"; then
          printf '[buildslot] unable to use lock directory: %s\n' "$lock_dir" >&2
          exit 73
        fi
        if flock -n "$lock_fd" && fd_holds_path "$lock_fd" "$lock_file"; then
          stamp_slot "$lock_file" "$@"
          release_ticket
          trap - EXIT
          mark_held
          write_status admitted
          if (( announced )); then
            printf '[buildslot] acquired build slot after %ss\n' "$((SECONDS - started))" >&2
          fi
          exec_admitted "$@"
        fi
        exec {lock_fd}>&-
      done
    fi
  fi

  elapsed=$((SECONDS - started))
  if [[ -n "$timeout" ]] && (( elapsed >= timeout )); then
    write_status timeout
    printf '[buildslot] timed out waiting for build slot after %ss\n' "$elapsed" >&2
    exit 75
  fi
  if (( elapsed > 2 && ! announced )); then
    printf '[buildslot] waiting for build slot\n' >&2
    announced=1
  fi
  # Only the head needs low admission latency; non-head waiters polling at
  # 10Hz kept fd churn on the slot files high enough to defeat ghost scans.
  if (( is_head )); then
    sleep 0.1
  else
    sleep 1
  fi
done