#!/usr/bin/env bash
# Buildbox-side recovery for seats interrupted by a reboot or a crash.
#
#   seat-reconcile.sh boot                 every seat/job left over from an EARLIER boot
#   seat-reconcile.sh guard <mirrorPath>   same, called from inside the transport's mirror lock
#   seat-reconcile.sh install <srcDir>     put this script + its unit in place and arm it
#
# A seat runs as a transient systemd unit around a podman container; neither survives a reboot.
# What survives is the disk: the seat record, the rb job dir, and the mirror working tree with the
# agent's uncommitted edits in it. Recovery turns those three into a truthful terminal state.
#
# `guard` is the load-bearing one. The transport destroys a mirror working tree
# (`git checkout -qf` + `git clean -qffd`) on the next dispatch, and that can beat the boot unit to
# the tree. So the destroyer itself calls in here first, under the same lock, and this runs
# fail-closed: no proof of reconcile for THIS boot means reconcile now.
set -uo pipefail

RB="$HOME/.rb"
SEATS="$RB/seats"
STAMP="$SEATS/.reconciled"
UNIT_NAME=harness-seat-reconcile.service

read -r BOOT_ID < /proc/sys/kernel/random/boot_id
BTIME="$(awk '/^btime/{print $2}' /proc/stat)"

log() { printf 'seat-reconcile: %s\n' "$1" >&2; }

# Deliberately no node and no jq. This runs from a systemd --user unit, whose PATH has neither
# (node reaches jobs through mise, which runner.sh sources and a unit does not), and from the
# transport's mirror lock, where a missing interpreter would mean silently skipping the rescue.
# The state file is flat, single-line, and written by seat-run.sh, so sed is sufficient — and
# every field the reconciler reads is a string.
jread() { # jread <file> <key>
  sed -n "s/.*\"$2\":\"\([^\"]*\)\".*/\1/p" "$1" 2>/dev/null | head -1
}
mark_settled() { # mark_settled <file> <rc>
  local f="$1" rc="$2" tmp="$1.tmp"
  sed -e 's/"phase":"[^"]*"/"phase":"rebooted"/' \
      -e "s/\"rc\":[^,}]*/\"rc\":$rc/" \
      -e "s/}[[:space:]]*$/,\"reconciledAt\":\"$(date -Is)\"}/" "$f" > "$tmp" \
    && mv -f "$tmp" "$f"
}

# A job dir the watcher polls is only finalizable when it can never progress again: no rc, no unit,
# and last touched before this boot. Writing rc into a job that is mid-dispatch would fake a result.
finalize_job() { # finalize_job <jobId> <rc> <reason>
  local dir="$RB/jobs/$1" tmp
  [[ -d "$dir" ]] || return 0
  [[ -f "$dir/rc" || -f "$dir/rc.tmp" ]] && return 0
  systemctl --user is-active --quiet "rb-$1.service" && return 0
  printf '\n=== seat-reconcile: %s — the buildbox rebooted while this job was in flight ===\n' "$3" >> "$dir/log" 2>/dev/null
  tmp="$dir/.rc.reconcile.$$"
  printf '%s\n' "$2" > "$tmp" && mv -f "$tmp" "$dir/rc"
  log "job $1 finalized rc=$2 ($3)"
}

# The agent's uncommitted edits are the one thing that exists nowhere else. Park them in the
# mirror's own object store under a ref, where `git clean` cannot reach them and the laptop can
# fetch them back. Never touches HEAD or the index of the caller's repo.
# HELD_MIRROR is the mirror whose mat lock our caller already owns; re-taking it would deadlock.
# Every OTHER mirror must be locked here, or a rescue can read a tree a concurrent dispatch is
# halfway through rewriting.
HELD_MIRROR=""
SKIPPED=0
rescue_tree() { # rescue_tree <mirrorPath> <runId>
  local mirror="$1" run="$2"
  [[ -d "$mirror/.git" ]] || return 0
  if [[ "$mirror" != "$HELD_MIRROR" ]]; then
    # NON-blocking on purpose. We may already hold one mat lock, so waiting on a second could
    # deadlock against a peer holding them the other way round. A busy mirror is left for the peer
    # that holds it — and the pass then refuses to stamp, so that peer's own guard redoes it.
    local fd rc
    exec {fd}>"$RB/mat-$(basename "$mirror").lock" || return 1
    if ! flock -n "$fd"; then
      log "seat $run: mirror $(basename "$mirror") locked by another dispatch — its guard will rescue"
      SKIPPED=$((SKIPPED + 1)); exec {fd}>&-; return 1
    fi
    rescue_tree_locked "$mirror" "$run"; rc=$?
    exec {fd}>&-
    return $rc
  fi
  rescue_tree_locked "$mirror" "$run"
}
rescue_tree_locked() {
  local mirror="$1" run="$2" ref="refs/harness-seat/$2" tree parent commit
  git -C "$mirror" rev-parse --verify -q "$ref" >/dev/null && return 0
  [[ -n "$(git -C "$mirror" status --porcelain 2>/dev/null)" ]] || { log "seat $run left a clean tree — nothing to rescue"; return 0; }
  # a private index: `git add -A` must not disturb the mirror's own index
  tree="$(GIT_INDEX_FILE="$mirror/.git/seat-rescue-$run.idx" bash -c '
    git -C "$1" read-tree HEAD 2>/dev/null || git -C "$1" read-tree --empty
    git -C "$1" add -A 2>/dev/null
    git -C "$1" write-tree' _ "$mirror" 2>/dev/null | tail -1)"
  rm -f "$mirror/.git/seat-rescue-$run.idx"
  [[ "$tree" =~ ^[0-9a-f]{40}$ ]] || { log "seat $run rescue failed: no tree"; return 1; }
  parent="$(git -C "$mirror" rev-parse -q --verify HEAD 2>/dev/null)"
  # An explicit identity, not the box's git config: commit-tree refuses to run without one, and a
  # box with no user.name would silently lose the rescue. It is also honestly not the agent's
  # commit — the agent never committed this, the reconciler did.
  commit="$(GIT_AUTHOR_NAME="harness seat reconciler" GIT_AUTHOR_EMAIL="seat-reconcile@localhost" \
            GIT_COMMITTER_NAME="harness seat reconciler" GIT_COMMITTER_EMAIL="seat-reconcile@localhost" \
            git -C "$mirror" commit-tree "$tree" ${parent:+-p "$parent"} \
              -m "harness seat $run: uncommitted work rescued after a buildbox reboot")"
  [[ "$commit" =~ ^[0-9a-f]{40}$ ]] || { log "seat $run rescue failed: no commit"; return 1; }
  git -C "$mirror" update-ref "$ref" "$commit" || return 1
  log "seat $run rescued uncommitted work to $ref ($commit)"
}

reconcile_seats() {
  local state run phase boot mirror job
  shopt -s nullglob
  for state in "$SEATS"/*/state.json; do
    run="$(basename "$(dirname "$state")")"
    phase="$(jread "$state" phase)"
    boot="$(jread "$state" bootId)"
    [[ "$phase" == done || "$phase" == rebooted ]] && continue
    [[ -n "$boot" && "$boot" == "$BOOT_ID" ]] && continue   # this boot's seat: still live, leave it
    mirror="$(jread "$state" mirrorPath)"
    # The seat is only settled once its work is safe. A skipped rescue leaves the record untouched
    # so the pass that can take the lock still finds it.
    if [[ -n "$mirror" ]] && ! rescue_tree "$mirror" "$run"; then continue; fi
    job="$(jread "$state" jobId)"
    [[ -n "$job" ]] && finalize_job "$job" 124 "seat $run"
    mark_settled "$state" 124
  done
  shopt -u nullglob
}

# --rm reaps a seat that EXITS. A reboot exits nothing — every harness-seat-* container on disk
# after a boot is a corpse. The name filter is deliberately narrow: other projects' containers on
# these boxes are not ours to touch.
reap_containers() {
  command -v podman >/dev/null 2>&1 || return 0
  local ids
  ids="$(podman ps -a --filter 'name=harness-seat-' -q 2>/dev/null)"
  [[ -n "$ids" ]] || return 0
  printf '%s\n' "$ids" | xargs -r podman rm -f >/dev/null 2>&1
  log "reaped $(printf '%s\n' "$ids" | grep -c .) seat container(s) left by the previous boot"
}

# Plain (non-seat) rb jobs from an earlier boot: their transient unit is gone for good, so the
# laptop's watcher would sit on a job that can never finish. 254 is the runner's own abnormal code.
sweep_orphan_jobs() {
  local dir id
  shopt -s nullglob
  for dir in "$RB"/jobs/*/; do
    id="$(basename "$dir")"
    [[ -f "$dir/meta.json" ]] || continue
    [[ "$(stat -c %Y "$dir" 2>/dev/null || echo 0)" -lt "$BTIME" ]] || continue
    finalize_job "$id" 254 "job $id"
  done
  shopt -u nullglob
}

cmd_boot() {
  mkdir -p "$SEATS" "$RB/jobs"
  local fd
  exec {fd}>"$SEATS/.reconcile.lock" || return 1
  flock -w 600 "$fd" || { log "another reconcile pass held the lock 600s"; return 1; }
  # a peer that finished the whole pass while we queued leaves nothing to do
  local seen=""; [[ -f "$STAMP" ]] && read -r seen < "$STAMP"
  if [[ "$seen" != "$BOOT_ID" ]]; then
    reconcile_seats
    reap_containers
    sweep_orphan_jobs
    # Stamping means "every seat from an earlier boot is settled". A skipped rescue makes that
    # false, so the stamp is withheld and the next guard repeats the pass.
    if [[ "$SKIPPED" -eq 0 ]]; then
      printf '%s\n' "$BOOT_ID" > "$STAMP.tmp" && mv -f "$STAMP.tmp" "$STAMP"
    else
      log "$SKIPPED seat(s) left for a peer — not stamping this boot as reconciled"
    fi
  fi
  exec {fd}>&-
}

# Called from inside the transport's ~/.rb/mat-<mirror>.lock, so it must never take that lock.
# Cheap on the hot path: one file read when the stamp already matches this boot.
cmd_guard() {
  local seen=""
  [[ -f "$STAMP" ]] && read -r seen < "$STAMP"
  [[ "$seen" == "$BOOT_ID" ]] && return 0
  HELD_MIRROR="$1"
  log "no reconcile recorded for this boot — reconciling before the mirror is rewritten"
  cmd_boot
}

cmd_install() { # cmd_install <srcDir>  (the seat dir inside a repo mirror)
  local src="$1" unitdir="$HOME/.config/systemd/user"
  [[ -f "$src/seat-reconcile.sh" ]] || return 1
  mkdir -p "$RB" "$unitdir"
  if ! cmp -s "$src/seat-reconcile.sh" "$RB/seat-reconcile.sh"; then
    install -m 0755 "$src/seat-reconcile.sh" "$RB/seat-reconcile.sh" || return 1
  fi
  if ! cmp -s "$src/systemd/$UNIT_NAME" "$unitdir/$UNIT_NAME"; then
    install -m 0644 "$src/systemd/$UNIT_NAME" "$unitdir/$UNIT_NAME" || return 1
    systemctl --user daemon-reload >/dev/null 2>&1
  fi
  systemctl --user is-enabled --quiet "$UNIT_NAME" 2>/dev/null \
    || systemctl --user enable "$UNIT_NAME" >/dev/null 2>&1
}

case "${1:-}" in
  boot)    cmd_boot;;
  guard)   [[ -n "${2:-}" ]] || { log "guard needs a mirror path"; exit 2; }; cmd_guard "$2";;
  install) [[ -n "${2:-}" ]] || { log "install needs a source dir"; exit 2; }; cmd_install "$2";;
  *) printf 'usage: seat-reconcile.sh boot|guard <mirrorPath>|install <srcDir>\n' >&2; exit 2;;
esac
