#!/usr/bin/env bash # Buildbox-side recovery for seats interrupted by a reboot or a crash. # # seat-reconcile.sh boot every seat/job left over from an EARLIER boot # seat-reconcile.sh guard same, called from inside the transport's mirror lock # seat-reconcile.sh install put this script + its unit in place and arm it # # A seat runs as a transient systemd unit around a podman container; neither survives a reboot. # What survives is the disk: the seat record, the rb job dir, and the mirror working tree with the # agent's uncommitted edits in it. Recovery turns those three into a truthful terminal state. # # `guard` is the load-bearing one. The transport destroys a mirror working tree # (`git checkout -qf` + `git clean -qffd`) on the next dispatch, and that can beat the boot unit to # the tree. So the destroyer itself calls in here first, under the same lock, and this runs # fail-closed: no proof of reconcile for THIS boot means reconcile now. set -uo pipefail RB="$HOME/.rb" SEATS="$RB/seats" STAMP="$SEATS/.reconciled" UNIT_NAME=harness-seat-reconcile.service read -r BOOT_ID < /proc/sys/kernel/random/boot_id BTIME="$(awk '/^btime/{print $2}' /proc/stat)" log() { printf 'seat-reconcile: %s\n' "$1" >&2; } # Deliberately no node and no jq. This runs from a systemd --user unit, whose PATH has neither # (node reaches jobs through mise, which runner.sh sources and a unit does not), and from the # transport's mirror lock, where a missing interpreter would mean silently skipping the rescue. # The state file is flat, single-line, and written by seat-run.sh, so sed is sufficient — and # every field the reconciler reads is a string. jread() { # jread sed -n "s/.*\"$2\":\"\([^\"]*\)\".*/\1/p" "$1" 2>/dev/null | head -1 } mark_settled() { # mark_settled local f="$1" rc="$2" tmp="$1.tmp" sed -e 's/"phase":"[^"]*"/"phase":"rebooted"/' \ -e "s/\"rc\":[^,}]*/\"rc\":$rc/" \ -e "s/}[[:space:]]*$/,\"reconciledAt\":\"$(date -Is)\"}/" "$f" > "$tmp" \ && mv -f "$tmp" "$f" } # A job dir the watcher polls is only finalizable when it can never progress again: no rc, no unit, # and last touched before this boot. Writing rc into a job that is mid-dispatch would fake a result. finalize_job() { # finalize_job local dir="$RB/jobs/$1" tmp [[ -d "$dir" ]] || return 0 [[ -f "$dir/rc" || -f "$dir/rc.tmp" ]] && return 0 systemctl --user is-active --quiet "rb-$1.service" && return 0 printf '\n=== seat-reconcile: %s — the buildbox rebooted while this job was in flight ===\n' "$3" >> "$dir/log" 2>/dev/null tmp="$dir/.rc.reconcile.$$" printf '%s\n' "$2" > "$tmp" && mv -f "$tmp" "$dir/rc" log "job $1 finalized rc=$2 ($3)" } # The agent's uncommitted edits are the one thing that exists nowhere else. Park them in the # mirror's own object store under a ref, where `git clean` cannot reach them and the laptop can # fetch them back. Never touches HEAD or the index of the caller's repo. # HELD_MIRROR is the mirror whose mat lock our caller already owns; re-taking it would deadlock. # Every OTHER mirror must be locked here, or a rescue can read a tree a concurrent dispatch is # halfway through rewriting. HELD_MIRROR="" SKIPPED=0 rescue_tree() { # rescue_tree local mirror="$1" run="$2" [[ -d "$mirror/.git" ]] || return 0 if [[ "$mirror" != "$HELD_MIRROR" ]]; then # NON-blocking on purpose. We may already hold one mat lock, so waiting on a second could # deadlock against a peer holding them the other way round. A busy mirror is left for the peer # that holds it — and the pass then refuses to stamp, so that peer's own guard redoes it. local fd rc exec {fd}>"$RB/mat-$(basename "$mirror").lock" || return 1 if ! flock -n "$fd"; then log "seat $run: mirror $(basename "$mirror") locked by another dispatch — its guard will rescue" SKIPPED=$((SKIPPED + 1)); exec {fd}>&-; return 1 fi rescue_tree_locked "$mirror" "$run"; rc=$? exec {fd}>&- return $rc fi rescue_tree_locked "$mirror" "$run" } rescue_tree_locked() { local mirror="$1" run="$2" ref="refs/harness-seat/$2" tree parent commit git -C "$mirror" rev-parse --verify -q "$ref" >/dev/null && return 0 [[ -n "$(git -C "$mirror" status --porcelain 2>/dev/null)" ]] || { log "seat $run left a clean tree — nothing to rescue"; return 0; } # a private index: `git add -A` must not disturb the mirror's own index tree="$(GIT_INDEX_FILE="$mirror/.git/seat-rescue-$run.idx" bash -c ' git -C "$1" read-tree HEAD 2>/dev/null || git -C "$1" read-tree --empty git -C "$1" add -A 2>/dev/null git -C "$1" write-tree' _ "$mirror" 2>/dev/null | tail -1)" rm -f "$mirror/.git/seat-rescue-$run.idx" [[ "$tree" =~ ^[0-9a-f]{40}$ ]] || { log "seat $run rescue failed: no tree"; return 1; } parent="$(git -C "$mirror" rev-parse -q --verify HEAD 2>/dev/null)" # An explicit identity, not the box's git config: commit-tree refuses to run without one, and a # box with no user.name would silently lose the rescue. It is also honestly not the agent's # commit — the agent never committed this, the reconciler did. commit="$(GIT_AUTHOR_NAME="harness seat reconciler" GIT_AUTHOR_EMAIL="seat-reconcile@localhost" \ GIT_COMMITTER_NAME="harness seat reconciler" GIT_COMMITTER_EMAIL="seat-reconcile@localhost" \ git -C "$mirror" commit-tree "$tree" ${parent:+-p "$parent"} \ -m "harness seat $run: uncommitted work rescued after a buildbox reboot")" [[ "$commit" =~ ^[0-9a-f]{40}$ ]] || { log "seat $run rescue failed: no commit"; return 1; } git -C "$mirror" update-ref "$ref" "$commit" || return 1 log "seat $run rescued uncommitted work to $ref ($commit)" } reconcile_seats() { local state run phase boot mirror job shopt -s nullglob for state in "$SEATS"/*/state.json; do run="$(basename "$(dirname "$state")")" phase="$(jread "$state" phase)" boot="$(jread "$state" bootId)" [[ "$phase" == done || "$phase" == rebooted ]] && continue [[ -n "$boot" && "$boot" == "$BOOT_ID" ]] && continue # this boot's seat: still live, leave it mirror="$(jread "$state" mirrorPath)" # The seat is only settled once its work is safe. A skipped rescue leaves the record untouched # so the pass that can take the lock still finds it. if [[ -n "$mirror" ]] && ! rescue_tree "$mirror" "$run"; then continue; fi job="$(jread "$state" jobId)" [[ -n "$job" ]] && finalize_job "$job" 124 "seat $run" mark_settled "$state" 124 done shopt -u nullglob } # --rm reaps a seat that EXITS. A reboot exits nothing — every harness-seat-* container on disk # after a boot is a corpse. The name filter is deliberately narrow: other projects' containers on # these boxes are not ours to touch. reap_containers() { command -v podman >/dev/null 2>&1 || return 0 local ids ids="$(podman ps -a --filter 'name=harness-seat-' -q 2>/dev/null)" [[ -n "$ids" ]] || return 0 printf '%s\n' "$ids" | xargs -r podman rm -f >/dev/null 2>&1 log "reaped $(printf '%s\n' "$ids" | grep -c .) seat container(s) left by the previous boot" } # Plain (non-seat) rb jobs from an earlier boot: their transient unit is gone for good, so the # laptop's watcher would sit on a job that can never finish. 254 is the runner's own abnormal code. sweep_orphan_jobs() { local dir id shopt -s nullglob for dir in "$RB"/jobs/*/; do id="$(basename "$dir")" [[ -f "$dir/meta.json" ]] || continue [[ "$(stat -c %Y "$dir" 2>/dev/null || echo 0)" -lt "$BTIME" ]] || continue finalize_job "$id" 254 "job $id" done shopt -u nullglob } cmd_boot() { mkdir -p "$SEATS" "$RB/jobs" local fd exec {fd}>"$SEATS/.reconcile.lock" || return 1 flock -w 600 "$fd" || { log "another reconcile pass held the lock 600s"; return 1; } # a peer that finished the whole pass while we queued leaves nothing to do local seen=""; [[ -f "$STAMP" ]] && read -r seen < "$STAMP" if [[ "$seen" != "$BOOT_ID" ]]; then reconcile_seats reap_containers sweep_orphan_jobs # Stamping means "every seat from an earlier boot is settled". A skipped rescue makes that # false, so the stamp is withheld and the next guard repeats the pass. if [[ "$SKIPPED" -eq 0 ]]; then printf '%s\n' "$BOOT_ID" > "$STAMP.tmp" && mv -f "$STAMP.tmp" "$STAMP" else log "$SKIPPED seat(s) left for a peer — not stamping this boot as reconciled" fi fi exec {fd}>&- } # Called from inside the transport's ~/.rb/mat-.lock, so it must never take that lock. # Cheap on the hot path: one file read when the stamp already matches this boot. cmd_guard() { local seen="" [[ -f "$STAMP" ]] && read -r seen < "$STAMP" [[ "$seen" == "$BOOT_ID" ]] && return 0 HELD_MIRROR="$1" log "no reconcile recorded for this boot — reconciling before the mirror is rewritten" cmd_boot } cmd_install() { # cmd_install (the seat dir inside a repo mirror) local src="$1" unitdir="$HOME/.config/systemd/user" [[ -f "$src/seat-reconcile.sh" ]] || return 1 mkdir -p "$RB" "$unitdir" if ! cmp -s "$src/seat-reconcile.sh" "$RB/seat-reconcile.sh"; then install -m 0755 "$src/seat-reconcile.sh" "$RB/seat-reconcile.sh" || return 1 fi if ! cmp -s "$src/systemd/$UNIT_NAME" "$unitdir/$UNIT_NAME"; then install -m 0644 "$src/systemd/$UNIT_NAME" "$unitdir/$UNIT_NAME" || return 1 systemctl --user daemon-reload >/dev/null 2>&1 fi systemctl --user is-enabled --quiet "$UNIT_NAME" 2>/dev/null \ || systemctl --user enable "$UNIT_NAME" >/dev/null 2>&1 } # On-demand rescue for a mirror that is still alive (not a reboot recovery pass): a resident # session's dirty tree, parked to refs/harness-seat/ the same way a rebooted seat's is, so # the laptop can fetch it home with the SAME fetchRescueRef() path either way. Takes the mirror's # own mat lock (blocking — this is a deliberate one-off call, not the fail-closed boot guard, so # waiting behind a concurrent dispatch on the SAME mirror is correct instead of skipping). cmd_rescue() { # cmd_rescue local mirror="$1" run="$2" fd [[ -d "$mirror/.git" ]] || { log "rescue: not a git mirror: $mirror"; return 1; } exec {fd}>"$RB/mat-$(basename "$mirror").lock" || return 1 flock -w 60 "$fd" || { log "rescue: mirror $(basename "$mirror") lock held 60s"; exec {fd}>&-; return 1; } HELD_MIRROR="$mirror" rescue_tree_locked "$mirror" "$run" local rc=$? exec {fd}>&- return $rc } case "${1:-}" in boot) cmd_boot;; guard) [[ -n "${2:-}" ]] || { log "guard needs a mirror path"; exit 2; }; cmd_guard "$2";; install) [[ -n "${2:-}" ]] || { log "install needs a source dir"; exit 2; }; cmd_install "$2";; rescue) [[ -n "${2:-}" && -n "${3:-}" ]] || { log "rescue needs "; exit 2; }; cmd_rescue "$2" "$3";; *) printf 'usage: seat-reconcile.sh boot|guard |install |rescue \n' >&2; exit 2;; esac