#!/usr/bin/env bash
# Runs AS ROOT on a build box, from the directory this file lives in (shipped there
# by bin/buildbox harden). Installs the unattended-failsafe host configuration and
# reads the effective state back. Idempotent.
#
# Deliberately NOT under modules/*/systemd/system or modules/*/system-sbin: those
# globs are what lib/deckctl/system-units.sh copies onto the workstation, and none
# of this belongs on a machine that has a keyboard.
set -euo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=change-id.sh
. "$HERE/change-id.sh"

SCRATCH_LABEL=buildbox-scratch
SCRATCH_MNT=/var/lib/buildbox
SCRATCH_USER=user
# debian1's factory NTFS carries 112 MB of System Volume Information and nothing else.
# The threshold passes that and fails anything a person would miss.
SCRATCH_DATA_MAX=$((512 * 1024 * 1024))
SCRATCH_BEGIN='# >>> buildbox scratch disk, declared by host-config/apply.sh >>>'
SCRATCH_END='# <<< buildbox scratch disk <<<'

# Declared state, byte-identical on every box. LABEL= not UUID= so the text does not
# differ per host; nofail so a dead scratch disk degrades the box instead of stopping
# its boot with nobody there to press a key.
scratch_fstab() {
  cat <<'FSTAB'
LABEL=buildbox-scratch  /var/lib/buildbox  ext4  defaults,noatime,nofail,x-systemd.device-timeout=10s  0  2
/var/lib/buildbox/builds        /home/user/builds                  none  bind,nofail,x-systemd.requires=/var/lib/buildbox  0 0
/var/lib/buildbox/pnpm-store    /home/user/.local/share/pnpm       none  bind,nofail,x-systemd.requires=/var/lib/buildbox  0 0
/var/lib/buildbox/npm           /home/user/.npm                    none  bind,nofail,x-systemd.requires=/var/lib/buildbox  0 0
/var/lib/buildbox/cargo-registry /home/user/.cargo/registry        none  bind,nofail,x-systemd.requires=/var/lib/buildbox  0 0
/var/lib/buildbox/dev-tools     /home/user/.dev-tools              none  bind,nofail,x-systemd.requires=/var/lib/buildbox  0 0
/var/lib/buildbox/runs          /home/user/runs                    none  bind,nofail,x-systemd.requires=/var/lib/buildbox  0 0
/var/lib/buildbox/playwright    /home/user/.cache/ms-playwright    none  bind,nofail,x-systemd.requires=/var/lib/buildbox  0 0
FSTAB
  local runner target name runner_home="${SCRATCH_RUNNER_HOME:-/home/user}"
  for runner in "$runner_home"/actions-runner-*; do
    [ -d "$runner" ] || continue
    [ ! -L "$runner" ] || { echo "apply.sh: runner path is a symlink: $runner" >&2; return 1; }
    target="$runner/_work"
    [ ! -L "$target" ] || { echo "apply.sh: runner work path is a symlink: $target" >&2; return 1; }
    name="${runner##*/}"
    [[ "$name" =~ ^actions-runner-[A-Za-z0-9._-]+$ ]] \
      || { printf 'apply.sh: invalid runner name: %q\n' "$name" >&2; return 1; }
    printf '/var/lib/buildbox/runner-work/%s  %s  none  bind,nofail,x-systemd.requires=/var/lib/buildbox  0 0\n' "$name" "$target"
  done
}

scratch_binds() { scratch_fstab | awk 'NR > 1 { print $1, $2 }'; }

# Used bytes across every filesystem on a disk. A filesystem that will not mount
# read-only is reported as unreadable, never as empty.
scratch_probe_used() {
  local dev="$1" name fstype used total=0 mp
  mp=$(mktemp -d)
  trap 'mountpoint -q "$mp" && umount "$mp"; rmdir "$mp"' RETURN
  while read -r name fstype; do
    [ -n "$fstype" ] || continue
    case "$fstype" in
      swap|crypto_LUKS|LVM2_member|linux_raid_member)
        echo "/dev/$name is $fstype, whose contents cannot be read here"; return 1;;
    esac
    mount -o ro "/dev/$name" "$mp" 2>/dev/null \
      || { echo "/dev/$name holds $fstype and will not mount read-only"; return 1; }
    used=$(df -B1 --output=used "$mp" | tail -1 | tr -dc '0-9')
    umount "$mp"
    total=$((total + ${used:-0}))
  done < <(lsblk -nr -o NAME,FSTYPE "$dev")
  echo "$total"
}

# Prints "CANDIDATE <dev> used=<bytes>", or "REFUSE <why>" with a non-zero status.
# Zero candidates and more than one candidate are both refusals: these boxes are
# headless with no console, so a wrong device here has no undo.
scratch_discover() {
  local root_disk name dev used types cands=()
  root_disk=$(lsblk -no PKNAME "$(findmnt -no SOURCE /)" 2>/dev/null | awk 'NR == 1')
  [ -n "$root_disk" ] || { echo "REFUSE cannot resolve which disk carries /"; return 1; }

  while read -r name; do
    dev="/dev/$name"
    [ "$name" = "$root_disk" ] && continue
    [ "$(lsblk -nr -o MOUNTPOINTS "$dev" | grep -c .)" = 0 ] || continue
    lsblk -nr -o LABEL "$dev" | grep -qx "$SCRATCH_LABEL" && continue
    cands+=("$dev")
  done < <(lsblk -dn -o NAME,TYPE | awk '$2 == "disk" { print $1 }')

  if [ "${#cands[@]}" -eq 0 ]; then
    echo "REFUSE no candidate: every disk carries /, is mounted, or is already $SCRATCH_LABEL"
    return 1
  fi
  if [ "${#cands[@]}" -gt 1 ]; then
    echo "REFUSE ${#cands[@]} candidates (${cands[*]}) — never guess which one is scratch"
    return 1
  fi

  dev="${cands[0]}"
  used=$(scratch_probe_used "$dev") || { echo "REFUSE $dev: $used"; return 1; }
  if [ "$used" -gt "$SCRATCH_DATA_MAX" ]; then
    types=$(lsblk -nr -o FSTYPE "$dev" | grep -v '^$' | paste -sd, -)
    echo "REFUSE $dev holds $used bytes of $types, above the $SCRATCH_DATA_MAX byte threshold"
    return 1
  fi
  echo "CANDIDATE $dev used=$used"
}

if [ "${1:-}" = --discover-scratch ]; then scratch_discover; exit; fi
if [ "${1:-}" = --scratch-fstab ]; then scratch_fstab; exit; fi

[ "$(id -u)" = 0 ] || { echo "apply.sh: must run as root" >&2; exit 2; }

install_tree() { # $1=source subdir  $2=target dir  $3=mode
  local src="$HERE/$1" dst="$2" mode="$3" f rel
  [ -d "$src" ] || return 0
  while IFS= read -r -d '' f; do
    rel="${f#"$src"/}"
    install -D -m "$mode" -o root -g root "$f" "$dst/$rel"
    echo "INSTALL $dst/$rel"
  done < <(find "$src" -type f -print0 | sort -z)
}

install_tree sysctl.d           /etc/sysctl.d                 0644
install_tree system-conf.d      /etc/systemd/system.conf.d    0644
install_tree journald.conf.d    /etc/systemd/journald.conf.d  0644
install_tree systemd-system     /etc/systemd/system           0644
install_tree k3s/config.yaml.d /etc/rancher/k3s/config.yaml.d 0644
install -D -m 0644 -o root -g root "$HERE/k3s/registries.yaml" /etc/rancher/k3s/registries.yaml
echo "INSTALL /etc/rancher/k3s/registries.yaml"
install -D -m 0644 -o root -g root "$HERE/lib/disk-admission.sh" /usr/local/lib/buildbox/disk-admission.sh
echo "INSTALL /usr/local/lib/buildbox/disk-admission.sh"

# A hand-run `systemctl set-property --runtime user@1000.service ManagedOOM…` left a
# drop-in under /run that shadows the declared config and dies at the next reboot.
# The /etc drop-in installed above replaces it; drop the runtime copy so effective
# state equals declared state.
rm -rf /run/systemd/system.control/user@1000.service.d

# superseded by sysctl.d/99-buildbox-failsafe.conf, and it sorts AFTER it ('-' < '.'),
# so a hand-edited copy would silently win over the declared one
rm -f /etc/sysctl.d/99-buildbox.conf

# Rootless-podman enablement for agent seat users (ods-*): each needs a subordinate
# uid/gid range (user namespaces) and a lingering user manager (runtime dir without a
# login session). Ranges are allocated 64K-aligned above every existing allocation, so
# re-runs and seat users added later converge without overlap.
next_sub_base() {
  local max=$((1000000 - 65536)) f start count
  for f in /etc/subuid /etc/subgid; do
    [ -r "$f" ] || continue
    while IFS=: read -r _ start count; do
      [ $((start + count)) -gt "$max" ] && max=$((start + count))
    done < "$f"
  done
  echo $(( (max + 65535) / 65536 * 65536 ))
}
while IFS=: read -r seat _; do
  if ! grep -q "^$seat:" /etc/subuid 2>/dev/null; then
    base=$(next_sub_base)
    usermod --add-subuids "$base-$((base + 65535))" "$seat"
    echo "SUBUID $seat $base"
  fi
  if ! grep -q "^$seat:" /etc/subgid 2>/dev/null; then
    base=$(next_sub_base)
    usermod --add-subgids "$base-$((base + 65535))" "$seat"
    echo "SUBGID $seat $base"
  fi
  loginctl enable-linger "$seat"
done < <(getent passwd | awk -F: '$1 ~ /^ods-/')

# Every launcher reaches a box with `ssh -F /dev/null` on the single port in
# build-remote.json, so the port is fleet state, not a per-box choice. sshd binds the
# wildcards rather than this node's tailnet addresses: a ListenAddress that stops
# existing takes the daemon down with it, and these boxes have no console. The timer
# re-runs bin/buildbox-sshd-access so a box that loses its listener repairs itself.
RESCUE_DOOR_DIR=/etc/buildbox-rescue-door

# The recovery door: a second ssh listener on 2223 that shares no state with the
# converged one. Its config is outside /etc/ssh, its host key is its own, and PID 1
# owns the bind, so the change class that took debian2 and debian3 off the network —
# a rewrite under /etc/ssh/sshd_config.d that sshd cannot bind at the next cold boot —
# cannot reach it. Converged BEFORE converge_sshd_access so the new door is already up
# before anything touches the door in use.
converge_rescue_door() {
  install -D -m 0755 -o root -g root "$HERE/bin/buildbox-rescue-door" /usr/local/sbin/buildbox-rescue-door
  echo "INSTALL /usr/local/sbin/buildbox-rescue-door"
  /usr/local/sbin/buildbox-rescue-door "$HERE/rescue-door/sshd_config"
}

converge_tailscale_ssh() {
  local rejected
  systemctl enable --now buildbox-tailscale-ssh.timer >/dev/null
  # Not fatal: on a box being enrolled, harden runs before tailscaled necessarily holds
  # an authenticated node, and aborting the whole converge here would leave the box
  # without the sshd door that converge_sshd_access installs below. The timer owns the
  # guarantee — it retries at OnBootSec=120s and every 15min, which is also what
  # recovers this door after a node re-registration resets tailscaled's prefs.
  if ! rejected=$(systemctl start buildbox-tailscale-ssh.service 2>&1); then
    echo "apply.sh: Tailscale SSH not enabled yet — one door fewer until the timer converges it" >&2
    printf '%s\n' "$rejected" >&2
  fi
}

converge_sshd_access() {
  # Superseded by ssh.service.d/51-bind-retry.conf, which carries the same restart
  # policy without ordering sshd behind tailscaled. It sorts BEFORE 51, so its
  # After=/Wants= would still delay every sshd start behind a daemon sshd no longer
  # needs to bind.
  rm -f /etc/systemd/system/ssh.service.d/10-tailscale-ordering.conf
  install -D -m 0755 -o root -g root "$HERE/bin/buildbox-sshd-access" /usr/local/sbin/buildbox-sshd-access
  echo "INSTALL /usr/local/sbin/buildbox-sshd-access"
  # Vendor-preset enabled: were ssh.socket ever to activate, its ListenStream would
  # govern and the Port/ListenAddress drop-in would be silently ignored.
  systemctl mask ssh.socket >/dev/null 2>&1 || true
  systemctl enable --now buildbox-sshd-access.timer >/dev/null
  /usr/local/sbin/buildbox-sshd-access
}

scratch_format() {
  local dev="$1" part
  wipefs -a "$dev" >/dev/null
  printf 'label: gpt\ntype=0FC63DAF-8483-4772-8E79-3D69D8477DE4\n' | sfdisk --quiet "$dev"
  udevadm settle
  part=$(lsblk -nr -o NAME,TYPE "$dev" | awk '$2 == "part" && !seen++ { print "/dev/" $1 }')
  [ -n "$part" ] || { echo "apply.sh: sfdisk left no partition on $dev" >&2; return 1; }
  # -m 0: this is scratch, and the default 5% reserve is ~5.5G of pure loss
  mkfs.ext4 -q -F -m 0 -L "$SCRATCH_LABEL" "$part"
  udevadm settle
  echo "SCRATCH format $part ext4 LABEL=$SCRATCH_LABEL"
}

scratch_fstab_converge() {
  local new verify
  new=$(mktemp)
  awk -v b="$SCRATCH_BEGIN" -v e="$SCRATCH_END" '
    $0 == b { skip = 1 } !skip { print } $0 == e { skip = 0 }' /etc/fstab >"$new"
  { printf '%s\n' "$SCRATCH_BEGIN"; scratch_fstab; printf '%s\n' "$SCRATCH_END"; } >>"$new"
  if ! verify=$(findmnt --verify --tab-file "$new" 2>&1); then
    echo "apply.sh: rejected fstab candidate" >&2; printf '%s\n' "$verify" >&2
    rm -f "$new"; return 1
  fi
  install -m 0644 -o root -g root "$new" /etc/fstab
  rm -f "$new"
  systemctl daemon-reload
  echo "SCRATCH fstab $(scratch_fstab | grep -c .) entries"
}

# Runner services and agent seats write into every bind target, so they are stopped
# for the one-time copy and started again afterwards.
scratch_quiesce_units=""
scratch_user_units=""
scratch_user_systemctl() {
  runuser -u "$SCRATCH_USER" -- env "XDG_RUNTIME_DIR=/run/user/$(id -u "$SCRATCH_USER")" systemctl --user "$@"
}
scratch_quiesce() {
  scratch_quiesce_units=$(systemctl list-units --type=service --state=active --no-legend --plain 'actions.runner.*' | awk '{print $1}')
  scratch_user_units=$(scratch_user_systemctl list-units --type=service --state=active --no-legend --plain '*runner*' | awk '{print $1}')
  if [ -n "$scratch_quiesce_units" ]; then systemctl stop $scratch_quiesce_units; fi
  if [ -n "$scratch_user_units" ]; then scratch_user_systemctl stop $scratch_user_units; fi
}
scratch_resume() {
  if [ -n "$scratch_user_units" ]; then scratch_user_systemctl start $scratch_user_units; fi
  if [ -n "$scratch_quiesce_units" ]; then systemctl start $scratch_quiesce_units; fi
}

scratch_missing_paths() {
  LC_ALL=C comm -23 <(cd "$1" && find . -mindepth 1 -printf '%P\n' | LC_ALL=C sort) \
                    <(cd "$2" && find . -mindepth 1 -printf '%P\n' | LC_ALL=C sort)
}

scratch_mkdir_user() {
  local d="$1"
  [ -d "$d" ] && return 0
  scratch_mkdir_user "$(dirname "$d")"
  install -d -o "$SCRATCH_USER" -g "$SCRATCH_USER" -m 0755 "$d"
}

# The home directory is renamed rather than emptied in place: once the bind is mounted
# the original content is unreachable through that path. An interrupted run therefore
# leaves the copy under <target>.migrated — wasteful, never lost — and re-runs resume.
scratch_migrate_one() {
  local src="$1" target="$2" stash="$2.migrated"
  if [ ! -e "$stash" ]; then
    mountpoint -q "$target" && return 0
    if [ -d "$target" ] && [ -n "$(ls -A "$target" 2>/dev/null)" ]; then
      mv "$target" "$stash"
    else
      rmdir "$target" 2>/dev/null || true
      scratch_mkdir_user "$target"
      mount --bind "$src" "$target"
      echo "SCRATCH bind  $target (nothing to migrate)"
      return 0
    fi
  fi
  if ! mountpoint -q "$target"; then
    rsync -aHAX --delete "$stash/" "$src/"
    scratch_mkdir_user "$target"
    mount --bind "$src" "$target"
  fi
  # The stash is deleted only once every path in it also exists on the scratch disk.
  # Presence, not content: remote-build reaches these directories over ssh from the
  # workstation and cannot be quiesced from here, so a path that exists in both is the
  # live copy being newer, while a path that exists only in the stash is a loss.
  local missing
  missing=$(scratch_missing_paths "$stash" "$target")
  if [ -n "$missing" ]; then
    rsync -aHAX --ignore-existing "$stash/" "$target/"
    missing=$(scratch_missing_paths "$stash" "$target")
  fi
  if [ -n "$missing" ]; then
    echo "apply.sh: $stash holds paths missing from $target; leaving both in place" >&2
    head -20 <<<"$missing" >&2
    return 1
  fi
  rm -rf "$stash"
  echo "SCRATCH bind  $target (migrated)"
}

converge_scratch_disk() {
  local verdict dev src target pending="" rc=0
  if ! blkid -L "$SCRATCH_LABEL" >/dev/null 2>&1; then
    verdict=$(scratch_discover) || { printf 'apply.sh: %s\n' "$verdict" >&2; return 1; }
    dev="${verdict#CANDIDATE }"; dev="${dev%% *}"
    scratch_format "$dev"
  fi

  install -d -o root -g root -m 0755 "$SCRATCH_MNT"
  mountpoint -q "$SCRATCH_MNT" || mount -L "$SCRATCH_LABEL" "$SCRATCH_MNT"

  while read -r src target; do
    scratch_mkdir_user "$src"
    # every source and target exists before fstab is rewritten: findmnt --verify rejects a
    # line whose source or target is missing, nofail or not
    scratch_mkdir_user "$target"
    if ! mountpoint -q "$target" || [ -e "$target.migrated" ]; then
      pending+="$src $target"$'\n'
    fi
  done < <(scratch_binds)

  scratch_fstab_converge

  if [ -n "$pending" ]; then
    scratch_quiesce
    while read -r src target; do
      [ -n "$src" ] || continue
      scratch_migrate_one "$src" "$target" || rc=1
    done <<<"$pending"
    scratch_resume
  fi
  return "$rc"
}

sysctl --system >/dev/null
# daemon-reload re-reads system.conf.d (re-arming the hardware watchdog) and
# re-applies cgroup attributes to running units, so the new MemoryMin reserves take
# effect without restarting sshd out from under the session applying this. It precedes
# converge_sshd_access because that reloads ssh, and reloading a unit whose drop-ins were
# just rewritten warns until the manager has re-read them.
systemctl daemon-reload

converge_rescue_door
converge_tailscale_ssh
converge_sshd_access
converge_scratch_disk

systemctl restart systemd-journald.service
systemctl restart systemd-oomd.service

# Written only after the whole apply succeeded, so a host that half-converged reports the
# older id and audit calls it drifted rather than current.
install -d -m 0755 /etc/buildbox
printf 'change=%s\napplied=%s\n' \
  "$(buildbox_change_id "$HERE")" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >/etc/buildbox/host-config.stamp
chmod 0644 /etc/buildbox/host-config.stamp

echo "--- readback"
sysctl -n kernel.panic kernel.panic_on_oops kernel.sysrq vm.swappiness vm.panic_on_oom \
  | paste -d' ' <(printf 'kernel.panic\nkernel.panic_on_oops\nkernel.sysrq\nvm.swappiness\nvm.panic_on_oom\n') -
systemctl show -p RuntimeWatchdogUSec -p RebootWatchdogUSec
echo "watchdog0.state=$(cat /sys/class/watchdog/watchdog0/state 2>/dev/null || echo none)"
while IFS=: read -r seat _; do
  echo "seat $seat subuid=$(grep -c "^$seat:" /etc/subuid) subgid=$(grep -c "^$seat:" /etc/subgid) linger=$(loginctl show-user "$seat" -p Linger --value 2>/dev/null || echo no)"
done < <(getent passwd | awk -F: '$1 ~ /^ods-/')
echo "watchdog0.timeout=$(cat /sys/class/watchdog/watchdog0/timeout 2>/dev/null || echo none)"
ss -H -lnt 'sport = :2222' | awk '{print "sshd listening " $4}'
ss -H -lnt 'sport = :2223' | awk '{print "rescue door listening " $4}'
echo "rescue door key $(ssh-keygen -lf "$RESCUE_DOOR_DIR/ssh_host_ed25519_key.pub" 2>/dev/null || echo missing)"
echo "tailscale-ssh RunSSH=$(tailscale debug prefs 2>/dev/null | sed -n 's/.*"RunSSH": *\([a-z]*\).*/\1/p')"
systemctl show ssh.service -p MemoryMin -p OOMScoreAdjust
echo "ssh.service memory.min=$(cat /sys/fs/cgroup/system.slice/ssh.service/memory.min)"
echo "system.slice memory.min=$(cat /sys/fs/cgroup/system.slice/memory.min)"
echo "user.slice memory.min=$(cat /sys/fs/cgroup/user.slice/memory.min)"
echo "user-1000.slice memory.min=$(cat /sys/fs/cgroup/user.slice/user-1000.slice/memory.min)"
find /sys/fs/cgroup/user.slice/user-1000.slice -maxdepth 1 -name 'session-*.scope' \
  -exec sh -c 'echo "$1 memory.min=$(cat "$1/memory.min")"' _ {} \;
systemctl show user@1000.service -p ManagedOOMSwap -p ManagedOOMMemoryPressure -p ManagedOOMMemoryPressureLimit
findmnt -no SOURCE,TARGET,FSTYPE,SIZE,AVAIL --mountpoint "$SCRATCH_MNT" \
  | awk '{print "scratch " $0}'
while read -r _ target; do
  printf 'scratch bind %s %s\n' "$target" "$(findmnt -no SOURCE --mountpoint "$target" 2>/dev/null || echo UNMOUNTED)"
done < <(scratch_binds)
journalctl --disk-usage
for slice in agent.slice build.slice; do
  echo "oomd monitored lists containing $slice: $(oomctl | grep -c "/user@1000.service/$slice\$") (0 until a seat loads the slice)"
done