#!/usr/bin/env bash
# buildbox-headless.sh — convert a Debian workstation into a headless build-cluster node.
# Idempotent, fail-closed, re-runnable. RUNS ON THE TARGET BOX (not the laptop).
#
# Provision a node:
#   scp -F /dev/null -o BatchMode=yes -i <key> -P <port> buildbox-headless.sh root@<host>:/tmp/
#   ssh -F /dev/null -o BatchMode=yes -i <key> -p <port> root@<host> 'bash /tmp/buildbox-headless.sh'
#
# Does NOT uninstall packages (chromium kept for headless playwright).
# Does NOT reboot. Applies every change LIVE + persistent-for-boot.
# REBOOT ACCEPTANCE TEST IS DEFERRED: run it only with a human near the box
# (failed boot on a remote node with no console = node lost). After the first
# reboot, verify: multi-user.target reached, no Xorg/lightdm procs, sshd:2222 up,
# tailscale up, linger yes, GH runner active, crontab intact, local-gate --remote-doctor PASS.

set -euo pipefail
export PATH=/usr/sbin:/usr/bin:/sbin:/bin:$PATH

log()  { printf '  %s\n' "$*"; }
sect() { printf '\n===== %s =====\n' "$*"; }

[ "$(id -u)" -eq 0 ] || { echo "FATAL: must run as root" >&2; exit 1; }

# Detect the build user (linger-enabled non-system uid running rb-* units).
BUILD_USER="${BUILD_USER:-user}"
BUILD_UID="$(id -u "$BUILD_USER" 2>/dev/null || echo '')"
[ -n "$BUILD_UID" ] || { echo "FATAL: build user '$BUILD_USER' not found" >&2; exit 1; }
USER_RUNTIME="/run/user/${BUILD_UID}"

uctl() { runuser -l "$BUILD_USER" -c "XDG_RUNTIME_DIR=${USER_RUNTIME} systemctl --user $*"; }

# ---------------------------------------------------------------------------
sect "1. KILL GRAPHICAL STACK"
systemctl set-default multi-user.target >/dev/null
log "default target -> $(systemctl get-default)"

# display-manager.service is an alias -> the concrete DM unit; disable whichever exists.
for dm in lightdm gdm3 gdm sddm; do
  if systemctl cat "${dm}.service" >/dev/null 2>&1; then
    systemctl disable "${dm}.service" >/dev/null 2>&1 || true
    systemctl stop    "${dm}.service" >/dev/null 2>&1 || true
    log "disabled+stopped ${dm}.service"
  fi
done

# ---------------------------------------------------------------------------
sect "2. TRIM DESKTOP-ONLY SERVICES"
# Conservative: only obvious desktop/print/discovery/remote-GUI cruft.
# Left running on purpose: NetworkManager, networking, wpa_supplicant, tailscaled,
# ssh, cron, udisks2, fwupd, lm-sensors, apparmor, systemd-oomd, timesyncd, upower.
CRUFT=(
  bluetooth.service blueman-mechanism.service
  cups.service cups.socket cups-browsed.service
  avahi-daemon.service avahi-daemon.socket
  power-profiles-daemon.service
  ModemManager.service
  rustdesk.service
  switcheroo-control.service
  accounts-daemon.service
)
for u in "${CRUFT[@]}"; do
  if systemctl cat "$u" >/dev/null 2>&1; then
    systemctl disable --now "$u" >/dev/null 2>&1 || true
    log "disabled+stopped $u"
  fi
done

# ---------------------------------------------------------------------------
sect "3. CONTROL-CHANNEL PRIORITY (ssh/tailscaled win under build load)"
# Determine the real ssh unit (Debian: ssh.service; sshd.service is an alias).
SSH_UNIT=ssh.service
systemctl cat ssh.service >/dev/null 2>&1 || SSH_UNIT=sshd.service

write_dropin() {  # $1=unit-dir $2=file $3=content
  install -d -m 0755 "$1"
  printf '%s\n' "$3" > "${1}/$2"
}

PRIO_BODY=$'[Service]\nCPUWeight=10000\nIOWeight=10000\nOOMScoreAdjust=-900'
write_dropin "/etc/systemd/system/${SSH_UNIT}.d" 20-buildbox-priority.conf "$PRIO_BODY"
write_dropin "/etc/systemd/system/tailscaled.service.d" 20-buildbox-priority.conf "$PRIO_BODY"
log "high-priority drop-ins: ${SSH_UNIT}, tailscaled.service"

# Deprioritise the whole user build tree (root-level sibling of system.slice).
write_dropin "/etc/systemd/system/user.slice.d" 20-buildbox-priority.conf \
  $'[Slice]\nCPUWeight=30\nIOWeight=30'
# Within the user manager, builds land in app.slice -> yield to interactive session.slice.
write_dropin "/etc/systemd/user/app.slice.d" 20-buildbox-priority.conf \
  $'[Slice]\nCPUWeight=30\nIOWeight=30'
# GH Actions runner also spawns heavy builds (in system.slice) -> yield to ssh too.
GH_UNIT="$(systemctl list-unit-files 'actions.runner.*.service' --no-legend 2>/dev/null | awk '{print $1; exit}')"
if [ -n "${GH_UNIT:-}" ]; then
  write_dropin "/etc/systemd/system/${GH_UNIT}.d" 20-buildbox-priority.conf \
    $'[Service]\nCPUWeight=30\nIOWeight=30'
  log "low-priority drop-ins: user.slice, app.slice(user), ${GH_UNIT}"
fi

systemctl daemon-reload
# Resource-control weights apply LIVE on daemon-reload (no unit restart => no ssh lockout risk).
uctl daemon-reload || true
# OOMScoreAdjust is per-exec; nudge the already-running control-plane pids live.
if command -v choom >/dev/null 2>&1; then
  for p in $(pgrep -x sshd) $(pgrep -x tailscaled); do choom -n -900 -p "$p" >/dev/null 2>&1 || true; done
fi

# ---------------------------------------------------------------------------
sect "4. BUILD-SERVER TUNING"
# swappiness low (protect page cache / build throughput).
write_dropin /etc/sysctl.d 99-buildbox.conf 'vm.swappiness=10'
sysctl -q -w vm.swappiness=10
log "vm.swappiness=$(cat /proc/sys/vm/swappiness)"

# noatime on / (fewer metadata writes). fstab root line carries the unique token
# 'errors=remount-ro'; add noatime once, then remount live.
if ! findmnt -no OPTIONS / | grep -qw noatime; then
  if grep -q 'errors=remount-ro' /etc/fstab && ! grep -q 'noatime,errors=remount-ro' /etc/fstab; then
    cp -a /etc/fstab /etc/fstab.buildbox.bak
    sed -i 's/\berrors=remount-ro\b/noatime,errors=remount-ro/' /etc/fstab
  fi
  mount -o remount / || true
fi
log "root mount opts: $(findmnt -no OPTIONS /)"

# CPU governor: switch off powersave (intel_pstate exposes performance/powersave only).
# power-profiles-daemon was disabled above, so pin via a boot oneshot (single source of truth).
GOV_SVC=/etc/systemd/system/cpu-governor-performance.service
cat > "$GOV_SVC" <<'UNIT'
[Unit]
Description=Pin CPU governor to performance (build server)
After=sysinit.target
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/bin/sh -c 'for g in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo performance > "$g" 2>/dev/null || true; done'
[Install]
WantedBy=multi-user.target
UNIT
systemctl daemon-reload
systemctl enable --now cpu-governor-performance.service >/dev/null 2>&1 || true
log "cpu governor -> $(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null)"

# ---------------------------------------------------------------------------
sect "5. STATIC BOOT-PERSISTENCE VERIFY (no reboot)"
log "get-default: $(systemctl get-default)"
for u in "${SSH_UNIT}" tailscaled.service "${GH_UNIT:-}" cron.service cpu-governor-performance.service; do
  [ -n "$u" ] && log "is-enabled $u: $(systemctl is-enabled "$u" 2>/dev/null || echo n/a)"
done
log "linger($BUILD_USER): $(loginctl show-user "$BUILD_USER" 2>/dev/null | sed -n 's/^Linger=//p')"
CHANGED_UNITS=("${SSH_UNIT}" tailscaled.service user.slice cpu-governor-performance.service)
[ -n "${GH_UNIT:-}" ] && CHANGED_UNITS+=("${GH_UNIT}")
if systemd-analyze verify "${CHANGED_UNITS[@]}" 2>&1 | grep -v '^$' ; then
  log "systemd-analyze verify: emitted warnings above (review)"
else
  log "systemd-analyze verify: CLEAN"
fi

echo
echo "BUILDBOX-HEADLESS DONE. Reboot acceptance test still PENDING (human near box required)."