#!/usr/bin/env bash
# buildbox-watch.sh — notify on remote build-pool transitions, and converge any
# builder that comes back unhardened.
# Run by systemd-user timer (buildbox-watch.timer). State survives runs;
# first run baselines silently.
#
# Anti-spam: debian2 SSH flakes used to fire DEGRADED+ONLINE pairs every few
# minutes. Partial and full collapse into one "available" tier, and a tier counts
# as reached only after STABLE_CHECKS consecutive identical probes. One message per
# crossing, edge-triggered: a tier that has already been announced never announces
# again, so nothing else has to hold a message back.
set -uo pipefail

SELF="$(readlink -f "${BASH_SOURCE[0]}")"
REGISTRY_LIB="${SELF%/bin/*}/lib/buildbox-registry.mjs"
if ! HOSTS_CSV="$(CPU_GUARD_ACTIVE=1 node "$REGISTRY_LIB" hosts --order build)"; then
  echo "buildbox-watch: buildbox registry unusable — not probing any host" >&2
  exit 3
fi
IFS=',' read -r -a HOSTS <<<"$HOSTS_CSV"
PORT=2222
KEY="$HOME/.ssh/id_ed25519_buildbox"
STATE_DIR="$HOME/.claude/run/buildbox-watch"
STATE_FILE="$STATE_DIR/state.json"
LOG_FILE="$HOME/.claude/buildbox-watch.log"
HARDEN_BACKOFF="${HARDEN_BACKOFF:-3600}"  # seconds before re-hardening a host that failed
STABLE_CHECKS="${STABLE_CHECKS:-3}"       # consecutive identical tiers (~3 min at 1/min)
BUILDBOX="${SELF%/modules/workstation/claude/bin/*}/modules/buildbox/bin/buildbox"
mkdir -p "$STATE_DIR"

# A harden pass outlives the one-minute timer interval, so a second run must not
# start one on top of it.
exec 9>"$STATE_DIR/watch.lock"
flock -n 9 || exit 0

bbssh() {
  local host="$1"; shift
  ssh -F /dev/null -o BatchMode=yes -o ConnectTimeout=6 -o StrictHostKeyChecking=accept-new \
    -o IdentitiesOnly=yes -i "$KEY" -p "$PORT" "user@$host" "$@" 2>/dev/null
}

converged() {
  bbssh "$1" 'mountpoint -q /var/lib/buildbox \
    && systemctl is-active --quiet buildbox-sshd-access.timer \
    && systemctl is-enabled --quiet ssh'
}

online=()
for host in "${HOSTS[@]}"; do
  if bbssh "$host" true; then
    online+=("$host")
  fi
done

# One host per pass: `buildbox harden` refuses batches, and a builder that has just
# booted is the one case where the box is reachable but not yet declared-state.
for host in "${online[@]}"; do
  converged "$host" && { rm -f "$STATE_DIR/harden-fail.$host"; continue; }
  # A host that cannot converge would otherwise be hardened once a minute forever.
  # The stamp holds the boot it failed under, so a reboot retries immediately.
  boot=$(bbssh "$host" 'cat /proc/sys/kernel/random/boot_id')
  stamp="$STATE_DIR/harden-fail.$host"
  if [ -f "$stamp" ] && [ "$(cat "$stamp")" = "$boot" ]; then
    [ "$(( $(date +%s) - $(stat -c %Y "$stamp") ))" -lt "$HARDEN_BACKOFF" ] && continue
  fi
  echo "$(date -Is) harden $host: start" >>"$LOG_FILE"
  if [ -x "$BUILDBOX" ]; then
    "$BUILDBOX" harden "$host" >>"$LOG_FILE" 2>&1; harden_rc=$?
    "$BUILDBOX" audit "$host" >>"$LOG_FILE" 2>&1; audit_rc=$?
    echo "$(date -Is) harden $host: harden=$harden_rc audit=$audit_rc" >>"$LOG_FILE"
    if [ "$harden_rc" -eq 0 ] && [ "$audit_rc" -eq 0 ]; then
      rm -f "$stamp"
    else
      printf '%s' "$boot" >"$stamp"
    fi
  else
    echo "$(date -Is) harden $host: skipped, $BUILDBOX not executable" >>"$LOG_FILE"
  fi
  break
done

raw_state="offline"
[ "${#online[@]}" -eq "${#HOSTS[@]}" ] && raw_state="online:${online[*]}"
[ "${#online[@]}" -gt 0 ] && [ "${#online[@]}" -lt "${#HOSTS[@]}" ] && raw_state="degraded:${online[*]}"

notify_tier="unavailable"
[ "${#online[@]}" -gt 0 ] && notify_tier="available"

prev_json="$(cat "$STATE_FILE" 2>/dev/null || echo "")"
if [ -z "$prev_json" ]; then
  python3 - "$STATE_FILE" "$raw_state" "$notify_tier" <<'PY'
import json, sys
path, raw, tier = sys.argv[1:4]
json.dump({
    "raw": raw,
    "announced_tier": tier,
    "pending_tier": tier,
    "pending_count": 1,
}, open(path, "w"))
PY
  exit 0
fi

eval "$(python3 - "$STATE_FILE" "$raw_state" "$notify_tier" \
  "$STABLE_CHECKS" "${#HOSTS[@]}" "${online[*]}" <<'PY'
import json, sys

path, raw, tier, stable_s, total_s, *online = sys.argv[1:]
total = int(total_s)
stable = int(stable_s)
online_hosts = online

with open(path) as f:
    st = json.load(f)

prev_raw = st.get("raw", "")
announced = st.get("announced_tier", "")
pending = st.get("pending_tier", tier)
count = int(st.get("pending_count", 0))

if tier == pending:
    count += 1
else:
    pending = tier
    count = 1

notify = False
summary = ""
body = ""
urgency = "normal"

if count >= stable and pending != announced:
    notify = True
    if pending == "available":
        if len(online_hosts) == total:
            summary = "Build pool ONLINE"
            body = f"{' + '.join(online_hosts)} available"
        else:
            summary = "Build pool available"
            body = f"Remote builds on: {' '.join(online_hosts)} (partial pool)"
        urgency = "normal"
    else:
        summary = "Build pool OFFLINE"
        body = "No remote builders reachable; builds wait or run locally"
        urgency = "critical"
    announced = pending

st.update({
    "raw": raw,
    "announced_tier": announced,
    "pending_tier": pending,
    "pending_count": count,
})
with open(path, "w") as f:
    json.dump(st, f)

if prev_raw != raw:
    print(f'LOG_TRANSITION=1')
    print(f'PREV_RAW={prev_raw!r}')
    print(f'RAW={raw!r}')
print(f'NOTIFY={1 if notify else 0}')
print(f'SUMMARY={summary!r}')
print(f'BODY={body!r}')
print(f'URGENCY={urgency!r}')
PY
)"

if [ "${LOG_TRANSITION:-0}" = "1" ]; then
  ts="$(date -Is)"
  echo "$ts ${PREV_RAW:-?} -> ${RAW:-?}" >> "$LOG_FILE"
fi

if [ "${NOTIFY:-0}" = "1" ]; then
  icon="network-server"
  [ "${URGENCY:-normal}" = "critical" ] && icon="network-error"
  notify-send -u "${URGENCY:-normal}" -i "$icon" "${SUMMARY:-Build pool}" "${BODY:-}" 2>/dev/null || true
fi