#!/usr/bin/env bash
# buildbox-watch.sh — notify on remote build-pool transitions, and converge any
# builder that comes back unhardened.
# Run by systemd-user timer (buildbox-watch.timer). State survives runs;
# first run baselines silently.
#
# Anti-spam: debian2 SSH flakes used to fire DEGRADED+ONLINE pairs every few
# minutes. We now (a) collapse partial+full into one "available" tier for
# notifications, (b) require STABLE_CHECKS consecutive identical tiers before
# announcing, and (c) enforce NOTIFY_COOLDOWN between popups.
set -uo pipefail

HOSTS=(debian1 debian2 debian3)
PORT=2222
KEY="$HOME/.ssh/id_ed25519_buildbox"
STATE_DIR="$HOME/.claude/run/buildbox-watch"
STATE_FILE="$STATE_DIR/state.json"
LOG_FILE="$HOME/.claude/buildbox-watch.log"
HARDEN_BACKOFF="${HARDEN_BACKOFF:-3600}"  # seconds before re-hardening a host that failed
STABLE_CHECKS="${STABLE_CHECKS:-3}"       # consecutive identical tiers (~3 min at 1/min)
NOTIFY_COOLDOWN="${NOTIFY_COOLDOWN:-1800}" # seconds between desktop popups
SELF="$(readlink -f "${BASH_SOURCE[0]}")"
BUILDBOX="${SELF%/modules/workstation/claude/bin/*}/modules/buildbox/bin/buildbox"
mkdir -p "$STATE_DIR"

# A harden pass outlives the one-minute timer interval, so a second run must not
# start one on top of it.
exec 9>"$STATE_DIR/watch.lock"
flock -n 9 || exit 0

bbssh() {
  local host="$1"; shift
  ssh -F /dev/null -o BatchMode=yes -o ConnectTimeout=6 -o StrictHostKeyChecking=accept-new \
    -o IdentitiesOnly=yes -i "$KEY" -p "$PORT" "user@$host" "$@" 2>/dev/null
}

converged() {
  bbssh "$1" 'mountpoint -q /var/lib/buildbox \
    && systemctl is-active --quiet buildbox-sshd-access.timer \
    && systemctl is-enabled --quiet ssh'
}

online=()
for host in "${HOSTS[@]}"; do
  if bbssh "$host" true; then
    online+=("$host")
  fi
done

# One host per pass: `buildbox harden` refuses batches, and a builder that has just
# booted is the one case where the box is reachable but not yet declared-state.
for host in "${online[@]}"; do
  converged "$host" && { rm -f "$STATE_DIR/harden-fail.$host"; continue; }
  # A host that cannot converge would otherwise be hardened once a minute forever.
  # The stamp holds the boot it failed under, so a reboot retries immediately.
  boot=$(bbssh "$host" 'cat /proc/sys/kernel/random/boot_id')
  stamp="$STATE_DIR/harden-fail.$host"
  if [ -f "$stamp" ] && [ "$(cat "$stamp")" = "$boot" ]; then
    [ "$(( $(date +%s) - $(stat -c %Y "$stamp") ))" -lt "$HARDEN_BACKOFF" ] && continue
  fi
  echo "$(date -Is) harden $host: start" >>"$LOG_FILE"
  if [ -x "$BUILDBOX" ]; then
    "$BUILDBOX" harden "$host" >>"$LOG_FILE" 2>&1; harden_rc=$?
    "$BUILDBOX" audit "$host" >>"$LOG_FILE" 2>&1; audit_rc=$?
    echo "$(date -Is) harden $host: harden=$harden_rc audit=$audit_rc" >>"$LOG_FILE"
    if [ "$harden_rc" -eq 0 ] && [ "$audit_rc" -eq 0 ]; then
      rm -f "$stamp"
    else
      printf '%s' "$boot" >"$stamp"
    fi
  else
    echo "$(date -Is) harden $host: skipped, $BUILDBOX not executable" >>"$LOG_FILE"
  fi
  break
done

raw_state="offline"
[ "${#online[@]}" -eq "${#HOSTS[@]}" ] && raw_state="online:${online[*]}"
[ "${#online[@]}" -gt 0 ] && [ "${#online[@]}" -lt "${#HOSTS[@]}" ] && raw_state="degraded:${online[*]}"

notify_tier="unavailable"
[ "${#online[@]}" -gt 0 ] && notify_tier="available"

now_epoch="$(date +%s)"
prev_json="$(cat "$STATE_FILE" 2>/dev/null || echo "")"
if [ -z "$prev_json" ]; then
  python3 - "$STATE_FILE" "$raw_state" "$notify_tier" <<'PY'
import json, sys
path, raw, tier = sys.argv[1:4]
json.dump({
    "raw": raw,
    "announced_tier": tier,
    "pending_tier": tier,
    "pending_count": 1,
    "last_notify": 0,
}, open(path, "w"))
PY
  exit 0
fi

eval "$(python3 - "$STATE_FILE" "$raw_state" "$notify_tier" "$now_epoch" \
  "$STABLE_CHECKS" "$NOTIFY_COOLDOWN" "${#HOSTS[@]}" "${online[*]}" <<'PY'
import json, sys

path, raw, tier, now_s, stable_s, cooldown_s, total_s, *online = sys.argv[1:]
total = int(total_s)
stable = int(stable_s)
cooldown = int(cooldown_s)
now = int(now_s)
online_hosts = online

with open(path) as f:
    st = json.load(f)

prev_raw = st.get("raw", "")
announced = st.get("announced_tier", "")
pending = st.get("pending_tier", tier)
count = int(st.get("pending_count", 0))
last_notify = int(st.get("last_notify", 0))

if tier == pending:
    count += 1
else:
    pending = tier
    count = 1

notify = False
summary = ""
body = ""
urgency = "normal"

if count >= stable and pending != announced:
    if now - last_notify >= cooldown:
        notify = True
        if pending == "available":
            if len(online_hosts) == total:
                summary = "Build pool ONLINE"
                body = f"{' + '.join(online_hosts)} available"
            else:
                summary = "Build pool available"
                body = f"Remote builds on: {' '.join(online_hosts)} (partial pool)"
            urgency = "normal"
        else:
            summary = "Build pool OFFLINE"
            body = "No remote builders reachable; builds wait or run locally"
            urgency = "critical"
        announced = pending
        last_notify = now

st.update({
    "raw": raw,
    "announced_tier": announced,
    "pending_tier": pending,
    "pending_count": count,
    "last_notify": last_notify,
})
with open(path, "w") as f:
    json.dump(st, f)

if prev_raw != raw:
    print(f'LOG_TRANSITION=1')
    print(f'PREV_RAW={prev_raw!r}')
    print(f'RAW={raw!r}')
print(f'NOTIFY={1 if notify else 0}')
print(f'SUMMARY={summary!r}')
print(f'BODY={body!r}')
print(f'URGENCY={urgency!r}')
PY
)"

if [ "${LOG_TRANSITION:-0}" = "1" ]; then
  ts="$(date -Is)"
  echo "$ts ${PREV_RAW:-?} -> ${RAW:-?}" >> "$LOG_FILE"
fi

if [ "${NOTIFY:-0}" = "1" ]; then
  icon="network-server"
  [ "${URGENCY:-normal}" = "critical" ] && icon="network-error"
  notify-send -u "${URGENCY:-normal}" -i "$icon" "${SUMMARY:-Build pool}" "${BODY:-}" 2>/dev/null || true
fi