#!/usr/bin/env bash
# debian1-primary failover for github.com/platform-modules/mod.
#
# This repo has ONE registered runner identity (platform-devbox) and a GitHub
# session is held by exactly one host: a second host presenting the same
# credentials loses with "A session for this runner already exists" ->
# SessionConflict, and its unit exits. So this is a failover, not a fallback pool
# — the identity serves from debian1 and returns to the laptop only while debian1
# cannot serve at all. A merely BUSY debian1 keeps the identity: jobs queue behind
# it, which is the point of a queue.
#
# The runner-list/label APIs are closed to this account (push-level collaborator on
# that org), so liveness is probed over ssh rather than read from GitHub.
set -uo pipefail

UNIT="${PLATFORM_FALLBACK_UNIT:-actions.runner.platform-modules-mod.platform-devbox.service}"
REGISTRY_LIB="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/../lib/buildbox-registry.mjs"
REMOTE_NAME=""
REMOTE=""
REMOTE_PORT=""
REMOTE_KEY=""

# Sourcing this file for its decide() logic must not touch the registry, so
# resolution happens on the main path only.
resolve_remote() {
  REMOTE_NAME="${PLATFORM_FALLBACK_REMOTE:-}"
  if [ -n "$REMOTE_NAME" ]; then
    REMOTE_NAME="$(CPU_GUARD_ACTIVE=1 FT_FROM_HOOK=1 FT_HINTS=1 ft node "$REGISTRY_LIB" check "$REMOTE_NAME")" || {
      echo "platform-fallback: PLATFORM_FALLBACK_REMOTE rejected by the buildbox registry" >&2
      return 3
    }
  else
    REMOTE_NAME="$(CPU_GUARD_ACTIVE=1 FT_FROM_HOOK=1 FT_HINTS=1 ft node "$REGISTRY_LIB" hosts --order build | cut -d, -f1)" || {
      echo "platform-fallback: no reachable buildbox in the registry" >&2
      return 3
    }
  fi
  local target port key
  read -r target port key < <(CPU_GUARD_ACTIVE=1 FT_FROM_HOOK=1 FT_HINTS=1 ft node "$REGISTRY_LIB" access "$REMOTE_NAME" --path tailscale_ip) || {
    echo "platform-fallback: no tailscale_ip access for $REMOTE_NAME" >&2
    return 3
  }
  REMOTE="${target#*@}"
  REMOTE_PORT="${PLATFORM_FALLBACK_PORT:-$port}"
  REMOTE_KEY="${PLATFORM_FALLBACK_KEY:-${key/#\~/$HOME}}"
  if [ "$REMOTE_KEY" = "-" ]; then
    echo "platform-fallback: $REMOTE_NAME has no identity file for tailscale_ip" >&2
    return 3
  fi
}

STATE_DIR="${PLATFORM_FALLBACK_STATE:-$HOME/.claude/state/platform-fallback}"
LOG="${PLATFORM_FALLBACK_LOG:-$HOME/.claude/platform-fallback.log}"
FAILOVER_AFTER="${PLATFORM_FALLBACK_FAILOVER_AFTER:-2}"

mkdir -p "$STATE_DIR"

bump() { local f="$STATE_DIR/$1" n; n=$(( $(cat "$f" 2>/dev/null || echo 0) + 1 )); echo "$n" > "$f"; echo "$n"; }
reset() { echo 0 > "$STATE_DIR/$1"; }
log() { printf '%s %s\n' "$(date -Is)" "$*" >> "$LOG"; }

rssh() {
  ssh -F /dev/null -o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new \
      -o IdentitiesOnly=yes -i "$REMOTE_KEY" -p "$REMOTE_PORT" "root@$REMOTE" "$@" 2>/dev/null
}

# decide <reachable> <remote_active> <local_active> <local_busy> <failstreak>
#   -> failover | recover | start-remote | hold
#
# recover outranks everything reachable: while the laptop holds the session debian1
# CANNOT start (it would lose the conflict), so the laptop must release it first.
decide() {
  local reachable="$1" remote_active="$2" local_active="$3" local_busy="$4" streak="$5"
  if [ "$reachable" = 1 ]; then
    if [ "$local_active" = 1 ]; then
      [ "$local_busy" = 1 ] && { echo hold; return; }
      echo recover; return
    fi
    [ "$remote_active" = 1 ] || { echo start-remote; return; }
    echo hold; return
  fi
  if [ "$local_active" = 0 ] && [ "$streak" -ge "$FAILOVER_AFTER" ]; then echo failover; return; fi
  echo hold
}

# Liveness is the listener process, never the unit: these units are KillMode=process,
# so a stopped unit routinely leaves a listener alive still holding the GitHub session
# (that orphan is what silently kept this laptop serving CI for 7h). The bracket keeps
# the pattern from matching this script's own command line.
LOCAL_LISTENER='/opt/actions-runner-platform/bin/[R]unner\.Listener'
REMOTE_LISTENER='/home/user/actions-runner-platform/bin/[R]unner\.Listener'

local_serving() { pgrep -f "$LOCAL_LISTENER" >/dev/null 2>&1 && echo 1 || echo 0; }
local_busy() { pgrep -f '/opt/actions-runner-platform/bin/[R]unner\.Worker' >/dev/null 2>&1 && echo 1 || echo 0; }

# Stop the unit, then reap what KillMode=process leaves behind. The session is not
# released until the listener is gone, and debian1 cannot take it back until it is.
release_local() {
  sudo -n systemctl stop "$UNIT" 2>/dev/null || { log "RECOVER failed: needs root"; return 1; }
  local pid tries=0
  while [ "$(local_serving)" = 1 ] && [ "$tries" -lt 10 ]; do
    pid=$(pgrep -f "$LOCAL_LISTENER" | head -1)
    [ -n "$pid" ] || break
    if [ "$tries" -lt 5 ]; then kill -TERM "$pid" 2>/dev/null
    else log "REAP SIGKILL orphan listener pid=$pid"; kill -KILL "$pid" 2>/dev/null; fi
    tries=$((tries + 1)); sleep 1
  done
  [ "$(local_serving)" = 0 ] || { log "RECOVER blocked: laptop listener will not die"; return 1; }
  return 0
}

main() {
  resolve_remote || exit 3
  local reachable=0 remote_active=0 local_active=0 busy streak action
  if rssh true; then
    reachable=1
    rssh "pgrep -f '$REMOTE_LISTENER' >/dev/null" && remote_active=1
  fi
  local_active=$(local_serving)
  busy=$(local_busy)
  if [ "$reachable" = 1 ]; then reset failstreak; streak=0; else streak=$(bump failstreak); fi

  action=$(decide "$reachable" "$remote_active" "$local_active" "$busy" "$streak")
  case "$action" in
    failover)
      log "FAILOVER -> laptop ($REMOTE_NAME unreachable x$streak)"
      sudo -n systemctl start "$UNIT" 2>/dev/null || log "FAILOVER failed: needs root"
      ;;
    recover)
      log "RECOVER -> $REMOTE_NAME (laptop idle, remote back)"
      release_local || exit 0
      rssh systemctl restart "$UNIT" || log "RECOVER: remote restart failed"
      ;;
    start-remote)
      log "START-REMOTE ($REMOTE_NAME up, its runner was down)"
      rssh systemctl start "$UNIT" || log "START-REMOTE failed"
      ;;
  esac
}

[ "${PLATFORM_FALLBACK_LIB:-0}" = 1 ] || main "$@"