#!/usr/bin/env bash
# debian1-primary failover for github.com/platform-modules/mod.
#
# This repo has ONE registered runner identity (platform-devbox) and a GitHub
# session is held by exactly one host: a second host presenting the same
# credentials loses with "A session for this runner already exists" ->
# SessionConflict, and its unit exits. So this is a failover, not a fallback pool
# — the identity serves from debian1 and returns to the laptop only while debian1
# cannot serve at all. A merely BUSY debian1 keeps the identity: jobs queue behind
# it, which is the point of a queue.
#
# The runner-list/label APIs are closed to this account (push-level collaborator on
# that org), so liveness is probed over ssh rather than read from GitHub.
set -uo pipefail

UNIT="${PLATFORM_FALLBACK_UNIT:-actions.runner.platform-modules-mod.platform-devbox.service}"
REMOTE="${PLATFORM_FALLBACK_REMOTE:-debian1}"
REMOTE_PORT="${PLATFORM_FALLBACK_PORT:-2222}"
REMOTE_KEY="${PLATFORM_FALLBACK_KEY:-$HOME/.ssh/id_ed25519_buildbox}"
STATE_DIR="${PLATFORM_FALLBACK_STATE:-$HOME/.claude/state/platform-fallback}"
LOG="${PLATFORM_FALLBACK_LOG:-$HOME/.claude/platform-fallback.log}"
FAILOVER_AFTER="${PLATFORM_FALLBACK_FAILOVER_AFTER:-2}"

mkdir -p "$STATE_DIR"

bump() { local f="$STATE_DIR/$1" n; n=$(( $(cat "$f" 2>/dev/null || echo 0) + 1 )); echo "$n" > "$f"; echo "$n"; }
reset() { echo 0 > "$STATE_DIR/$1"; }
log() { printf '%s %s\n' "$(date -Is)" "$*" >> "$LOG"; }

rssh() {
  ssh -F /dev/null -o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new \
      -o IdentitiesOnly=yes -i "$REMOTE_KEY" -p "$REMOTE_PORT" "root@$REMOTE" "$@" 2>/dev/null
}

# decide <reachable> <remote_active> <local_active> <local_busy> <failstreak>
#   -> failover | recover | start-remote | hold
#
# recover outranks everything reachable: while the laptop holds the session debian1
# CANNOT start (it would lose the conflict), so the laptop must release it first.
decide() {
  local reachable="$1" remote_active="$2" local_active="$3" local_busy="$4" streak="$5"
  if [ "$reachable" = 1 ]; then
    if [ "$local_active" = 1 ]; then
      [ "$local_busy" = 1 ] && { echo hold; return; }
      echo recover; return
    fi
    [ "$remote_active" = 1 ] || { echo start-remote; return; }
    echo hold; return
  fi
  if [ "$local_active" = 0 ] && [ "$streak" -ge "$FAILOVER_AFTER" ]; then echo failover; return; fi
  echo hold
}

# Liveness is the listener process, never the unit: these units are KillMode=process,
# so a stopped unit routinely leaves a listener alive still holding the GitHub session
# (that orphan is what silently kept this laptop serving CI for 7h). The bracket keeps
# the pattern from matching this script's own command line.
LOCAL_LISTENER='/opt/actions-runner-platform/bin/[R]unner\.Listener'
REMOTE_LISTENER='/home/user/actions-runner-platform/bin/[R]unner\.Listener'

local_serving() { pgrep -f "$LOCAL_LISTENER" >/dev/null 2>&1 && echo 1 || echo 0; }
local_busy() { pgrep -f '/opt/actions-runner-platform/bin/[R]unner\.Worker' >/dev/null 2>&1 && echo 1 || echo 0; }

# Stop the unit, then reap what KillMode=process leaves behind. The session is not
# released until the listener is gone, and debian1 cannot take it back until it is.
release_local() {
  sudo -n systemctl stop "$UNIT" 2>/dev/null || { log "RECOVER failed: needs root"; return 1; }
  local pid tries=0
  while [ "$(local_serving)" = 1 ] && [ "$tries" -lt 10 ]; do
    pid=$(pgrep -f "$LOCAL_LISTENER" | head -1)
    [ -n "$pid" ] || break
    if [ "$tries" -lt 5 ]; then kill -TERM "$pid" 2>/dev/null
    else log "REAP SIGKILL orphan listener pid=$pid"; kill -KILL "$pid" 2>/dev/null; fi
    tries=$((tries + 1)); sleep 1
  done
  [ "$(local_serving)" = 0 ] || { log "RECOVER blocked: laptop listener will not die"; return 1; }
  return 0
}

main() {
  local reachable=0 remote_active=0 local_active=0 busy streak action
  if rssh true; then
    reachable=1
    rssh "pgrep -f '$REMOTE_LISTENER' >/dev/null" && remote_active=1
  fi
  local_active=$(local_serving)
  busy=$(local_busy)
  if [ "$reachable" = 1 ]; then reset failstreak; streak=0; else streak=$(bump failstreak); fi

  action=$(decide "$reachable" "$remote_active" "$local_active" "$busy" "$streak")
  case "$action" in
    failover)
      log "FAILOVER -> laptop ($REMOTE unreachable x$streak)"
      sudo -n systemctl start "$UNIT" 2>/dev/null || log "FAILOVER failed: needs root"
      ;;
    recover)
      log "RECOVER -> $REMOTE (laptop idle, remote back)"
      release_local || exit 0
      rssh systemctl restart "$UNIT" || log "RECOVER: remote restart failed"
      ;;
    start-remote)
      log "START-REMOTE ($REMOTE up, its runner was down)"
      rssh systemctl start "$UNIT" || log "START-REMOTE failed"
      ;;
  esac
}

[ "${PLATFORM_FALLBACK_LIB:-0}" = 1 ] || main "$@"