#!/usr/bin/env bash
# packaging/web-watchdog.sh — self-healing supervisor for the overdeck web deck.
# audience: AI agents. Fail-closed: an unusable probe configuration exits non-zero
# rather than reporting health it did not verify.
#
# Sweeps EVERY static page route, not just "/". A rebuild of dist/ under a running
# server leaves the process holding chunk names the new build no longer emits, so
# only the routes whose chunks were never resident 500 — "/" keeps answering 200.
# A single-route probe is blind to that failure class; the sweep is the point.
#
# Runs as a plain bash+curl loop with no dependency on the artifact it watches:
# a node/bun watchdog living in the deploy clone would be torn by the same rebuild.
set -uo pipefail

WEB_URL="${OVERDECK_WEB_URL:-http://127.0.0.1:31337}"
PAGES_DIR="${OVERDECK_WEB_PAGES_DIR:-${HOME}/.local/share/overdeck/deploy/apps/web/src/pages}"
UNIT="${OVERDECK_WEB_UNIT:-overdeck-web.service}"
INTERVAL="${OVERDECK_WEB_WATCHDOG_INTERVAL:-30}"
THRESHOLD="${OVERDECK_WEB_WATCHDOG_FAILURES:-2}"
SETTLE="${OVERDECK_WEB_WATCHDOG_SETTLE:-20}"
PROBE_TIMEOUT="${OVERDECK_WEB_WATCHDOG_PROBE_TIMEOUT:-10}"
MAX_HEALS="${OVERDECK_WEB_WATCHDOG_MAX_HEALS:-3}"
BACKOFF="${OVERDECK_WEB_WATCHDOG_BACKOFF:-300}"
RESTART_CMD="${OVERDECK_WEB_WATCHDOG_RESTART_CMD:-}"
ROLLBACK_CMD="${OVERDECK_WEB_WATCHDOG_ROLLBACK_CMD:-$HOME/.local/bin/overdeck-web-release rollback}"
RELEASE_LOCK="${OVERDECK_WEB_RELEASE_LOCK:-$HOME/.local/share/overdeck/deploy/apps/web/.releases/.release.lock}"
RELEASE_LOCK_WAIT="${OVERDECK_WEB_RELEASE_LOCK_WAIT:-120}"

log() { # $1=event, rest=key=value pairs
  local event="$1"
  shift
  local fields="" pair
  for pair in "$@"; do
    fields+=",\"${pair%%=*}\":\"${pair#*=}\""
  done
  printf '{"component":"web-watchdog","event":"%s"%s}\n' "$event" "$fields" >&2
}

die() {
  log fatal "detail=$1"
  exit 1
}

assert_positive_int() { # $1=value $2=name
  [[ "$1" =~ ^[0-9]+$ ]] && ((10#$1 > 0)) || die "$2 must be a positive integer, got '$1'"
}

routes() { # → every static page route served by the deck, one per line
  [[ -d "$PAGES_DIR" ]] || die "pages dir not found: $PAGES_DIR"
  local found=0 file rel route
  while IFS= read -r -d '' file; do
    rel="${file#"$PAGES_DIR"/}"
    # dynamic segments have no probe-able concrete value; api/ is not a page
    [[ "$rel" == *'['* || "$rel" == api/* || "$(basename "$rel")" == _* ]] && continue
    route="/${rel%.astro}"
    [[ "$route" == /index ]] && route=/
    route="${route%/index}"
    printf '%s\n' "$route"
    found=1
  done < <(find "$PAGES_DIR" -type f -name '*.astro' -print0 | sort -z)
  ((found == 1)) || die "no probe-able routes under $PAGES_DIR"
}

sweep() { # 0 = every route answered 200; 1 = at least one did not; 2 = probe unusable
  local route code bad="" list
  # routes() must run in a command substitution, not a process substitution: its
  # fail-closed exit has to reach this caller instead of dying in a background shell.
  list=$(routes) || return 2
  while IFS= read -r route; do
    code=$(curl -sS -o /dev/null -w '%{http_code}' --max-time "$PROBE_TIMEOUT" "${WEB_URL}${route}" 2>/dev/null) || code=000
    [[ "$code" == 200 ]] || bad+="${route}=${code} "
  done <<<"$list"
  [[ -z "$bad" ]] && return 0
  printf '%s\n' "${bad% }"
  return 1
}

heal() { # $1=zero-based attempt; later attempts restore the last known good release
  local attempt="$1" release_fd status result
  mkdir -p "$(dirname "$RELEASE_LOCK")" || return 1
  exec {release_fd}>"$RELEASE_LOCK" || return 1
  flock -w "$RELEASE_LOCK_WAIT" "$release_fd" || { exec {release_fd}>&-; return 1; }
  sweep >/dev/null 2>&1
  status=$?
  if ((status == 0)); then
    flock -u "$release_fd"
    exec {release_fd}>&-
    return 0
  fi
  if ((status == 2)); then
    flock -u "$release_fd"
    exec {release_fd}>&-
    return 1
  fi
  result=0
  if ((attempt > 0)); then
    OVERDECK_WEB_RELEASE_LOCK_FD="$release_fd" eval "$ROLLBACK_CMD" || result=$?
    ((result == 0)) && log rollback "attempt=$((attempt + 1))"
  fi
  if ((result == 0)); then
    if [[ -n "$RESTART_CMD" ]]; then
      eval "$RESTART_CMD" || result=$?
    else
      systemctl --user reset-failed "$UNIT" >/dev/null 2>&1
      systemctl --user restart "$UNIT" || result=$?
    fi
  fi
  flock -u "$release_fd"
  exec {release_fd}>&-
  return "$result"
}

daemon() {
  assert_positive_int "$INTERVAL" OVERDECK_WEB_WATCHDOG_INTERVAL
  assert_positive_int "$THRESHOLD" OVERDECK_WEB_WATCHDOG_FAILURES
  assert_positive_int "$PROBE_TIMEOUT" OVERDECK_WEB_WATCHDOG_PROBE_TIMEOUT
  assert_positive_int "$MAX_HEALS" OVERDECK_WEB_WATCHDOG_MAX_HEALS
  ((PROBE_TIMEOUT < INTERVAL)) || die "probe timeout must be below the sweep interval"

  local list
  list=$(routes) || exit 1
  log started "url=$WEB_URL" "unit=$UNIT" "interval=$INTERVAL" "threshold=$THRESHOLD" \
    "routes=$(printf '%s\n' "$list" | wc -l)"

  local fails=0 heals=0 failing status
  while true; do
    failing=$(sweep)
    status=$?
    # a probe that cannot run is never evidence the deck is broken — never restart on it
    ((status == 2)) && die "probe became unusable: $PAGES_DIR"
    if ((status == 0)); then
      ((fails > 0)) && log recovered
      fails=0
      heals=0
    else
      fails=$((fails + 1))
      log unhealthy "failing=$failing" "consecutive=$fails"
      if ((fails >= THRESHOLD)); then
        if ((heals >= MAX_HEALS)); then
          log giving-up "detail=restarts are not restoring health; backing off ${BACKOFF}s"
          sleep "$BACKOFF"
          fails=0
          heals=0
          continue
        fi
        if heal "$heals"; then
          heals=$((heals + 1))
          log healed "attempt=$heals"
        else
          log heal-failed "attempt=$((heals + 1))"
          heals=$((heals + 1))
        fi
        fails=0
        sleep "$SETTLE"
        continue
      fi
    fi
    sleep "$INTERVAL"
  done
}

case "${1:-daemon}" in
  daemon) daemon ;;
  sweep)
    failing=$(sweep)
    status=$?
    ((status == 0)) && { printf '{"ok":true}\n'; exit 0; }
    ((status == 2)) && exit 2
    printf '{"ok":false,"failing":"%s"}\n' "$failing" >&2
    exit 1
    ;;
  routes) routes ;;
  --help | -h)
    cat <<'USAGE'
web-watchdog — self-healing supervisor for the overdeck web deck

Usage:
  web-watchdog [daemon]   sweep every page route on a loop; restart the unit on
                          OVERDECK_WEB_WATCHDOG_FAILURES consecutive bad sweeps
  web-watchdog sweep      one sweep; exit 0 healthy, 1 unhealthy
  web-watchdog routes     print the routes that would be probed

Environment (defaults):
  OVERDECK_WEB_URL                     http://127.0.0.1:31337
  OVERDECK_WEB_PAGES_DIR               ~/.local/share/overdeck/deploy/apps/web/src/pages
  OVERDECK_WEB_UNIT                    overdeck-web.service
  OVERDECK_WEB_WATCHDOG_INTERVAL       30    seconds between sweeps
  OVERDECK_WEB_WATCHDOG_FAILURES       2     consecutive bad sweeps before a restart
  OVERDECK_WEB_WATCHDOG_SETTLE         20    seconds to let a restart settle
  OVERDECK_WEB_WATCHDOG_PROBE_TIMEOUT  10    per-route curl timeout
  OVERDECK_WEB_WATCHDOG_MAX_HEALS      3     restarts before backing off
  OVERDECK_WEB_WATCHDOG_BACKOFF        300   backoff seconds once restarts stop helping
  OVERDECK_WEB_WATCHDOG_RESTART_CMD          override the restart command (tests)
  OVERDECK_WEB_WATCHDOG_ROLLBACK_CMD         override the release rollback command
USAGE
    ;;
  *)
    printf '{"ok":false,"code":"USAGE","detail":"unknown command: %s"}\n' "$1" >&2
    exit 2
    ;;
esac