#!/usr/bin/env bash
set -Eeuo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
TEXTFILE_DIR="/var/lib/node_exporter/textfile"
LIB_DIR="/usr/local/lib/system-monitor"
PROM_RULE_DIR="/etc/prometheus/rules"
GRAFANA_PROV_DIR="/etc/grafana/provisioning"
GRAFANA_AUTH="${GRAFANA_AUTH:-admin:admin}"
PROM_UID_PLACEHOLDER="__PROMETHEUS_DS_UID__"
TARGET_USER="${SUDO_USER:-}"
TARGET_UID=""
TARGET_HOME=""
# 2x the slowest producer timer (node-textfile@backup_age.py at OnUnitActiveSec=1h).
TEXTFILE_MAX_AGE=7200

log() { printf '\n==> %s\n' "$*"; }
ok() { printf '[OK] %s\n' "$*"; }
fail() { printf '[FAIL] %s\n' "$*" >&2; exit 1; }

# Evaluates a PromQL expression that must yield at most one sample; prints its
# value, or "empty" when the expression matched nothing.
prom_scalar() {
  local body
  body="$(curl -fsSG --max-time 10 --data-urlencode "query=$1" http://127.0.0.1:9090/api/v1/query)" \
    || fail "Prometheus query failed: $1"
  PROM_QUERY="$1" PROM_BODY="$body" python3 <<'PY'
import json, os, sys
query = os.environ["PROM_QUERY"]
payload = json.loads(os.environ["PROM_BODY"])
if payload.get("status") != "success":
    sys.exit(f"Prometheus rejected query {query!r}: {payload.get('error')}")
result = payload["data"]["result"]
if len(result) > 1:
    sys.exit(f"query {query!r} returned {len(result)} series, expected at most 1")
print(result[0]["value"][1] if result else "empty")
PY
}

# User units are symlinked to the repo body, matching `deckctl units apply`.
# Copying here would replace that link and let the live unit drift from the repo.
link_user_unit() {
  local src="$1" dst="$2"
  install -d -m0755 -o "$TARGET_USER" -g "$TARGET_USER" "$(dirname "$dst")"
  ln -sfn "$src" "$dst"
  chown -h "$TARGET_USER":"$TARGET_USER" "$dst"
}

[[ "${EUID}" -eq 0 ]] || fail "run as root, for example: sudo -A bash grafana/install-dashboards.sh"

resolve_guard_user() {
  [[ -n "$TARGET_USER" && "$TARGET_USER" != "root" ]] || fail "run via sudo -A from the desktop login user so agent-guard can install as a systemd --user service"
  TARGET_UID="$(id -u "$TARGET_USER")"
  TARGET_HOME="$(getent passwd "$TARGET_USER" | cut -d: -f6)"
  [[ -n "$TARGET_HOME" && -d "$TARGET_HOME" ]] || fail "could not resolve home directory for $TARGET_USER"
  [[ -d "/run/user/$TARGET_UID" ]] || fail "no active user runtime dir for $TARGET_USER; log into the desktop session before running this installer"
}

install_node_exporter_args() {
  # Declare the FULL desired ARGS line (idempotent by construction — no in-place
  # mutate/dedup, which duplicated flags and wedged the service on start-limit-hit).
  local defaults="/etc/default/prometheus-node-exporter"
  cat >"$defaults" <<'EOF'
ARGS="--web.listen-address=127.0.0.1:9100 --collector.processes --collector.textfile.directory=/var/lib/node_exporter/textfile"
EOF
}

install_textfile_producers() {
  install -d -m0755 "$TEXTFILE_DIR" "$LIB_DIR"
  find "$ROOT/grafana/node_exporter/textfile-producers" -maxdepth 1 -type f \
    -exec install -m0755 {} "$LIB_DIR/" \;
  install -m0644 "$ROOT/systemd/system/node-textfile@.service" /etc/systemd/system/
  install -m0644 "$ROOT/systemd/system/node-textfile@.timer" /etc/systemd/system/

  systemctl daemon-reload
  systemctl reset-failed prometheus-node-exporter || true
  systemctl restart prometheus-node-exporter
  bash "$ROOT/grafana/install-textfile-timers.sh" || fail "could not arm the textfile producers"
}

install_prometheus_rules() {
  install -d -m0755 "$PROM_RULE_DIR"
  install -m0644 "$ROOT/grafana/prometheus/rules/system-monitor.rules.yml" "$PROM_RULE_DIR/system-monitor.rules.yml"
  python3 - /etc/prometheus/prometheus.yml <<'PY'
import pathlib, sys
path = pathlib.Path(sys.argv[1])
text = path.read_text()
needle = "/etc/prometheus/rules/*.yml"
if needle in text:
    raise SystemExit(0)
if "rule_files:" in text:
    lines = text.splitlines()
    out = []
    inserted = False
    in_rules = False
    for line in lines:
        out.append(line)
        if line.startswith("rule_files:"):
            in_rules = True
            continue
        if in_rules and line and not line.startswith((" ", "-")):
            out.insert(len(out) - 1, f"  - {needle!r}")
            inserted = True
            in_rules = False
    if in_rules and not inserted:
        out.append(f"  - {needle!r}")
    path.write_text("\n".join(out) + "\n")
else:
    path.write_text(text.rstrip() + f"\n\nrule_files:\n  - {needle!r}\n")
PY
  if command -v promtool >/dev/null 2>&1; then
    promtool check config /etc/prometheus/prometheus.yml >/dev/null
  elif [[ -x /opt/prometheus/promtool ]]; then
    /opt/prometheus/promtool check config /etc/prometheus/prometheus.yml >/dev/null
  fi
  curl -fsS -X POST http://127.0.0.1:9090/-/reload >/dev/null
}

install_grafana_provisioning() {
  local uid tmp alert_src
  # Fixed, provisioned datasource UID — no admin creds / API needed (survives
  # the admin password change every fresh Grafana forces on first login).
  uid="${PROM_DS_UID:-prometheus}"
  install -d -m0755 \
    "$GRAFANA_PROV_DIR/dashboards/json" \
    "$GRAFANA_PROV_DIR/alerting"
  install -m0644 "$ROOT/grafana/provisioning/dashboards/system-monitor.yaml" "$GRAFANA_PROV_DIR/dashboards/system-monitor.yaml"
  install -m0644 "$ROOT/grafana/provisioning/dashboards/json/overview.json" "$GRAFANA_PROV_DIR/dashboards/json/overview.json"
  install -m0644 "$ROOT/grafana/provisioning/alerting/contactpoints.yaml" "$GRAFANA_PROV_DIR/alerting/contactpoints.yaml"
  alert_src="$ROOT/grafana/provisioning/alerting/system-monitor.yaml"
  tmp="$(mktemp)"
  sed "s/${PROM_UID_PLACEHOLDER}/${uid}/g" "$alert_src" >"$tmp"
  install -m0644 "$tmp" "$GRAFANA_PROV_DIR/alerting/system-monitor.yaml"
  rm -f "$tmp"
  chown -R grafana:grafana "$GRAFANA_PROV_DIR"
  GRAFANA_RESTART_TS="$(date --iso-8601=seconds)"
  systemctl restart grafana-server
}

install_agent_guard_user_service() {
  local user_systemd_dir runtime_dir
  resolve_guard_user
  user_systemd_dir="$TARGET_HOME/.config/systemd/user"
  runtime_dir="/run/user/$TARGET_UID"
  local lib_dir="$TARGET_HOME/.local/share/system-monitor/lib"
  # Deploy the pure-stdlib guard code to a stable user-owned path (zero deps ->
  # no venv/pip/PyPI needed; systemd runs it as a module via PYTHONPATH).
  install -d -m0755 -o "$TARGET_USER" -g "$TARGET_USER" \
    "$TARGET_HOME/.local" "$TARGET_HOME/.local/share" "$TARGET_HOME/.local/share/system-monitor" "$lib_dir"
  rm -rf "$lib_dir/agent_guard"
  cp -a "$ROOT/agent-guard/src/agent_guard" "$lib_dir/agent_guard"
  chown -R "$TARGET_USER":"$TARGET_USER" "$lib_dir/agent_guard"
  install -d -m0755 -o "$TARGET_USER" -g "$TARGET_USER" "$TARGET_HOME/.config" "$TARGET_HOME/.config/systemd" "$user_systemd_dir"
  link_user_unit "$ROOT/systemd/user/agent-guard.service" "$user_systemd_dir/agent-guard.service"
  link_user_unit "$ROOT/systemd/user/agent-guard-failure-notify.service" "$user_systemd_dir/agent-guard-failure-notify.service"
  runuser -u "$TARGET_USER" -- env XDG_RUNTIME_DIR="$runtime_dir" systemctl --user daemon-reload
  runuser -u "$TARGET_USER" -- env XDG_RUNTIME_DIR="$runtime_dir" systemctl --user enable agent-guard.service
  runuser -u "$TARGET_USER" -- env XDG_RUNTIME_DIR="$runtime_dir" systemctl --user restart agent-guard.service
}

verify() {
  local start="${1}"
  local metrics i oldest
  for i in $(seq 1 15); do
    curl -fsS http://127.0.0.1:9100/metrics >/dev/null 2>&1 && break
    [[ $i -eq 15 ]] && fail "node_exporter not reachable on 127.0.0.1:9100 after 15s"
    sleep 1
  done
  metrics="$(curl -fsS http://127.0.0.1:9100/metrics)"
  grep -q 'node_mem_exhaustion_eta_seconds' <<<"$metrics" && ok "node_mem_exhaustion_eta_seconds served"
  grep -q 'node_cpu_nonbuild_busy_percent' <<<"$metrics" && ok "node_cpu_nonbuild_busy_percent served"
  grep -q 'node_tmp_used_percent' <<<"$metrics" && ok "node_tmp_used_percent served"
  grep -q 'node_textfile_scrape_error 0' <<<"$metrics" && ok "node_exporter textfile scrape clean"
  # Presence of node_textfile_mtime_seconds proves nothing: a producer that stopped
  # writing months ago still leaves its .prom on disk and still exports an mtime.
  # The oldest file must be younger than TEXTFILE_MAX_AGE.
  for i in $(seq 1 15); do
    oldest="$(prom_scalar 'max(time() - node_textfile_mtime_seconds)')"
    if [[ "$oldest" != empty ]]; then
      awk -v a="$oldest" -v m="$TEXTFILE_MAX_AGE" 'BEGIN{exit !(a<m)}' \
        || fail "oldest textfile .prom in Prometheus is ${oldest}s old (limit ${TEXTFILE_MAX_AGE}s) — a producer is writing nothing"
      break
    fi
    [[ $i -eq 15 ]] && fail "Prometheus scraped no node_textfile_mtime_seconds within 15s"
    sleep 1
  done
  ok "textfile producers fresh in Prometheus (oldest ${oldest}s, limit ${TEXTFILE_MAX_AGE}s)"
  curl -fsS http://127.0.0.1:9090/api/v1/rules | grep -q 'system-monitor' && ok "Prometheus system-monitor rules loaded"
  # Grafana provisioning is file-based (no API/admin creds); confirm files landed
  # and inspect ONLY the current process's provisioning cycle (since the restart in
  # install_grafana_provisioning) so a prior process's shutdown error can't false-fail.
  [[ -f "$GRAFANA_PROV_DIR/dashboards/json/overview.json" ]] && ok "Grafana dashboard provisioned (file present)"
  [[ -f "$GRAFANA_PROV_DIR/alerting/system-monitor.yaml" ]] && ok "Grafana alert rules provisioned (file present)"
  local since="${GRAFANA_RESTART_TS:-$start}"
  for i in $(seq 1 30); do
    journalctl -u grafana-server --since "$since" --no-pager | grep -q 'finished to provision alerting' && break
    [[ $i -eq 30 ]] && fail "Grafana did not finish provisioning alerting within 30s"
    sleep 1
  done
  if journalctl -u grafana-server --since "$since" --no-pager | grep -Ei 'failed to provision|invalid alert rule' >/dev/null; then
    fail "Grafana provisioning error found in journal (current process)"
  fi
  ok "Grafana provisioned cleanly (current process, no errors)"
}

START_TS="$(date --iso-8601=seconds)"
log "node_exporter args and textfile producers"
install_node_exporter_args
install_textfile_producers
ok "node_exporter localhost args, producers, timers installed"

log "Prometheus recording rules"
install_prometheus_rules
ok "Prometheus rules installed and reload requested"

log "Grafana provisioning"
install_grafana_provisioning
ok "Grafana provisioning installed"

log "agent-guard user service"
install_agent_guard_user_service
ok "agent-guard user service installed and restarted"

log "Verification"
verify "$START_TS"
ok "system-monitor Grafana/Prometheus provisioning complete"