#!/usr/bin/env bash
# ============================================================================
# install-prometheus-grafana.sh — FOSS monitoring stack, localhost-only,
# Grafana cloud/upsell/phone-home disabled. Replaces netdata.
#
#   node_exporter : Debian apt (prometheus-node-exporter), bind 127.0.0.1:9100
#   prometheus    : upstream static binary (not in forky apt), bind 127.0.0.1:9090
#   grafana       : grafana OSS apt repo (codename-agnostic), bind 127.0.0.1:3000
#
# Idempotent, fail-closed. Run:  sudo -A bash install-prometheus-grafana.sh
# ============================================================================
set -euo pipefail
[[ $EUID -eq 0 ]] || { echo "run as root (sudo -A)"; exit 1; }

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROM_VER=3.13.0
PROM_TARBALL="prometheus-${PROM_VER}.linux-amd64.tar.gz"
PROM_BASE="https://github.com/prometheus/prometheus/releases/download/v${PROM_VER}"
TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT
log(){ printf '\n=== %s ===\n' "$1"; }

# ---------------------------------------------------------------------------
log "node_exporter (apt), bind localhost"
apt-get install -y prometheus-node-exporter
install -d /etc/default
cat > /etc/default/prometheus-node-exporter <<'EOF'
ARGS="--web.listen-address=127.0.0.1:9100"
EOF
systemctl enable --now prometheus-node-exporter
systemctl restart prometheus-node-exporter

# ---------------------------------------------------------------------------
log "prometheus (verified static binary), bind localhost"
cd "$TMP"
curl -fsSL -o "$PROM_TARBALL" "${PROM_BASE}/${PROM_TARBALL}"
curl -fsSL -o sha256sums.txt "${PROM_BASE}/sha256sums.txt"
EXPECT="$(grep " ${PROM_TARBALL}\$" sha256sums.txt | awk '{print $1}')"
[[ -n "$EXPECT" ]] || { echo "no expected sha for ${PROM_TARBALL}"; exit 1; }
echo "${EXPECT}  ${PROM_TARBALL}" | sha256sum -c - || { echo "CHECKSUM MISMATCH — abort"; exit 1; }
tar xzf "$PROM_TARBALL"
SRC="${TMP}/prometheus-${PROM_VER}.linux-amd64"
install -d /opt/prometheus /etc/prometheus /var/lib/prometheus
install -m0755 "${SRC}/prometheus" "${SRC}/promtool" /opt/prometheus/
cp -r "${SRC}/consoles" "${SRC}/console_libraries" /opt/prometheus/ 2>/dev/null || true
id -u prometheus &>/dev/null || useradd --system --no-create-home --shell /usr/sbin/nologin prometheus
chown -R prometheus:prometheus /var/lib/prometheus /etc/prometheus

cat > /etc/prometheus/prometheus.yml <<'EOF'
global:
  scrape_interval: 15s
  evaluation_interval: 15s
scrape_configs:
  - job_name: node
    static_configs:
      - targets: ['127.0.0.1:9100']
  - job_name: prometheus
    static_configs:
      - targets: ['127.0.0.1:9090']
EOF
chown prometheus:prometheus /etc/prometheus/prometheus.yml
/opt/prometheus/promtool check config /etc/prometheus/prometheus.yml

install -m0644 "$ROOT/systemd/system/prometheus.service" /etc/systemd/system/prometheus.service
systemctl daemon-reload
systemctl enable --now prometheus
systemctl restart prometheus

# Retention lives in the unit's argv, so deckctl's daemon-reload alone leaves it
# inert; only this restart applies it. Read it back off the running process.
for i in $(seq 1 30); do
  curl -fsS -m 5 http://127.0.0.1:9090/-/ready >/dev/null 2>&1 && break
  [[ $i -eq 30 ]] && { echo "prometheus not ready on 127.0.0.1:9090 after 30s"; exit 1; }
  sleep 1
done
/usr/bin/python3 - <<'PY'
import json, sys, urllib.request
want = {"storage.tsdb.retention.time": "30d", "storage.tsdb.retention.size": "2GiB"}
with urllib.request.urlopen("http://127.0.0.1:9090/api/v1/status/flags", timeout=10) as r:
    live = json.load(r)["data"]
wrong = {k: live.get(k) for k, v in want.items() if live.get(k) != v}
if wrong:
    sys.exit(f"prometheus live retention flags are {wrong}, want {want}")
print(f"[OK] prometheus retention live: {want}")
PY

# ---------------------------------------------------------------------------
log "grafana OSS (apt repo), localhost + cloud/upsell/phone-home OFF"
install -d /etc/apt/keyrings
curl -fsSL https://apt.grafana.com/gpg.key | gpg --dearmor --yes -o /etc/apt/keyrings/grafana.gpg
cat > /etc/apt/sources.list.d/grafana.sources <<'EOF'
Types: deb
URIs: https://apt.grafana.com
Suites: stable
Components: main
Signed-By: /etc/apt/keyrings/grafana.gpg
EOF
# tolerate unrelated broken third-party repos (e.g. tailscale keyring); grafana
# index still refreshes. The apt-get install below fails loudly if it did not.
apt-get update || true
apt-get install -y grafana

# minimal grafana.ini = only our overrides; grafana uses built-in defaults elsewhere
cat > /etc/grafana/grafana.ini <<'EOF'
[server]
http_addr = 127.0.0.1
http_port = 3000

# --- kill all phone-home / update-check / SaaS-nag ---
[analytics]
reporting_enabled = false
check_for_updates = false
check_for_plugin_updates = false
feedback_links_enabled = false

[news]
news_feed_enabled = false

[plugins]
plugin_admin_enabled = false

[profile]
enabled = false
EOF

# provision Prometheus as the default datasource (no click-through setup)
install -d /etc/grafana/provisioning/datasources
cat > /etc/grafana/provisioning/datasources/prometheus.yml <<'EOF'
apiVersion: 1
deleteDatasources:
  - name: Prometheus
    orgId: 1
datasources:
  - name: Prometheus
    type: prometheus
    uid: prometheus
    access: proxy
    url: http://127.0.0.1:9090
    isDefault: true
    editable: true
EOF
chown -R grafana:grafana /etc/grafana/provisioning 2>/dev/null || true
systemctl daemon-reload
systemctl enable --now grafana-server
systemctl restart grafana-server

# ---------------------------------------------------------------------------
log "verify listeners (all must be 127.0.0.1 only)"
sleep 1
ss -ltn | grep -E ':(9090|9100|3000)\b' || true
echo
echo "DONE."
echo "  Prometheus  : http://127.0.0.1:9090"
echo "  Grafana     : http://127.0.0.1:3000   (login admin/admin, set password on first login)"
echo "  node metrics: http://127.0.0.1:9100/metrics"