#!/usr/bin/env bash
# grafana-port-forward.sh — e2e smoke test: Grafana installs via Helm,
#                           port-forward is accessible, Loki datasource
#                           is pre-wired and reachable.
#
# Usage: scripts/e2e/grafana-port-forward.sh
#
# Pre-conditions:
#   - kubectl context is set to a live k8s cluster (does NOT spin up k3d)
#   - helm binary available
#   - jq binary available
#   - grafana Helm repo added (helm repo add grafana https://grafana.github.io/helm-charts)
#   - Loki is already installed (scripts/e2e/loki-ingest.sh ran successfully
#     OR `helm status olam-loki -n monitoring` is healthy)
#
# Idempotency: `helm upgrade --install` is idempotent; re-runs succeed on an
#              existing cluster. The Secret is applied via --dry-run | kubectl apply
#              so re-runs update the password (useful for rotation testing).
#              The olam-dashboards ConfigMap is applied before helm install so
#              Grafana's volume mount finds the ConfigMap on first boot.
#
# Cleanup: port-forward is killed on exit; Helm release is left in place so
#          downstream tasks can reuse the same cluster.
#
# Refs: docs/plans/k3s-ingress-observability/phase-b-tasks.md — Task B2, B3
#       Chart: grafana/grafana 8.5.2 (pinned; latest stable 2026-05-20)

set -euo pipefail

NAMESPACE="monitoring"
GRAFANA_RELEASE="olam-grafana"
GRAFANA_CHART_VERSION="8.5.2"
LOCAL_PORT="3000"
GRAFANA_SVC_PORT="80"
PF_BIND_SECONDS=5

log()  { printf '[grafana-port-forward] %s\n' "$*" >&2; }
fail() { printf '[grafana-port-forward] FAIL: %s\n' "$*" >&2; exit 1; }

# -------------------------------------------------------------------------
# Cleanup trap — kill port-forward on exit; leave Helm release in place
# -------------------------------------------------------------------------
PF_PID=""
cleanup() {
  if [[ -n "$PF_PID" ]] && kill -0 "$PF_PID" 2>/dev/null; then
    kill "$PF_PID" 2>/dev/null || true
  fi
}
trap cleanup EXIT

# -------------------------------------------------------------------------
# Pre-flight
# -------------------------------------------------------------------------
command -v helm    >/dev/null 2>&1 || fail "helm not installed"
command -v kubectl >/dev/null 2>&1 || fail "kubectl not installed"
command -v curl    >/dev/null 2>&1 || fail "curl not installed"
command -v openssl >/dev/null 2>&1 || fail "openssl not installed"
command -v jq      >/dev/null 2>&1 || fail "jq not installed (required for B3 dashboard assertion)"
kubectl cluster-info >/dev/null 2>&1 || fail "kubectl: no reachable cluster; set KUBECONFIG"

log "pre-flight checks passed"

# -------------------------------------------------------------------------
# Ensure grafana Helm repo is present (idempotent — safe to re-run)
# -------------------------------------------------------------------------
helm repo add grafana https://grafana.github.io/helm-charts 2>/dev/null || true
helm repo update grafana

# Verify Loki is already installed (B2 depends on B1)
if ! helm status "olam-loki" -n "$NAMESPACE" >/dev/null 2>&1; then
  fail "olam-loki Helm release not found in namespace $NAMESPACE — run scripts/e2e/loki-ingest.sh first"
fi
log "Loki pre-condition satisfied (olam-loki release found)"

# -------------------------------------------------------------------------
# Step 1: Resolve admin password (preserve existing on idempotent re-run)
# -------------------------------------------------------------------------
# Grafana persists the admin password in its internal SQLite on first
# deploy. Subsequent helm upgrades do NOT re-read GF_SECURITY_ADMIN_PASSWORD
# from the env (env value is set once at pod-start and not refreshed). So
# on a re-run, rotating the Secret leaves the in-Grafana password stale
# and breaks API auth.
#
# Idempotency contract: if the Secret already exists, reuse its current
# password. The Secret's value matches Grafana's stored value (set in
# concert on first install). Only generate a new password when the
# Secret doesn't exist yet — i.e. true first deploy.
if kubectl get secret olam-grafana-admin -n "$NAMESPACE" >/dev/null 2>&1; then
  log "reusing existing admin password from Secret olam-grafana-admin"
  GRAFANA_ADMIN_PW=$(kubectl get secret olam-grafana-admin -n "$NAMESPACE" \
    -o jsonpath='{.data.admin-password}' | base64 -d)
else
  log "generating fresh admin password (first deploy)"
  GRAFANA_ADMIN_PW=$(openssl rand -base64 24)
fi
export GRAFANA_ADMIN_PW

# -------------------------------------------------------------------------
# Step 2: Create / update the admin Secret idempotently
# -------------------------------------------------------------------------
log "applying Secret olam-grafana-admin in namespace $NAMESPACE"
kubectl create secret generic olam-grafana-admin \
  --from-literal=admin-user=admin \
  --from-literal=admin-password="$GRAFANA_ADMIN_PW" \
  -n "$NAMESPACE" \
  --dry-run=client -o yaml \
  | kubectl apply -f -

log "Secret applied"

# -------------------------------------------------------------------------
# Step 3a: Apply olam-dashboards ConfigMap BEFORE helm install
#          so Grafana's volume mount finds it on first boot (B3).
#          The ConfigMap is generated from grafana-dashboards/*.json by
#          packages/peripheral-services/scripts/sync-grafana-dashboards.sh.
# -------------------------------------------------------------------------
REPO_ROOT="$(git -C "$(dirname "$0")" rev-parse --show-toplevel 2>/dev/null || pwd)"
# When invoked from a published @pleri/olam-cli install (no monorepo), `olam
# setup` exports OLAM_BUNDLE_ROOT=<install>/host-cp so the bundled
# peripheral-services/{helm-values,manifests} directory is reachable.
# Monorepo callers leave it unset; the script falls back to the source dir
# under packages/peripheral-services/.
if [[ -n "${OLAM_BUNDLE_ROOT:-}" ]]; then
  PERIPHERAL_SERVICES_DIR="$OLAM_BUNDLE_ROOT/peripheral-services"
else
  PERIPHERAL_SERVICES_DIR="$REPO_ROOT/packages/peripheral-services"
fi
CONFIGMAP_MANIFEST="$PERIPHERAL_SERVICES_DIR/manifests/80-grafana-dashboard-configmap.yaml"

if [[ -f "$CONFIGMAP_MANIFEST" ]]; then
  log "applying olam-dashboards ConfigMap from $CONFIGMAP_MANIFEST"
  kubectl apply -f "$CONFIGMAP_MANIFEST"
  log "ConfigMap applied"
else
  log "WARN: $CONFIGMAP_MANIFEST not found — Grafana will warn 'ConfigMap not found' until B3 is deployed"
fi

# -------------------------------------------------------------------------
# Step 3: Helm upgrade --install
# -------------------------------------------------------------------------
log "installing grafana/grafana ($GRAFANA_RELEASE) in namespace $NAMESPACE"
helm upgrade --install "$GRAFANA_RELEASE" grafana/grafana \
  --version "$GRAFANA_CHART_VERSION" \
  --namespace "$NAMESPACE" \
  --create-namespace \
  -f "$PERIPHERAL_SERVICES_DIR/helm-values/grafana-values.yaml" \
  --wait \
  --timeout "${OLAM_HELM_TIMEOUT:-600s}"

log "Grafana Helm install complete"

# -------------------------------------------------------------------------
# Step 4: Wait for Grafana pod Ready
# -------------------------------------------------------------------------
log "waiting for Grafana pod Ready (120s)"
kubectl wait \
  --for=condition=ready pod \
  -l "app.kubernetes.io/name=grafana" \
  -n "$NAMESPACE" \
  --timeout=120s

log "Grafana pod Ready"

# -------------------------------------------------------------------------
# Step 5: Start port-forward in background
# -------------------------------------------------------------------------
log "port-forwarding svc/$GRAFANA_RELEASE $LOCAL_PORT:$GRAFANA_SVC_PORT in namespace $NAMESPACE"
kubectl port-forward \
  -n "$NAMESPACE" \
  "svc/$GRAFANA_RELEASE" \
  "${LOCAL_PORT}:${GRAFANA_SVC_PORT}" &
PF_PID=$!

log "port-forward PID $PF_PID; waiting ${PF_BIND_SECONDS}s for bind"
sleep "$PF_BIND_SECONDS"

# Verify the port-forward process is still alive after sleep
kill -0 "$PF_PID" 2>/dev/null || fail "port-forward process exited prematurely"

# -------------------------------------------------------------------------
# Diagnostic helper — called on assertion failure
# -------------------------------------------------------------------------
dump_diagnostics() {
  log "DIAGNOSTIC: last 50 lines of Grafana pod logs:"
  kubectl logs -n "$NAMESPACE" \
    -l "app.kubernetes.io/name=grafana" \
    --tail=50 2>&1 >&2 || true
}

# -------------------------------------------------------------------------
# Step 6: Assertion 1 — /api/health returns 200 with database: ok
# -------------------------------------------------------------------------
log "asserting Grafana health (GET /api/health)"
HEALTH_RESPONSE=$(
  curl -sf \
    -u "admin:${GRAFANA_ADMIN_PW}" \
    "http://localhost:${LOCAL_PORT}/api/health" \
  || { dump_diagnostics; fail "GET /api/health failed — Grafana not reachable on port $LOCAL_PORT"; }
)

if ! echo "$HEALTH_RESPONSE" | jq -e '.database == "ok"' >/dev/null 2>&1; then
  log "DIAGNOSTIC: /api/health response:"
  echo "$HEALTH_RESPONSE" >&2
  dump_diagnostics
  fail '/api/health returned database != "ok" — Grafana DB layer not healthy'
fi

log "PASS: /api/health → database: ok"

# -------------------------------------------------------------------------
# Step 7: Assertion 2 — /api/datasources includes Loki entry with cluster URL
# -------------------------------------------------------------------------
log "asserting Loki datasource pre-wired (GET /api/datasources)"
DS_RESPONSE=$(
  curl -sf \
    -u "admin:${GRAFANA_ADMIN_PW}" \
    "http://localhost:${LOCAL_PORT}/api/datasources" \
  || { dump_diagnostics; fail "GET /api/datasources failed"; }
)

EXPECTED_URL="olam-loki.monitoring.svc.cluster.local:3100"

if ! echo "$DS_RESPONSE" | jq -e 'map(select(.type == "loki")) | length >= 1' >/dev/null 2>&1; then
  log "DIAGNOSTIC: /api/datasources response:"
  echo "$DS_RESPONSE" >&2
  dump_diagnostics
  fail "datasources response contains no 'loki' type entry — datasource not provisioned"
fi

if ! echo "$DS_RESPONSE" | jq -e --arg url "$EXPECTED_URL" 'map(select(.type == "loki" and (.url | contains($url)))) | length >= 1' >/dev/null 2>&1; then
  log "DIAGNOSTIC: /api/datasources response:"
  echo "$DS_RESPONSE" >&2
  dump_diagnostics
  fail "Loki datasource URL does not contain '$EXPECTED_URL' — check grafana-values.yaml datasources block"
fi

log "PASS: Loki datasource found with cluster-local URL $EXPECTED_URL"

# -------------------------------------------------------------------------
# Step 7b: Assertion 2b — dashboard provider loaded olam-home (catches mount-path bugs)
# -------------------------------------------------------------------------
log "asserting olam-home dashboard visible in /api/search (catches ConfigMap mount failures)"
DASHBOARDS=$(
  curl -sf \
    -u "admin:${GRAFANA_ADMIN_PW}" \
    "http://localhost:${LOCAL_PORT}/api/search?type=dash-db&query=olam" \
  || true
)

if ! echo "$DASHBOARDS" | jq -e 'map(select(.uid == "olam-home")) | length == 1' >/dev/null 2>&1; then
  log "DIAGNOSTIC: /api/search response:"
  echo "$DASHBOARDS" >&2
  dump_diagnostics
  fail "olam-home dashboard not found in /api/search — check ConfigMap mount path and dashboard provider config"
fi

log "PASS: olam-home dashboard found via /api/search"

# -------------------------------------------------------------------------
# Step 8: Assertion 3 — olam-home dashboard present (B3)
# -------------------------------------------------------------------------
log "asserting olam-home dashboard present (GET /api/dashboards/uid/olam-home)"
DASHBOARD_RESPONSE=$(
  curl -sf \
    -u "admin:${GRAFANA_ADMIN_PW}" \
    "http://localhost:${LOCAL_PORT}/api/dashboards/uid/olam-home" \
  || { dump_diagnostics; fail "GET /api/dashboards/uid/olam-home failed — dashboard not found or Grafana unreachable"; }
)

if ! echo "$DASHBOARD_RESPONSE" | jq -e '.dashboard.uid == "olam-home"' >/dev/null 2>&1; then
  log "DIAGNOSTIC: /api/dashboards/uid/olam-home response:"
  echo "$DASHBOARD_RESPONSE" >&2
  dump_diagnostics
  fail "olam-home dashboard uid mismatch or missing — check ConfigMap provisioning and Grafana provider config"
fi

log "PASS: olam-home dashboard present with uid=olam-home"

# -------------------------------------------------------------------------
# Final
# -------------------------------------------------------------------------
log "PASS: Grafana port-forward accessible; Loki datasource pre-wired; olam-home dashboard provisioned — Tasks B2+B3 verified"
exit 0
