#!/usr/bin/env bash
# prom-no-double-grafana.sh — Phase C Task C1 e2e smoke test.
#
# Verifies:
#   1. kube-prometheus-stack installs (Prometheus pod becomes Ready).
#   2. ServiceMonitor CRD is Established before Phase B charts are upgraded.
#   3. Phase B charts (Loki + Promtail + Grafana) are helm-upgraded to pick up
#      serviceMonitor.enabled: true now that the CRD exists.
#   4. Exactly one Grafana Deployment is running in the cluster (no double-Grafana).
#   5. Phase B's Grafana (olam-grafana) has exactly one Prometheus datasource
#      provisioned (from grafana-values.yaml datasources block added in C1).
#   6. Prometheus is scraping at least one active target.
#
# Pre-conditions:
#   - kubectl context is set to a live k8s cluster.
#   - Phase B e2e (loki-ingest.sh + grafana-port-forward.sh + grafana-dashboard-persistence.sh)
#     has already run: olam-loki, olam-promtail, and olam-grafana releases are installed.
#   - The olam-grafana-admin Secret exists (created by grafana-port-forward.sh).
#   - helm, kubectl, curl, jq binaries available.
#
# Chart: prometheus-community/kube-prometheus-stack 85.2.0 (pinned; latest stable 2026-05-21).
#
# Idempotency: helm upgrade --install is idempotent; re-runs on an existing
# cluster succeed. Port-forwards are killed on exit via trap.
#
# Refs: docs/plans/k3s-ingress-observability/phase-c-tasks.md — Task C1

set -euo pipefail

NAMESPACE="monitoring"
PROM_RELEASE="olam-prom"
PROM_CHART_VERSION="85.2.0"
GRAFANA_RELEASE="olam-grafana"
GRAFANA_LOCAL_PORT="3001"   # avoid collision if phase-b-e2e left a port-forward on 3000
GRAFANA_SVC_PORT="80"
PROM_LOCAL_PORT="9090"
PF_BIND_SECONDS=5

log()  { printf '[prom-no-double-grafana] %s\n' "$*" >&2; }
fail() { printf '[prom-no-double-grafana] FAIL: %s\n' "$*" >&2; exit 1; }

# -------------------------------------------------------------------------
# Resolve repo root so helm -f paths work regardless of invocation cwd
# -------------------------------------------------------------------------
REPO_ROOT="$(git -C "$(dirname "$0")" rev-parse --show-toplevel 2>/dev/null || pwd)"
# When invoked from a published @pleri/olam-cli install (no monorepo), `olam
# setup` exports OLAM_BUNDLE_ROOT=<install>/host-cp so the bundled
# peripheral-services/{helm-values,manifests} directory is reachable.
# Monorepo callers leave it unset; the script falls back to the source dir
# under packages/peripheral-services/.
if [[ -n "${OLAM_BUNDLE_ROOT:-}" ]]; then
  PERIPHERAL_SERVICES_DIR="$OLAM_BUNDLE_ROOT/peripheral-services"
else
  PERIPHERAL_SERVICES_DIR="$REPO_ROOT/packages/peripheral-services"
fi

# -------------------------------------------------------------------------
# Cleanup trap — kill port-forwards on exit; leave Helm releases in place
# -------------------------------------------------------------------------
GRAFANA_PF_PID=""
PROM_PF_PID=""
cleanup() {
  [[ -n "$GRAFANA_PF_PID" ]] && kill "$GRAFANA_PF_PID" 2>/dev/null || true
  [[ -n "$PROM_PF_PID"   ]] && kill "$PROM_PF_PID"   2>/dev/null || true
}
trap cleanup EXIT

# -------------------------------------------------------------------------
# Pre-flight
# -------------------------------------------------------------------------
command -v helm    >/dev/null 2>&1 || fail "helm not installed"
command -v kubectl >/dev/null 2>&1 || fail "kubectl not installed"
command -v curl    >/dev/null 2>&1 || fail "curl not installed"
command -v jq      >/dev/null 2>&1 || fail "jq not installed"
kubectl cluster-info >/dev/null 2>&1 || fail "kubectl: no reachable cluster; set KUBECONFIG"

log "pre-flight checks passed"

# Verify Phase B pre-conditions
for release in olam-loki olam-promtail "$GRAFANA_RELEASE"; do
  helm status "$release" -n "$NAMESPACE" >/dev/null 2>&1 \
    || fail "Phase B release '$release' not found in namespace $NAMESPACE — run phase-b-e2e first"
done
log "Phase B pre-conditions satisfied (olam-loki, olam-promtail, olam-grafana releases found)"

# -------------------------------------------------------------------------
# Step 1: Add prometheus-community repo and install kube-prometheus-stack
# -------------------------------------------------------------------------
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts 2>/dev/null || true
helm repo update prometheus-community

log "installing prometheus-community/kube-prometheus-stack ($PROM_RELEASE) version $PROM_CHART_VERSION"
helm upgrade --install "$PROM_RELEASE" prometheus-community/kube-prometheus-stack \
  --version "$PROM_CHART_VERSION" \
  --namespace "$NAMESPACE" \
  --create-namespace \
  -f "$PERIPHERAL_SERVICES_DIR/helm-values/kube-prom-stack-values.yaml" \
  --wait \
  --timeout "${OLAM_HELM_TIMEOUT:-600s}"

log "kube-prometheus-stack helm install complete"

# -------------------------------------------------------------------------
# Step 2: Wait for ServiceMonitor CRD to be Established
# This is the gate before upgrading Phase B charts — the CRD must exist
# for serviceMonitor.enabled: true to produce a valid ServiceMonitor object.
# -------------------------------------------------------------------------
log "waiting for ServiceMonitor CRD to be Established (60s)"
kubectl wait \
  --for=condition=established \
  crd/servicemonitors.monitoring.coreos.com \
  --timeout=60s

log "ServiceMonitor CRD Established"

# -------------------------------------------------------------------------
# Step 3: Helm-upgrade Phase B charts to enable ServiceMonitor at RUNTIME
#
# The source-of-truth values files keep serviceMonitor.enabled: false so a
# standalone Phase B install (without kube-prometheus-stack) does not
# hard-fail with "no matches for kind ServiceMonitor". We flip the toggle
# at runtime here, AFTER the CRD is Established, via --set overrides. This
# preserves Phase B's standalone-installability invariant while wiring
# Prometheus discovery when kube-prom-stack is present.
#
# NOTE: Loki 6.7.4 uses monitoring.serviceMonitor (not top-level serviceMonitor)
# — chart-version-specific path.
# -------------------------------------------------------------------------
# Chart version pins MUST match the ones in phase-b-e2e's loki-ingest.sh +
# grafana-port-forward.sh. Without --version, helm pulls latest from the repo;
# the latest charts may reference new template values not present in our
# values files (e.g., Loki 6.8.x references .Values.loki.ui.enabled which is
# nil in our 6.7.4-shaped values, producing a nil-pointer template error
# during upgrade).
LOKI_CHART_VERSION="6.7.4"
PROMTAIL_CHART_VERSION="6.16.6"
GRAFANA_CHART_VERSION="8.5.2"

log "upgrading Phase B charts with runtime --set serviceMonitor.enabled=true (pinned versions)"

helm upgrade olam-loki grafana/loki \
  --version "$LOKI_CHART_VERSION" \
  --namespace "$NAMESPACE" \
  -f "$PERIPHERAL_SERVICES_DIR/helm-values/loki-values.yaml" \
  --wait \
  --timeout "${OLAM_HELM_TIMEOUT:-600s}" \
  --reuse-values \
  --set monitoring.serviceMonitor.enabled=true

log "olam-loki upgraded (ServiceMonitor enabled)"

helm upgrade olam-promtail grafana/promtail \
  --version "$PROMTAIL_CHART_VERSION" \
  --namespace "$NAMESPACE" \
  -f "$PERIPHERAL_SERVICES_DIR/helm-values/promtail-values.yaml" \
  --wait \
  --timeout "${OLAM_HELM_TIMEOUT:-600s}" \
  --reuse-values \
  --set serviceMonitor.enabled=true

log "olam-promtail upgraded (ServiceMonitor enabled)"

helm upgrade "$GRAFANA_RELEASE" grafana/grafana \
  --version "$GRAFANA_CHART_VERSION" \
  --namespace "$NAMESPACE" \
  -f "$PERIPHERAL_SERVICES_DIR/helm-values/grafana-values.yaml" \
  --wait \
  --timeout "${OLAM_HELM_TIMEOUT:-600s}" \
  --reuse-values \
  --set serviceMonitor.enabled=true

log "$GRAFANA_RELEASE upgraded (ServiceMonitor enabled; Prometheus datasource provisioned)"

# -------------------------------------------------------------------------
# Step 4: Wait for Prometheus pod Ready
# -------------------------------------------------------------------------
log "waiting for Prometheus pod Ready (300s)"
kubectl wait \
  --for=condition=ready pod \
  -l "app.kubernetes.io/name=prometheus" \
  -n "$NAMESPACE" \
  --timeout=300s

log "Prometheus pod Ready"

# -------------------------------------------------------------------------
# Step 5: Assertion — exactly one Grafana Deployment in the cluster
# This catches any regression where kube-prometheus-stack's bundled Grafana
# sub-chart accidentally gets enabled.
# -------------------------------------------------------------------------
log "asserting exactly 1 Grafana Deployment in namespace $NAMESPACE"
GRAFANA_DEPS=$(kubectl get deployment \
  -n "$NAMESPACE" \
  -l "app.kubernetes.io/name=grafana" \
  -o name \
  | wc -l \
  | tr -d ' ')

if [ "$GRAFANA_DEPS" != "1" ]; then
  log "FAIL: expected exactly 1 Grafana Deployment, found $GRAFANA_DEPS"
  kubectl get deployment -n "$NAMESPACE" -l "app.kubernetes.io/name=grafana" >&2
  fail "double-Grafana detected — kube-prometheus-stack's grafana.enabled must be false"
fi

log "PASS: exactly 1 Grafana Deployment found"

# -------------------------------------------------------------------------
# Step 6: Assertion — Grafana has exactly one Prometheus datasource
# Re-read the admin password from the Secret (grafana-port-forward.sh created it).
# Use port 3001 to avoid colliding with any live phase-b-e2e port-forward on 3000.
# -------------------------------------------------------------------------
log "reading admin password from Secret olam-grafana-admin"
GRAFANA_ADMIN_PW=$(kubectl get secret olam-grafana-admin \
  -n "$NAMESPACE" \
  -o jsonpath='{.data.admin-password}' \
  | base64 -d)

log "port-forwarding svc/$GRAFANA_RELEASE $GRAFANA_LOCAL_PORT:$GRAFANA_SVC_PORT"
kubectl port-forward \
  -n "$NAMESPACE" \
  "svc/$GRAFANA_RELEASE" \
  "${GRAFANA_LOCAL_PORT}:${GRAFANA_SVC_PORT}" &
GRAFANA_PF_PID=$!

log "waiting ${PF_BIND_SECONDS}s for Grafana port-forward to bind"
sleep "$PF_BIND_SECONDS"
kill -0 "$GRAFANA_PF_PID" 2>/dev/null \
  || fail "Grafana port-forward process exited prematurely"

log "asserting exactly 1 Prometheus datasource in Grafana (GET /api/datasources)"
DATASOURCES=$(curl -sf \
  -u "admin:${GRAFANA_ADMIN_PW}" \
  "http://localhost:${GRAFANA_LOCAL_PORT}/api/datasources" \
  || { kubectl logs -n "$NAMESPACE" -l "app.kubernetes.io/name=grafana" --tail=30 >&2 || true
       fail "GET /api/datasources failed — Grafana not reachable on port $GRAFANA_LOCAL_PORT"; })

if ! echo "$DATASOURCES" | jq -e 'map(select(.type == "prometheus")) | length == 1' >/dev/null 2>&1; then
  log "FAIL: Grafana does not have exactly 1 Prometheus datasource"
  echo "$DATASOURCES" | jq . >&2
  fail "Prometheus datasource not provisioned — check datasources block in grafana-values.yaml"
fi

PROM_URL=$(echo "$DATASOURCES" | jq -r 'map(select(.type == "prometheus")) | .[0].url')
log "PASS: Grafana has exactly 1 Prometheus datasource (url=$PROM_URL)"

# -------------------------------------------------------------------------
# Step 7: Assertion — Prometheus is scraping at least one active target
# -------------------------------------------------------------------------
log "port-forwarding svc/prometheus-operated $PROM_LOCAL_PORT:9090"
kubectl port-forward \
  -n "$NAMESPACE" \
  "svc/prometheus-operated" \
  "${PROM_LOCAL_PORT}:9090" &
PROM_PF_PID=$!

log "waiting ${PF_BIND_SECONDS}s for Prometheus port-forward to bind"
sleep "$PF_BIND_SECONDS"
kill -0 "$PROM_PF_PID" 2>/dev/null \
  || fail "Prometheus port-forward process exited prematurely"

log "querying Prometheus /api/v1/targets for active targets"
TARGETS=$(curl -sf "http://localhost:${PROM_LOCAL_PORT}/api/v1/targets" \
  || fail "GET /api/v1/targets failed — Prometheus not reachable on port $PROM_LOCAL_PORT")

ACTIVE=$(echo "$TARGETS" | jq '.data.activeTargets | length')
if [ "$ACTIVE" -lt 1 ]; then
  log "FAIL: Prometheus has 0 active scrape targets"
  echo "$TARGETS" | jq '.data.activeTargets' >&2
  fail "Prometheus has no active targets — check ServiceMonitor CRD and scrapeConfig"
fi

log "PASS: $ACTIVE active scrape target(s) found in Prometheus"

# -------------------------------------------------------------------------
# Assertion C4: Recording rules from 95-prom-recording-rules.yaml are loaded
#
# The 9[0-9]-prom-* glob in apply-manifests.sh skips this file (requires
# kube-prom-stack CRDs to exist). We kubectl apply it here, then poll
# /api/v1/rules until the olam-http-aggregations group appears.
# The port-forward on PROM_LOCAL_PORT is already open from Step 7 above.
# -------------------------------------------------------------------------
PROM_URL="http://localhost:${PROM_LOCAL_PORT}"

log "applying 95-prom-recording-rules.yaml (skipped by apply-manifests due to 9[0-9]-prom-* filter)"
kubectl apply -f "$PERIPHERAL_SERVICES_DIR/manifests/95-prom-recording-rules.yaml"

# Prometheus operator reconcile + config reload can take ~60-90s (C2 lesson).
# Poll /api/v1/rules until our group appears (up to 180s).
RECORDING_RULES_TIMEOUT=180
log "polling ${PROM_URL}/api/v1/rules for olam-http-aggregations group (up to ${RECORDING_RULES_TIMEOUT}s)"
elapsed=0
while [ "$elapsed" -lt "$RECORDING_RULES_TIMEOUT" ]; do
  if curl -sf "${PROM_URL}/api/v1/rules" 2>/dev/null \
       | jq -e '.data.groups[] | select(.name == "olam-http-aggregations") | .rules[] | select(.name == "olam:http_requests:rate5m_by_service")' >/dev/null 2>&1; then
    log "PASS: olam-http-aggregations rule group loaded after ${elapsed}s"
    break
  fi
  sleep 10
  elapsed=$((elapsed + 10))
done
if [ "$elapsed" -ge "$RECORDING_RULES_TIMEOUT" ]; then
  log "FAIL: olam-http-aggregations rule group not found in /api/v1/rules within ${RECORDING_RULES_TIMEOUT}s"
  curl -sf "${PROM_URL}/api/v1/rules" | jq '.data.groups[] | .name' >&2 || true
  fail "PrometheusRule not loaded by operator"
fi

# -------------------------------------------------------------------------
# Final
# -------------------------------------------------------------------------
log "PASS: kube-prometheus-stack installed; single Grafana confirmed; Prometheus datasource provisioned; $ACTIVE active target(s); recording rules loaded — Tasks C1+C4 verified"
exit 0
