#!/usr/bin/env bash
# loki-ingest.sh — e2e smoke test: Loki single-binary installs, Promtail tails,
#                  OAuth query-param scrubbing verified (code=REDACTED, no raw token).
#
# Usage: scripts/e2e/loki-ingest.sh
#
# Pre-conditions:
#   - kubectl context is set to a live k8s cluster (does NOT spin up k3d)
#   - helm binary available
#   - grafana Helm repo added (helm repo add grafana https://grafana.github.io/helm-charts)
#
# This script is invoked by the A12 harness (scripts/test-ingress-integration/)
# after cluster-up.sh. It can also be run manually against any live cluster.
#
# Idempotency: `helm upgrade --install` is idempotent; re-runs succeed on an
#              existing cluster. The synthetic pod is cleaned up regardless of
#              pass/fail via a trap.
#
# Refs: docs/plans/k3s-ingress-observability/phase-b-tasks.md — Task B1
#       Chart: grafana/loki 6.7.4 (pinned; latest stable 2026-05-20)
#       Chart: grafana/promtail 6.16.6 (latest stable 2026-05-20)

set -euo pipefail

NAMESPACE="monitoring"
LOKI_RELEASE="olam-loki"
PROMTAIL_RELEASE="olam-promtail"
SYNTHETIC_POD="loki-e2e-synthetic"
LOKI_PORT="3100"
LOCAL_PORT="13100"   # avoid conflict with any host-level Loki

# Magic-number commentary: Promtail's tail → ingest cycle involves:
#   - inotify event (near-instant)
#   - Promtail pipeline processing (~1s)
#   - Loki write path (ingester chunk idle period: default 30m, but flush on
#     query pressure; typically <5s in practice)
# 10s is conservative for a single log line in a lightly loaded cluster.
INGEST_LAG_SECONDS=10

log()  { printf '[loki-ingest] %s\n' "$*" >&2; }
fail() { printf '[loki-ingest] FAIL: %s\n' "$*" >&2; exit 1; }

# -------------------------------------------------------------------------
# Cleanup trap — remove synthetic pod and port-forward on exit
# -------------------------------------------------------------------------
PF_PID=""
cleanup() {
  if [[ -n "$PF_PID" ]] && kill -0 "$PF_PID" 2>/dev/null; then
    kill "$PF_PID" 2>/dev/null || true
  fi
  kubectl delete pod "$SYNTHETIC_POD" -n default --ignore-not-found=true 2>/dev/null || true
}
trap cleanup EXIT

# -------------------------------------------------------------------------
# Pre-flight
# -------------------------------------------------------------------------
command -v helm    >/dev/null 2>&1 || fail "helm not installed"
command -v kubectl >/dev/null 2>&1 || fail "kubectl not installed"
command -v curl    >/dev/null 2>&1 || fail "curl not installed"
kubectl cluster-info >/dev/null 2>&1 || fail "kubectl: no reachable cluster; set KUBECONFIG"

log "pre-flight checks passed"

# -------------------------------------------------------------------------
# Resolve repo root so helm -f paths work regardless of invocation cwd
# -------------------------------------------------------------------------
REPO_ROOT="$(git -C "$(dirname "$0")" rev-parse --show-toplevel 2>/dev/null || pwd)"
# When invoked from a published @pleri/olam-cli install (no monorepo), `olam
# setup` exports OLAM_BUNDLE_ROOT=<install>/host-cp so the bundled
# peripheral-services/{helm-values,manifests} directory is reachable.
# Monorepo callers leave it unset; the script resolves the source dir under
# packages/peripheral-services/.
if [[ -n "${OLAM_BUNDLE_ROOT:-}" ]]; then
  PERIPHERAL_SERVICES_DIR="$OLAM_BUNDLE_ROOT/peripheral-services"
else
  PERIPHERAL_SERVICES_DIR="$REPO_ROOT/packages/peripheral-services"
fi

# -------------------------------------------------------------------------
# Ensure grafana Helm repo is present (idempotent — safe to re-run)
# -------------------------------------------------------------------------
helm repo add grafana https://grafana.github.io/helm-charts 2>/dev/null || true
helm repo update grafana

# -------------------------------------------------------------------------
# Step 1: Install / upgrade Loki (single-binary mode)
# -------------------------------------------------------------------------
log "installing grafana/loki ($LOKI_RELEASE) in namespace $NAMESPACE"
helm upgrade --install "$LOKI_RELEASE" grafana/loki \
  --version 6.7.4 \
  --namespace "$NAMESPACE" \
  --create-namespace \
  -f "$PERIPHERAL_SERVICES_DIR/helm-values/loki-values.yaml" \
  --wait \
  --timeout "${OLAM_HELM_TIMEOUT:-600s}"

log "loki helm install complete"

# -------------------------------------------------------------------------
# Step 2: Install / upgrade Promtail
# -------------------------------------------------------------------------
log "installing grafana/promtail ($PROMTAIL_RELEASE) in namespace $NAMESPACE"
helm upgrade --install "$PROMTAIL_RELEASE" grafana/promtail \
  --version 6.16.6 \
  --namespace "$NAMESPACE" \
  -f "$PERIPHERAL_SERVICES_DIR/helm-values/promtail-values.yaml" \
  --wait \
  --timeout 120s

log "promtail helm install complete"

# -------------------------------------------------------------------------
# Step 3: Wait for Loki pod Ready
# -------------------------------------------------------------------------
log "waiting for Loki pod Ready (120s)"
kubectl wait \
  --for=condition=ready pod \
  -l app.kubernetes.io/name=loki \
  -n "$NAMESPACE" \
  --timeout=120s

log "loki pod Ready"

# -------------------------------------------------------------------------
# Step 4: Generate synthetic log line with raw OAuth tokens in URL and headers.
#
# The pod prints a single log line containing all 4 scrub patterns:
#   ?code=SECRETTOKEN123        → code=REDACTED
#   &access_token=SECRETTOKEN456 → access_token=REDACTED
#   &state=SESSION789            → state=REDACTED
#   Authorization: Bearer SECRETBEARER000 → Authorization: Bearer REDACTED
#
# Promtail tails it, runs the scrubbing pipeline, and pushes to Loki with all
# 4 raw tokens absent and all 4 REDACTED markers present.
# -------------------------------------------------------------------------
log "launching synthetic pod (prints all 4 raw token patterns)"
kubectl run "$SYNTHETIC_POD" \
  --image=busybox \
  --restart=Never \
  -n default \
  -- sh -c 'echo "GET http://example.com/callback?code=SECRETTOKEN123&access_token=SECRETTOKEN456&state=SESSION789 HTTP/1.1 Authorization: Bearer SECRETBEARER000"'

# -------------------------------------------------------------------------
# Step 5: Wait for Promtail tail + ingest lag
# -------------------------------------------------------------------------
log "waiting ${INGEST_LAG_SECONDS}s for Promtail to tail and ingest synthetic log"
sleep "$INGEST_LAG_SECONDS"

# -------------------------------------------------------------------------
# Step 6: Port-forward Loki and query
# -------------------------------------------------------------------------
log "port-forwarding Loki svc to localhost:${LOCAL_PORT}"
kubectl port-forward \
  "svc/${LOKI_RELEASE}" \
  "${LOCAL_PORT}:${LOKI_PORT}" \
  -n "$NAMESPACE" &
PF_PID=$!

# Give port-forward a moment to establish
sleep 2

# Query Loki for log lines from the default namespace within the last 5 minutes.
# We search broadly for "SECRETTOKEN" to catch any raw token that leaked through,
# and separately verify all 4 REDACTED markers are present.
log "querying Loki for scrubbed entries"
QUERY_RESPONSE=$(
  curl -s -G \
    "http://localhost:${LOCAL_PORT}/loki/api/v1/query_range" \
    --data-urlencode 'query={namespace="default"} |= "REDACTED"' \
    --data-urlencode "start=$(date -u -v-5M +%s 2>/dev/null || date -u -d '5 minutes ago' +%s)000000000" \
    --data-urlencode "end=$(date -u +%s)000000000" \
    --data-urlencode 'limit=50'
)

# -------------------------------------------------------------------------
# Step 7: Assertions — verify all 4 scrub patterns
#
# Contract (matches Phase B spec + promtail-values.yaml):
#   ?code=SECRETTOKEN123        → code=REDACTED       (absent: SECRETTOKEN123)
#   &access_token=SECRETTOKEN456 → access_token=REDACTED (absent: SECRETTOKEN456)
#   &state=SESSION789            → state=REDACTED      (absent: SESSION789)
#   Authorization: Bearer SECRETBEARER000 → Bearer REDACTED (absent: SECRETBEARER000)
# -------------------------------------------------------------------------
log "asserting scrubbing correctness (all 4 patterns)"

diag() {
  log "DIAGNOSTIC: Loki query response:"
  echo "$QUERY_RESPONSE" >&2
  log "DIAGNOSTIC: last 50 lines of Promtail logs:"
  kubectl logs -n "$NAMESPACE" -l app.kubernetes.io/name=promtail --tail=50 2>&1 >&2 || true
}

# Assertion 1: query response is non-empty (Loki returned results)
if ! echo "$QUERY_RESPONSE" | grep -q '"result"'; then
  diag
  fail "Loki returned no result block — Promtail may not have ingested the synthetic log yet"
fi

# --- Scrubbed markers present ---

# Assertion 2a: code= is scrubbed
if ! echo "$QUERY_RESPONSE" | grep -q 'code=REDACTED'; then
  diag
  fail "'code=REDACTED' not found in Loki response — code= scrub stage not working"
fi

# Assertion 2b: access_token= is scrubbed
if ! echo "$QUERY_RESPONSE" | grep -q 'access_token=REDACTED'; then
  diag
  fail "'access_token=REDACTED' not found in Loki response — access_token= scrub stage not working"
fi

# Assertion 2c: state= is scrubbed
if ! echo "$QUERY_RESPONSE" | grep -q 'state=REDACTED'; then
  diag
  fail "'state=REDACTED' not found in Loki response — state= scrub stage not working"
fi

# Assertion 2d: Authorization Bearer is scrubbed
if ! echo "$QUERY_RESPONSE" | grep -q 'Bearer REDACTED'; then
  diag
  fail "'Bearer REDACTED' not found in Loki response — Authorization Bearer scrub stage not working"
fi

# --- Raw tokens absent ---

# Assertion 3a: raw code= token is absent
if echo "$QUERY_RESPONSE" | grep -q 'SECRETTOKEN123'; then
  diag
  fail "raw token 'SECRETTOKEN123' (code=) found in Loki response — scrubbing pipeline is NOT working"
fi

# Assertion 3b: raw access_token= token is absent
if echo "$QUERY_RESPONSE" | grep -q 'SECRETTOKEN456'; then
  diag
  fail "raw token 'SECRETTOKEN456' (access_token=) found in Loki response — scrubbing pipeline is NOT working"
fi

# Assertion 3c: raw state= token is absent
if echo "$QUERY_RESPONSE" | grep -q 'SESSION789'; then
  diag
  fail "raw token 'SESSION789' (state=) found in Loki response — scrubbing pipeline is NOT working"
fi

# Assertion 3d: raw Bearer token is absent
if echo "$QUERY_RESPONSE" | grep -q 'SECRETBEARER000'; then
  diag
  fail "raw token 'SECRETBEARER000' (Authorization Bearer) found in Loki response — scrubbing pipeline is NOT working"
fi

log "PASS: all 4 scrub patterns verified — code=REDACTED, access_token=REDACTED, state=REDACTED, Bearer REDACTED present; all raw tokens absent"
exit 0
