#!/usr/bin/env bash
# kyverno-cardinality-mutate.sh — Phase C C8 follow-up e2e smoke test.
#
# Verifies that the Kyverno ClusterPolicy
# `enforce-cardinality-labeldrop` mutates incoming ServiceMonitor and
# PodMonitor objects at admission time, regardless of authorship,
# closing codex's "policy by convention" gap on PR #783.
#
# Test approach:
#   1. helm-install Kyverno (pinned 3.8.1) into the `kyverno` namespace.
#   2. Apply the ClusterPolicy.
#   3. POSITIVE test: apply ServiceMonitor `kyverno-mutate-positive-test`
#      with selector `app: kyverno-mutate-positive-test` (no backing Service)
#      and NO metricRelabelings; assert Kyverno mutated it; delete immediately.
#   4. IDEMPOTENCY test: apply ServiceMonitor `kyverno-mutate-idempotency-test`
#      with selector `app: kyverno-mutate-idempotency-test` (different non-existent
#      label) and the labeldrop already present; assert count stays at 1; delete.
#   5. SCRAPE-VERIFICATION test: deploy synthetic `kyverno-emitter` (Service +
#      Deployment + ConfigMap) + dedicated ServiceMonitor `kyverno-emitter-sm`
#      applied WITHOUT metricRelabelings; assert Kyverno mutates the SM at admission;
#      wait for pod Ready; poll Prometheus for http_requests_total; assert
#      world_id label is ABSENT.
#
# Key design decision: POSITIVE and IDEMPOTENCY tests use selectors that match
# no real Service, so they are isolated from each other and from the SCRAPE test.
# A single dedicated SM (`kyverno-emitter-sm`) owns the emitter endpoint, so
# prometheus-operator can reliably reconcile exactly one scrape config for it.
# Root cause of the prior failure (PR #828 CI run 26239574154): two SMs
# (naive-violator + pre-armoured-violator) competed for the same
# `app: kyverno-emitter` Endpoints; operator never reconciled either.
#
# Pre-conditions:
#   - kube-prometheus-stack installed (cardinality-drop.sh ran).
#   - kubectl context set to a live cluster; helm + jq + curl available.
#
# Idempotency: kubectl apply is idempotent; helm upgrade --install is
# idempotent. Cleanup trap removes synthetic resources on exit. The
# ClusterPolicy + Kyverno install are LEFT in the cluster (permanent
# C8 fixtures).
#
# Refs: docs/plans/k3s-ingress-observability/phase-c-tasks.md — C8
#       codex review on PR #783 ("policy by convention" finding)
#       PR #828 CI run 26239574154 (competing-SM root cause)

set -euo pipefail

KYVERNO_VERSION="3.8.1"
KYVERNO_NAMESPACE="kyverno"
TEST_NAMESPACE="monitoring"
PROM_LOCAL_PORT="9092"   # 9090, 9091 may be in use by sibling Phase C scripts
PF_BIND_SECONDS=5
TARGET_DISCOVERY_TIMEOUT="${OLAM_PROM_DISCOVERY_TIMEOUT:-240}"   # bumped from 180s; one CI attempt observed kyverno-emitter still not scraped at 180s
SCRAPE_POLL_INTERVAL=10

log()  { printf '[kyverno-mutate] %s\n' "$*" >&2; }
fail() { printf '[kyverno-mutate] FAIL: %s\n' "$*" >&2; exit 1; }

REPO_ROOT="$(git -C "$(dirname "$0")" rev-parse --show-toplevel 2>/dev/null || pwd)"
# When invoked from a published @pleri/olam-cli install (no monorepo), `olam
# setup` exports OLAM_BUNDLE_ROOT=<install>/host-cp so the bundled
# peripheral-services/{helm-values,manifests} directory is reachable.
# Monorepo callers leave it unset; the script falls back to the source dir
# under packages/peripheral-services/.
if [[ -n "${OLAM_BUNDLE_ROOT:-}" ]]; then
  PERIPHERAL_SERVICES_DIR="$OLAM_BUNDLE_ROOT/peripheral-services"
else
  PERIPHERAL_SERVICES_DIR="$REPO_ROOT/packages/peripheral-services"
fi

# -------------------------------------------------------------------------
# Cleanup trap — kill port-forwards; remove synthetic resources on exit.
# Kyverno chart + ClusterPolicy stay (permanent C8 fixtures).
# -------------------------------------------------------------------------
PROM_PF_PID=""
cleanup() {
  [[ -n "$PROM_PF_PID" ]] && kill "$PROM_PF_PID" 2>/dev/null || true
  log "removing synthetic resources (idempotent)"
  # Mutation-test SMs (already deleted inline, but --ignore-not-found makes this safe)
  kubectl delete servicemonitor kyverno-mutate-positive-test    -n "$TEST_NAMESPACE" --ignore-not-found=true 2>/dev/null || true
  kubectl delete servicemonitor kyverno-mutate-idempotency-test -n "$TEST_NAMESPACE" --ignore-not-found=true 2>/dev/null || true
  # Scrape-verification resources
  kubectl delete servicemonitor kyverno-emitter-sm     -n "$TEST_NAMESPACE" --ignore-not-found=true 2>/dev/null || true
  kubectl delete deployment     kyverno-emitter        -n "$TEST_NAMESPACE" --ignore-not-found=true 2>/dev/null || true
  kubectl delete service        kyverno-emitter-svc    -n "$TEST_NAMESPACE" --ignore-not-found=true 2>/dev/null || true
  kubectl delete configmap      kyverno-emitter-config -n "$TEST_NAMESPACE" --ignore-not-found=true 2>/dev/null || true
}
trap cleanup EXIT

# -------------------------------------------------------------------------
# Pre-flight
# -------------------------------------------------------------------------
command -v helm    >/dev/null 2>&1 || fail "helm not installed"
command -v kubectl >/dev/null 2>&1 || fail "kubectl not installed"
command -v curl    >/dev/null 2>&1 || fail "curl not installed"
command -v jq      >/dev/null 2>&1 || fail "jq not installed"
kubectl cluster-info >/dev/null 2>&1 || fail "kubectl: no reachable cluster; set KUBECONFIG"

# kube-prom-stack must already be up — we rely on Prometheus + the
# ServiceMonitor CRD existing.
kubectl get crd servicemonitors.monitoring.coreos.com >/dev/null 2>&1 \
  || fail "ServiceMonitor CRD not present — run prom-no-double-grafana.sh first"
kubectl get deployment -n "$TEST_NAMESPACE" -l "app.kubernetes.io/name=prometheus-operator" \
  >/dev/null 2>&1 \
  || fail "prometheus-operator not found in $TEST_NAMESPACE — run prom-no-double-grafana.sh first"

log "pre-flight checks passed"

# -------------------------------------------------------------------------
# Step 1: helm-install Kyverno
#
# Repo add is idempotent; helm upgrade --install handles fresh install + upgrade.
# `--wait` blocks until pods are Ready; admission webhook needs to be live
# before we apply the ClusterPolicy or our test ServiceMonitors.
# -------------------------------------------------------------------------
log "ensuring kyverno helm repo is configured"
helm repo add kyverno https://kyverno.github.io/kyverno/ >/dev/null 2>&1 || true
helm repo update kyverno >/dev/null 2>&1 || true

log "installing kyverno chart $KYVERNO_VERSION (waits for admission webhook Ready)"
helm upgrade --install olam-kyverno kyverno/kyverno \
  --version "$KYVERNO_VERSION" \
  --namespace "$KYVERNO_NAMESPACE" \
  --create-namespace \
  -f "$PERIPHERAL_SERVICES_DIR/helm-values/kyverno-values.yaml" \
  --wait --timeout "${OLAM_HELM_TIMEOUT:-600s}" 2>&1 | tail -8

# Sanity: kyverno-admission-controller Deployment Ready.
kubectl get deployment -n "$KYVERNO_NAMESPACE" -l "app.kubernetes.io/component=admission-controller" \
  >/dev/null 2>&1 \
  || fail "kyverno admission controller not found in $KYVERNO_NAMESPACE"

log "waiting for kyverno admission webhook to be registered with apiserver"
# The webhook registration is the LAST thing kyverno does after pod-Ready;
# poll until our ClusterPolicy can be admitted.
elapsed=0
while [ "$elapsed" -lt 120 ]; do
  if kubectl get validatingwebhookconfiguration kyverno-policy-validating-webhook-cfg \
       >/dev/null 2>&1; then
    log "kyverno webhooks registered after ${elapsed}s"
    break
  fi
  sleep 5
  elapsed=$((elapsed + 5))
done
if [ "$elapsed" -ge 120 ]; then
  fail "kyverno webhook registration timed out after 120s"
fi

# -------------------------------------------------------------------------
# Step 2: Apply the ClusterPolicy
# -------------------------------------------------------------------------
log "applying ClusterPolicy enforce-cardinality-labeldrop"
kubectl apply -f "$PERIPHERAL_SERVICES_DIR/manifests/96-kyverno-cardinality-mutate.yaml"

# Wait for policy to be Ready (Kyverno controller picks it up and reports
# readiness in status.ready / .conditions).
log "waiting up to 60s for ClusterPolicy to be Ready"
elapsed=0
while [ "$elapsed" -lt 60 ]; do
  READY=$(kubectl get clusterpolicy enforce-cardinality-labeldrop \
    -o jsonpath='{.status.ready}' 2>/dev/null || echo "")
  if [ "$READY" = "true" ]; then
    log "ClusterPolicy Ready after ${elapsed}s"
    break
  fi
  sleep 3
  elapsed=$((elapsed + 3))
done
if [ "$elapsed" -ge 60 ]; then
  log "WARN: ClusterPolicy status.ready not observed within 60s; proceeding (status field can lag)"
fi

# -------------------------------------------------------------------------
# Step 3: POSITIVE test — mutation only, no backing Service
#
# Uses selector `app: kyverno-mutate-positive-test` — a label that no
# real Service carries, so this SM never competes with anything for
# Endpoints. Its sole job is to exercise the Kyverno admission webhook.
#
# Deleted immediately after assertion so the SM space is clean when
# the scrape test runs.
# -------------------------------------------------------------------------
log "POSITIVE test: applying naive ServiceMonitor (no metricRelabelings, non-Service-backed selector)"
kubectl apply -f - <<'EOF'
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: kyverno-mutate-positive-test
  namespace: monitoring
  labels:
    release: olam-prom
spec:
  namespaceSelector:
    matchNames:
      - monitoring
  selector:
    matchLabels:
      app: kyverno-mutate-positive-test
  endpoints:
    - port: metrics
      interval: 15s
      # NOTE: deliberately NO metricRelabelings — Kyverno must inject it.
EOF

# Read back and assert.
ACTUAL=$(kubectl get servicemonitor kyverno-mutate-positive-test -n "$TEST_NAMESPACE" -o json \
  | jq -r '.spec.endpoints[0].metricRelabelings // [] | tojson')
log "kyverno-mutate-positive-test metricRelabelings after admission: $ACTUAL"

INJECTED_COUNT=$(echo "$ACTUAL" | jq '[ .[] | select(.action == "labeldrop" and (.regex | contains("world_id"))) ] | length')
if [ "$INJECTED_COUNT" -lt 1 ]; then
  log "actual policy state:"
  kubectl get clusterpolicy enforce-cardinality-labeldrop -o yaml >&2 || true
  fail "POSITIVE test FAILED: Kyverno did not inject labeldrop into naive ServiceMonitor — third-party bypass gap NOT closed"
fi
log "PASS: naive ServiceMonitor was mutated at admission (labeldrop injected)"

log "deleting kyverno-mutate-positive-test (mutation-only test; SM space clean for scrape test)"
kubectl delete servicemonitor kyverno-mutate-positive-test -n "$TEST_NAMESPACE" --ignore-not-found=true

# -------------------------------------------------------------------------
# Step 4: IDEMPOTENCY test — mutation only, no backing Service
#
# Uses selector `app: kyverno-mutate-idempotency-test` — different from
# the positive test and from the scrape test label. No real Service.
# Deleted immediately after assertion.
# -------------------------------------------------------------------------
log "IDEMPOTENCY test: applying pre-armoured ServiceMonitor (labeldrop already present)"
kubectl apply -f - <<'EOF'
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: kyverno-mutate-idempotency-test
  namespace: monitoring
  labels:
    release: olam-prom
spec:
  namespaceSelector:
    matchNames:
      - monitoring
  selector:
    matchLabels:
      app: kyverno-mutate-idempotency-test
  endpoints:
    - port: metrics
      interval: 15s
      metricRelabelings:
        - action: labeldrop
          regex: 'world_id|trace_id|user_id|request_id|operator_id'
EOF

DUP_COUNT=$(kubectl get servicemonitor kyverno-mutate-idempotency-test -n "$TEST_NAMESPACE" -o json \
  | jq '[ .spec.endpoints[0].metricRelabelings[] | select(.action == "labeldrop" and (.regex | contains("world_id"))) ] | length')
log "kyverno-mutate-idempotency-test labeldrop count: $DUP_COUNT"
if [ "$DUP_COUNT" -ne 1 ]; then
  kubectl get servicemonitor kyverno-mutate-idempotency-test -n "$TEST_NAMESPACE" -o yaml >&2
  fail "IDEMPOTENCY test FAILED: expected 1 labeldrop entry, got $DUP_COUNT — policy double-adds"
fi
log "PASS: pre-armoured ServiceMonitor has exactly 1 labeldrop (no double-add)"

log "deleting kyverno-mutate-idempotency-test (mutation-only test; SM space clean for scrape test)"
kubectl delete servicemonitor kyverno-mutate-idempotency-test -n "$TEST_NAMESPACE" --ignore-not-found=true

# -------------------------------------------------------------------------
# Step 5: SCRAPE-VERIFICATION test — dedicated SM + Service + Pod
#
# One SM (`kyverno-emitter-sm`) selects exactly one Service (`kyverno-emitter-svc`).
# No other SM in the cluster selects `app: kyverno-emitter`, so prometheus-operator
# reconciles a single clean scrape config.
#
# The SM is applied WITHOUT metricRelabelings so Kyverno's admission webhook
# fires — this is the load-bearing check that the policy applies during real
# scrape setup, not just on test fixtures.
#
# After admission we verify the spec has the labeldrop, then wait for the pod
# to be Ready and poll Prometheus for http_requests_total. We assert
# world_id is absent from all returned series.
#
# Mirrors the working pattern from dashboards-have-data.sh (single dedicated
# SM + co-located Service in `monitoring` namespace).
# -------------------------------------------------------------------------
log "SCRAPE-VERIFICATION test: deploying synthetic kyverno-emitter (emits http_requests_total{world_id})"
kubectl apply -f - <<'EOF'
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: kyverno-emitter-config
  namespace: monitoring
data:
  metrics: |
    # HELP http_requests_total Synthetic counter; world_id is the cardinality bomb
    # TYPE http_requests_total counter
    http_requests_total{world_id="kyverno-world",route="/api",method="GET",status_code="200"} 1
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kyverno-emitter
  namespace: monitoring
  labels:
    app: kyverno-emitter
spec:
  replicas: 1
  selector:
    matchLabels:
      app: kyverno-emitter
  template:
    metadata:
      labels:
        app: kyverno-emitter
    spec:
      containers:
        - name: emitter
          image: python:3.11-alpine
          ports:
            - containerPort: 8080
          command: ["python3", "-c"]
          args:
            - |
              import http.server
              with open('/config/metrics') as f: METRICS = f.read().encode()
              class H(http.server.BaseHTTPRequestHandler):
                  def do_GET(self):
                      if self.path != '/metrics':
                          self.send_response(404); self.end_headers(); return
                      self.send_response(200)
                      self.send_header('Content-Type', 'text/plain; version=0.0.4; charset=utf-8')
                      self.end_headers()
                      self.wfile.write(METRICS)
                  def log_message(self, *a): pass
              http.server.HTTPServer(('0.0.0.0', 8080), H).serve_forever()
          volumeMounts:
            - name: config
              mountPath: /config
      volumes:
        - name: config
          configMap:
            name: kyverno-emitter-config
---
apiVersion: v1
kind: Service
metadata:
  name: kyverno-emitter-svc
  namespace: monitoring
  labels:
    app: kyverno-emitter
spec:
  selector:
    app: kyverno-emitter
  ports:
    - name: metrics
      port: 8080
      targetPort: 8080
EOF

log "waiting for kyverno-emitter deployment Ready"
kubectl rollout status deployment/kyverno-emitter -n "$TEST_NAMESPACE" --timeout=120s

# Apply the dedicated ServiceMonitor WITHOUT metricRelabelings so Kyverno
# mutates it at admission — this proves the policy fires on real SM objects,
# not just on the POSITIVE test fixture.
log "applying kyverno-emitter-sm (no metricRelabelings — Kyverno must inject)"
kubectl apply -f - <<'EOF'
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: kyverno-emitter-sm
  namespace: monitoring
  labels:
    release: olam-prom
spec:
  namespaceSelector:
    matchNames:
      - monitoring
  selector:
    matchLabels:
      app: kyverno-emitter
  endpoints:
    - port: metrics
      interval: 15s
      # NOTE: NO metricRelabelings — Kyverno must inject the labeldrop at admission.
EOF

# Verify Kyverno mutated this SM too (belt-and-suspenders: proves the policy
# applies to the SM that actually drives the scrape, not just the test fixtures).
SCRAPE_SM_ACTUAL=$(kubectl get servicemonitor kyverno-emitter-sm -n "$TEST_NAMESPACE" -o json \
  | jq -r '.spec.endpoints[0].metricRelabelings // [] | tojson')
log "kyverno-emitter-sm metricRelabelings after admission: $SCRAPE_SM_ACTUAL"

SCRAPE_SM_INJECTED=$(echo "$SCRAPE_SM_ACTUAL" | jq '[ .[] | select(.action == "labeldrop" and (.regex | contains("world_id"))) ] | length')
if [ "$SCRAPE_SM_INJECTED" -lt 1 ]; then
  log "actual policy state:"
  kubectl get clusterpolicy enforce-cardinality-labeldrop -o yaml >&2 || true
  fail "SCRAPE-VERIFICATION test FAILED: Kyverno did not mutate kyverno-emitter-sm at admission"
fi
log "PASS: kyverno-emitter-sm was mutated at admission (labeldrop injected)"

# Port-forward Prometheus and poll for metric samples.
log "port-forwarding svc/prometheus-operated $PROM_LOCAL_PORT:9090"
kubectl port-forward \
  -n "$TEST_NAMESPACE" \
  "svc/prometheus-operated" \
  "${PROM_LOCAL_PORT}:9090" &
PROM_PF_PID=$!
sleep "$PF_BIND_SECONDS"
kill -0 "$PROM_PF_PID" 2>/dev/null \
  || fail "Prometheus port-forward exited prematurely"

PROM_URL="http://localhost:${PROM_LOCAL_PORT}"

# Direct-metric polling rather than target-discovery polling.
#
# Rationale: kube-prometheus-stack's default relabel sets the `job` label
# from the k8s Service name. Polling by job-name is brittle — operator
# reconciliation races, dropped-target filtering, and rare CRD revision
# lag have all surfaced as "target not in activeTargets" flakes during
# earlier ingress-integration runs. What we ACTUALLY care about is
# whether the mutated relabel was applied to a real scrape sample. So
# poll for the metric directly. With a single SM selecting on
# `app=kyverno-emitter`, any http_requests_total series returned
# necessarily came through kyverno-emitter-sm.
log "polling Prometheus for http_requests_total samples (up to ${TARGET_DISCOVERY_TIMEOUT}s)"
elapsed=0
RESULT=""
while [ "$elapsed" -lt "$TARGET_DISCOVERY_TIMEOUT" ]; do
  RESULT=$(curl -sf "${PROM_URL}/api/v1/query?query=http_requests_total" 2>/dev/null || echo "")
  if [ -n "$RESULT" ]; then
    SERIES_COUNT=$(echo "$RESULT" | jq '.data.result | length' 2>/dev/null || echo "0")
    if [ "$SERIES_COUNT" -ge 1 ]; then
      log "http_requests_total returned $SERIES_COUNT series after ${elapsed}s"
      break
    fi
  fi
  sleep "$SCRAPE_POLL_INTERVAL"
  elapsed=$((elapsed + SCRAPE_POLL_INTERVAL))
done

if [ "$elapsed" -ge "$TARGET_DISCOVERY_TIMEOUT" ]; then
  log "Active targets snapshot for diagnosis:"
  curl -sf "${PROM_URL}/api/v1/targets" | jq '.data.activeTargets[] | {job: .labels.job, service: .labels.service, namespace: .labels.namespace, health: .health, lastError: .lastError}' >&2 || true
  log "ServiceMonitor kyverno-emitter-sm status:"
  kubectl get servicemonitor kyverno-emitter-sm -n "$TEST_NAMESPACE" -o yaml >&2 || true
  log "prometheus-operator log tail (last 50 lines):"
  kubectl logs -n "$TEST_NAMESPACE" -l "app.kubernetes.io/name=prometheus-operator" --tail=50 >&2 || true
  fail "Prometheus did not scrape kyverno-emitter within ${TARGET_DISCOVERY_TIMEOUT}s"
fi

SERIES_COUNT=$(echo "$RESULT" | jq '.data.result | length')

LEAKED=$(echo "$RESULT" | jq '[.data.result[] | .metric | has("world_id")] | any')
if [ "$LEAKED" = "true" ]; then
  echo "$RESULT" | jq '.data.result[] | .metric' >&2
  fail "world_id label leaked into Prometheus — Kyverno-mutated relabel did NOT take effect at scrape time"
fi

log "PASS: kyverno-emitter scraped via kyverno-emitter-sm; world_id absent at scrape time"
log "PASS: C8 verified — Kyverno mutates third-party-shaped ServiceMonitors at admission and the mutation takes effect at scrape time"
exit 0
