import Fastify from 'fastify';
import { env, initializeNoPersistenceMode, isMemoryMode, memoryLogger, getBlockedWarnings, createLogger, Logger } from '@4runr/shared';
import { getPersistenceMode } from './config/persistence.js';
import { getRunStore } from './runs/index.js';
import { requireAuth } from './middleware/auth.js';
import { readRateLimit, writeRateLimit, strictRateLimit } from './middleware/rateLimit.js';
import { validateBody, validateQuery, validateParams } from './middleware/validate.js';
import { addCorrelationId, getCorrelationId } from './middleware/correlationId.js';
import { setSecurityHeaders, validateSecrets, logSecurityValidation } from './middleware/security.js';
import { applyShieldConfigFromEnv, logShieldConfigValidation } from './security/shield-config.js';
import { applyRunKill, isTerminalRunStatus, setRunKillSideEffects } from './runs/run-kill.js';
import { createGatewayCancelAdapter } from './adapters/gateway-cancel-adapter.js';
import { publishSentinelPolicyViolationFireAndForget } from './adapters/redis-sentinel-publisher.js';
import { setCancelAdapter, setPolicyViolationPublisher } from '@4runr/sentinel';
import { errorHandler, handleValidationError, handleNotFound } from './middleware/errorHandler.js';
import { performHealthChecks, isAlive, computeReady } from './health/index.js';
import { 
  createRunSchema, 
  listRunsQuerySchema, 
  runIdParamSchema, 
  updateRunStatusSchema,
  workspacePlanSchema,
  registryPublishSchema,
  privacyToggleSchema
} from './schemas/runs.js';
import { disconnectPrisma } from './db/prisma.js';
import { getRedisClient, disconnectRedis, isRedisAvailable } from './db/redis.js';
import {
  createIdempotencyMiddleware,
  captureIdempotencyResponse,
} from '../../../packages/middleware/idempotency/idempotency.js';
import type { IdempotencyStore } from '../../../packages/adapters/redis/IdempotencyStore.js';

type IdempotencyPreHandler = ReturnType<typeof createIdempotencyMiddleware>;
// Idempotency stores - will be imported dynamically where needed
import { Sentinel } from '@4runr/sentinel';
import { Shield } from '@4runr/shield';
import { GatewayEventStream } from './adapters/sentinel-event-stream.js';
import {
  initializeQueue,
  queueRunExecution,
  shutdownQueue,
  getQueue,
} from './queue/index.js';
import { setProcessorContext } from './queue/processor.js';
import {
  getMetrics,
  httpRequestDuration,
  httpRequestTotal,
  httpRequestErrors,
  runsCreated,
  runsStarted,
  runsCompleted,
  runsDuration,
  runsActive,
  sseConnectionsOpened,
  sseConnectionsClosed,
  sseActiveConnections,
  sseMessagesTotal,
  idempotencyHits,
  idempotencyMisses,
  idempotencyConflicts,
  idempotencyStoreSize,
  rateLimitHits,
  authSuccess,
  authFailures,
  queueJobsQueued,
  queueJobsCompleted,
  queueJobsFailed,
  queueJobDuration,
  queueJobsWaiting,
  queueJobsActive,
  healthCheckTotal,
} from './metrics/index.js';
import path from 'path';
import { registerDevKitRoutes } from './devkit/routes.js';
import { mfaRoutes } from './routes/mfa.js';
import { gdprRoutes } from './routes/gdpr.js';
import { registerMonitoringRoutes } from './routes/monitoring.js';
import { ddosProtection } from './middleware/ddos-protection.js';
import { wrapLoggerForMonitoring } from './middleware/log-capture.js';
import { initializeDatabase, shutdownDatabase } from './db/init.js';
import type { PrismaClient } from '@prisma/client';
import { applyPersistedSentinelConfig } from './security/sentinel-config-store.js';

// Initialize no-persistence mode BEFORE any other initialization (only in memory mode)
// This must happen before any filesystem writes or database connections
const projectRoot = process.cwd();
const persistenceMode = getPersistenceMode();

if (persistenceMode === 'memory') {
  initializeNoPersistenceMode(projectRoot);
}

// Create structured logger (will be updated per-request with correlation ID)
let baseLogger = isMemoryMode ? memoryLogger : createLogger('Gateway');

// Wrap logger to capture logs for monitoring endpoints (Phase 0)
baseLogger = wrapLoggerForMonitoring(baseLogger);

const PORT = env.PORT;
const HOST = env.HOST;
/** Monitoring-style routes skip `requireAuth` only when Gateway binds to loopback (`AUTH_ENABLED` still applies elsewhere in `requireAuth`). */
const isLocalBind = HOST === '127.0.0.1' || HOST === 'localhost';

// Initialize run store lazily (memory or postgres based on GATEWAY_PERSISTENCE)
// Don't initialize PostgresRunStore here as it requires Prisma client which may not be generated yet
let runStore: ReturnType<typeof getRunStore> | null = null;
const getStore = () => {
  if (!runStore) {
    runStore = getRunStore();
  }
  return runStore;
};

// Database client (will be initialized during startup)
let dbClient: PrismaClient | null = null;

// Initialize idempotency store
// CRITICAL: Initialize synchronously - this MUST complete before routes register
const ttlSeconds = 24 * 60 * 60; // 24 hours
let idempotencyStore: IdempotencyStore | null = null;
let idempotencyMiddleware: IdempotencyPreHandler | null = null;

// Synchronous initialization - wrapped in IIFE to ensure execution
(async function initializeIdempotencySync() {
  try {
    // Dynamic import to handle module resolution
    const { InMemoryIdempotencyStore } = await import('../../../packages/adapters/redis/IdempotencyStore.js');
    idempotencyStore = new InMemoryIdempotencyStore(ttlSeconds);
    idempotencyMiddleware = createIdempotencyMiddleware({
      enabled: true,
      ttlSeconds,
      store: idempotencyStore,
    });
    baseLogger.info('[IDEMPOTENCY] Middleware initialized synchronously');
  } catch (error) {
    const errorMsg = error instanceof Error ? error.message : String(error);
    const errorStack = error instanceof Error ? error.stack : undefined;
    baseLogger.error('[IDEMPOTENCY] Failed to initialize', { 
      error: errorMsg,
      stack: errorStack
    });
    // Continue without idempotency - better than crashing
  }
})();

// Try to upgrade to Redis if available (non-blocking)
async function initializeIdempotency() {
  try {
    const redisClient = await getRedisClient();
    if (redisClient) {
      const { RedisIdempotencyStore } = await import('../../../packages/adapters/redis/IdempotencyStore.js');
      const redisStore = new RedisIdempotencyStore(redisClient, ttlSeconds);
      idempotencyStore = redisStore;
      idempotencyMiddleware = createIdempotencyMiddleware({
        enabled: true,
        ttlSeconds,
        store: redisStore,
      });
      baseLogger.info('Idempotency store upgraded to Redis');
    } else {
      baseLogger.warn('Idempotency store using in-memory fallback (Redis not available)');
    }
  } catch (err) {
    baseLogger.error('Failed to initialize Redis idempotency store', { error: err instanceof Error ? err.message : String(err) });
    baseLogger.warn('Continuing with in-memory idempotency store');
  }
}

// Initialize Redis upgrade in background (non-blocking)
initializeIdempotency().catch(() => {
  // Already handled in function
});

// Create Fastify instance
const fastify = Fastify({
  logger: false, // We use our own structured logger
  trustProxy: true
});

// Add security headers middleware (before other hooks)
fastify.addHook('onRequest', async (request, reply) => {
  await setSecurityHeaders(request, reply);
});

// Add correlation ID middleware globally
fastify.addHook('onRequest', async (request, reply) => {
  await addCorrelationId(request, reply);
  
  const correlationId = getCorrelationId(request);
  if (correlationId) {
    const requestLogger = wrapLoggerForMonitoring(createLogger('Gateway'));
    requestLogger.setCorrelationId(correlationId);
    (request as any).logger = requestLogger;
  } else {
    (request as any).logger = baseLogger;
  }
});

// Add request timing and metrics
fastify.addHook('onRequest', async (request, reply) => {
  (request as any).startTime = Date.now();
});

fastify.addHook('onResponse', async (request, reply) => {
  const startTime = (request as any).startTime;
  if (startTime) {
    const duration = (Date.now() - startTime) / 1000; // Convert to seconds
    const method = request.method;
    const route = (request as any).routeOptions?.url || request.url.split('?')[0] || 'unknown';
    const statusCode = reply.statusCode;
    
    // Record metrics
    httpRequestDuration.observe({ method, route, status_code: statusCode }, duration);
    httpRequestTotal.inc({ method, route, status_code: statusCode });
    
    if (statusCode >= 400) {
      httpRequestErrors.inc({ method, route, error_type: statusCode >= 500 ? 'server_error' : 'client_error' });
    }
  }
});

// Configure content type parser to handle empty bodies
fastify.addContentTypeParser('application/json', { parseAs: 'string' }, function (req, body, done) {
  try {
    if (body === '') {
      done(null, {});
    } else {
      const json = JSON.parse(body as string);
      done(null, json);
    }
  } catch (err) {
    const error = err as Error;
    (error as any).statusCode = 400;
    done(error, undefined);
  }
});

// Register global error handler
fastify.setErrorHandler((error, request, reply) => {
  if ((error as any).validation) {
    handleValidationError(error as any, request, reply);
  } else {
    errorHandler(error as Error, request, reply);
  }
});

// Register 404 handler
fastify.setNotFoundHandler((request, reply) => {
  handleNotFound(request, reply);
});

// === DATABASE INITIALIZATION ===
// Initialize database (Docker auto-start or existing connection)
// Must happen before DevKit routes (they require Prisma)
baseLogger.info('Initializing database...');
const dbResult = await initializeDatabase(baseLogger);

if (dbResult.mode === 'memory') {
  baseLogger.warn('⚠️  Running in memory-only mode. Data will not persist across restarts.');
  if (dbResult.error) {
    baseLogger.warn(`Database initialization failed: ${dbResult.error}`);
  }
  if (dbResult.warnings) {
    dbResult.warnings.forEach(w => baseLogger.warn(`  • ${w}`));
  }
} else {
  baseLogger.info(`✓ Database ready (${dbResult.mode} mode)`);
  dbClient = dbResult.client;
}

// Register DevKit routes (dev-only API namespace)
// Must be registered before other routes to ensure proper priority
// Only available when we have a real database (not memory mode)
if (dbResult.client && persistenceMode !== 'memory') {
  baseLogger.info('Registering DevKit routes...');
  registerDevKitRoutes(fastify);
  baseLogger.info('✓ DevKit routes registered');
} else {
  baseLogger.info('DevKit routes disabled (memory mode)');
}

// Health endpoint (liveness probe - no auth required)
fastify.get('/health', async (request, reply) => {
  const alive = isAlive();
  const { getMetricsSummary } = await import('./metrics/index.js');
  const summary = await getMetricsSummary();
  const healthChecks = await performHealthChecks(); // S5: include Shield + all checks
  
  const statusCode = alive ? 200 : 503;
  reply.status(statusCode);
  
  // Track actual HTTP status (not just 200)
  healthCheckTotal.inc({ endpoint: '/health', status: String(statusCode) });
  
  return {
    ok: alive,
    status: alive ? 'alive' : 'dead',
    time: new Date().toISOString(),
    uptime: summary.uptime,
    memory: summary.memory,
    cpu: summary.cpu,
    persistence: persistenceMode,
    /** 4Runr platform DB: docker = Postgres via Docker; memory = no DB */
    fourRunrDb: {
      mode: dbResult.mode,
      prismaConnected: !!dbClient,
    },
    auth: {
      enabled: env.AUTH_ENABLED,
      mode: env.AUTH_MODE
    },
    rateLimit: {
      enabled: env.RATE_LIMIT_ENABLED,
      max: env.RATE_LIMIT_MAX,
      window_ms: env.RATE_LIMIT_WINDOW_MS
    },
    checks: healthChecks.checks, // S5: dependency + Shield health
  };
});

// Readiness endpoint (checks all dependencies)
fastify.get('/ready', async (request, reply) => {
  const health = await performHealthChecks();
  const ready = computeReady(health);

  const statusCode = ready ? 200 : 503;
  reply.status(statusCode);

  healthCheckTotal.inc({ endpoint: '/ready', status: String(statusCode) });

  return {
    ready,
    status: health.status,
    timestamp: new Date().toISOString(),
    checks: health.checks
  };
});

// Liveness endpoint (simple alive check)
fastify.get('/liveness', async (request, reply) => {
  const alive = isAlive();
  const statusCode = alive ? 200 : 503;
  reply.status(statusCode);
  
  return {
    alive,
    timestamp: new Date().toISOString()
  };
});

// Startup probe (check if server has started)
fastify.get('/startup', async (request, reply) => {
  // Server is started if this endpoint is accessible
  return {
    started: true,
    timestamp: new Date().toISOString()
  };
});

// In-memory storage for SSE connections
const sseConnections = new Map<string, Set<any>>();

// Sentinel integration - event streams for each run
const sentinelEventStreams = new Map<string, GatewayEventStream>();
const sentinel = Sentinel.getInstance();
applyPersistedSentinelConfig(sentinel, baseLogger);

// Shield integration - AI Safety Layer (mode from SENTINEL_SHIELD_MODE — see shield-config.ts)
const shield = Shield.getInstance();
applyShieldConfigFromEnv(shield, env.SENTINEL_SHIELD_MODE);

// Initialize queue system
// Queue will be initialized during server startup
let queueInitialized = false;

// Metrics endpoint (no auth required)
fastify.get('/metrics', async (request, reply) => {
  reply.header('X-Gateway', 'main');
  
  // Update gauge metrics
  const activeConnections = Array.from(sseConnections.values()).reduce((sum, connections) => sum + connections.size, 0);
  sseActiveConnections.set(activeConnections);

  const runQueue = getQueue();
  if (runQueue) {
    try {
      // BullMQ v4+ getJobCounts: pass state names as separate args, returns exact keys
      const counts = await runQueue.getJobCounts(
        'waiting',
        'active',
        'completed',
        'failed',
        'delayed',
        'paused'
      ) as Record<string, number>;
      const waiting = counts['waiting'] ?? 0;
      const activeJobs = counts['active'] ?? 0;
      queueJobsWaiting.set({ queue: 'run-execution' }, waiting);
      queueJobsActive.set({ queue: 'run-execution' }, activeJobs);
    } catch (err) {
      baseLogger.warn('Failed to refresh queue depth metrics', {
        error: err instanceof Error ? err.message : String(err),
      });
    }
  }
  
  // Get Prometheus metrics
  const prometheusMetrics = await getMetrics();
  
  reply.type('text/plain');
  return prometheusMetrics;
});

// Enhanced monitoring endpoint for dashboards
fastify.get(
  '/api/monitoring/summary',
  {
    ...(!isLocalBind && { preHandler: [requireAuth, readRateLimit] }),
  },
  async (request, reply) => {
    const { getMetricsSummary } = await import('./metrics/index.js');
    const summary = await getMetricsSummary();

    const dependencies = await performHealthChecks();
    const readyStatus = computeReady(dependencies);

    return {
      status: dependencies.status,
      alive: isAlive(),
      ready: readyStatus,
      uptime: summary.uptime,
      memory: summary.memory,
      cpu: summary.cpu,
      dependencies,
      timestamp: summary.timestamp,
      persistence: persistenceMode,
      database: {
        mode: dbResult.mode,
        connected: !!dbClient,
      },
    };
  }
);

// Register advanced monitoring routes (Phase 0)
// SECURITY: In production, these require auth. For local dev (127.0.0.1), consider skipping auth.
// Deployment guidance: use network policy to restrict access or enable requireAuth.
registerMonitoringRoutes(fastify, {
  requireAuth: isLocalBind ? false : requireAuth, // Explicitly false for local bind, function for non-loopback
  readRateLimit: readRateLimit,
});

// Apply DDoS protection globally (before other middleware)
fastify.addHook('onRequest', ddosProtection);

// Workspace plan endpoint (requires auth)
fastify.post('/api/workspace/plan', { 
  preHandler: [requireAuth, writeRateLimit, validateBody(workspacePlanSchema)],
}, async (request, reply) => {
  const body = request.body as any;
  const logger = (request as any).logger || baseLogger;
  logger.info('Setting workspace plan', { prompt: body.prompt?.substring(0, 50) });
  
  return {
    success: true,
    prompt: body.prompt,
    updated_at: new Date().toISOString()
  };
});

// Create run endpoint (requires auth)
const preHandlers = [
  requireAuth, 
  writeRateLimit, 
  validateBody(createRunSchema),
];

// Add idempotency middleware if available

if (idempotencyMiddleware) {
  preHandlers.push(idempotencyMiddleware);
  baseLogger.info('[IDEMPOTENCY] Middleware added to /api/runs route');
} else {
  baseLogger.warn('[IDEMPOTENCY] Middleware not available for /api/runs route');
}

// AI Provider routes
try {
  const { aiProviderRoutes } = await import('./routes/ai-providers-simple.js');
  await aiProviderRoutes(fastify);
  baseLogger.info('AI Provider routes registered');
  
  // Load stored credentials from Redis on startup (only if Redis available)
  // Don't block server startup or spam retry logs when Redis is down
  Promise.resolve().then(async () => {
    try {
      // Check if Redis is actually available before instantiating client
      const redisAvailable = await isRedisAvailable();
      if (!redisAvailable) {
        baseLogger.info('Redis not available, skipping credential store initialization');
        return;
      }
      
      const { getCredentialsStore } = await import('./ai-providers/redis-credentials-store.js');
      const credStore = getCredentialsStore();
      
      // Race the loadAllToEnv() against a 5-second timeout
      const timeoutPromise = new Promise((_, reject) => 
        setTimeout(() => reject(new Error('Redis credential load timed out after 5s')), 5000)
      );
      
      await Promise.race([
        credStore.loadAllToEnv(),
        timeoutPromise
      ]);
      
      baseLogger.info('AI Provider credentials loaded from Redis');
    } catch (credError) {
      baseLogger.warn('Failed to load AI credentials from Redis (non-critical)', { 
        error: credError instanceof Error ? credError.message : String(credError) 
      });
    }
  }).catch((err) => {
    baseLogger.warn('Redis credential loading failed, continuing without stored credentials');
  });
} catch (error) {
  baseLogger.warn('AI Provider routes not available', { error });
}

// Chat history routes
try {
  const { chatRoutes } = await import('./routes/chats.js');
  await chatRoutes(fastify);
  baseLogger.info('Chat history routes registered');
} catch (error) {
  baseLogger.warn('Chat history routes not available', { error });
}

// LAYER 2: Tool Credentials Management (Zero-Trust Tool Access)
try {
  const { toolCredentialRoutes } = await import('./routes/tool-credentials.js');
  await toolCredentialRoutes(fastify);
  baseLogger.info('Tool credential routes registered (Layer 2 - Storage)');
} catch (error) {
  baseLogger.warn('Tool credential routes not available', { error: error instanceof Error ? error.message : String(error) });
}

// LAYER 2: Tool Proxy (Zero-Trust Execution)
try {
  const { toolProxyRoutes } = await import('./routes/tool-proxy.js');
  await toolProxyRoutes(fastify);
  baseLogger.info('Tool proxy routes registered (Layer 2 - Execution)');
} catch (error) {
  baseLogger.warn('Tool proxy routes not available', { error: error instanceof Error ? error.message : String(error) });
}

// SENTINEL: Policy Management Routes
try {
  const { sentinelPolicyRoutes } = await import('./routes/sentinel-policies.js');
  await sentinelPolicyRoutes(fastify);
  baseLogger.info('Sentinel policy routes registered');
} catch (error) {
  baseLogger.warn('Sentinel policy routes not available', { error: error instanceof Error ? error.message : String(error) });
}

// SENTINEL: Predictive Routes
try {
  const { sentinelPredictiveRoutes } = await import('./routes/sentinel-predictive.js');
  await sentinelPredictiveRoutes(fastify);
  baseLogger.info('Sentinel predictive routes registered');
} catch (error) {
  baseLogger.warn('Sentinel predictive routes not available', { error: error instanceof Error ? error.message : String(error) });
}

// SHIELD: AI Safety Layer Routes
try {
  baseLogger.info('Attempting to register Shield routes...');
  const { shieldRoutes } = await import('./routes/shield.js');
  baseLogger.info('Shield routes module imported successfully');
  await shieldRoutes(fastify);
  baseLogger.info('Shield routes registered successfully');
} catch (error) {
  baseLogger.error('Shield routes registration failed', { 
    error: error instanceof Error ? error.message : String(error),
    stack: error instanceof Error ? error.stack : undefined
  });
}

// MFA and GDPR routes will be registered in start() function

fastify.post('/api/runs', { 
  preHandler: preHandlers,
  onSend: [captureIdempotencyResponse],
}, async (request, reply) => {
  const body = request.body as any;
  const logger = (request as any).logger || baseLogger;
  
  // Check for idempotency token (clientToken) - legacy support
  if (body.clientToken && !request.headers['idempotency-key']) {
    const existingRun = await getStore().getRunByClientToken(body.clientToken);
    if (existingRun) {
      idempotencyHits.inc();
      reply.status(200);
      return {
        success: true,
        run: existingRun,
        idempotent: true
      };
    }
  }
  
  // Create new run
  const runStartTime = Date.now();
  const run = await getStore().createRun({
    id: body.id,
    name: body.name || 'Test Run',
    input: body.input || null,
    clientToken: body.clientToken || null,
    tags: body.tags || []
  });

  runsCreated.inc({ status: run.status });
  runsActive.inc();
  logger.info('Run created', { runId: run.id, name: run.name });

  const { scheduleRunRetentionEnforcement } = await import('./runs/run-retention.js');
  scheduleRunRetentionEnforcement();

  // Create Sentinel event stream for this run
  const eventStream = new GatewayEventStream();
  sentinelEventStreams.set(run.id, eventStream);
  
  // Emit created event
  eventStream.emitCreated(run.id);
  
  // Emit events to any waiting SSE connections
  const connections = sseConnections.get(run.id);
  if (connections) {
    connections.forEach(connection => {
      connection.write(`event: log\ndata: ${JSON.stringify({
        type: 'run_created',
        runId: run.id,
        timestamp: new Date().toISOString()
      })}\n\n`);
      sseMessagesTotal.inc();
    });
  }
  
  reply.status(201);
  return {
    success: true,
    run: run
  };
});

// List all runs endpoint (requires auth)
fastify.get('/api/runs', { 
  preHandler: [requireAuth, readRateLimit, validateQuery(listRunsQuerySchema)],
}, async (request, reply) => {
  const query = request.query as any;
  const filter: any = {};
  
  if (query.status) {
    filter.status = query.status;
  }
  if (query.tag) {
    filter.tag = query.tag;
  }
  if (query.limit) {
    filter.limit = parseInt(query.limit, 10);
  }
  
  const runs = await getStore().listRuns(filter);
  return { runs };
});

// Get run endpoint (requires auth)
fastify.get('/api/runs/:id', { 
  preHandler: [requireAuth, readRateLimit, validateParams(runIdParamSchema)],
}, async (request, reply) => {
  const { id } = request.params as { id: string };
  const run = await getStore().getRunById(id);
  
  if (!run) {
    return reply.status(404).send({ error: 'Run not found' });
  }
  
  // Return the run data directly (not wrapped in success/run)
  return run;
});

// Start run endpoint (requires auth)
fastify.post('/api/runs/:id/start', { 
  preHandler: [requireAuth, writeRateLimit, validateParams(runIdParamSchema)],
}, async (request, reply) => {
  const { id } = request.params as { id: string };
  const body = request.body as any;
  const run = await getStore().getRunById(id);
  
  if (!run) {
    return reply.status(404).send({ error: 'Run not found' });
  }
  
  // Check if run is already running, queued, completed, or failed
  if (run.status === 'running' || run.status === 'queued' || run.status === 'completed' || run.status === 'failed') {
    return reply.status(400).send({ 
      error: `Run is already ${run.status}`,
      run 
    });
  }
  
  const logger = (request as any).logger || baseLogger;
  
  // Get priority from request body (default: normal)
  const priority = body?.priority || 'normal';
  if (!['high', 'normal', 'low'].includes(priority)) {
    return reply.status(400).send({ 
      error: 'Invalid priority. Must be: high, normal, or low' 
    });
  }
  
  // Update run status to queued if queue is available, otherwise running
  const queue = getQueue();
  if (queue) {
    // Queue is available - queue the job
    try {
      await getStore().updateRunStatus(id, 'queued', {
        startedAt: null // Will be set when job starts processing
      });
      
      await queueRunExecution(id, priority);
      queueJobsQueued.inc({ queue: 'run-execution', priority });
      
      logger.info('Run queued for execution', { runId: id, priority });
      
      const updatedRun = await getStore().getRunById(id);
      
      // Emit queued event to SSE connections
      const connections = sseConnections.get(id);
      if (connections) {
        connections.forEach(connection => {
          connection.write(`event: log\ndata: ${JSON.stringify({
            type: 'run_queued',
            runId: id,
            status: 'queued',
            priority,
            timestamp: new Date().toISOString()
          })}\n\n`);
          sseMessagesTotal.inc();
        });
      }
      
      return {
        success: true,
        run: updatedRun,
        queued: true
      };
    } catch (error) {
      logger.error('Failed to queue run execution', { 
        runId: id, 
        error: error instanceof Error ? error.message : String(error) 
      });
      return reply.status(500).send({ 
        error: 'Failed to queue run execution',
        message: error instanceof Error ? error.message : String(error)
      });
    }
  } else {
    // Queue not available - fall back to synchronous execution
    logger.warn('Queue not available, using synchronous execution', { runId: id });
    
    await getStore().updateRunStatus(id, 'running', {
      startedAt: new Date().toISOString()
    });

    runsStarted.inc();
    const updatedRun = await getStore().getRunById(id);

    // Start Sentinel monitoring for this run
    const eventStream = sentinelEventStreams.get(id);
    if (eventStream) {
      if (sentinel.isEnabled()) {
        sentinel.watchRun(id, eventStream);
        logger.info('Sentinel watching run', { runId: id });
      }
      eventStream.emitStarted(id);
    }
    
    // Emit events to SSE connections
    const connections = sseConnections.get(id);
    if (connections) {
      connections.forEach(connection => {
        connection.write(`event: log\ndata: ${JSON.stringify({
          type: 'run_started',
          runId: id,
          status: 'running',
          timestamp: new Date().toISOString()
        })}\n\n`);
        sseMessagesTotal.inc();
      });
    }
    
    // Fallback to setTimeout for synchronous execution
    const runStartTime = Date.now();
    setTimeout(async () => {
      const currentRun = await getStore().getRunById(id);
      if (currentRun && currentRun.status === 'running') {
        const runDuration = (Date.now() - runStartTime) / 1000;
        await getStore().updateRunStatus(id, 'completed', {
          completedAt: new Date().toISOString(),
          output: { result: 'Run completed successfully' },
          logs: [
            ...(currentRun.logs || []),
            {
              timestamp: new Date().toISOString(),
              level: 'info',
              message: 'Run completed'
            }
          ]
        });
        
        runsCompleted.inc({ status: 'completed' });
        runsDuration.observe({ status: 'completed' }, runDuration);
        runsActive.dec();
        
        const eventStream = sentinelEventStreams.get(id);
        if (eventStream) {
          eventStream.emitFinish(id, 'ok');
        }
        
        sentinelEventStreams.delete(id);
        
        const connections = sseConnections.get(id);
        if (connections) {
          connections.forEach(connection => {
            connection.write(`event: end\ndata: ${JSON.stringify({
              type: 'run_completed',
              runId: id,
              status: 'completed',
              timestamp: new Date().toISOString()
            })}\n\n`);
            sseMessagesTotal.inc();
          });
        }
      }
    }, 3000);
    
    logger.info('Run started (synchronous)', { runId: id });
    
    return {
      success: true,
      run: updatedRun,
      queued: false
    };
  }
});

// Cancel/kill run endpoint (requires auth) - Sentinel kill switch
fastify.post('/api/runs/:id/cancel', { 
  preHandler: [requireAuth, writeRateLimit, validateParams(runIdParamSchema)],
}, async (request, reply) => {
  const { id } = request.params as { id: string };
  const body = request.body as { reason?: string };
  const run = await getStore().getRunById(id);
  
  if (!run) {
    return reply.status(404).send({ error: 'Run not found' });
  }

  const logger = (request as any).logger || baseLogger;
  const reason = body.reason || 'manual_cancellation';

  if (isTerminalRunStatus(run.status)) {
    return {
      success: true,
      idempotent: true,
      runId: id,
      status: run.status,
      message: `Run already ${run.status}`,
    };
  }

  try {
    if (sentinel.isEnabled()) {
      sentinel.syncExternalKill(id, reason);
    } else {
      const eventStream = sentinelEventStreams.get(id);
      eventStream?.emitFinish(id, 'canceled');
    }

    const result = await applyRunKill(id, reason);

    return {
      success: true,
      idempotent: result.idempotent,
      message: result.idempotent ? 'Run already terminal' : 'Run canceled successfully',
      runId: id,
      status: result.status,
    };
  } catch (error) {
    logger.error('Failed to cancel run', { 
      runId: id, 
      error: error instanceof Error ? error.message : String(error) 
    });
    return reply.status(500).send({ 
      error: 'Failed to cancel run',
      message: error instanceof Error ? error.message : String(error)
    });
  }
});

// SSE endpoint for run logs - TOLERANT of pre-run subscriptions (requires auth)
fastify.get('/api/runs/:id/logs/stream', { 
  preHandler: [requireAuth, readRateLimit, validateParams(runIdParamSchema)],
}, async (request, reply) => {
  const { id } = request.params as { id: string };
  const run = await getStore().getRunById(id);
  
  // Don't 404 if run doesn't exist - keep connection open for pre-run subscriptions
  reply.raw.writeHead(200, {
    'Content-Type': 'text/event-stream',
    'Cache-Control': 'no-cache',
    'Connection': 'keep-alive',
    'Access-Control-Allow-Origin': '*'
  });

  // Track this connection
  if (!sseConnections.has(id)) {
    sseConnections.set(id, new Set());
  }
  sseConnections.get(id)!.add(reply.raw);
  sseConnectionsOpened.inc();
  sseActiveConnections.inc();
  sseMessagesTotal.inc();

  let messageId = 1;
  const lastEventId = request.headers['last-event-id'];
  if (lastEventId && !isNaN(Number(lastEventId))) {
    messageId = Number(lastEventId) + 1;
  }
  
  const sendEvent = (data: any) => {
    reply.raw.write(`id: ${messageId++}\n`);
    reply.raw.write(`data: ${JSON.stringify(data)}\n\n`);
    sseMessagesTotal.inc();
  };

  // Send initial connection event
  sendEvent({
    type: 'connected',
    runId: id,
    timestamp: new Date().toISOString()
  });

  // Send heartbeat every 15 seconds to keep connection alive
  const heartbeatInterval = setInterval(() => {
    reply.raw.write(': keepalive\n\n');
  }, 15000);

  // Send run status updates every 5 seconds (if run exists)
  const statusInterval = setInterval(async () => {
    const currentRun = await getStore().getRunById(id);
    if (currentRun) {
      sendEvent({
        type: 'status_update',
        runId: id,
        status: currentRun.status,
        timestamp: new Date().toISOString()
      });
    }
  }, 5000);

  // Clean up on disconnect
  request.raw.on('close', () => {
    clearInterval(heartbeatInterval);
    clearInterval(statusInterval);
    
    // Remove from tracking
    const connections = sseConnections.get(id);
    if (connections) {
      connections.delete(reply.raw);
      sseConnectionsClosed.inc();
      sseActiveConnections.dec();
      if (connections.size === 0) {
        sseConnections.delete(id);
      }
    }
  });
});

// SSE endpoint for S2 harness pattern (without run ID)
fastify.get(
  '/api/runs/logs/stream',
  {
    ...(!isLocalBind && { preHandler: [requireAuth, readRateLimit] }),
  },
  async (request, reply) => {
  // Don't 404 - keep connection open for S2 harness
  reply.raw.writeHead(200, {
    'Content-Type': 'text/event-stream',
    'Cache-Control': 'no-cache',
    'Connection': 'keep-alive',
    'Access-Control-Allow-Origin': '*'
  });

  // Track this connection with a generic ID
  const genericId = 's2-harness-generic';
  if (!sseConnections.has(genericId)) {
    sseConnections.set(genericId, new Set());
  }
  sseConnections.get(genericId)!.add(reply.raw);
  sseConnectionsOpened.inc();
  sseActiveConnections.inc();
  sseMessagesTotal.inc();

  let messageId = 1;
  const lastEventId = request.headers['last-event-id'];
  if (lastEventId && !isNaN(Number(lastEventId))) {
    messageId = Number(lastEventId) + 1;
  }
  
  const sendEvent = (data: any) => {
    reply.raw.write(`id: ${messageId++}\n`);
    reply.raw.write(`data: ${JSON.stringify(data)}\n\n`);
    sseMessagesTotal.inc();
  };

  // Send initial connection event
  sendEvent({
    type: 'connected',
    timestamp: new Date().toISOString()
  });

  // Send heartbeat every 15 seconds to keep connection alive
  const heartbeatInterval = setInterval(() => {
    reply.raw.write(': keepalive\n\n');
  }, 15000);

  // Clean up on disconnect
  request.raw.on('close', () => {
    clearInterval(heartbeatInterval);
    
    // Remove from tracking
    const connections = sseConnections.get(genericId);
    if (connections) {
      connections.delete(reply.raw);
      sseConnectionsClosed.inc();
      sseActiveConnections.dec();
      if (connections.size === 0) {
        sseConnections.delete(genericId);
      }
    }
  });
});

// Registry publish endpoint (requires auth)
fastify.post('/api/registry/publish', { 
  preHandler: [requireAuth, writeRateLimit, validateBody(registryPublishSchema)],
}, async (request, reply) => {
  const body = request.body as any;
  const logger = (request as any).logger || baseLogger;
  logger.info('Registry publish', { name: body.name, version: body.version });
  
  reply.status(201);
  return {
    success: true,
    name: body.name,
    version: body.version,
    published_at: new Date().toISOString()
  };
});

// Safety check endpoint
fastify.get(
  '/api/safety/check',
  {
    ...(!isLocalBind && { preHandler: [requireAuth, readRateLimit] }),
  },
  async (request, reply) => {
  return {
    status: 'safe',
    timestamp: new Date().toISOString()
  };
});

// Serve Sentinel dashboard
fastify.get(
  '/sentinel/dashboard',
  {
    ...(!isLocalBind && { preHandler: [requireAuth, readRateLimit] }),
  },
  async (request, reply) => {
  const fs = await import('fs/promises');
  const path = await import('path');
  
  try {
    const dashboardPath = path.join(process.cwd(), 'apps/gateway/public/sentinel-dashboard.html');
    const html = await fs.readFile(dashboardPath, 'utf-8');
    return reply.type('text/html').send(html);
  } catch (error) {
    return reply.status(404).send({
      error: 'Dashboard not found',
      message: 'Sentinel dashboard HTML file not found'
    });
  }
});

// Sentinel health endpoint
fastify.get(
  '/sentinel/health',
  {
    ...(!isLocalBind && { preHandler: [requireAuth, readRateLimit] }),
  },
  async (request, reply) => {
  const sentinelEnabled = env.SENTINEL_ENABLED;
  const sentinelHealthy = sentinel ? sentinel.isEnabled() : false;
  
  return {
    enabled: sentinelEnabled,
    healthy: sentinelHealthy,
    timestamp: new Date().toISOString(),
    status: sentinelHealthy ? 'healthy' : 'disabled',
    watchedRuns: sentinel ? sentinel.getWatchedRunIds().length : 0
  };
});

// Sentinel metrics endpoint
fastify.get(
  '/sentinel/metrics',
  {
    ...(!isLocalBind && { preHandler: [requireAuth, readRateLimit] }),
  },
  async (request, reply) => {
  if (!sentinel) {
    return {
      error: 'Sentinel not initialized',
      timestamp: new Date().toISOString()
    };
  }

  const activeRuns = sentinel.getActiveRuns();
  const watchedRunIds = sentinel.getWatchedRunIds();
  
  // Calculate real metrics from active runs
  let totalTokenUsage = 0;
  let totalCost = 0;
  let totalEvents = 0;
  const killReasons: Record<string, number> = {};
  const policyViolations: Record<string, number> = {};
  
      // Aggregate data from all active runs
      for (const runState of activeRuns) {
        totalTokenUsage += runState.tokenCount;
        totalCost += runState.cost || 0;
        totalEvents += runState.events.length;
        
        if (runState.killReason) {
          const reason = runState.killReason.split(':')[0]; // Extract policy name
          if (reason) {
            killReasons[reason] = (killReasons[reason] || 0) + 1;
          }
          
          // Get policy violations for this run
          const violations = sentinel.getPolicyViolations(runState.runId);
          for (const violation of violations) {
            policyViolations[violation.policy] = (policyViolations[violation.policy] || 0) + 1;
          }
        }
      }
  
  // Calculate average latency (time from start to now for active runs)
  let totalLatency = 0;
  let latencyCount = 0;
  for (const runState of activeRuns) {
    if (runState.startedAt) {
      totalLatency += Date.now() - runState.startedAt;
      latencyCount++;
    }
  }
  const avgLatency = latencyCount > 0 ? Math.round(totalLatency / latencyCount) : 0;
  
  // Get all watched runs (including finished ones in buffer)
  const allRuns = activeRuns.length;
  
  const config = sentinel.getConfig();
  
  return {
    // Status
    enabled: config.enabled,
    healthy: true,
    
    // Real-time metrics
    activeRuns: allRuns,
    watchedRuns: watchedRunIds.length,
    totalEvents,
    totalTokenUsage,
    totalCost: parseFloat(totalCost.toFixed(4)),
    avgLatency,
    
    // Policy violations
    policyViolations,
    totalViolations: Object.values(policyViolations).reduce((sum, count) => sum + count, 0),
    
    // Kill statistics
    killReasons,
    totalKills: Object.values(killReasons).reduce((sum, count) => sum + count, 0),
    
    // Configuration
    config: config,
    
    // Legacy placeholders (for backward compatibility)
    totalSpans: 0, // Not implemented yet
    totalVerdicts: 0, // Not implemented yet
    flaggedHallucinations: 0, // Not implemented yet
    flaggedInjections: 0, // Not implemented yet
    flaggedPII: 0, // Not implemented yet
    
    timestamp: new Date().toISOString()
  };
});

// Privacy toggle endpoint (requires auth)
fastify.post('/api/privacy/toggle', { 
  preHandler: [requireAuth, writeRateLimit, validateBody(privacyToggleSchema)],
}, async (request, reply) => {
  const body = request.body as any;
  const logger = (request as any).logger || baseLogger;
  logger.info('Privacy toggle', { storePlain: body.storePlain });
  
  return {
    success: true,
    storePlain: body.storePlain,
    updated_at: new Date().toISOString()
  };
});

// Demo run endpoint for testing
fastify.post(
  '/api/diagnostics/emit-demo-run',
  {
    ...(!isLocalBind && { preHandler: [requireAuth, writeRateLimit] }),
  },
  async (request, reply) => {
  const runId = `demo-run-${Date.now()}`;
  const logger = (request as any).logger || baseLogger;
  logger.info('Demo run created', { runId });
  
  return {
    success: true,
    runId,
    message: 'Demo run created successfully'
  };
});

// SSE test endpoint
fastify.get(
  '/diagnostics/sse-test',
  {
    ...(!isLocalBind && { preHandler: [requireAuth, readRateLimit] }),
  },
  async (request, reply) => {
  reply.raw.writeHead(200, {
    'Content-Type': 'text/event-stream',
    'Cache-Control': 'no-cache',
    'Connection': 'keep-alive',
    'Access-Control-Allow-Origin': '*'
  });

  const sendEvent = (data: any) => {
    reply.raw.write(`data: ${JSON.stringify(data)}\n\n`);
  };

  // Send test events every 2 seconds
  const interval = setInterval(() => {
    sendEvent({
      type: 'test',
      timestamp: new Date().toISOString(),
      message: 'SSE test event'
    });
  }, 2000);

  // Clean up on disconnect
  request.raw.on('close', () => {
    clearInterval(interval);
  });
});

// Debug endpoint for logs (memory mode only)
if (isMemoryMode) {
  fastify.get('/debug/logs', async (request, reply) => {
    const limit = parseInt((request.query as any)?.limit || '50', 10);
    const logs = memoryLogger.getLogs(limit);
    return {
      logs,
      count: logs.length,
      mode: 'memory'
    };
  });
  
  fastify.get('/debug/blocked-writes', async (request, reply) => {
    const warnings = getBlockedWarnings();
    return {
      blockedWrites: warnings,
      count: warnings.length,
      mode: 'memory'
    };
  });
}

// Start server
const start = async () => {
  try {
    // Validate secrets configuration on startup
    logSecurityValidation();
    logShieldConfigValidation(shield, env.SENTINEL_SHIELD_MODE, env.NODE_ENV);

    setCancelAdapter(createGatewayCancelAdapter());
    setPolicyViolationPublisher((violation) => {
      publishSentinelPolicyViolationFireAndForget(violation.runId, {
        policy: violation.policy,
        reason: violation.reason,
        threshold: violation.threshold,
        actual: violation.actual,
        action: 'deny',
      });
    });
    setRunKillSideEffects({
      decRunsActive: () => runsActive.dec(),
      getSseConnections: (runId) => sseConnections.get(runId),
      onSseMessage: () => sseMessagesTotal.inc(),
      deleteEventStream: (runId) => { sentinelEventStreams.delete(runId); },
    });
    
    // Initialize queue system (non-blocking)
    // Queue will fall back to synchronous execution if Redis is not available
    const queue = await initializeQueue();
    if (queue) {
      queueInitialized = true;
      // Set processor context for queue workers (including SSE connections)
      setProcessorContext(sentinelEventStreams, sentinel, shield, sseConnections, sseMessagesTotal);
      baseLogger.info('Queue system initialized successfully');
    } else {
      baseLogger.warn('Queue system not available (Redis required). Using synchronous execution.');
    }
    
    // Initialize Prisma if in postgres mode (non-blocking)
    // Don't fail startup if Prisma client isn't generated yet
    if (persistenceMode === 'postgres') {
      // Run Prisma connection in background, don't block startup
      Promise.resolve().then(async () => {
        try {
          const { getPrismaClient } = await import('./db/prisma.js');
          const prisma = await getPrismaClient();
          // Test connection
          await prisma.$connect();
          baseLogger.info('Connected to PostgreSQL database');
        } catch (err: any) {
          // Log error but don't crash - Prisma client may not be generated yet
          if (err?.code === 'MODULE_NOT_FOUND' || err?.message?.includes('Cannot find module')) {
            baseLogger.warn('Prisma client not generated yet. Run: pnpm db:generate');
            baseLogger.warn('Gateway will start in degraded mode until Prisma client is generated.');
          } else {
            baseLogger.error('Failed to connect to PostgreSQL:', err?.message || err);
            if (env.NODE_ENV === 'production') {
              baseLogger.error('Production mode: database connection required');
            } else {
              baseLogger.warn('Continuing in degraded mode (no database)');
            }
          }
        }
      }).catch((err) => {
        // Silently handle promise rejection - already logged above
        baseLogger.warn('Prisma initialization failed, continuing without database');
      });
    }
    
    // MFA: Multi-Factor Authentication Routes
    baseLogger.info('=== STARTING MFA/GDPR ROUTE REGISTRATION ===');
    baseLogger.info('About to attempt MFA route registration...');
    try {
      // Check if dependencies are available before importing
      try {
        await import('otplib');
        await import('qrcode');
      } catch (depError) {
        baseLogger.warn('MFA dependencies not available, skipping MFA routes', {
          error: depError instanceof Error ? depError.message : String(depError)
        });
        throw new Error('MFA dependencies not installed');
      }
      
      baseLogger.info('Attempting to register MFA routes...');
      baseLogger.info('Calling mfaRoutes function...');
      await mfaRoutes(fastify);
      baseLogger.info('MFA routes registered successfully');
    } catch (error) {
      baseLogger.error('MFA routes registration failed', { 
        error: error instanceof Error ? error.message : String(error),
        stack: error instanceof Error ? error.stack : undefined,
        name: error instanceof Error ? error.name : typeof error
      });
      // Don't fail startup, but log the error
    }

    // GDPR: Compliance Routes
    baseLogger.info('About to attempt GDPR route registration...');
    try {
      baseLogger.info('Attempting to register GDPR routes...');
      baseLogger.info('Calling gdprRoutes function...');
      await gdprRoutes(fastify);
      baseLogger.info('GDPR routes registered successfully');
    } catch (error) {
      baseLogger.error('GDPR routes registration failed', { 
        error: error instanceof Error ? error.message : String(error),
        stack: error instanceof Error ? error.stack : undefined,
        name: error instanceof Error ? error.name : typeof error
      });
      // Don't fail startup, but log the error
    }
    
    await fastify.listen({ 
      port: PORT, 
      host: HOST 
    });
    baseLogger.info(`Gateway server listening on ${HOST}:${PORT}`);
    baseLogger.info(`Persistence mode: ${persistenceMode}`);
    baseLogger.info(`Authentication: ${env.AUTH_ENABLED ? 'enabled' : 'disabled'}`);

    if (persistenceMode !== 'memory') {
      const { enforceRunRetention, getRunRetentionConfig } = await import('./runs/run-retention.js');
      const retentionCfg = getRunRetentionConfig();
      if (retentionCfg.enabled) {
        baseLogger.info('Run retention enabled', {
          maxCount: retentionCfg.maxCount,
          maxAgeDays: retentionCfg.maxAgeDays,
        });
        enforceRunRetention(retentionCfg).catch((err: unknown) => {
          baseLogger.warn('Startup run retention failed', {
            error: err instanceof Error ? err.message : String(err),
          });
        });
      }
    }
    
    if (persistenceMode === 'memory') {
      baseLogger.warn('[Gateway] Running in NO-PERSISTENCE / MEMORY mode. All state is ephemeral.');
    } else if (persistenceMode === 'postgres') {
      baseLogger.info('[Gateway] PostgreSQL mode enabled. Generate Prisma client with: pnpm db:generate');
    }
  } catch (err) {
    baseLogger.error('Error starting server:', err);
    process.exit(1);
  }
};

// Graceful shutdown
const shutdown = async () => {
  baseLogger.info('Shutting down gateway...');
  try {
    // Close Fastify server first to stop accepting new connections
    baseLogger.info('Closing Fastify server...');
    await fastify.close();
    
    // Shutdown queue system (let jobs complete)
    if (queueInitialized) {
      baseLogger.info('Shutting down queue...');
      await shutdownQueue();
    }
    
    // Shutdown Sentinel
    if (sentinel) {
      baseLogger.info('Shutting down Sentinel...');
      sentinel.shutdown();
    }
    
    // Disconnect databases
    baseLogger.info('Disconnecting databases...');
    await shutdownDatabase(dbClient, baseLogger);
    await disconnectPrisma();
    await disconnectRedis();
    
    baseLogger.info('Gateway shut down successfully');
    process.exit(0);
  } catch (err) {
    baseLogger.error('Error during shutdown:', err);
    process.exit(1);
  }
};

// Handle uncaught errors
process.on('uncaughtException', (err) => {
  baseLogger.error('Uncaught exception:', err);
  shutdown();
});

process.on('unhandledRejection', (reason, promise) => {
  baseLogger.error('Unhandled rejection:', { reason, promise });
  shutdown();
});

process.on('SIGTERM', shutdown);
process.on('SIGINT', shutdown);

start();
