"use strict";
/**
 * MidnightCrawler, Blockchain Crawler Orchestrator
 *
 * Two-phase operation:
 * 1. Catch-Up: Sync historical blocks from lastIndexedHeight to chain tip
 * 2. Live: Subscribe to new block headers and process in real-time
 *
 */
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.MidnightCrawler = void 0;
const cds_1 = __importDefault(require("@sap/cds"));
const { SELECT, INSERT, UPDATE, DELETE } = cds_1.default.ql;
const MidnightNodeProvider_1 = require("../providers/MidnightNodeProvider");
const BlockProcessor_1 = require("./BlockProcessor");
const cds_model_1 = require("../utils/cds-model");
const retry_1 = require("../utils/retry");
const sync_state_1 = require("../utils/sync-state");
const midnight_1 = require("#cds-models/midnight");
// ============================================================================
// Crawler Orchestrator
// ============================================================================
class MidnightCrawler {
    nodeProvider;
    isRunning = false;
    isCatchingUp = false;
    processing = false; // Mutex: prevent concurrent block processing
    pendingHeights = []; // Queued live block heights received during processing
    subscriptionId = null;
    db;
    processor;
    startTime = 0;
    blocksProcessed = 0;
    config;
    constructor(nodeProvider, config) {
        this.nodeProvider = nodeProvider;
        this.config = {
            enabled: config.enabled,
            nodeUrl: config.nodeUrl || 'ws://localhost:9944',
            batchSize: config.batchSize || 10,
            maxRetries: config.maxRetries || 3,
            retryDelay: config.retryDelay || 2000,
            requestTimeout: config.requestTimeout || 30000,
            fetchConcurrency: config.fetchConcurrency ?? 8,
            rpcBatchSize: config.rpcBatchSize ?? 32
        };
    }
    // ========================================================================
    // Lifecycle
    // ========================================================================
    async start() {
        if (this.isRunning) {
            console.warn('[Crawler] Already running');
            return;
        }
        this.isRunning = true;
        this.startTime = Date.now();
        try {
            await (0, cds_model_1.ensureNightgateModelLoaded)();
            this.db = await cds_1.default.connect.to('db');
            // Ensure SyncState singleton exists (shared utility)
            await (0, sync_state_1.ensureSyncStateSingleton)(this.db, this.config.nodeUrl);
            // Connect to node FIRST
            if (!this.nodeProvider.isConnected()) {
                await this.nodeProvider.connect();
            }
            // Initialize block processor (after node is connected)
            this.processor = new BlockProcessor_1.BlockProcessor(this.nodeProvider);
            await this.processor.init();
            console.log('[Crawler] Starting...');
            // Run the catch-up + live-subscription pipeline in the background.
            // Awaiting it here would block the caller (cds.on('served') callback)
            // until catch-up completes, which can take hours on a fresh DB. That
            // prevents CAP's HTTP server from binding to its port. Fire-and-forget
            // so the OData services come online immediately; the crawler keeps
            // ingesting in parallel.
            this.runIngestPipeline().catch(err => {
                console.error('[Crawler] Ingest pipeline failed:', err);
                this.isRunning = false;
            });
        }
        catch (err) {
            this.isRunning = false;
            throw err;
        }
    }
    async runIngestPipeline() {
        // Phase 1: Catch-up
        await this.catchUp();
        // Phase 2: Live subscription
        if (this.isRunning) {
            await this.subscribeLive();
        }
        // Phase 3: Second catch-up to cover blocks finalized between
        // end of Phase 1 and subscription establishment in Phase 2
        if (this.isRunning) {
            const gapBlocks = await this.catchUp();
            if (gapBlocks > 0) {
                console.log(`[Crawler] Gap catch-up: ${gapBlocks} blocks indexed`);
            }
        }
    }
    async stop() {
        console.log('[Crawler] Stopping...');
        this.isRunning = false;
        // Unsubscribe from live updates
        if (this.subscriptionId) {
            try {
                await this.nodeProvider.unsubscribeFinalizedHeads(this.subscriptionId);
            }
            catch {
                // Ignore unsubscribe errors during shutdown
            }
            this.subscriptionId = null;
        }
        // Update sync state
        try {
            await this.db.run(UPDATE.entity(midnight_1.SyncState).set({
                syncStatus: 'stopped'
            }).where({ ID: 'SINGLETON' }));
        }
        catch {
            // DB might be closed
        }
        console.log('[Crawler] Stopped');
    }
    // ========================================================================
    // Phase 1: Catch-Up (Historical Blocks)
    // ========================================================================
    async catchUp() {
        this.isCatchingUp = true;
        try {
            const syncState = await this.getSyncState();
            const startHeight = this.getCatchUpStartHeight(syncState);
            // Target finalized head (not chain tip), avoids ingesting soon-reverted blocks
            const finalizedHash = await this.nodeProvider.getFinalizedHead();
            const finalizedHeader = await this.nodeProvider.getHeader(finalizedHash);
            const tipHeight = MidnightNodeProvider_1.MidnightNodeProvider.parseBlockNumber(finalizedHeader.number);
            if (startHeight > tipHeight) {
                console.log(`[Crawler] Already synced to finalized head (height ${tipHeight})`);
                return 0;
            }
            const totalBlocks = tipHeight - startHeight + 1;
            console.log(`[Crawler] Catch-up: ${startHeight} → ${tipHeight} (${totalBlocks} blocks, finalized)`);
            await this.db.run(UPDATE.entity(midnight_1.SyncState).set({
                syncStatus: 'syncing',
                chainHeight: tipHeight,
                lastFinalizedHeight: tipHeight,
                lastFinalizedHash: finalizedHash
            }).where({ ID: 'SINGLETON' }));
            const processed = await this.runCatchUpPipeline(startHeight, tipHeight, totalBlocks);
            return processed;
        }
        finally {
            this.isCatchingUp = false;
        }
    }
    /**
     * Pipelined catch-up with JSON-RPC batching.
     *
     * Each in-flight unit is now a BATCH of K consecutive heights. Each batch
     * does exactly 2 WSS round-trips (one for hashes, one for blocks+timestamps).
     * `fetchConcurrency` batches stay in flight at once.
     *
     * Throughput model: bps ≈ K × concurrency × (1 / (2 × RTT)). For RTT=200ms,
     * K=32, concurrency=8: 32×8/0.4 = 640 bps theoretical. Real-world ceiling
     * comes from public-endpoint rate limits.
     *
     * Persist is serial in height order (reorg detection requires monotonic
     * progression). With persist at ~15ms/block we get a write ceiling of
     * ~65 bps per persister thread. If fetch outpaces persist, `wait=0` in the
     * diagnostic line and persist becomes the floor.
     */
    async runCatchUpPipeline(startHeight, tipHeight, totalBlocks) {
        const concurrency = Math.max(1, this.config.fetchConcurrency);
        const rpcBatchSize = Math.max(1, this.config.rpcBatchSize ?? 32);
        const batchStart = Date.now();
        let processed = 0;
        let nextHeightToFetch = startHeight;
        let nextHeightToPersist = startHeight;
        // Diagnostic accumulators (reset per progress log).
        let acc = { fetchMsTotal: 0, persistMsTotal: 0, waitedForFetchMs: 0, samples: 0 };
        // Queue of in-flight batches; each is a Promise<PreparedBlock[]> covering
        // a contiguous height range. Drained in submission order.
        const queue = [];
        const pumpFetches = () => {
            while (queue.length < concurrency &&
                nextHeightToFetch <= tipHeight &&
                this.isRunning) {
                const from = nextHeightToFetch;
                const to = Math.min(from + rpcBatchSize - 1, tipHeight);
                const heights = [];
                for (let h = from; h <= to; h++)
                    heights.push(h);
                queue.push({ from, to, data: this.fetchBlockBatchWithRetry(heights) });
                nextHeightToFetch = to + 1;
            }
        };
        pumpFetches();
        outer: while (nextHeightToPersist <= tipHeight && this.isRunning) {
            const head = queue.shift();
            if (!head)
                break;
            let preps;
            try {
                const waitStart = Date.now();
                preps = await head.data;
                const waitedForFetchMs = preps[0]?.fetchCompletedAt
                    ? Math.max(0, Date.now() - preps[0].fetchCompletedAt)
                    : Math.max(0, Date.now() - waitStart);
                for (const prep of preps) {
                    if (!this.isRunning)
                        break;
                    const h = prep.height;
                    const fetchMs = (prep.fetchCompletedAt ?? Date.now()) - (prep.fetchStartedAt ?? Date.now());
                    const fetchMsPerBlock = preps.length > 0 ? fetchMs / preps.length : fetchMs;
                    const persistStart = Date.now();
                    await this.processor.persistPreparedBlock(prep);
                    const persistMs = Date.now() - persistStart;
                    acc.fetchMsTotal += fetchMsPerBlock;
                    acc.persistMsTotal += persistMs;
                    acc.waitedForFetchMs += waitedForFetchMs / preps.length;
                    acc.samples++;
                    processed++;
                    this.blocksProcessed++;
                    nextHeightToPersist = h + 1;
                    if (processed % this.config.batchSize === 0 || h === tipHeight) {
                        const elapsed = (Date.now() - batchStart) / 1000;
                        const bps = elapsed > 0 ? processed / elapsed : 0;
                        const remaining = tipHeight - h;
                        const eta = bps > 0 ? remaining / bps : 0;
                        const avgFetch = acc.samples ? (acc.fetchMsTotal / acc.samples).toFixed(0) : '0';
                        const avgPersist = acc.samples ? (acc.persistMsTotal / acc.samples).toFixed(0) : '0';
                        const avgWait = acc.samples ? (acc.waitedForFetchMs / acc.samples).toFixed(0) : '0';
                        console.log(`[Crawler] Catch-up: ${h}/${tipHeight} ` +
                            `(${((h - startHeight + 1) / totalBlocks * 100).toFixed(1)}%) ` +
                            `${bps.toFixed(1)} bps, ETA: ${Math.ceil(eta)}s ` +
                            `[fetch=${avgFetch}ms persist=${avgPersist}ms wait=${avgWait}ms batch=${rpcBatchSize}]`);
                        acc = { fetchMsTotal: 0, persistMsTotal: 0, waitedForFetchMs: 0, samples: 0 };
                        await this.db.run(UPDATE.entity(midnight_1.SyncState).set({
                            syncProgress: ((h - startHeight + 1) / totalBlocks * 100),
                            blocksPerSecond: bps,
                            consecutiveErrors: 0
                        }).where({ ID: 'SINGLETON' }));
                    }
                }
            }
            catch (err) {
                console.error(`[Crawler] Failed to process batch ${head.from}-${head.to}:`, err.message);
                await this.recordError(err.message);
                const state = await this.getSyncState();
                if ((state?.consecutiveErrors || 0) > 10) {
                    console.error('[Crawler] Too many consecutive errors, stopping catch-up');
                    break outer;
                }
                nextHeightToPersist = head.to + 1;
            }
            pumpFetches();
        }
        // Drain in-flight prefetches we don't intend to persist (best effort).
        for (const item of queue) {
            item.data.catch(() => { });
        }
        console.log(`[Crawler] Catch-up complete: ${processed} blocks in ${((Date.now() - batchStart) / 1000).toFixed(1)}s`);
        return processed;
    }
    /**
     * Batch fetch with the same transient-error retry policy as
     * `fetchBlockWithRetry`. On retry, the entire batch is re-fetched.
     */
    async fetchBlockBatchWithRetry(heights) {
        let lastError = null;
        for (let attempt = 1; attempt <= this.config.maxRetries; attempt++) {
            try {
                return await this.processor.fetchBlockBatch(heights);
            }
            catch (err) {
                lastError = err;
                const transient = (0, retry_1.isTransientError)(lastError);
                if (!transient) {
                    console.error(`[Crawler] Batch ${heights[0]}-${heights[heights.length - 1]} permanent error: ${lastError.message}`);
                    break;
                }
                if (attempt < this.config.maxRetries) {
                    const delay = (0, retry_1.calcBackoff)(attempt, this.config.retryDelay);
                    console.warn(`[Crawler] Batch ${heights[0]}-${heights[heights.length - 1]} attempt ${attempt} failed (transient): ` +
                        `${lastError.message}, retrying in ${Math.round(delay)}ms`);
                    await this.sleep(delay);
                }
            }
        }
        throw lastError || new Error(`Failed to fetch batch starting at ${heights[0]}`);
    }
    /**
     * Fetch a block's data (no DB write) with the same transient-error retry
     * policy used by the legacy `processBlockWithRetry`.
     */
    async fetchBlockWithRetry(height) {
        let lastError = null;
        for (let attempt = 1; attempt <= this.config.maxRetries; attempt++) {
            try {
                return await this.processor.fetchBlockData(height);
            }
            catch (err) {
                lastError = err;
                const transient = (0, retry_1.isTransientError)(lastError);
                if (!transient) {
                    console.error(`[Crawler] Block ${height} permanent error: ${lastError.message}`);
                    break;
                }
                if (attempt < this.config.maxRetries) {
                    const delay = (0, retry_1.calcBackoff)(attempt, this.config.retryDelay);
                    console.warn(`[Crawler] Block ${height} fetch attempt ${attempt} failed (transient): ` +
                        `${lastError.message}, retrying in ${Math.round(delay)}ms`);
                    await this.sleep(delay);
                }
            }
        }
        throw lastError || new Error(`Failed to fetch block ${height}`);
    }
    // ========================================================================
    // Phase 2: Live Subscription
    // ========================================================================
    async subscribeLive() {
        console.log('[Crawler] Starting live subscription...');
        // Register reconnect handler to re-subscribe after connection loss
        if (typeof this.nodeProvider.setOnReconnect === 'function') {
            this.nodeProvider.setOnReconnect(async () => {
                if (this.isRunning) {
                    console.log('[Crawler] Re-subscribing after reconnect...');
                    this.subscriptionId = null;
                    await this.subscribeLive();
                }
            });
        }
        this.subscriptionId = await this.nodeProvider.subscribeFinalizedHeads(async (header) => {
            if (!this.isRunning || this.isCatchingUp)
                return;
            const height = MidnightNodeProvider_1.MidnightNodeProvider.parseBlockNumber(header.number);
            // Queue block height if already processing, don't drop it
            if (this.processing) {
                this.pendingHeights.push(height);
                return;
            }
            this.processing = true;
            try {
                await this.processLiveBlock(header, height);
                // Drain queued heights (process highest, then catch-up gaps)
                while (this.pendingHeights.length > 0 && this.isRunning) {
                    const maxHeight = Math.max(...this.pendingHeights);
                    this.pendingHeights = [];
                    // Catch up any gaps between current tip and maxHeight
                    const gapBlocks = await this.catchUp();
                    if (gapBlocks > 0) {
                        console.log(`[Crawler] Drained ${gapBlocks} queued blocks`);
                    }
                }
            }
            finally {
                this.processing = false;
            }
        });
        // Update sync state
        await this.db.run(UPDATE.entity(midnight_1.SyncState).set({
            syncStatus: 'synced'
        }).where({ ID: 'SINGLETON' }));
        console.log('[Crawler] Live subscription active');
    }
    async processLiveBlock(header, height) {
        try {
            // Update chain height
            await this.db.run(UPDATE.entity(midnight_1.SyncState).set({
                chainHeight: height
            }).where({ ID: 'SINGLETON' }));
            // Check for reorg
            const reorg = await this.checkForReorg(header);
            if (reorg) {
                const reorgLogId = await this.handleReorg(reorg);
                const reIndexedCount = await this.catchUp();
                await this.db.run(UPDATE.entity(midnight_1.ReorgLog).set({
                    blocksReIndexed: reIndexedCount,
                    status: 'completed'
                }).where({ ID: reorgLogId }));
                return;
            }
            // Process new block (processBlockByHeight already fetches the hash)
            const result = await this.processBlockWithRetry(height);
            this.blocksProcessed++;
            // Update sync state to synced
            const elapsed = (Date.now() - this.startTime) / 1000;
            const syncState = await this.getSyncState();
            await this.db.run(UPDATE.entity(midnight_1.SyncState).set({
                syncStatus: 'synced',
                syncProgress: 100,
                blocksPerSecond: this.blocksProcessed / elapsed,
                consecutiveErrors: 0,
                lastFinalizedHeight: height,
                lastFinalizedHash: syncState?.lastIndexedHash || result.blockHash
            }).where({ ID: 'SINGLETON' }));
            console.log(`[Crawler] Live: block ${height} ` +
                `(${result.transactionCount} txs, ${result.processingTimeMs}ms)`);
        }
        catch (err) {
            const error = err;
            const transient = (0, retry_1.isTransientError)(error);
            console.error(`[Crawler] Live: failed to process block ${height} ` +
                `(${transient ? 'transient' : 'permanent'}): ${error.message}`);
            await this.recordError(error.message);
            const state = await this.getSyncState();
            if ((state?.consecutiveErrors || 0) > 10) {
                console.error('[Crawler] Too many consecutive errors in live mode, pausing...');
            }
        }
    }
    // ========================================================================
    // Reorg Detection & Recovery
    // ========================================================================
    async checkForReorg(header) {
        const syncState = await this.getSyncState();
        if (!syncState?.lastIndexedHash)
            return null;
        // If parent hash doesn't match our tip, we have a reorg
        if (header.parentHash !== syncState.lastIndexedHash) {
            const newHeight = MidnightNodeProvider_1.MidnightNodeProvider.parseBlockNumber(header.number);
            console.warn(`[Crawler] Reorg detected at height ${newHeight}: parent ${header.parentHash} != tip ${syncState.lastIndexedHash}`);
            const forkHeight = await this.findForkPoint(header);
            return {
                forkHeight,
                oldTipHash: syncState.lastIndexedHash,
                newTipHash: header.parentHash
            };
        }
        return null;
    }
    async findForkPoint(header) {
        let currentHash = header.parentHash;
        let height = MidnightNodeProvider_1.MidnightNodeProvider.parseBlockNumber(header.number) - 1;
        while (height > 0) {
            const localBlock = await this.db.run(SELECT.one.from(midnight_1.Blocks).where({ hash: currentHash }));
            if (localBlock) {
                return height + 1;
            }
            try {
                const prevHeader = await this.nodeProvider.getHeader(currentHash);
                currentHash = prevHeader.parentHash;
                height--;
            }
            catch {
                return height;
            }
            if (MidnightNodeProvider_1.MidnightNodeProvider.parseBlockNumber(header.number) - height > 100) {
                console.error('[Crawler] Reorg depth > 100 blocks, stopping search');
                return height;
            }
        }
        return 0;
    }
    async handleReorg(reorg) {
        console.warn(`[Crawler] Handling reorg: rolling back from height ${reorg.forkHeight}`);
        const startTime = Date.now();
        const reorgLogId = cds_1.default.utils.uuid();
        await this.db.tx(async (tx) => {
            const blocksToRollback = await tx.run(SELECT.from(midnight_1.Blocks).columns('ID', 'height')
                .where({ height: { '>=': reorg.forkHeight } })) || [];
            const rollbackCount = blocksToRollback.length;
            const blockIds = blocksToRollback.map((b) => b.ID);
            if (blockIds.length === 0)
                return;
            const txsToDelete = await tx.run(SELECT.from(midnight_1.Transactions).columns('ID')
                .where({ block_ID: { in: blockIds } })) || [];
            if (txsToDelete.length > 0) {
                const txIds = txsToDelete.map((t) => t.ID);
                // Batch delete contract balances via action IDs
                const actionsToDelete = await tx.run(SELECT.from(midnight_1.ContractActions).columns('ID')
                    .where({ transaction_ID: { in: txIds } })) || [];
                if (actionsToDelete.length > 0) {
                    const actionIds = actionsToDelete.map((a) => a.ID);
                    await tx.run(DELETE.from(midnight_1.ContractBalances).where({ contractAction_ID: { in: actionIds } }));
                }
                // Batch delete all tx-child tables
                await tx.run(DELETE.from(midnight_1.ContractActions).where({ transaction_ID: { in: txIds } }));
                await tx.run(DELETE.from(midnight_1.UnshieldedUtxos).where({ createdAtTransaction_ID: { in: txIds } }));
                await tx.run(DELETE.from(midnight_1.ZswapLedgerEvents).where({ transaction_ID: { in: txIds } }));
                await tx.run(DELETE.from(midnight_1.DustLedgerEvents).where({ transaction_ID: { in: txIds } }));
                await tx.run(DELETE.from(midnight_1.TransactionFees).where({ transaction_ID: { in: txIds } }));
                // Batch delete transaction segments via result IDs
                const resultsToDelete = await tx.run(SELECT.from(midnight_1.TransactionResults).columns('ID').where({ transaction_ID: { in: txIds } })) || [];
                if (resultsToDelete.length > 0) {
                    const resultIds = resultsToDelete.map((r) => r.ID);
                    await tx.run(DELETE.from(midnight_1.TransactionSegments).where({ transactionResult_ID: { in: resultIds } }));
                }
                await tx.run(DELETE.from(midnight_1.TransactionResults).where({ transaction_ID: { in: txIds } }));
                // Unlink spent UTXOs
                await tx.run(UPDATE.entity(midnight_1.UnshieldedUtxos)
                    .set({ spentAtTransaction_ID: null })
                    .where({ spentAtTransaction_ID: { in: txIds } }));
            }
            // Batch delete transactions and blocks
            await tx.run(DELETE.from(midnight_1.Transactions).where({ block_ID: { in: blockIds } }));
            await tx.run(DELETE.from(midnight_1.Blocks).where({ ID: { in: blockIds } }));
            const forkBlock = await tx.run(SELECT.one.from(midnight_1.Blocks)
                .where({ height: { '<': reorg.forkHeight } })
                .orderBy('height desc'));
            await tx.run(UPDATE.entity(midnight_1.SyncState).set({
                lastIndexedHeight: forkBlock?.height ?? 0,
                lastIndexedHash: forkBlock?.hash ?? null,
                lastIndexedAt: new Date().toISOString(),
                syncStatus: 'syncing'
            }).where({ ID: 'SINGLETON' }));
            await tx.run(INSERT.into(midnight_1.ReorgLog).entries({
                ID: reorgLogId,
                detectedAt: new Date().toISOString(),
                forkHeight: reorg.forkHeight,
                oldTipHash: reorg.oldTipHash,
                newTipHash: reorg.newTipHash,
                blocksRolledBack: rollbackCount,
                blocksReIndexed: 0,
                status: 'in_progress'
            }));
        });
        const elapsed = Date.now() - startTime;
        console.log(`[Crawler] Reorg handled: rolled back to height ${reorg.forkHeight - 1} in ${elapsed}ms`);
        return reorgLogId;
    }
    // ========================================================================
    // Helpers
    // ========================================================================
    async processBlockWithRetry(height) {
        let lastError = null;
        for (let attempt = 1; attempt <= this.config.maxRetries; attempt++) {
            try {
                return await this.processor.processBlockByHeight(height);
            }
            catch (err) {
                lastError = err;
                const transient = (0, retry_1.isTransientError)(lastError);
                if (!transient) {
                    // Permanent error, don't retry
                    console.error(`[Crawler] Block ${height} permanent error: ${lastError.message}`);
                    break;
                }
                if (attempt < this.config.maxRetries) {
                    const delay = (0, retry_1.calcBackoff)(attempt, this.config.retryDelay);
                    console.warn(`[Crawler] Block ${height} attempt ${attempt} failed (transient): ` +
                        `${lastError.message}, retrying in ${Math.round(delay)}ms`);
                    await this.sleep(delay);
                }
            }
        }
        throw lastError || new Error(`Failed to process block ${height}`);
    }
    async getSyncState() {
        return this.db.run(SELECT.one.from(midnight_1.SyncState).where({ ID: 'SINGLETON' }));
    }
    async recordError(message) {
        try {
            const state = await this.getSyncState();
            await this.db.run(UPDATE.entity(midnight_1.SyncState).set({
                lastError: message.slice(0, 500),
                lastErrorAt: new Date().toISOString(),
                consecutiveErrors: (state?.consecutiveErrors || 0) + 1,
                syncStatus: 'error'
            }).where({ ID: 'SINGLETON' }));
        }
        catch {
            // Don't fail on error recording
        }
    }
    getCatchUpStartHeight(syncState) {
        if (!syncState?.lastIndexedHash) {
            return 0;
        }
        return (syncState.lastIndexedHeight ?? -1) + 1;
    }
    sleep(ms) {
        return new Promise(resolve => setTimeout(resolve, ms));
    }
}
exports.MidnightCrawler = MidnightCrawler;
