import { ErrorAborted, toRootHex } from "@lodestar/utils";
import { isBlockInputBlobs, isBlockInputColumns } from "../../chain/blocks/blockInput/blockInput.js";
import { BlockInputErrorCode } from "../../chain/blocks/blockInput/errors.js";
import { BlobSidecarErrorCode } from "../../chain/errors/blobSidecarError.js";
import { DataColumnSidecarErrorCode } from "../../chain/errors/dataColumnSidecarError.js";
import { PeerAction, prettyPrintPeerIdStr } from "../../network/index.js";
import { ItTrigger } from "../../util/itTrigger.js";
import { wrapError } from "../../util/wrapError.js";
import { BATCH_BUFFER_SIZE, EPOCHS_PER_BATCH, MAX_LOOK_AHEAD_EPOCHS } from "../constants.js";
import { DownloadByRangeErrorCode } from "../utils/downloadByRange.js";
import { RangeSyncType } from "../utils/remoteSyncType.js";
import { Batch, BatchError, BatchErrorCode, BatchStatus } from "./batch.js";
import { ChainPeersBalancer, batchStartEpochIsAfterSlot, computeHighestTarget, getBatchSlotRange, getNextBatchToProcess, isSyncChainDone, toArr, toBeDownloadedStartEpoch, validateBatchesStatus, } from "./utils/index.js";
export class SyncChainStartError extends Error {
}
export { SyncChainStatus };
var SyncChainStatus;
(function (SyncChainStatus) {
    SyncChainStatus["Stopped"] = "Stopped";
    SyncChainStatus["Syncing"] = "Syncing";
    SyncChainStatus["Done"] = "Done";
    SyncChainStatus["Error"] = "Error";
})(SyncChainStatus || (SyncChainStatus = {}));
// this global chain id is used to identify the chain over time, increase it every time a new chain is created
// a chain type could be Finalized or Head, so it should be appended with this id to make the log unique
let nextChainId = 0;
/**
 * Dynamic target sync chain. Peers with multiple targets but with the same syncType are added
 * through the `addPeer()` hook.
 *
 * A chain of blocks that need to be downloaded. Peers who claim to contain the target head
 * root are grouped into the peer pool and queried for batches when downloading the chain.
 */
export class SyncChain {
    /** Short string id to identify this SyncChain in logs */
    logId;
    syncType;
    /**
     * Should sync up until this slot, then stop.
     * Finalized SyncChains have a dynamic target, so if this chain has no peers the target can become null
     */
    target;
    /** Number of validated epochs. For the SyncRange to prevent switching chains too fast */
    validatedEpochs = 0;
    firstBatchEpoch;
    /**
     * The start of the chain segment. Any epoch previous to this one has been validated.
     * Note: lastEpochWithProcessBlocks` signals the epoch at which 1 or more blocks have been processed
     * successfully. So that epoch itself may or may not be valid.
     */
    lastEpochWithProcessBlocks;
    status = SyncChainStatus.Stopped;
    processChainSegment;
    downloadByRange;
    reportPeer;
    getConnectedPeerSyncMeta;
    pruneBlockInputs;
    /** AsyncIterable that guarantees processChainSegment is run only at once at anytime */
    batchProcessor = new ItTrigger();
    /** Sorted map of batches undergoing some kind of processing. */
    batches = new Map();
    peerset = new Map();
    logger;
    config;
    clock;
    metrics;
    custodyConfig;
    constructor(initialBatchEpoch, initialTarget, syncType, fns, modules) {
        const { config, clock, custodyConfig, logger, metrics } = modules;
        this.firstBatchEpoch = initialBatchEpoch;
        this.lastEpochWithProcessBlocks = initialBatchEpoch;
        this.target = initialTarget;
        this.syncType = syncType;
        this.processChainSegment = fns.processChainSegment;
        this.downloadByRange = fns.downloadByRange;
        this.reportPeer = fns.reportPeer;
        this.pruneBlockInputs = fns.pruneBlockInputs;
        this.getConnectedPeerSyncMeta = fns.getConnectedPeerSyncMeta;
        this.config = config;
        this.clock = clock;
        this.metrics = metrics;
        this.custodyConfig = custodyConfig;
        this.logger = logger;
        this.logId = `${syncType}-${nextChainId++}`;
        if (metrics) {
            metrics.syncRange.headSyncPeers.addCollect(() => this.scrapeMetrics(metrics));
        }
        // Trigger event on parent class
        this.sync().then(() => fns.onEnd(null, this.target), (e) => fns.onEnd(e, null));
    }
    /**
     * Start syncing a new chain or an old one with an existing peer list
     * In the same call, advance the chain if localFinalizedEpoch >
     */
    startSyncing(localFinalizedEpoch) {
        switch (this.status) {
            case SyncChainStatus.Stopped:
                break; // Ok, continue
            case SyncChainStatus.Syncing:
                return; // Skip, already started
            case SyncChainStatus.Error:
            case SyncChainStatus.Done:
                throw new SyncChainStartError(`Attempted to start an ended SyncChain ${this.status}`);
        }
        this.status = SyncChainStatus.Syncing;
        this.logger.debug("SyncChain startSyncing", {
            localFinalizedEpoch,
            lastEpochWithProcessBlocks: this.lastEpochWithProcessBlocks,
            targetSlot: this.target.slot,
        });
        // to avoid dropping local progress, we advance the chain with its batch boundaries.
        // get the aligned epoch that produces a batch containing the `localFinalizedEpoch`
        const lastEpochWithProcessBlocksAligned = this.lastEpochWithProcessBlocks +
            Math.floor((localFinalizedEpoch - this.lastEpochWithProcessBlocks) / EPOCHS_PER_BATCH) * EPOCHS_PER_BATCH;
        this.advanceChain(lastEpochWithProcessBlocksAligned);
        // Potentially download new batches and process pending
        this.triggerBatchDownloader();
        this.triggerBatchProcessor();
    }
    /**
     * Temporarily stop the chain. Will prevent batches from being processed
     */
    stopSyncing() {
        this.status = SyncChainStatus.Stopped;
    }
    /**
     * Permanently remove this chain. Throws the main AsyncIterable
     */
    remove() {
        this.batchProcessor.end(new ErrorAborted("SyncChain"));
    }
    /**
     * Add peer to the chain and request batches if active
     */
    addPeer(peer, target) {
        this.peerset.set(peer, target);
        this.computeTarget();
        this.triggerBatchDownloader();
    }
    /**
     * Returns true if the peer existed and has been removed
     * NOTE: The RangeSync will take care of deleting the SyncChain if peers = 0
     */
    removePeer(peerId) {
        const deleted = this.peerset.delete(peerId);
        this.computeTarget();
        return deleted;
    }
    /**
     * Helper to print internal state for debugging when chain gets stuck
     */
    getBatchesState() {
        return toArr(this.batches).map((batch) => batch.getMetadata());
    }
    get lastValidatedSlot() {
        // Last epoch of the batch after the last one validated
        return getBatchSlotRange(this.lastEpochWithProcessBlocks + EPOCHS_PER_BATCH).startSlot - 1;
    }
    get isSyncing() {
        return this.status === SyncChainStatus.Syncing;
    }
    get isRemovable() {
        return this.status === SyncChainStatus.Error || this.status === SyncChainStatus.Done;
    }
    get peers() {
        return this.peerset.size;
    }
    getPeers() {
        return Array.from(this.peerset.keys());
    }
    /** Full debug state for lodestar API */
    getDebugState() {
        return {
            targetRoot: toRootHex(this.target.root),
            targetSlot: this.target.slot,
            syncType: this.syncType,
            status: this.status,
            startEpoch: this.lastEpochWithProcessBlocks,
            peers: this.peers,
            batches: this.getBatchesState(),
        };
    }
    computeTarget() {
        if (this.peerset.size > 0) {
            const targets = Array.from(this.peerset.values());
            this.target = computeHighestTarget(targets);
        }
    }
    /**
     * Main Promise that handles the sync process. Will resolve when initial sync completes
     * i.e. when it successfully processes a epoch >= than this chain `targetEpoch`
     */
    async sync() {
        try {
            // Start processing batches on demand in strict sequence
            for await (const _ of this.batchProcessor) {
                if (this.status !== SyncChainStatus.Syncing) {
                    continue;
                }
                // TODO: Consider running this check less often after the sync is well tested
                validateBatchesStatus(toArr(this.batches));
                // Returns true if SyncChain has processed all possible blocks with slot <= target.slot
                if (isSyncChainDone(toArr(this.batches), this.lastEpochWithProcessBlocks, this.target.slot)) {
                    break;
                }
                // Processes the next batch if ready
                const batch = getNextBatchToProcess(toArr(this.batches));
                if (batch)
                    await this.processBatch(batch);
            }
            this.status = SyncChainStatus.Done;
            this.logger.verbose("SyncChain Done", { id: this.logId });
        }
        catch (e) {
            if (e instanceof ErrorAborted) {
                return; // Ignore
            }
            for (const batch of this.batches.values()) {
                this.pruneBlockInputs(batch.getBlocks());
            }
            this.status = SyncChainStatus.Error;
            this.logger.verbose("SyncChain Error", { id: this.logId }, e);
            // If a batch exceeds it's retry limit, maybe downscore peers.
            // shouldDownscoreOnBatchError() functions enforces that all BatchErrorCode values are covered
            if (e instanceof BatchError) {
                const shouldReportPeer = shouldReportPeerOnBatchError(e.type.code);
                if (shouldReportPeer) {
                    for (const peer of this.peerset.keys()) {
                        this.reportPeer(peer, shouldReportPeer.action, shouldReportPeer.reason);
                    }
                }
            }
            throw e;
        }
    }
    /**
     * Request to process batches if possible
     */
    triggerBatchProcessor() {
        this.batchProcessor.trigger();
    }
    /**
     * Request to download batches if possible
     * Backlogs requests into a single pending request
     */
    triggerBatchDownloader() {
        try {
            this.requestBatches();
        }
        catch (e) {
            // bubble the error up to the main async iterable loop
            this.batchProcessor.end(e);
        }
    }
    /**
     * Attempts to request the next required batches from the peer pool if the chain is syncing.
     * It will exhaust the peer pool and left over batches until the batch buffer is reached.
     */
    requestBatches() {
        if (this.status !== SyncChainStatus.Syncing) {
            return;
        }
        const peersSyncInfo = [];
        for (const [peerId, target] of this.peerset.entries()) {
            try {
                peersSyncInfo.push({ ...this.getConnectedPeerSyncMeta(peerId), target });
            }
            catch (e) {
                this.logger.debug("Failed to get peer sync meta", { peerId }, e);
            }
        }
        const peerBalancer = new ChainPeersBalancer(peersSyncInfo, toArr(this.batches), this.custodyConfig, this.syncType);
        // Retry download of existing batches
        for (const batch of this.batches.values()) {
            if (batch.state.status !== BatchStatus.AwaitingDownload) {
                continue;
            }
            const peer = peerBalancer.bestPeerToRetryBatch(batch);
            if (peer) {
                void this.sendBatch(batch, peer);
            }
        }
        // find the next pending batch and request it from the peer
        let batch = this.includeNextBatch();
        while (batch != null) {
            const peer = peerBalancer.idlePeerForBatch(batch);
            if (!peer) {
                // if there is no peer available, we stop requesting batches because next batches will have greater startEpoch with the same sampling groups
                break;
            }
            void this.sendBatch(batch, peer);
            batch = this.includeNextBatch();
        }
    }
    /**
     * Creates the next required batch from the chain. If there are no more batches required, returns `null`.
     */
    includeNextBatch() {
        const batches = toArr(this.batches);
        // Only request batches up to the buffer size limit
        // Note: Don't count batches in the AwaitingValidation state, to prevent stalling sync
        // if the current processing window is contained in a long range of skip slots.
        const batchesInBuffer = batches.filter((batch) => {
            return batch.state.status === BatchStatus.Downloading || batch.state.status === BatchStatus.AwaitingProcessing;
        });
        if (batchesInBuffer.length > BATCH_BUFFER_SIZE) {
            return null;
        }
        // if last processed epoch is n, we don't want to request batches with epoch > n + MAX_LOOK_AHEAD_EPOCHS
        // we should have enough batches to process in the buffer: n + 1, ..., n + MAX_LOOK_AHEAD_EPOCHS
        // let's focus on redownloading these batches first because it may have to reach different peers to get enough sampled columns
        if (batches.length > 0 &&
            Math.max(...batches.map((b) => b.startEpoch)) >= this.lastEpochWithProcessBlocks + MAX_LOOK_AHEAD_EPOCHS) {
            return null;
        }
        // This line decides the starting epoch of the next batch. MUST ensure no duplicate batch for the same startEpoch
        const startEpoch = toBeDownloadedStartEpoch(batches, this.lastEpochWithProcessBlocks);
        // Don't request batches beyond the target head slot. The to-be-downloaded batch must be strictly after target.slot
        if (batchStartEpochIsAfterSlot(startEpoch, this.target.slot)) {
            return null;
        }
        if (this.batches.has(startEpoch)) {
            this.logger.error("Attempting to add existing Batch to SyncChain", { id: this.logId, startEpoch });
            return null;
        }
        const batch = new Batch(startEpoch, this.config, this.clock, this.custodyConfig);
        this.batches.set(startEpoch, batch);
        return batch;
    }
    /**
     * Requests the batch assigned to the given id from a given peer.
     */
    async sendBatch(batch, peer) {
        this.logger.verbose("Downloading batch", {
            id: this.logId,
            ...batch.getMetadata(),
            peer: prettyPrintPeerIdStr(peer.peerId),
        });
        try {
            batch.startDownloading(peer.peerId);
            // wrapError ensures to never call both batch success() and batch error()
            const res = await wrapError(this.downloadByRange(peer, batch, this.syncType));
            if (res.err) {
                // There's several known error cases where we want to take action on the peer
                const errCode = res.err.type?.code;
                this.metrics?.syncRange.downloadByRange.error.inc({ client: peer.client, code: errCode ?? "UNKNOWN" });
                if (this.syncType === RangeSyncType.Finalized) {
                    // For finalized sync, we are stricter with peers as there is no ambiguity about which chain we're syncing.
                    // The below cases indicate the peer may be on a different chain, so are not penalized during head sync.
                    switch (errCode) {
                        case BlockInputErrorCode.MISMATCHED_ROOT_HEX:
                        case DownloadByRangeErrorCode.MISSING_BLOBS:
                        case DownloadByRangeErrorCode.EXTRA_BLOBS:
                        case DownloadByRangeErrorCode.MISSING_COLUMNS:
                        case DownloadByRangeErrorCode.EXTRA_COLUMNS:
                        case BlobSidecarErrorCode.INCORRECT_SIDECAR_COUNT:
                        case BlobSidecarErrorCode.INCORRECT_BLOCK:
                        case DataColumnSidecarErrorCode.INCORRECT_SIDECAR_COUNT:
                        case DataColumnSidecarErrorCode.INCORRECT_BLOCK:
                            this.reportPeer(peer.peerId, PeerAction.LowToleranceError, res.err.message);
                    }
                }
                switch (errCode) {
                    case DownloadByRangeErrorCode.EXTRA_BLOCKS:
                    case DownloadByRangeErrorCode.OUT_OF_ORDER_BLOCKS:
                    case DownloadByRangeErrorCode.OUT_OF_RANGE_BLOCKS:
                    case DownloadByRangeErrorCode.PARENT_ROOT_MISMATCH:
                    case BlobSidecarErrorCode.INCLUSION_PROOF_INVALID:
                    case BlobSidecarErrorCode.INVALID_KZG_PROOF_BATCH:
                    case DataColumnSidecarErrorCode.INCORRECT_KZG_COMMITMENTS_COUNT:
                    case DataColumnSidecarErrorCode.INCORRECT_KZG_PROOF_COUNT:
                    case DataColumnSidecarErrorCode.INVALID_KZG_PROOF_BATCH:
                    case DataColumnSidecarErrorCode.INCLUSION_PROOF_INVALID:
                        this.reportPeer(peer.peerId, PeerAction.LowToleranceError, res.err.message);
                }
                this.logger.verbose("Batch download error", { id: this.logId, ...batch.getMetadata(), peer: prettyPrintPeerIdStr(peer.peerId) }, res.err);
                batch.downloadingError(peer.peerId); // Throws after MAX_DOWNLOAD_ATTEMPTS
            }
            else {
                this.logger.verbose("Batch download success", {
                    id: this.logId,
                    ...batch.getMetadata(),
                    peer: prettyPrintPeerIdStr(peer.peerId),
                });
                this.metrics?.syncRange.downloadByRange.success.inc();
                const { warnings, result } = res.result;
                const downloadSuccessOutput = batch.downloadingSuccess(peer.peerId, result);
                const logMeta = {
                    blockCount: downloadSuccessOutput.blocks.length,
                };
                if (warnings && warnings.length > 0) {
                    for (const warning of warnings) {
                        this.metrics?.syncRange.downloadByRange.warn.inc({ client: peer.client, code: warning.type.code });
                        this.logger.debug("Batch downloaded with warning", { id: this.logId, epoch: batch.startEpoch, ...logMeta, peer: prettyPrintPeerIdStr(peer.peerId) }, warning);
                    }
                }
                for (const block of downloadSuccessOutput.blocks) {
                    if (isBlockInputBlobs(block)) {
                        const blockLogMeta = block.getLogMeta();
                        const expectedBlobs = typeof blockLogMeta.expectedBlobs === "number" ? blockLogMeta.expectedBlobs : 0;
                        logMeta.expectedBlobCount = (logMeta.expectedBlobCount ?? 0) + expectedBlobs;
                        logMeta.receivedBlobCount = (logMeta.receivedBlobCount ?? 0) + blockLogMeta.receivedBlobs;
                    }
                    else if (isBlockInputColumns(block)) {
                        logMeta.columnCount = (logMeta.columnCount ?? 0) + block.getLogMeta().receivedColumns;
                    }
                }
                let logMessage;
                if (downloadSuccessOutput.status === BatchStatus.AwaitingProcessing) {
                    logMessage = "Finished downloading batch by range";
                    this.triggerBatchProcessor();
                }
                else {
                    logMessage = "Partially downloaded batch by range. Attempting another round of downloads";
                    // the flow will continue to call triggerBatchDownloader() below
                }
                this.logger.debug(logMessage, {
                    id: this.logId,
                    epoch: batch.startEpoch,
                    ...logMeta,
                    peer: prettyPrintPeerIdStr(peer.peerId),
                });
            }
            // Preemptively request more blocks from peers whilst we process current blocks
            //
            // TODO(fulu): why is this second call here.  should fall through to the one below the catch block. commenting
            //      for now and will resolve during PR process
            // this.triggerBatchDownloader();
        }
        catch (e) {
            // bubble the error up to the main async iterable loop
            this.batchProcessor.end(e);
        }
        // Preemptively request more blocks from peers whilst we process current blocks
        this.triggerBatchDownloader();
    }
    /**
     * Sends `batch` to the processor. Note: batch may be empty
     */
    async processBatch(batch) {
        const blocks = batch.startProcessing();
        // wrapError ensures to never call both batch success() and batch error()
        const res = await wrapError(this.processChainSegment(blocks, this.syncType));
        if (!res.err) {
            batch.processingSuccess();
            // If the processed batch is not empty, validate previous AwaitingValidation blocks.
            if (blocks.length > 0) {
                this.advanceChain(batch.startEpoch);
            }
            // Potentially process next AwaitingProcessing batch
            this.triggerBatchProcessor();
        }
        else {
            this.logger.verbose("Batch process error", { id: this.logId, ...batch.getMetadata() }, res.err);
            batch.processingError(res.err); // Throws after MAX_BATCH_PROCESSING_ATTEMPTS
            // At least one block was successfully verified and imported, so we can be sure all
            // previous batches are valid and we only need to download the current failed batch.
            // TODO: Disabled for now
            // if (res.err instanceof ChainSegmentError && res.err.importedBlocks > 0) {
            //   this.advanceChain(batch.startEpoch);
            // }
            // The current batch could not be processed, so either this or previous batches are invalid.
            // All previous batches (AwaitingValidation) are potentially faulty and marked for retry.
            // Progress will be drop back to `this.startEpoch`
            for (const pendingBatch of this.batches.values()) {
                if (pendingBatch.startEpoch < batch.startEpoch) {
                    this.logger.verbose("Batch validation error", { id: this.logId, ...pendingBatch.getMetadata() });
                    pendingBatch.validationError(res.err); // Throws after MAX_BATCH_PROCESSING_ATTEMPTS
                }
            }
        }
        // A batch is no longer in Processing status, queue has an empty spot to download next batch
        this.triggerBatchDownloader();
    }
    /**
     * Drops any batches previous to `newLatestValidatedEpoch` and updates the chain boundaries
     */
    advanceChain(newLastEpochWithProcessBlocks) {
        // make sure this epoch produces an advancement
        if (newLastEpochWithProcessBlocks <= this.lastEpochWithProcessBlocks) {
            return;
        }
        for (const [batchKey, batch] of this.batches.entries()) {
            if (batch.startEpoch < newLastEpochWithProcessBlocks) {
                this.batches.delete(batchKey);
                this.validatedEpochs += EPOCHS_PER_BATCH;
                // The last batch attempt is right, all others are wrong. Penalize other peers
                const attemptOk = batch.validationSuccess();
                for (const attempt of batch.failedProcessingAttempts) {
                    if (attempt.hash !== attemptOk.hash) {
                        for (const badAttemptPeer of attempt.peers) {
                            if (attemptOk.peers.find((goodPeer) => goodPeer === badAttemptPeer)) {
                                // The same peer corrected its previous attempt
                                this.reportPeer(badAttemptPeer, PeerAction.MidToleranceError, "SyncChainInvalidBatchSelf");
                            }
                            else {
                                // A different peer sent an bad batch
                                this.reportPeer(badAttemptPeer, PeerAction.LowToleranceError, "SyncChainInvalidBatchOther");
                            }
                        }
                    }
                }
            }
        }
        this.lastEpochWithProcessBlocks = newLastEpochWithProcessBlocks;
        this.logger.verbose("Advanced chain", {
            id: this.logId,
            lastEpochWithProcessBlocks: this.lastEpochWithProcessBlocks,
        });
    }
    scrapeMetrics(metrics) {
        const syncPeersMetric = this.syncType === RangeSyncType.Finalized
            ? metrics.syncRange.finalizedSyncPeers
            : metrics.syncRange.headSyncPeers;
        const peersSyncMeta = new Map();
        for (const peerId of this.peerset.keys()) {
            try {
                peersSyncMeta.set(peerId, this.getConnectedPeerSyncMeta(peerId));
            }
            catch (_) {
                // ignore for metric as peer could be disconnected
            }
        }
        const peersByColumnIndex = new Map();
        for (const [columnIndex, column] of this.custodyConfig.sampledColumns.entries()) {
            for (const { custodyColumns } of peersSyncMeta.values()) {
                if (custodyColumns.includes(column)) {
                    peersByColumnIndex.set(columnIndex, (peersByColumnIndex.get(columnIndex) ?? 0) + 1);
                }
            }
        }
        for (let columnIndex = 0; columnIndex < this.custodyConfig.sampledColumns.length; columnIndex++) {
            const peerCount = peersByColumnIndex.get(columnIndex) ?? 0;
            syncPeersMetric.set({ columnIndex }, peerCount);
        }
    }
}
/**
 * Enforces that a report peer action is defined for all BatchErrorCode exhaustively.
 * If peer should not be downscored, returns null.
 */
export function shouldReportPeerOnBatchError(code) {
    switch (code) {
        // A batch could not be processed after max retry limit. It's likely that all peers
        // in this chain are sending invalid batches repeatedly so are either malicious or faulty.
        // We drop the chain and report all peers.
        // There are some edge cases with forks that could cause this situation, but it's unlikely.
        case BatchErrorCode.MAX_PROCESSING_ATTEMPTS:
            return { action: PeerAction.LowToleranceError, reason: "SyncChainMaxProcessingAttempts" };
        // TODO: Should peers be reported for MAX_DOWNLOAD_ATTEMPTS?
        case BatchErrorCode.MAX_DOWNLOAD_ATTEMPTS:
        case BatchErrorCode.INVALID_COUNT:
        case BatchErrorCode.WRONG_STATUS:
        case BatchErrorCode.MAX_EXECUTION_ENGINE_ERROR_ATTEMPTS:
            return null;
    }
}
//# sourceMappingURL=chain.js.map