import { ForkSeq } from "@lodestar/params";
import { RequestError, RequestErrorCode } from "@lodestar/reqresp";
import { computeTimeAtSlot } from "@lodestar/state-transition";
import { prettyPrintIndices, pruneSetToMax, sleep } from "@lodestar/utils";
import { isBlockInputBlobs, isBlockInputColumns } from "../chain/blocks/blockInput/blockInput.js";
import { BlockInputSource } from "../chain/blocks/blockInput/types.js";
import { BlockError, BlockErrorCode } from "../chain/errors/index.js";
import { ChainEvent } from "../chain/index.js";
import { NetworkEvent, prettyPrintPeerIdStr } from "../network/index.js";
import { shuffle } from "../util/shuffle.js";
import { sortBy } from "../util/sortBy.js";
import { wrapError } from "../util/wrapError.js";
import { MAX_CONCURRENT_REQUESTS } from "./constants.js";
import { PendingBlockInputStatus, PendingBlockType, getBlockInputSyncCacheItemRootHex, getBlockInputSyncCacheItemSlot, isPendingBlockInput, } from "./types.js";
import { DownloadByRootError, downloadByRoot } from "./utils/downloadByRoot.js";
import { getAllDescendantBlocks, getDescendantBlocks, getUnknownAndAncestorBlocks } from "./utils/pendingBlocksTree.js";
const MAX_ATTEMPTS_PER_BLOCK = 5;
const MAX_KNOWN_BAD_BLOCKS = 500;
const MAX_PENDING_BLOCKS = 100;
var FetchResult;
(function (FetchResult) {
    FetchResult["SuccessResolved"] = "success_resolved";
    FetchResult["SuccessMissingParent"] = "success_missing_parent";
    FetchResult["SuccessLate"] = "success_late";
    FetchResult["FailureTriedAllPeers"] = "failure_tried_all_peers";
    FetchResult["FailureMaxAttempts"] = "failure_max_attempts";
})(FetchResult || (FetchResult = {}));
/**
 * BlockInputSync is a class that handles ReqResp to find blocks and data related to a specific blockRoot.  The
 * blockRoot may have been found via object gossip, or the API.  Gossip objects that can trigger a search are block,
 * blobs, columns, attestations, etc.  In the case of blocks and data this is generally during the current slot but
 * can also be for items that are received late but are not fully verified and thus not in fork-choice (old blocks on
 * an unknown fork). It can also be triggered via an attestation (or sync committee message or any other item that
 * gets gossiped) that references a blockRoot that is not in fork-choice.  In rare (and realistically should not happen)
 * situations it can get triggered via the API when the validator attempts to publish a block, attestation, aggregate
 * and proof or a sync committee contribution that has unknown information included (parentRoot for instance).
 *
 * The goal of the class is to make sure that all information that is necessary for import into fork-choice is pulled
 * from peers so that the block and data can be processed, and thus the object that triggered the search can be
 * referenced and validated.
 *
 * The most common case for this search is a set of block/data that comes across gossip for the current slot, during
 * normal chain operation, but not everything was received before the gossip cutoff window happens so it is necessary
 * to pull remaining data via req/resp so that fork-choice can be updated prior to making an attestation for the
 * current slot.
 *
 * Event sources for old UnknownBlock
 *
 * - publishBlock
 * - gossipHandlers
 * - searchUnknownSlotRoot
 *    = produceSyncCommitteeContribution
 *    = validateGossipFnRetryUnknownRoot
 *        * submitPoolAttestationsV2
 *        * publishAggregateAndProofsV2
 *    = onPendingGossipsubMessage
 *        * NetworkEvent.pendingGossipsubMessage
 *            - onGossipsubMessage
 */
export class BlockInputSync {
    config;
    network;
    chain;
    logger;
    metrics;
    opts;
    /**
     * block RootHex -> PendingBlock. To avoid finding same root at the same time
     */
    pendingBlocks = new Map();
    knownBadBlocks = new Set();
    maxPendingBlocks;
    subscribedToNetworkEvents = false;
    peerBalancer;
    constructor(config, network, chain, logger, metrics, opts) {
        this.config = config;
        this.network = network;
        this.chain = chain;
        this.logger = logger;
        this.metrics = metrics;
        this.opts = opts;
        this.maxPendingBlocks = opts?.maxPendingBlocks ?? MAX_PENDING_BLOCKS;
        this.peerBalancer = new UnknownBlockPeerBalancer();
        if (metrics) {
            metrics.blockInputSync.pendingBlocks.addCollect(() => metrics.blockInputSync.pendingBlocks.set(this.pendingBlocks.size));
            metrics.blockInputSync.knownBadBlocks.addCollect(() => metrics.blockInputSync.knownBadBlocks.set(this.knownBadBlocks.size));
        }
    }
    subscribeToNetwork() {
        if (this.opts?.disableBlockInputSync) {
            this.logger.verbose("BlockInputSync disabled by disableBlockInputSync option.");
            return;
        }
        // cannot chain to the above if or the log will be incorrect
        if (!this.subscribedToNetworkEvents) {
            this.logger.verbose("BlockInputSync enabled.");
            this.chain.emitter.on(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
            this.chain.emitter.on(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
            this.chain.emitter.on(ChainEvent.unknownParent, this.onUnknownParent);
            this.network.events.on(NetworkEvent.peerConnected, this.onPeerConnected);
            this.network.events.on(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
            this.subscribedToNetworkEvents = true;
        }
    }
    unsubscribeFromNetwork() {
        this.logger.verbose("BlockInputSync disabled.");
        this.chain.emitter.off(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
        this.chain.emitter.off(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
        this.chain.emitter.off(ChainEvent.unknownParent, this.onUnknownParent);
        this.network.events.off(NetworkEvent.peerConnected, this.onPeerConnected);
        this.network.events.off(NetworkEvent.peerDisconnected, this.onPeerDisconnected);
        this.subscribedToNetworkEvents = false;
    }
    close() {
        this.unsubscribeFromNetwork();
    }
    isSubscribedToNetwork() {
        return this.subscribedToNetworkEvents;
    }
    /**
     * Process an unknownBlock event and register the block in `pendingBlocks` Map.
     */
    onUnknownBlockRoot = (data) => {
        try {
            this.addByRootHex(data.rootHex, data.peer);
            this.triggerUnknownBlockSearch();
            this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.UNKNOWN_BLOCK_ROOT });
            this.metrics?.blockInputSync.source.inc({ source: data.source });
        }
        catch (e) {
            this.logger.debug("Error handling unknownBlockRoot event", {}, e);
        }
    };
    /**
     * Process an unknownBlockInput event and register the block in `pendingBlocks` Map.
     */
    onIncompleteBlockInput = (data) => {
        try {
            this.addByBlockInput(data.blockInput, data.peer);
            this.triggerUnknownBlockSearch();
            this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.INCOMPLETE_BLOCK_INPUT });
            this.metrics?.blockInputSync.source.inc({ source: data.source });
        }
        catch (e) {
            this.logger.debug("Error handling incompleteBlockInput event", {}, e);
        }
    };
    /**
     * Process an unknownBlockParent event and register the block in `pendingBlocks` Map.
     */
    onUnknownParent = (data) => {
        try {
            this.addByRootHex(data.blockInput.parentRootHex, data.peer);
            this.addByBlockInput(data.blockInput, data.peer);
            this.triggerUnknownBlockSearch();
            this.metrics?.blockInputSync.requests.inc({ type: PendingBlockType.UNKNOWN_PARENT });
            this.metrics?.blockInputSync.source.inc({ source: data.source });
        }
        catch (e) {
            this.logger.debug("Error handling unknownParent event", {}, e);
        }
    };
    addByRootHex = (rootHex, peerIdStr) => {
        let pendingBlock = this.pendingBlocks.get(rootHex);
        if (!pendingBlock) {
            pendingBlock = {
                status: PendingBlockInputStatus.pending,
                rootHex: rootHex,
                peerIdStrings: new Set(),
                timeAddedSec: Date.now() / 1000,
            };
            this.pendingBlocks.set(rootHex, pendingBlock);
            this.logger.verbose("Added new rootHex to BlockInputSync.pendingBlocks", {
                root: pendingBlock.rootHex,
                peerIdStr: peerIdStr ?? "unknown peer",
            });
        }
        if (peerIdStr) {
            pendingBlock.peerIdStrings.add(peerIdStr);
        }
        // TODO: check this prune methodology
        // Limit pending blocks to prevent DOS attacks that cause OOM
        const prunedItemCount = pruneSetToMax(this.pendingBlocks, this.maxPendingBlocks);
        if (prunedItemCount > 0) {
            this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingBlocks`);
        }
    };
    addByBlockInput = (blockInput, peerIdStr) => {
        let pendingBlock = this.pendingBlocks.get(blockInput.blockRootHex);
        // if entry is missing or was added via rootHex and now we have more complete information overwrite
        // the existing information with the more complete cache entry
        if (!pendingBlock || !isPendingBlockInput(pendingBlock)) {
            pendingBlock = {
                // can be added via unknown parent and we may already have full block input. need to check and set correctly
                // so we pull the data if its missing or handle the block correctly in getIncompleteAndAncestorBlocks
                status: blockInput.hasBlockAndAllData() ? PendingBlockInputStatus.downloaded : PendingBlockInputStatus.pending,
                blockInput,
                peerIdStrings: new Set(),
                timeAddedSec: Date.now() / 1000,
            };
            this.pendingBlocks.set(blockInput.blockRootHex, pendingBlock);
            this.logger.verbose("Added blockInput to BlockInputSync.pendingBlocks", pendingBlock.blockInput.getLogMeta());
        }
        if (peerIdStr) {
            pendingBlock.peerIdStrings.add(peerIdStr);
        }
        // TODO: check this prune methodology
        // Limit pending blocks to prevent DOS attacks that cause OOM
        const prunedItemCount = pruneSetToMax(this.pendingBlocks, this.maxPendingBlocks);
        if (prunedItemCount > 0) {
            this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingBlocks`);
        }
    };
    onPeerConnected = (data) => {
        try {
            const peerId = data.peer;
            const peerSyncMeta = this.network.getConnectedPeerSyncMeta(peerId);
            this.peerBalancer.onPeerConnected(data.peer, peerSyncMeta);
            this.triggerUnknownBlockSearch();
        }
        catch (e) {
            this.logger.debug("Error handling peerConnected event", {}, e);
        }
    };
    onPeerDisconnected = (data) => {
        const peerId = data.peer;
        this.peerBalancer.onPeerDisconnected(peerId);
    };
    /**
     * Gather tip parent blocks with unknown parent and do a search for all of them
     */
    triggerUnknownBlockSearch = () => {
        // Cheap early stop to prevent calling the network.getConnectedPeers()
        if (this.pendingBlocks.size === 0) {
            return;
        }
        // If the node loses all peers with pending unknown blocks, the sync will stall
        const connectedPeers = this.network.getConnectedPeers();
        if (connectedPeers.length === 0) {
            this.logger.debug("No connected peers, skipping unknown block search.");
            return;
        }
        const { unknowns, ancestors } = getUnknownAndAncestorBlocks(this.pendingBlocks);
        // it's rare when there is no unknown block
        // see https://github.com/ChainSafe/lodestar/issues/5649#issuecomment-1594213550
        if (unknowns.length === 0) {
            let processedBlocks = 0;
            for (const block of ancestors) {
                // when this happens, it's likely the block and parent block are processed by head sync
                if (this.chain.forkChoice.hasBlockHex(block.blockInput.parentRootHex)) {
                    processedBlocks++;
                    this.processBlock(block).catch((e) => {
                        this.logger.debug("Unexpected error - process old downloaded block", {}, e);
                    });
                }
            }
            this.logger.verbose("No unknown block, process ancestor downloaded blocks", {
                pendingBlocks: this.pendingBlocks.size,
                ancestorBlocks: ancestors.length,
                processedBlocks,
            });
            return;
        }
        // most of the time there is exactly 1 unknown block
        for (const block of unknowns) {
            this.downloadBlock(block).catch((e) => {
                this.logger.debug("Unexpected error - downloadBlock", { root: getBlockInputSyncCacheItemRootHex(block) }, e);
            });
        }
    };
    async downloadBlock(block) {
        if (block.status !== PendingBlockInputStatus.pending) {
            return;
        }
        const rootHex = getBlockInputSyncCacheItemRootHex(block);
        const logCtx = {
            slot: getBlockInputSyncCacheItemSlot(block),
            root: rootHex,
            pendingBlocks: this.pendingBlocks.size,
        };
        this.logger.verbose("BlockInputSync.downloadBlock()", logCtx);
        block.status = PendingBlockInputStatus.fetching;
        const res = await wrapError(this.fetchBlockInput(block));
        if (!res.err) {
            this.metrics?.blockInputSync.downloadedBlocksSuccess.inc();
            const pending = res.result;
            this.pendingBlocks.set(pending.blockInput.blockRootHex, pending);
            const blockSlot = pending.blockInput.slot;
            const finalizedSlot = this.chain.forkChoice.getFinalizedBlock().slot;
            const delaySec = Date.now() / 1000 - computeTimeAtSlot(this.config, blockSlot, this.chain.genesisTime);
            this.metrics?.blockInputSync.elapsedTimeTillReceived.observe(delaySec);
            const parentInForkChoice = this.chain.forkChoice.hasBlockHex(pending.blockInput.parentRootHex);
            const logCtx2 = {
                ...logCtx,
                slot: blockSlot,
                parentInForkChoice,
            };
            this.logger.verbose("Downloaded unknown block", logCtx2);
            if (parentInForkChoice) {
                // Bingo! Process block. Add to pending blocks anyway for recycle the cache that prevents duplicate processing
                this.processBlock(pending).catch((e) => {
                    this.logger.debug("Unexpected error - process newly downloaded block", logCtx2, e);
                });
            }
            else if (blockSlot <= finalizedSlot) {
                // the common ancestor of the downloading chain and canonical chain should be at least the finalized slot and
                // we should found it through forkchoice. If not, we should penalize all peers sending us this block chain
                // 0 - 1 - ... - n - finalizedSlot
                //                \
                //                parent 1 - parent 2 - ... - unknownParent block
                this.logger.debug("Downloaded block is before finalized slot", {
                    ...logCtx2,
                    finalizedSlot,
                });
                this.removeAndDownScoreAllDescendants(block);
            }
            else {
                this.onUnknownBlockRoot({ rootHex: pending.blockInput.parentRootHex, source: BlockInputSource.byRoot });
            }
        }
        else {
            this.metrics?.blockInputSync.downloadedBlocksError.inc();
            this.logger.debug("Ignoring unknown block root after many failed downloads", logCtx, res.err);
            this.removeAndDownScoreAllDescendants(block);
        }
    }
    /**
     * Send block to the processor awaiting completition. If processed successfully, send all children to the processor.
     * On error, remove and downscore all descendants.
     * This function could run recursively for all descendant blocks
     */
    async processBlock(pendingBlock) {
        // pending block status is `downloaded` right after `downloadBlock`
        // but could be `pending` if added by `onUnknownBlockParent` event and this function is called recursively
        if (pendingBlock.status !== PendingBlockInputStatus.downloaded) {
            if (pendingBlock.status === PendingBlockInputStatus.pending) {
                const connectedPeers = this.network.getConnectedPeers();
                if (connectedPeers.length === 0) {
                    this.logger.debug("No connected peers, skipping download block", {
                        slot: pendingBlock.blockInput.slot,
                        blockRoot: pendingBlock.blockInput.blockRootHex,
                    });
                    return;
                }
                // if the download is a success we'll call `processBlock()` for this block
                await this.downloadBlock(pendingBlock);
            }
            return;
        }
        pendingBlock.status = PendingBlockInputStatus.processing;
        // this prevents unbundling attack
        // see https://lighthouse-blog.sigmaprime.io/mev-unbundling-rpc.html
        const { slot: blockSlot, proposerIndex } = pendingBlock.blockInput.getBlock().message;
        const fork = this.config.getForkName(blockSlot);
        const proposerBoostWindowMs = this.config.getAttestationDueMs(fork);
        if (this.chain.clock.msFromSlot(blockSlot) < proposerBoostWindowMs &&
            this.chain.seenBlockProposers.isKnown(blockSlot, proposerIndex)) {
            // proposer is known by a gossip block already, wait a bit to make sure this block is not
            // eligible for proposer boost to prevent unbundling attack
            this.logger.verbose("Avoid proposer boost for this block of known proposer", {
                slot: blockSlot,
                root: pendingBlock.blockInput.blockRootHex,
                proposerIndex,
            });
            await sleep(proposerBoostWindowMs);
        }
        // At gossip time, it's critical to keep a good number of mesh peers.
        // To do that, the Gossip Job Wait Time should be consistently <3s to avoid the behavior penalties in gossip
        // Gossip Job Wait Time depends on the BLS Job Wait Time
        // so `blsVerifyOnMainThread = true`: we want to verify signatures immediately without affecting the bls thread pool.
        // otherwise we can't utilize bls thread pool capacity and Gossip Job Wait Time can't be kept low consistently.
        // See https://github.com/ChainSafe/lodestar/issues/3792
        const res = await wrapError(this.chain.processBlock(pendingBlock.blockInput, {
            ignoreIfKnown: true,
            // there could be finalized/head sync at the same time so we need to ignore if finalized
            // see https://github.com/ChainSafe/lodestar/issues/5650
            ignoreIfFinalized: true,
            blsVerifyOnMainThread: true,
        }));
        if (res.err)
            this.metrics?.blockInputSync.processedBlocksError.inc();
        else
            this.metrics?.blockInputSync.processedBlocksSuccess.inc();
        if (!res.err) {
            // no need to update status to "processed", delete anyway
            this.pendingBlocks.delete(pendingBlock.blockInput.blockRootHex);
            // Send child blocks to the processor
            for (const descendantBlock of getDescendantBlocks(pendingBlock.blockInput.blockRootHex, this.pendingBlocks)) {
                if (isPendingBlockInput(descendantBlock)) {
                    this.processBlock(descendantBlock).catch((e) => {
                        this.logger.debug("Unexpected error - process descendant block", {}, e);
                    });
                }
            }
        }
        else {
            const errorData = { slot: pendingBlock.blockInput.slot, root: pendingBlock.blockInput.blockRootHex };
            if (res.err instanceof BlockError) {
                switch (res.err.type.code) {
                    // This cases are already handled with `{ignoreIfKnown: true}`
                    // case BlockErrorCode.ALREADY_KNOWN:
                    // case BlockErrorCode.GENESIS_BLOCK:
                    case BlockErrorCode.PARENT_UNKNOWN:
                    case BlockErrorCode.PRESTATE_MISSING:
                        // Should not happen, mark as downloaded to try again latter
                        this.logger.debug("Attempted to process block but its parent was still unknown", errorData, res.err);
                        pendingBlock.status = PendingBlockInputStatus.downloaded;
                        break;
                    case BlockErrorCode.EXECUTION_ENGINE_ERROR:
                        // Removing the block(s) without penalizing the peers, hoping for EL to
                        // recover on a latter download + verify attempt
                        this.removeAllDescendants(pendingBlock);
                        break;
                    default:
                        // Block is not correct with respect to our chain. Log error loudly
                        this.logger.debug("Error processing block from unknown parent sync", errorData, res.err);
                        this.removeAndDownScoreAllDescendants(pendingBlock);
                }
            }
            // Probably a queue error or something unwanted happened, mark as pending to try again latter
            else {
                this.logger.debug("Unknown error processing block from unknown block sync", errorData, res.err);
                pendingBlock.status = PendingBlockInputStatus.downloaded;
            }
        }
    }
    /**
     * From a set of shuffled peers:
     *   - fetch the block
     *   - from deneb, fetch all missing blobs
     *   - from peerDAS, fetch sampled colmns
     * TODO: this means we only have block root, and nothing else. Consider to reflect this in the function name
     * prefulu, will attempt a max of `MAX_ATTEMPTS_PER_BLOCK` on different peers, postfulu we may attempt more as defined in `getMaxDownloadAttempts()` function
     * Also verifies the received block root + returns the peer that provided the block for future downscoring.
     */
    async fetchBlockInput(cacheItem) {
        const rootHex = getBlockInputSyncCacheItemRootHex(cacheItem);
        const excludedPeers = new Set();
        const defaultPendingColumns = this.config.getForkSeq(this.chain.clock.currentSlot) >= ForkSeq.fulu
            ? new Set(this.network.custodyConfig.sampledColumns)
            : null;
        const fetchStartSec = Date.now() / 1000;
        let slot = isPendingBlockInput(cacheItem) ? cacheItem.blockInput.slot : undefined;
        if (slot !== undefined) {
            this.metrics?.blockInputSync.fetchBegin.observe(this.chain.clock.secFromSlot(slot, fetchStartSec));
        }
        let i = 0;
        while (i++ < this.getMaxDownloadAttempts()) {
            const pendingColumns = isPendingBlockInput(cacheItem) && isBlockInputColumns(cacheItem.blockInput)
                ? new Set(cacheItem.blockInput.getMissingSampledColumnMeta().missing)
                : defaultPendingColumns;
            // pendingDataColumns is null pre-fulu
            const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers);
            if (peerMeta === null) {
                // no more peer with needed columns to try, throw error
                let message = `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: cannot find peer`;
                if (pendingColumns) {
                    message += ` with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`;
                }
                this.metrics?.blockInputSync.fetchTimeSec.observe({ result: FetchResult.FailureTriedAllPeers }, Date.now() / 1000 - fetchStartSec);
                this.metrics?.blockInputSync.fetchPeers.set({ result: FetchResult.FailureTriedAllPeers }, i);
                throw Error(message);
            }
            const { peerId, client: peerClient } = peerMeta;
            cacheItem.peerIdStrings.add(peerId);
            try {
                const downloadResult = await downloadByRoot({
                    config: this.config,
                    network: this.network,
                    chain: this.chain,
                    emitter: this.chain.emitter,
                    peerMeta,
                    cacheItem,
                });
                cacheItem = downloadResult.result;
                if (slot === undefined) {
                    slot = cacheItem.blockInput.slot;
                    // we were not able to observe the time into slot when starting the fetch, do it now
                    this.metrics?.blockInputSync.fetchBegin.observe(this.chain.clock.secFromSlot(slot, fetchStartSec));
                }
                const logCtx = { slot, rootHex, peerId, peerClient };
                this.logger.verbose("BlockInputSync.fetchBlockInput: successful download", logCtx);
                this.metrics?.blockInputSync.downloadByRoot.success.inc();
                const warnings = downloadResult.warnings;
                if (warnings) {
                    for (const warning of warnings) {
                        this.logger.debug("BlockInputSync.fetchBlockInput: downloaded with warning", logCtx, warning);
                        this.metrics?.blockInputSync.downloadByRoot.warn.inc({ code: warning.type.code, client: peerClient });
                    }
                    // TODO: penalize peer?
                }
            }
            catch (e) {
                this.logger.debug("Error downloading in BlockInputSync.fetchBlockInput", { slot, rootHex, attempt: i, peer: peerId, peerClient }, e);
                const downloadByRootMetrics = this.metrics?.blockInputSync.downloadByRoot;
                // TODO: penalize peer?
                if (e instanceof DownloadByRootError) {
                    const errorCode = e.type.code;
                    downloadByRootMetrics?.error.inc({ code: errorCode, client: peerClient });
                    excludedPeers.add(peerId);
                }
                else if (e instanceof RequestError) {
                    // should look into req_resp metrics in this case
                    downloadByRootMetrics?.error.inc({ code: "req_resp", client: peerClient });
                    switch (e.type.code) {
                        case RequestErrorCode.REQUEST_RATE_LIMITED:
                        case RequestErrorCode.REQUEST_TIMEOUT:
                            // do not exclude peer for these errors
                            break;
                        default:
                            excludedPeers.add(peerId);
                            break;
                    }
                }
                else {
                    // investigate if this happens
                    downloadByRootMetrics?.error.inc({ code: "unknown", client: peerClient });
                    excludedPeers.add(peerId);
                }
            }
            finally {
                this.peerBalancer.onRequestCompleted(peerId);
            }
            this.pendingBlocks.set(getBlockInputSyncCacheItemRootHex(cacheItem), cacheItem);
            if (cacheItem.status === PendingBlockInputStatus.downloaded) {
                // download was successful, no need to go with another peer, return
                const result = this.chain.forkChoice.hasBlockHex(cacheItem.blockInput.blockRootHex)
                    ? FetchResult.SuccessLate
                    : this.chain.forkChoice.hasBlockHex(cacheItem.blockInput.parentRootHex)
                        ? FetchResult.SuccessResolved
                        : FetchResult.SuccessMissingParent;
                this.metrics?.blockInputSync.fetchTimeSec.observe({ result }, Date.now() / 1000 - fetchStartSec);
                this.metrics?.blockInputSync.fetchPeers.set({ result }, i);
                return cacheItem;
            }
        } // end while loop over peers
        const message = `Error fetching BlockInput with slot=${slot} root=${rootHex} after ${i - 1} attempts.`;
        if (!isPendingBlockInput(cacheItem)) {
            throw Error(`${message} No block and no data was found.`);
        }
        if (!cacheItem.blockInput.hasBlock()) {
            throw new Error(`${message} Block was not found.`);
        }
        if (isBlockInputBlobs(cacheItem.blockInput)) {
            const missing = cacheItem.blockInput.getMissingBlobMeta().map((b) => b.index);
            if (missing.length) {
                throw new Error(`${message} Missing blob indices=${prettyPrintIndices(missing)}.`);
            }
        }
        if (isBlockInputColumns(cacheItem.blockInput)) {
            const missing = cacheItem.blockInput.getMissingSampledColumnMeta().missing;
            if (missing.length) {
                throw new Error(`${message} Missing column indices=${prettyPrintIndices(missing)}.`);
            }
        }
        this.metrics?.blockInputSync.fetchTimeSec.observe({ result: FetchResult.FailureMaxAttempts }, Date.now() / 1000 - fetchStartSec);
        this.metrics?.blockInputSync.fetchPeers.set({ result: FetchResult.FailureMaxAttempts }, i - 1);
        throw Error(message);
    }
    /**
     * Gets all descendant blocks of `block` recursively from `pendingBlocks`.
     * Assumes that if a parent block does not exist or is not processable, all descendant blocks are bad too.
     * Downscore all peers that have referenced any of this bad blocks. May report peers multiple times if they have
     * referenced more than one bad block.
     */
    removeAndDownScoreAllDescendants(block) {
        // Get all blocks that are a descendant of this one
        const badPendingBlocks = this.removeAllDescendants(block);
        // just console log and do not penalize on pending/bad blocks for debugging
        // console.log("removeAndDownscoreAllDescendants", {block});
        for (const block of badPendingBlocks) {
            //
            // TODO(fulu): why is this commented out here?
            //
            //   this.knownBadBlocks.add(block.blockRootHex);
            //   for (const peerIdStr of block.peerIdStrs) {
            //     // TODO: Refactor peerRpcScores to work with peerIdStr only
            //     this.network.reportPeer(peerIdStr, PeerAction.LowToleranceError, "BadBlockByRoot");
            //   }
            this.logger.debug("ignored Banning unknown block", {
                slot: getBlockInputSyncCacheItemSlot(block),
                root: getBlockInputSyncCacheItemRootHex(block),
                peerIdStrings: Array.from(block.peerIdStrings)
                    .map((id) => prettyPrintPeerIdStr(id))
                    .join(","),
            });
        }
        // Prune knownBadBlocks
        pruneSetToMax(this.knownBadBlocks, MAX_KNOWN_BAD_BLOCKS);
    }
    removeAllDescendants(block) {
        const rootHex = getBlockInputSyncCacheItemRootHex(block);
        const slot = getBlockInputSyncCacheItemSlot(block);
        // Get all blocks that are a descendant of this one
        const badPendingBlocks = [block, ...getAllDescendantBlocks(rootHex, this.pendingBlocks)];
        this.metrics?.blockInputSync.removedBlocks.inc(badPendingBlocks.length);
        for (const block of badPendingBlocks) {
            const rootHex = getBlockInputSyncCacheItemRootHex(block);
            this.pendingBlocks.delete(rootHex);
            this.chain.seenBlockInputCache.prune(rootHex);
            this.logger.debug("Removing bad/unknown/incomplete BlockInputSyncCacheItem", {
                slot,
                blockRoot: rootHex,
            });
        }
        return badPendingBlocks;
    }
    getMaxDownloadAttempts() {
        if (this.config.getForkSeq(this.chain.clock.currentSlot) < ForkSeq.fulu) {
            return MAX_ATTEMPTS_PER_BLOCK;
        }
        // TODO: I consider max 20 downloads per block for a supernode is enough for devnets
        // review this computation for public testnets or mainnet
        return Math.min(20, (MAX_ATTEMPTS_PER_BLOCK * this.network.custodyConfig.sampleGroups.length) / this.config.SAMPLES_PER_SLOT);
    }
}
/**
 * Class to track active byRoots requests and balance them across eligible peers.
 */
export class UnknownBlockPeerBalancer {
    peersMeta;
    activeRequests;
    constructor() {
        this.peersMeta = new Map();
        this.activeRequests = new Map();
    }
    /** Trigger on each peer re-status */
    onPeerConnected(peerId, syncMeta) {
        this.peersMeta.set(peerId, syncMeta);
        if (!this.activeRequests.has(peerId)) {
            this.activeRequests.set(peerId, 0);
        }
    }
    onPeerDisconnected(peerId) {
        this.peersMeta.delete(peerId);
        this.activeRequests.delete(peerId);
    }
    /**
     * called from fetchUnknownBlockRoot() where we only have block root and nothing else
     * excludedPeers are the peers that we requested already so we don't want to try again
     * pendingColumns is empty for prefulu, or the 1st time we we download a block by root
     */
    bestPeerForPendingColumns(pendingColumns, excludedPeers) {
        const eligiblePeers = this.filterPeers(pendingColumns, excludedPeers);
        if (eligiblePeers.length === 0) {
            return null;
        }
        const sortedEligiblePeers = sortBy(shuffle(eligiblePeers), 
        // prefer peers with least active req
        (peerId) => this.activeRequests.get(peerId) ?? 0);
        const bestPeerId = sortedEligiblePeers[0];
        this.onRequest(bestPeerId);
        return this.peersMeta.get(bestPeerId) ?? null;
    }
    /**
     * called from fetchUnavailableBlockInput() where we have either BlockInput or NullBlockInput
     * excludedPeers are the peers that we requested already so we don't want to try again
     */
    bestPeerForBlockInput(blockInput, excludedPeers) {
        const eligiblePeers = [];
        if (isBlockInputColumns(blockInput)) {
            const pendingDataColumns = new Set(blockInput.getMissingSampledColumnMeta().missing);
            // there could be no pending column in case when block is still missing
            eligiblePeers.push(...this.filterPeers(pendingDataColumns, excludedPeers));
        }
        else {
            // prefulu
            eligiblePeers.push(...this.filterPeers(null, excludedPeers));
        }
        if (eligiblePeers.length === 0) {
            return null;
        }
        const sortedEligiblePeers = sortBy(shuffle(eligiblePeers), 
        // prefer peers with least active req
        (peerId) => this.activeRequests.get(peerId) ?? 0);
        const bestPeerId = sortedEligiblePeers[0];
        this.onRequest(bestPeerId);
        return this.peersMeta.get(bestPeerId) ?? null;
    }
    /**
     * Consumers don't need to call this method directly, it is called internally by bestPeer*() methods
     * make this public for testing
     */
    onRequest(peerId) {
        this.activeRequests.set(peerId, (this.activeRequests.get(peerId) ?? 0) + 1);
    }
    /**
     * Consumers should call this method when a request is completed for a peer.
     */
    onRequestCompleted(peerId) {
        this.activeRequests.set(peerId, Math.max(0, (this.activeRequests.get(peerId) ?? 1) - 1));
    }
    getTotalActiveRequests() {
        let totalActiveRequests = 0;
        for (const count of this.activeRequests.values()) {
            totalActiveRequests += count;
        }
        return totalActiveRequests;
    }
    // pendingDataColumns could be null for prefulu
    filterPeers(pendingDataColumns, excludedPeers) {
        let maxColumnCount = 0;
        const considerPeers = [];
        for (const [peerId, syncMeta] of this.peersMeta.entries()) {
            if (excludedPeers.has(peerId)) {
                // made request to this peer already
                continue;
            }
            const activeRequests = this.activeRequests.get(peerId) ?? 0;
            if (activeRequests >= MAX_CONCURRENT_REQUESTS) {
                // should return peer with no more than MAX_CONCURRENT_REQUESTS active requests
                continue;
            }
            if (pendingDataColumns === null || pendingDataColumns.size === 0) {
                // prefulu, no pending columns
                considerPeers.push({ peerId, columnCount: 0 });
                continue;
            }
            // postfulu, find peers that have custody columns that we need
            const { custodyColumns: peerColumns } = syncMeta;
            // check if the peer has all needed columns
            // get match
            const columns = peerColumns.reduce((acc, elem) => {
                if (pendingDataColumns.has(elem)) {
                    acc.push(elem);
                }
                return acc;
            }, []);
            if (columns.length > 0) {
                if (columns.length > maxColumnCount) {
                    maxColumnCount = columns.length;
                }
                considerPeers.push({ peerId, columnCount: columns.length });
            }
        } // end for
        const eligiblePeers = [];
        for (const { peerId, columnCount } of considerPeers) {
            if (columnCount === maxColumnCount) {
                eligiblePeers.push(peerId);
            }
        }
        return eligiblePeers;
    }
}
//# sourceMappingURL=unknownBlock.js.map