import { ForkSeq, MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH } from "@lodestar/params";
import { computeEpochAtSlot, computeStartSlotAtEpoch, computeTimeAtSlot, getBlockRootAtSlot, getCurrentSlot, parseAttesterFlags, parseParticipationFlags, } from "@lodestar/state-transition";
import { LogLevel, MapDef, MapDefMax, prettyPrintIndices, toRootHex } from "@lodestar/utils";
import { GENESIS_SLOT } from "../constants/constants.js";
/** The validator monitor collects per-epoch data about each monitored validator.
 * Historical data will be kept around for `HISTORIC_EPOCHS` before it is pruned.
 */
const MAX_CACHED_EPOCHS = 4;
const MAX_CACHED_DISTINCT_TARGETS = 4;
// TODO GLOAS: re-evaluate these timings
const LATE_ATTESTATION_SUBMISSION_BPS = 5000;
const LATE_BLOCK_SUBMISSION_BPS = 2500;
/** Number of epochs to retain registered validators after their last registration */
const RETAIN_REGISTERED_VALIDATORS_EPOCHS = 2;
export { OpSource };
var OpSource;
(function (OpSource) {
    OpSource["api"] = "api";
    OpSource["gossip"] = "gossip";
})(OpSource || (OpSource = {}));
export const defaultValidatorMonitorOpts = {
    validatorMonitorLogs: false,
};
function statusToSummary(inclusionDelay, flag, isActiveInCurrentEpoch, isActiveInPreviousEpoch) {
    const flags = parseAttesterFlags(flag);
    return {
        isSlashed: flags.unslashed,
        isActiveInCurrentEpoch,
        isActiveInPreviousEpoch,
        // TODO: Implement
        currentEpochEffectiveBalance: 0,
        isPrevSourceAttester: flags.prevSourceAttester,
        isPrevTargetAttester: flags.prevTargetAttester,
        isPrevHeadAttester: flags.prevHeadAttester,
        isCurrSourceAttester: flags.currSourceAttester,
        isCurrTargetAttester: flags.currTargetAttester,
        isCurrHeadAttester: flags.currHeadAttester,
        inclusionDistance: inclusionDelay,
    };
}
function getEpochSummary(validator, epoch) {
    let summary = validator.summaries.get(epoch);
    if (!summary) {
        summary = {
            attestations: 0,
            attestationMinDelay: null,
            attestationAggregateInclusions: 0,
            attestationBlockInclusions: 0,
            attestationMinBlockInclusionDistance: null,
            blocks: 0,
            blockMinDelay: null,
            aggregates: 0,
            aggregateMinDelay: null,
            attestationCorrectHead: null,
            syncCommitteeHits: 0,
            syncCommitteeMisses: 0,
            syncSignatureAggregateInclusions: 0,
            blockProposals: [],
        };
        validator.summaries.set(epoch, summary);
    }
    // Prune
    const toPrune = validator.summaries.size - MAX_CACHED_EPOCHS;
    if (toPrune > 0) {
        let pruned = 0;
        for (const idx of validator.summaries.keys()) {
            validator.summaries.delete(idx);
            if (++pruned >= toPrune)
                break;
        }
    }
    return summary;
}
export function createValidatorMonitor(metricsRegister, config, genesisTime, logger, opts) {
    const logLevel = opts.validatorMonitorLogs ? LogLevel.info : LogLevel.debug;
    const log = (message, context) => {
        logger[logLevel](message, context);
    };
    // Calculate retain time dynamically based on slot duration (2 epochs)
    const retainRegisteredValidatorsMs = SLOTS_PER_EPOCH * config.SLOT_DURATION_MS * RETAIN_REGISTERED_VALIDATORS_EPOCHS;
    /** The validators that require additional monitoring. */
    const validators = new MapDef(() => ({
        summaries: new Map(),
        inSyncCommitteeUntilEpoch: -1,
        attestations: new MapDefMax(() => new MapDefMax(() => ({
            poolSubmitDelayMinSec: null,
            poolSubmitSentPeers: null,
            aggregateInclusionDelaysSec: [],
            blockInclusions: [],
        }), MAX_CACHED_DISTINCT_TARGETS), MAX_CACHED_EPOCHS),
        lastRegisteredTimeMs: 0,
    }));
    let lastRegisteredStatusEpoch = -1;
    // Track validator additions/removals per epoch for logging
    const addedValidatorsInEpoch = new Set();
    const removedValidatorsInEpoch = new Set();
    const validatorMonitorMetrics = metricsRegister ? createValidatorMonitorMetrics(metricsRegister) : null;
    const validatorMonitor = {
        registerLocalValidator(index) {
            const isNewValidator = !validators.has(index);
            validators.getOrDefault(index).lastRegisteredTimeMs = Date.now();
            if (isNewValidator) {
                addedValidatorsInEpoch.add(index);
            }
        },
        registerLocalValidatorInSyncCommittee(index, untilEpoch) {
            const validator = validators.get(index);
            if (validator) {
                validator.inSyncCommitteeUntilEpoch = Math.max(untilEpoch, validator.inSyncCommitteeUntilEpoch ?? -1);
            }
        },
        registerValidatorStatuses(currentEpoch, inclusionDelays, flags, isActiveCurrEpoch, isActiveInPrevEpoch, balances) {
            // Prevent registering status for the same epoch twice. processEpoch() may be ran more than once for the same epoch.
            if (currentEpoch <= lastRegisteredStatusEpoch) {
                return;
            }
            lastRegisteredStatusEpoch = currentEpoch;
            const previousEpoch = currentEpoch - 1;
            // There won't be any validator activity in epoch -1
            if (previousEpoch === -1) {
                return;
            }
            // Track total balance instead of per-validator balance to reduce metric cardinality
            let totalBalance = 0;
            for (const [index, monitoredValidator] of validators.entries()) {
                // We subtract two from the state of the epoch that generated these summaries.
                //
                // - One to account for it being the previous epoch.
                // - One to account for the state advancing an epoch whilst generating the validator
                //     statuses.
                const summary = statusToSummary(inclusionDelays[index], flags[index], isActiveCurrEpoch[index], isActiveInPrevEpoch[index]);
                if (summary.isPrevSourceAttester) {
                    validatorMonitorMetrics?.prevEpochOnChainSourceAttesterHit.inc();
                }
                else {
                    validatorMonitorMetrics?.prevEpochOnChainSourceAttesterMiss.inc();
                }
                if (summary.isPrevHeadAttester) {
                    validatorMonitorMetrics?.prevEpochOnChainHeadAttesterHit.inc();
                }
                else {
                    validatorMonitorMetrics?.prevEpochOnChainHeadAttesterMiss.inc();
                }
                if (summary.isPrevTargetAttester) {
                    validatorMonitorMetrics?.prevEpochOnChainTargetAttesterHit.inc();
                }
                else {
                    validatorMonitorMetrics?.prevEpochOnChainTargetAttesterMiss.inc();
                }
                const prevEpochSummary = monitoredValidator.summaries.get(previousEpoch);
                const attestationCorrectHead = prevEpochSummary?.attestationCorrectHead;
                if (attestationCorrectHead !== null && attestationCorrectHead !== undefined) {
                    if (attestationCorrectHead) {
                        validatorMonitorMetrics?.prevOnChainAttesterCorrectHead.inc();
                    }
                    else {
                        validatorMonitorMetrics?.prevOnChainAttesterIncorrectHead.inc();
                    }
                }
                const attestationMinBlockInclusionDistance = prevEpochSummary?.attestationMinBlockInclusionDistance;
                const inclusionDistance = attestationMinBlockInclusionDistance != null && attestationMinBlockInclusionDistance > 0
                    ? // altair, attestation is not missed
                        attestationMinBlockInclusionDistance
                    : summary.inclusionDistance
                        ? // phase0, this is from the state transition
                            summary.inclusionDistance
                        : null;
                if (inclusionDistance !== null) {
                    validatorMonitorMetrics?.prevEpochOnChainInclusionDistance.observe(inclusionDistance);
                    validatorMonitorMetrics?.prevEpochOnChainAttesterHit.inc();
                }
                else {
                    validatorMonitorMetrics?.prevEpochOnChainAttesterMiss.inc();
                }
                const balance = balances?.[index];
                if (balance !== undefined) {
                    totalBalance += balance;
                }
                if (!summary.isPrevSourceAttester || !summary.isPrevTargetAttester || !summary.isPrevHeadAttester) {
                    log("Failed attestation in previous epoch", {
                        validator: index,
                        prevEpoch: currentEpoch - 1,
                        isPrevSourceAttester: summary.isPrevSourceAttester,
                        isPrevHeadAttester: summary.isPrevHeadAttester,
                        isPrevTargetAttester: summary.isPrevTargetAttester,
                        // inclusionDistance is not available in summary since altair
                        inclusionDistance,
                    });
                }
            }
            if (balances !== undefined) {
                validatorMonitorMetrics?.prevEpochOnChainBalance.set(totalBalance);
            }
        },
        registerBeaconBlock(src, delaySec, block) {
            const validator = validators.get(block.proposerIndex);
            // Returns the delay between the start of `block.slot` and `seenTimestamp`.
            if (validator) {
                validatorMonitorMetrics?.beaconBlockTotal.inc({ src });
                validatorMonitorMetrics?.beaconBlockDelaySeconds.observe({ src }, delaySec);
                const summary = getEpochSummary(validator, computeEpochAtSlot(block.slot));
                summary.blockProposals.push({
                    blockRoot: toRootHex(config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block)),
                    blockSlot: block.slot,
                    poolSubmitDelaySec: delaySec,
                    successfullyImported: false,
                });
            }
        },
        registerBlobSidecar(_src, _seenTimestampSec, _blob) {
            //TODO: freetheblobs
        },
        registerImportedBlock(block, { proposerBalanceDelta }) {
            const validator = validators.get(block.proposerIndex);
            if (validator) {
                validatorMonitorMetrics?.proposerBalanceDeltaKnown.observe(proposerBalanceDelta);
                // There should be alredy a summary for the block. Could be missing when using one VC multiple BNs
                const summary = getEpochSummary(validator, computeEpochAtSlot(block.slot));
                const proposal = summary.blockProposals.find((p) => p.blockSlot === block.slot);
                if (proposal) {
                    proposal.successfullyImported = true;
                }
                else {
                    summary.blockProposals.push({
                        blockRoot: toRootHex(config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block)),
                        blockSlot: block.slot,
                        poolSubmitDelaySec: null,
                        successfullyImported: true,
                    });
                }
            }
        },
        onPoolSubmitUnaggregatedAttestation(seenTimestampSec, indexedAttestation, subnet, sentPeers) {
            const data = indexedAttestation.data;
            const fork = config.getForkName(data.slot);
            // Returns the duration between when the attestation `data` could be produced (ATTESTATION_DUE_BPS through the slot) and `seenTimestamp`.
            const delaySec = seenTimestampSec -
                (computeTimeAtSlot(config, data.slot, genesisTime) + config.getAttestationDueMs(fork) / 1000);
            for (const index of indexedAttestation.attestingIndices) {
                const validator = validators.get(index);
                if (validator) {
                    validatorMonitorMetrics?.unaggregatedAttestationSubmittedSentPeers.observe(sentPeers);
                    validatorMonitorMetrics?.unaggregatedAttestationDelaySeconds.observe({ src: OpSource.api }, delaySec);
                    log("Published unaggregated attestation", {
                        validator: index,
                        slot: data.slot,
                        committeeIndex: data.index,
                        subnet,
                        sentPeers,
                        delaySec: delaySec.toFixed(4),
                    });
                    const attestationSummary = validator.attestations
                        .getOrDefault(indexedAttestation.data.target.epoch)
                        .getOrDefault(toRootHex(indexedAttestation.data.target.root));
                    if (attestationSummary.poolSubmitDelayMinSec === null ||
                        attestationSummary.poolSubmitDelayMinSec > delaySec) {
                        attestationSummary.poolSubmitDelayMinSec = delaySec;
                    }
                }
            }
        },
        registerGossipUnaggregatedAttestation(seenTimestampSec, indexedAttestation) {
            const src = OpSource.gossip;
            const data = indexedAttestation.data;
            const epoch = computeEpochAtSlot(data.slot);
            const fork = config.getForkName(data.slot);
            // Returns the duration between when the attestation `data` could be produced (ATTESTATION_DUE_BPS through the slot) and `seenTimestamp`.
            const delaySec = seenTimestampSec -
                (computeTimeAtSlot(config, data.slot, genesisTime) + config.getAttestationDueMs(fork) / 1000);
            for (const index of indexedAttestation.attestingIndices) {
                const validator = validators.get(index);
                if (validator) {
                    validatorMonitorMetrics?.unaggregatedAttestationTotal.inc({ src });
                    validatorMonitorMetrics?.unaggregatedAttestationDelaySeconds.observe({ src }, delaySec);
                    const summary = getEpochSummary(validator, epoch);
                    summary.attestations += 1;
                    summary.attestationMinDelay = Math.min(delaySec, summary.attestationMinDelay ?? Infinity);
                }
            }
        },
        onPoolSubmitAggregatedAttestation(seenTimestampSec, indexedAttestation, sentPeers) {
            const data = indexedAttestation.data;
            const fork = config.getForkName(data.slot);
            // Returns the duration between when a `AggregateAndproof` with `data` could be produced (AGGREGATE_DUE_BPS through the slot) and `seenTimestamp`.
            const delaySec = seenTimestampSec - (computeTimeAtSlot(config, data.slot, genesisTime) + config.getAggregateDueMs(fork) / 1000);
            for (const index of indexedAttestation.attestingIndices) {
                const validator = validators.get(index);
                if (validator) {
                    validatorMonitorMetrics?.aggregatedAttestationDelaySeconds.observe({ src: OpSource.api }, delaySec);
                    log("Published aggregated attestation", {
                        validator: index,
                        slot: data.slot,
                        committeeIndex: data.index,
                        sentPeers,
                        delaySec: delaySec.toFixed(4),
                    });
                    validator.attestations
                        .getOrDefault(indexedAttestation.data.target.epoch)
                        .getOrDefault(toRootHex(indexedAttestation.data.target.root))
                        .aggregateInclusionDelaysSec.push(delaySec);
                }
            }
        },
        registerGossipAggregatedAttestation(seenTimestampSec, signedAggregateAndProof, indexedAttestation) {
            const src = OpSource.gossip;
            const data = indexedAttestation.data;
            const epoch = computeEpochAtSlot(data.slot);
            const fork = config.getForkName(data.slot);
            // Returns the duration between when a `AggregateAndproof` with `data` could be produced (AGGREGATE_DUE_BPS through the slot) and `seenTimestamp`.
            const delaySec = seenTimestampSec - (computeTimeAtSlot(config, data.slot, genesisTime) + config.getAggregateDueMs(fork) / 1000);
            const aggregatorIndex = signedAggregateAndProof.message.aggregatorIndex;
            const validatorAggregator = validators.get(aggregatorIndex);
            if (validatorAggregator) {
                validatorMonitorMetrics?.aggregatedAttestationTotal.inc({ src });
                validatorMonitorMetrics?.aggregatedAttestationDelaySeconds.observe({ src }, delaySec);
                const summary = getEpochSummary(validatorAggregator, epoch);
                summary.aggregates += 1;
                summary.aggregateMinDelay = Math.min(delaySec, summary.aggregateMinDelay ?? Infinity);
            }
            for (const index of indexedAttestation.attestingIndices) {
                const validator = validators.get(index);
                if (validator) {
                    validatorMonitorMetrics?.attestationInAggregateTotal.inc({ src });
                    validatorMonitorMetrics?.attestationInAggregateDelaySeconds.observe({ src }, delaySec);
                    const summary = getEpochSummary(validator, epoch);
                    summary.attestationAggregateInclusions += 1;
                    log("Attestation is included in aggregate", {
                        validator: index,
                        slot: data.slot,
                        committeeIndex: data.index,
                        aggregatorIndex,
                    });
                    validator.attestations
                        .getOrDefault(indexedAttestation.data.target.epoch)
                        .getOrDefault(toRootHex(indexedAttestation.data.target.root))
                        .aggregateInclusionDelaysSec.push(delaySec);
                }
            }
        },
        // Register that the `indexed_attestation` was included in a *valid* `BeaconBlock`.
        registerAttestationInBlock(indexedAttestation, parentSlot, correctHead, missedSlotVote, inclusionBlockRoot, inclusionBlockSlot) {
            const data = indexedAttestation.data;
            // optimal inclusion distance, not to count skipped slots between data.slot and blockSlot
            const inclusionDistance = Math.max(parentSlot - data.slot, 0) + 1;
            const delay = inclusionDistance - MIN_ATTESTATION_INCLUSION_DELAY;
            const epoch = computeEpochAtSlot(data.slot);
            const participants = indexedAttestation.attestingIndices.length;
            for (const index of indexedAttestation.attestingIndices) {
                const validator = validators.get(index);
                if (validator) {
                    validatorMonitorMetrics?.attestationInBlockTotal.inc();
                    validatorMonitorMetrics?.attestationInBlockDelaySlots.observe(delay);
                    validatorMonitorMetrics?.attestationInBlockParticipants.observe(participants);
                    const summary = getEpochSummary(validator, epoch);
                    summary.attestationBlockInclusions += 1;
                    if (summary.attestationMinBlockInclusionDistance !== null) {
                        summary.attestationMinBlockInclusionDistance = Math.min(summary.attestationMinBlockInclusionDistance, inclusionDistance);
                    }
                    else {
                        summary.attestationMinBlockInclusionDistance = inclusionDistance;
                    }
                    summary.attestationCorrectHead = correctHead;
                    validator.attestations
                        .getOrDefault(indexedAttestation.data.target.epoch)
                        .getOrDefault(toRootHex(indexedAttestation.data.target.root))
                        .blockInclusions.push({
                        blockRoot: inclusionBlockRoot,
                        blockSlot: inclusionBlockSlot,
                        votedCorrectHeadRoot: correctHead,
                        votedForMissedSlot: missedSlotVote,
                        attestationSlot: indexedAttestation.data.slot,
                    });
                    log("Attestation is included in block", {
                        validator: index,
                        slot: data.slot,
                        committeeIndex: data.index,
                        inclusionDistance,
                        correctHead,
                        participants,
                    });
                }
            }
        },
        registerGossipSyncContributionAndProof(syncContributionAndProof, syncCommitteeParticipantIndices) {
            const epoch = computeEpochAtSlot(syncContributionAndProof.contribution.slot);
            for (const index of syncCommitteeParticipantIndices) {
                const validator = validators.get(index);
                if (validator) {
                    validatorMonitorMetrics?.syncSignatureInAggregateTotal.inc();
                    const summary = getEpochSummary(validator, epoch);
                    summary.syncSignatureAggregateInclusions += 1;
                }
            }
        },
        registerSyncAggregateInBlock(epoch, syncAggregate, syncCommitteeIndices) {
            for (let i = 0; i < syncCommitteeIndices.length; i++) {
                const validator = validators.get(syncCommitteeIndices[i]);
                if (validator) {
                    const summary = getEpochSummary(validator, epoch);
                    if (syncAggregate.syncCommitteeBits.get(i)) {
                        summary.syncCommitteeHits++;
                    }
                    else {
                        summary.syncCommitteeMisses++;
                    }
                }
            }
        },
        // Validator monitor tracks performance of validators in healthy network conditions.
        // It does not attempt to track correctly duties on forking conditions deeper than 1 epoch.
        // To guard against short re-orgs it will track the status of epoch N at the end of epoch N+1.
        // This function **SHOULD** be called at the last slot of an epoch to have max possible information.
        onceEveryEndOfEpoch(headState) {
            if (headState.slot <= GENESIS_SLOT) {
                // Before genesis, there won't be any validator activity
                return;
            }
            // Prune validators not seen in a while
            for (const [index, validator] of validators.entries()) {
                if (Date.now() - validator.lastRegisteredTimeMs > retainRegisteredValidatorsMs) {
                    validators.delete(index);
                    removedValidatorsInEpoch.add(index);
                }
            }
            // Log validator monitor status every epoch
            const allIndices = Array.from(validators.keys()).sort((a, b) => a - b);
            const addedIndices = Array.from(addedValidatorsInEpoch).sort((a, b) => a - b);
            const removedIndices = Array.from(removedValidatorsInEpoch).sort((a, b) => a - b);
            log("Validator monitor status", {
                epoch: computeEpochAtSlot(headState.slot),
                added: addedIndices.length > 0 ? prettyPrintIndices(addedIndices) : "none",
                removed: removedIndices.length > 0 ? prettyPrintIndices(removedIndices) : "none",
                total: validators.size,
                indices: prettyPrintIndices(allIndices),
            });
            // Clear tracking sets for next epoch
            addedValidatorsInEpoch.clear();
            removedValidatorsInEpoch.clear();
            // Compute summaries of previous epoch attestation performance
            const prevEpoch = computeEpochAtSlot(headState.slot) - 1;
            // During the end of first epoch, the prev epoch with be -1
            // Skip this as there is no attestation and block proposal summary in epoch -1
            if (prevEpoch === -1) {
                return;
            }
            const rootCache = new RootHexCache(headState);
            if (config.getForkSeq(headState.slot) >= ForkSeq.altair) {
                const { previousEpochParticipation } = headState;
                const prevEpochStartSlot = computeStartSlotAtEpoch(prevEpoch);
                const prevEpochTargetRoot = toRootHex(getBlockRootAtSlot(headState, prevEpochStartSlot));
                // Check attestation performance
                for (const [index, validator] of validators.entries()) {
                    const flags = parseParticipationFlags(previousEpochParticipation.get(index));
                    const attestationSummary = validator.attestations.get(prevEpoch)?.get(prevEpochTargetRoot);
                    const summary = renderAttestationSummary(config, rootCache, attestationSummary, flags);
                    validatorMonitorMetrics?.prevEpochAttestationSummary.inc({ summary });
                    log("Previous epoch attestation", {
                        validator: index,
                        epoch: prevEpoch,
                        summary,
                    });
                }
            }
            if (headState.epochCtx.proposersPrevEpoch !== null) {
                // proposersPrevEpoch is null on the first epoch of `headState` being generated
                for (const [slotIndex, validatorIndex] of headState.epochCtx.proposersPrevEpoch.entries()) {
                    const validator = validators.get(validatorIndex);
                    if (validator) {
                        // If expected proposer is a tracked validator
                        const epochSummary = validator.summaries.get(prevEpoch);
                        const proposalSlot = SLOTS_PER_EPOCH * prevEpoch + slotIndex;
                        const summary = renderBlockProposalSummary(config, rootCache, epochSummary, proposalSlot);
                        validatorMonitorMetrics?.prevEpochBlockProposalSummary.inc({ summary });
                        log("Previous epoch block proposal", {
                            validator: validatorIndex,
                            slot: proposalSlot,
                            epoch: prevEpoch,
                            summary,
                        });
                    }
                }
            }
        },
        /**
         * Scrape `self` for metrics.
         * Should be called whenever Prometheus is scraping.
         */
        scrapeMetrics(slotClock) {
            validatorMonitorMetrics?.validatorsConnected.set(validators.size);
            // Update static metric with connected validator indices
            if (validatorMonitorMetrics?.validatorsConnectedIndices) {
                validatorMonitorMetrics.validatorsConnectedIndices.reset();
                const allIndices = Array.from(validators.keys()).sort((a, b) => a - b);
                validatorMonitorMetrics.validatorsConnectedIndices.set({ indices: prettyPrintIndices(allIndices) }, 1);
            }
            const epoch = computeEpochAtSlot(slotClock);
            const slotInEpoch = slotClock % SLOTS_PER_EPOCH;
            // Only start to report on the current epoch once we've progressed past the point where
            // all attestation should be included in a block.
            //
            // This allows us to set alarms on Grafana to detect when an attestation has been
            // missed. If we didn't delay beyond the attestation inclusion period then we could
            // expect some occasional false-positives on attestation misses.
            //
            // I have chosen 3 as an arbitrary number where we *probably* shouldn't see that many
            // skip slots on mainnet.
            const previousEpoch = slotInEpoch > MIN_ATTESTATION_INCLUSION_DELAY + 3 ? epoch - 1 : epoch - 2;
            // reset() to mimic the behaviour of an aggregated .set({index})
            validatorMonitorMetrics?.prevEpochAttestations.reset();
            validatorMonitorMetrics?.prevEpochAttestationsMinDelaySeconds.reset();
            validatorMonitorMetrics?.prevEpochAttestationAggregateInclusions.reset();
            validatorMonitorMetrics?.prevEpochAttestationBlockInclusions.reset();
            validatorMonitorMetrics?.prevEpochAttestationBlockMinInclusionDistance.reset();
            validatorMonitorMetrics?.prevEpochSyncSignatureAggregateInclusions.reset();
            let validatorsInSyncCommittee = 0;
            let prevEpochSyncCommitteeHits = 0;
            let prevEpochSyncCommitteeMisses = 0;
            for (const validator of validators.values()) {
                // Participation in sync committee
                const validatorInSyncCommittee = validator.inSyncCommitteeUntilEpoch >= epoch;
                if (validatorInSyncCommittee) {
                    validatorsInSyncCommittee++;
                }
                // Prev-epoch summary
                const summary = validator.summaries.get(previousEpoch);
                if (!summary) {
                    continue;
                }
                // Attestations
                validatorMonitorMetrics?.prevEpochAttestations.observe(summary.attestations);
                if (summary.attestationMinDelay !== null)
                    validatorMonitorMetrics?.prevEpochAttestationsMinDelaySeconds.observe(summary.attestationMinDelay);
                validatorMonitorMetrics?.prevEpochAttestationAggregateInclusions.observe(summary.attestationAggregateInclusions);
                validatorMonitorMetrics?.prevEpochAttestationBlockInclusions.observe(summary.attestationBlockInclusions);
                if (summary.attestationMinBlockInclusionDistance !== null) {
                    validatorMonitorMetrics?.prevEpochAttestationBlockMinInclusionDistance.observe(summary.attestationMinBlockInclusionDistance);
                }
                // Blocks
                validatorMonitorMetrics?.prevEpochBeaconBlocks.observe(summary.blocks);
                if (summary.blockMinDelay !== null)
                    validatorMonitorMetrics?.prevEpochBeaconBlocksMinDelaySeconds.observe(summary.blockMinDelay);
                // Aggregates
                validatorMonitorMetrics?.prevEpochAggregatesTotal.observe(summary.aggregates);
                if (summary.aggregateMinDelay !== null)
                    validatorMonitorMetrics?.prevEpochAggregatesMinDelaySeconds.observe(summary.aggregateMinDelay);
                // Sync committee
                prevEpochSyncCommitteeHits += summary.syncCommitteeHits;
                prevEpochSyncCommitteeMisses += summary.syncCommitteeMisses;
                // Only observe if included in sync committee to prevent distorting metrics
                if (validatorInSyncCommittee) {
                    validatorMonitorMetrics?.prevEpochSyncSignatureAggregateInclusions.observe(summary.syncSignatureAggregateInclusions);
                }
            }
            validatorMonitorMetrics?.validatorsInSyncCommittee.set(validatorsInSyncCommittee);
            validatorMonitorMetrics?.prevEpochSyncCommitteeHits.set(prevEpochSyncCommitteeHits);
            validatorMonitorMetrics?.prevEpochSyncCommitteeMisses.set(prevEpochSyncCommitteeMisses);
        },
        getMonitoredValidatorIndices() {
            return Array.from(validators.keys()).sort((a, b) => a - b);
        },
    };
    // Register a single collect() function to run all validatorMonitor metrics
    validatorMonitorMetrics?.validatorsConnected.addCollect(() => {
        const clockSlot = getCurrentSlot(config, genesisTime);
        validatorMonitor.scrapeMetrics(clockSlot);
    });
    return validatorMonitor;
}
/**
 * Best guess to automatically debug why validators do not achieve expected rewards.
 * Tries to answer common questions such as:
 * - Did the validator submit the attestation to this block?
 * - Was the attestation seen in an aggregate?
 * - Was the attestation seen in a block?
 */
function renderAttestationSummary(config, rootCache, summary, flags) {
    // Reference https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/beacon-chain.md#get_attestation_participation_flag_indices
    //
    // is_matching_source = data.source == justified_checkpoint
    // is_matching_target = is_matching_source and data.target.root == get_block_root(state, data.target.epoch)
    // is_matching_head = is_matching_target and data.beacon_block_root == get_block_root_at_slot(state, data.slot)
    //
    // is_matching_source MUST be true for the attestation to be included in a block
    //
    // timely_source = is_matching_source and inclusion_delay <= integer_squareroot(SLOTS_PER_EPOCH):
    // timely_target = is_matching_target and inclusion_delay <= SLOTS_PER_EPOCH:
    // timely_head = is_matching_head and inclusion_delay == MIN_ATTESTATION_INCLUSION_DELAY:
    if (flags.timelyHead) {
        // NOTE: If timelyHead everything else MUST be true also
        return "timely_head";
    }
    //
    if (flags.timelyTarget) {
        // timelyHead == false, means at least one is true
        // - attestation voted incorrect head
        // - attestation was included late
        // Note: the same attestation can be included in multiple blocks. For example, block with parent A at slot N can
        // include the attestation. Then block as slot N+1 re-orgs slot N setting as parent A and includes the attestations
        // from block at slot N.
        //
        // TODO: Track block inclusions, and then check which ones are canonical
        if (!summary) {
            // In normal conditions should never happen, validator is expected to submit an attestation to the tracking node.
            // If the validator is using multiple beacon nodes as fallback, this condition may be triggered.
            return "unexpected_timely_target_without_summary";
        }
        const canonicalBlockInclusion = summary.blockInclusions.find((block) => isCanonical(rootCache, block));
        if (!canonicalBlockInclusion) {
            // Should never happen, because for a state to exist that registers a validator's participation this specific
            // beacon node must have imported a block with the attestation that caused the change in participation.
            return "unexpected_timely_target_without_canonical_inclusion";
        }
        const { votedCorrectHeadRoot, blockSlot, attestationSlot } = canonicalBlockInclusion;
        const inclusionDistance = Math.max(blockSlot - attestationSlot - MIN_ATTESTATION_INCLUSION_DELAY, 0);
        if (votedCorrectHeadRoot && inclusionDistance === 0) {
            // Should never happen, in this case timelyHead must be true
            return "unexpected_timely_head_as_timely_target";
        }
        // Why is the distance > 0?
        // - Block that should have included the attestation was missed
        // - Attestation was not included in any aggregate
        // - Attestation was sent late
        // Why is the head vote wrong?
        // - We processed a block late and voted for the parent
        // - We voted for a block that latter was missed
        // - We voted for a block that was re-org for another chain
        let out = "timely_target";
        if (!votedCorrectHeadRoot) {
            out += "_" + whyIsHeadVoteWrong(rootCache, canonicalBlockInclusion);
        }
        if (inclusionDistance > 0) {
            out += "_" + whyIsDistanceNotOk(rootCache, canonicalBlockInclusion, summary);
        }
        return out;
    }
    //
    if (flags.timelySource) {
        // timelyTarget == false && timelySource == true means that
        // - attestation voted the wrong target but distance is <= integer_squareroot(SLOTS_PER_EPOCH)
        return "wrong_target_timely_source";
    }
    //
    // timelySource == false, either:
    // - attestation was not included in the block
    // - included in block with wrong target (very unlikely)
    // - included in block with distance > SLOTS_PER_EPOCH (very unlikely)
    // Validator failed to submit an attestation for this epoch, validator client is probably offline
    if (!summary || summary.poolSubmitDelayMinSec === null) {
        return "no_submission";
    }
    const canonicalBlockInclusion = summary.blockInclusions.find((block) => isCanonical(rootCache, block));
    if (canonicalBlockInclusion) {
        // Canonical block inclusion with no participation flags set means wrong target + late source
        return "wrong_target_late_source";
    }
    const submittedLate = summary.poolSubmitDelayMinSec > config.getSlotComponentDurationMs(LATE_ATTESTATION_SUBMISSION_BPS) / 1000;
    const aggregateInclusion = summary.aggregateInclusionDelaysSec.length > 0;
    if (submittedLate && aggregateInclusion) {
        return "late_submit";
    }
    if (submittedLate && !aggregateInclusion) {
        return "late_submit_no_aggregate_inclusion";
    }
    if (!submittedLate && aggregateInclusion) {
        // TODO: Why was it missed then?
        if (summary.blockInclusions.length) {
            return "block_inclusion_but_orphan";
        }
        return "aggregate_inclusion_but_missed";
        // } else if (!submittedLate && !aggregateInclusion) {
    }
    // Did the node had enough peers?
    if (summary.poolSubmitSentPeers === 0) {
        return "sent_to_zero_peers";
    }
    return "no_aggregate_inclusion";
}
function whyIsHeadVoteWrong(rootCache, canonicalBlockInclusion) {
    const { votedForMissedSlot, attestationSlot } = canonicalBlockInclusion;
    const canonicalAttestationSlotMissed = isMissedSlot(rootCache, attestationSlot);
    // __A_______C
    //    \_B1
    //      ^^ attestation slot
    //
    // We vote for B1, but the next proposer skips our voted block.
    // This scenario happens sometimes when blocks are published late
    // __A____________E
    //    \_B1__C__D
    //      ^^ attestation slot
    //
    // We vote for B1, and due to some issue a longer reorg happens orphaning our vote.
    // This scenario is considered in the above
    if (!votedForMissedSlot && canonicalAttestationSlotMissed) {
        // TODO: Did the block arrive late?
        return "vote_orphaned";
    }
    // __A__B1___C
    //    \_(A)
    //      ^^ attestation slot
    //
    // We vote for A assuming skip block, next proposer's view differs
    // This scenario happens sometimes when blocks are published late
    if (votedForMissedSlot && !canonicalAttestationSlotMissed) {
        // TODO: Did the block arrive late?
        return "wrong_skip_vote";
    }
    // __A__B2___C
    //    \_B1
    //      ^^ attestation slot
    //
    // We vote for B1, but the next proposer continues the chain on a competing block
    // This scenario is unlikely to happen in short re-orgs given no slashings, won't consider.
    //
    // __A____B_______C
    //    \    \_(B)
    //     \_(A)_(A)
    //
    // Vote for different heads on skipped slot
    return "wrong_head_vote";
}
function whyIsDistanceNotOk(rootCache, canonicalBlockInclusion, summary) {
    // If the attestation is not included in any aggregate it's likely because it was sent late.
    if (summary.aggregateInclusionDelaysSec.length === 0) {
        return "no_aggregate_inclusion";
    }
    // If the next slot of an attestation is missed, distance will be > 0 even if everything else was timely
    if (isMissedSlot(rootCache, canonicalBlockInclusion.attestationSlot + 1)) {
        return "next_slot_missed";
    }
    //
    return "late_unknown";
}
/** Returns true if the state's root record includes `block` */
function isCanonical(rootCache, block) {
    return rootCache.getBlockRootAtSlot(block.blockSlot) === block.blockRoot;
}
/** Returns true if root at slot is the same at slot - 1 == there was no new block at slot */
function isMissedSlot(rootCache, slot) {
    return slot > 0 && rootCache.getBlockRootAtSlot(slot) === rootCache.getBlockRootAtSlot(slot - 1);
}
function renderBlockProposalSummary(config, rootCache, summary, proposalSlot) {
    const proposal = summary?.blockProposals.find((proposal) => proposal.blockSlot === proposalSlot);
    if (!proposal) {
        return "not_submitted";
    }
    if (rootCache.getBlockRootAtSlot(proposalSlot) === proposal.blockRoot) {
        // Canonical state includes our block
        return "canonical";
    }
    let out = "orphaned";
    if (isMissedSlot(rootCache, proposalSlot)) {
        out += "_missed";
    }
    if (proposal.poolSubmitDelaySec !== null &&
        proposal.poolSubmitDelaySec > config.getSlotComponentDurationMs(LATE_BLOCK_SUBMISSION_BPS) / 1000) {
        out += "_late";
    }
    return out;
}
/**
 * Cache to prevent accessing the state tree to fetch block roots repeteadly.
 * In normal network conditions the same root is read multiple times, specially the target.
 */
export class RootHexCache {
    state;
    blockRootSlotCache = new Map();
    constructor(state) {
        this.state = state;
    }
    getBlockRootAtSlot(slot) {
        let root = this.blockRootSlotCache.get(slot);
        if (!root) {
            root = toRootHex(getBlockRootAtSlot(this.state, slot));
            this.blockRootSlotCache.set(slot, root);
        }
        return root;
    }
}
function createValidatorMonitorMetrics(register) {
    return {
        validatorsConnected: register.gauge({
            name: "validator_monitor_validators",
            help: "Count of validators that are specifically monitored by this beacon node",
        }),
        validatorsConnectedIndices: register.gauge({
            name: "validator_monitor_indices",
            help: "Static metric with connected validator indices as label, value is always 1",
            labelNames: ["indices"],
        }),
        validatorsInSyncCommittee: register.gauge({
            name: "validator_monitor_validators_in_sync_committee",
            help: "Count of validators monitored by this beacon node that are part of sync committee",
        }),
        // Validator Monitor Metrics (per-epoch summaries)
        prevEpochOnChainBalance: register.gauge({
            name: "validator_monitor_prev_epoch_on_chain_balance",
            help: "Total balance of all monitored validators after an epoch",
        }),
        prevEpochOnChainAttesterHit: register.gauge({
            name: "validator_monitor_prev_epoch_on_chain_attester_hit_total",
            help: "Incremented if validator's submitted attestation is included in some blocks",
        }),
        prevEpochOnChainAttesterMiss: register.gauge({
            name: "validator_monitor_prev_epoch_on_chain_attester_miss_total",
            help: "Incremented if validator's submitted attestation is not included in any blocks",
        }),
        prevEpochOnChainSourceAttesterHit: register.gauge({
            name: "validator_monitor_prev_epoch_on_chain_source_attester_hit_total",
            help: "Incremented if the validator is flagged as a previous epoch source attester during per epoch processing",
        }),
        prevEpochOnChainSourceAttesterMiss: register.gauge({
            name: "validator_monitor_prev_epoch_on_chain_source_attester_miss_total",
            help: "Incremented if the validator is not flagged as a previous epoch source attester during per epoch processing",
        }),
        prevEpochOnChainHeadAttesterHit: register.gauge({
            name: "validator_monitor_prev_epoch_on_chain_head_attester_hit_total",
            help: "Incremented if the validator is flagged as a previous epoch head attester during per epoch processing",
        }),
        prevEpochOnChainHeadAttesterMiss: register.gauge({
            name: "validator_monitor_prev_epoch_on_chain_head_attester_miss_total",
            help: "Incremented if the validator is not flagged as a previous epoch head attester during per epoch processing",
        }),
        prevOnChainAttesterCorrectHead: register.gauge({
            name: "validator_monitor_prev_epoch_on_chain_attester_correct_head_total",
            help: "Total count of times a validator votes correct head",
        }),
        prevOnChainAttesterIncorrectHead: register.gauge({
            name: "validator_monitor_prev_epoch_on_chain_attester_incorrect_head_total",
            help: "Total count of times a validator votes incorrect head",
        }),
        prevEpochOnChainTargetAttesterHit: register.gauge({
            name: "validator_monitor_prev_epoch_on_chain_target_attester_hit_total",
            help: "Incremented if the validator is flagged as a previous epoch target attester during per epoch processing",
        }),
        prevEpochOnChainTargetAttesterMiss: register.gauge({
            name: "validator_monitor_prev_epoch_on_chain_target_attester_miss_total",
            help: "Incremented if the validator is not flagged as a previous epoch target attester during per epoch processing",
        }),
        prevEpochOnChainInclusionDistance: register.histogram({
            name: "validator_monitor_prev_epoch_on_chain_inclusion_distance",
            help: "The attestation inclusion distance calculated during per epoch processing",
            // min inclusion distance is 1, usual values are 1,2,3 max is 32 (1 epoch)
            buckets: [1, 2, 3, 5, 10, 32],
        }),
        prevEpochAttestations: register.histogram({
            name: "validator_monitor_prev_epoch_attestations",
            help: "The number of unagg. attestations seen in the previous epoch",
            buckets: [0, 1, 2, 3],
        }),
        prevEpochAttestationsMinDelaySeconds: register.histogram({
            name: "validator_monitor_prev_epoch_attestations_min_delay_seconds",
            help: "The min delay between when the validator should send the attestation and when it was received",
            buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10],
        }),
        prevEpochAttestationAggregateInclusions: register.histogram({
            name: "validator_monitor_prev_epoch_attestation_aggregate_inclusions",
            help: "The count of times an attestation was seen inside an aggregate",
            buckets: [0, 1, 2, 3, 5, 10],
        }),
        prevEpochAttestationBlockInclusions: register.histogram({
            name: "validator_monitor_prev_epoch_attestation_block_inclusions",
            help: "The count of times an attestation was seen inside a block",
            buckets: [0, 1, 2, 3, 5],
        }),
        prevEpochAttestationBlockMinInclusionDistance: register.histogram({
            name: "validator_monitor_prev_epoch_attestation_block_min_inclusion_distance",
            help: "The minimum inclusion distance observed for the inclusion of an attestation in a block",
            buckets: [1, 2, 3, 5, 10, 32],
        }),
        prevEpochBeaconBlocks: register.histogram({
            name: "validator_monitor_prev_epoch_beacon_blocks",
            help: "The number of beacon_blocks seen in the previous epoch",
            buckets: [0, 1, 2, 3, 5, 10],
        }),
        prevEpochBeaconBlocksMinDelaySeconds: register.histogram({
            name: "validator_monitor_prev_epoch_beacon_blocks_min_delay_seconds",
            help: "The min delay between when the validator should send the block and when it was received",
            buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10],
        }),
        prevEpochAggregatesTotal: register.histogram({
            name: "validator_monitor_prev_epoch_aggregates",
            help: "The number of aggregates seen in the previous epoch",
            buckets: [0, 1, 2, 3, 5, 10],
        }),
        prevEpochAggregatesMinDelaySeconds: register.histogram({
            name: "validator_monitor_prev_epoch_aggregates_min_delay_seconds",
            help: "The min delay between when the validator should send the aggregate and when it was received",
            buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10],
        }),
        prevEpochSyncCommitteeHits: register.gauge({
            name: "validator_monitor_prev_epoch_sync_committee_hits",
            help: "Count of times in prev epoch connected validators participated in imported block's syncAggregate",
        }),
        prevEpochSyncCommitteeMisses: register.gauge({
            name: "validator_monitor_prev_epoch_sync_committee_misses",
            help: "Count of times in prev epoch connected validators fail to participate in imported block's syncAggregate",
        }),
        prevEpochSyncSignatureAggregateInclusions: register.histogram({
            name: "validator_monitor_prev_epoch_sync_signature_aggregate_inclusions",
            help: "The count of times a sync signature was seen inside an aggregate",
            buckets: [0, 1, 2, 3, 5, 10],
        }),
        prevEpochAttestationSummary: register.gauge({
            name: "validator_monitor_prev_epoch_attestation_summary",
            help: "Best guess of the node of the result of previous epoch validators attestation actions and causality",
            labelNames: ["summary"],
        }),
        prevEpochBlockProposalSummary: register.gauge({
            name: "validator_monitor_prev_epoch_block_proposal_summary",
            help: "Best guess of the node of the result of previous epoch validators block proposal actions and causality",
            labelNames: ["summary"],
        }),
        // Validator Monitor Metrics (real-time)
        unaggregatedAttestationTotal: register.gauge({
            name: "validator_monitor_unaggregated_attestation_total",
            help: "Number of unaggregated attestations seen",
            labelNames: ["src"],
        }),
        unaggregatedAttestationDelaySeconds: register.histogram({
            name: "validator_monitor_unaggregated_attestation_delay_seconds",
            help: "The delay between when the validator should send the attestation and when it was received",
            labelNames: ["src"],
            buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10],
        }),
        unaggregatedAttestationSubmittedSentPeers: register.histogram({
            name: "validator_monitor_unaggregated_attestation_submitted_sent_peers_count",
            help: "Number of peers that an unaggregated attestation sent to",
            // as of Apr 2022, most of the time we sent to >30 peers per attestations
            // these bucket values just base on that fact to get equal range
            // refine if we want more reasonable values
            buckets: [0, 10, 20, 30],
        }),
        aggregatedAttestationTotal: register.gauge({
            name: "validator_monitor_aggregated_attestation_total",
            help: "Number of aggregated attestations seen",
            labelNames: ["src"],
        }),
        aggregatedAttestationDelaySeconds: register.histogram({
            name: "validator_monitor_aggregated_attestation_delay_seconds",
            help: "The delay between then the validator should send the aggregate and when it was received",
            labelNames: ["src"],
            buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10],
        }),
        attestationInAggregateTotal: register.gauge({
            name: "validator_monitor_attestation_in_aggregate_total",
            help: "Number of times an attestation has been seen in an aggregate",
            labelNames: ["src"],
        }),
        attestationInAggregateDelaySeconds: register.histogram({
            name: "validator_monitor_attestation_in_aggregate_delay_seconds",
            help: "The delay between when the validator should send the aggregate and when it was received",
            labelNames: ["src"],
            buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10],
        }),
        attestationInBlockTotal: register.gauge({
            name: "validator_monitor_attestation_in_block_total",
            help: "Number of times an attestation has been seen in a block",
        }),
        attestationInBlockDelaySlots: register.histogram({
            name: "validator_monitor_attestation_in_block_delay_slots",
            help: "The excess slots (beyond the minimum delay) between the attestation slot and the block slot",
            buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10],
        }),
        attestationInBlockParticipants: register.histogram({
            name: "validator_monitor_attestation_in_block_participants",
            help: "The total participants in attestations of monitored validators included in blocks",
            buckets: [1, 5, 20, 50, 100, 200],
        }),
        syncSignatureInAggregateTotal: register.gauge({
            name: "validator_monitor_sync_signature_in_aggregate_total",
            help: "Number of times a sync signature has been seen in an aggregate",
        }),
        beaconBlockTotal: register.gauge({
            name: "validator_monitor_beacon_block_total",
            help: "Total number of beacon blocks seen",
            labelNames: ["src"],
        }),
        beaconBlockDelaySeconds: register.histogram({
            name: "validator_monitor_beacon_block_delay_seconds",
            help: "The delay between when the validator should send the block and when it was received",
            labelNames: ["src"],
            // we also want other nodes to received our published before 4s so add bucket 3 and 3.5
            buckets: [0.1, 0.25, 0.5, 1, 2, 3, 4, 6, 10],
        }),
        // Only for known
        proposerBalanceDeltaKnown: register.histogram({
            name: "validator_monitor_proposer_balance_delta_known_gwei",
            help: "Balance delta of known block proposer after importing a valid block",
            // Jul22 mainnet block reward is consistently between 29,000,000-28,000,000 GWei
            buckets: [10_000, 100_000, 1e6, 10e6, 20e6, 50e6, 100e6, 1000e6],
        }),
    };
}
//# sourceMappingURL=validatorMonitor.js.map