"use strict";
/*
Nodes can be lost at anytime without notifiying the network. This is different than Apoptosis where
the node sends a message to peers before exiting. When a node notifies the network that it is exiting,
the peers can remove it from their node list within 2 cycles. If a node does not notifiy the network
before exiting it will take the peers about 3 cycles to remove the node from their node list.
The lost node detection process is described in the "Lost Node Detection" Google doc under Shardus
internal documents.
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
    var ownKeys = function(o) {
        ownKeys = Object.getOwnPropertyNames || function (o) {
            var ar = [];
            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
            return ar;
        };
        return ownKeys(o);
    };
    return function (mod) {
        if (mod && mod.__esModule) return mod;
        var result = {};
        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
        __setModuleDefault(result, mod);
        return result;
    };
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.isDown = void 0;
exports.init = init;
exports.reset = reset;
exports.getTxs = getTxs;
exports.validateRecordTypes = validateRecordTypes;
exports.dropInvalidTxs = dropInvalidTxs;
exports.updateRecord = updateRecord;
exports.parseRecord = parseRecord;
exports.sendRequests = sendRequests;
exports.scheduleLostReport = scheduleLostReport;
exports.setIsUpTs = setIsUpTs;
exports.isNodeUpRecent = isNodeUpRecent;
exports.isNodeDown = isNodeDown;
exports.isNodeLost = isNodeLost;
exports.removeNodeWithCertificiate = removeNodeWithCertificiate;
const http = __importStar(require("../http"));
const logger_1 = require("../logger");
const utils = __importStar(require("../utils"));
const utils_1 = require("../utils");
const getCallstack_1 = __importDefault(require("../utils/getCallstack"));
const nestedCounters_1 = require("../utils/nestedCounters");
const profiler_1 = require("../utils/profiler");
const Apoptosis_1 = require("./Apoptosis");
const Comms = __importStar(require("./Comms"));
const Context_1 = require("./Context");
const CycleCreator_1 = require("./CycleCreator");
const NodeList = __importStar(require("./NodeList"));
const NodeList_1 = require("./NodeList");
const Self = __importStar(require("./Self"));
const Utils_1 = require("./Utils");
const network_1 = require("../network");
const ApoptosisProposalResp_1 = require("../types/ApoptosisProposalResp");
const ApoptosisProposalReq_1 = require("../types/ApoptosisProposalReq");
const GetTrieHashesReq_1 = require("../types/GetTrieHashesReq");
const GetTrieHashesResp_1 = require("../types/GetTrieHashesResp");
const InternalRouteEnum_1 = require("../types/enum/InternalRouteEnum");
const lib_types_1 = require("@shardeum-foundation/lib-types");
const RequestErrorEnum_1 = require("../types/enum/RequestErrorEnum");
const Helpers_1 = require("../types/Helpers");
const TypeIdentifierEnum_1 = require("../types/enum/TypeIdentifierEnum");
const LostReportReq_1 = require("../types/LostReportReq");
const debugMiddleware_1 = require("../network/debugMiddleware");
const csvPerfEvents_1 = require("../logger/csvPerfEvents");
const promises_1 = require("../utils/functions/promises");
/** STATE */
let p2pLogger;
let lostReported = new Map();
let receivedLostRecordMap = new Map();
let checkedLostRecordMap = new Map();
let upGossipMap = new Map();
let appRemoved = new Map();
exports.isDown = {};
let isUp = {};
let isUpTs = {};
let stopReporting = {};
let sendRefute = -1;
// map of <node_id-cycle_counter>
let scheduledForLostReport = new Map();
let scheduledRemoveApp = new Map();
/** ROUTES */
const killExternalRoute = {
    method: 'GET',
    name: 'kill',
    authHandler: debugMiddleware_1.isDebugModeMiddlewareHigh,
    handler: (_req, res) => {
        res.json({ status: 'left the network without telling any peers' });
        killSelf('Apoptosis being called killExternalRoute()->killSelf()->emitter.emit(`apoptosized`) at src/p2p/Lost.ts');
    },
};
const killOtherExternalRoute = {
    method: 'GET',
    name: 'killother',
    authHandler: debugMiddleware_1.isDebugModeMiddlewareHigh,
    handler: (_req, res) => {
        res.json({ status: 'killing another node' });
        killOther();
    },
};
const isDownCheckRoute = {
    method: 'GET',
    name: 'down-check',
    handler: async (_req, res) => {
        const nodeId = _req.query.nodeId;
        const node = NodeList_1.nodes.get(nodeId.toString());
        const result = await isDownCheck(node);
        res.json({ status: result });
    },
};
// const lostReportRoute: P2P.P2PTypes.Route<P2P.P2PTypes.InternalHandler<P2P.LostTypes.SignedLostReport>> = {
//   name: 'lost-report',
//   handler: lostReportHandler,
// }
/**
note: we are not using the SignedObject part yet
FUTURE-SLASHING
we would not want to blindly check signatures, and may later need
a way to mark a node as bad if it spams the ping endpoint too much
 */
// const pingNodeRoute: P2P.P2PTypes.Route<P2P.P2PTypes.InternalHandler<SignedPingMessage>> = {
//   name: 'ping-node',
//   handler: (payload, response, sender) => {
//     profilerInstance.scopedProfileSectionStart('ping-node')
//     try {
//       //used by isNodeDown to test if a node can be reached on the internal protocol
//       if (payload?.m === 'ping') {
//         response({ s: 'ack', r: 1 })
//       }
//     } finally {
//       profilerInstance.scopedProfileSectionEnd('ping-node')
//     }
//   },
// }
const lostDownRoute = (payload, sender, tracker) => {
    profiler_1.profilerInstance.scopedProfileSectionStart('lost-down');
    try {
        downGossipHandler(payload, sender, tracker);
    }
    finally {
        profiler_1.profilerInstance.scopedProfileSectionStart('lost-down');
    }
};
const lostUpRoute = (payload, sender, tracker) => {
    profiler_1.profilerInstance.scopedProfileSectionStart('lost-up');
    try {
        upGossipHandler(payload, sender, tracker);
    }
    finally {
        profiler_1.profilerInstance.scopedProfileSectionStart('lost-up');
    }
};
const removeByAppRoute = (payload, sender, tracker) => {
    profiler_1.profilerInstance.scopedProfileSectionStart('remove-by-app');
    try {
        removeByAppHandler(payload, sender, tracker);
    }
    finally {
        profiler_1.profilerInstance.scopedProfileSectionStart('remove-by-app');
    }
};
const routes = {
    external: [killExternalRoute, killOtherExternalRoute],
    internal: [], //pingNodeRoute, lostReportRoute
    gossip: {
        'lost-down': lostDownRoute,
        'lost-up': lostUpRoute,
        'remove-by-app': removeByAppRoute,
    },
};
/** FUNCTIONS */
function init() {
    // p2pLogger = logger.getLogger('p2p')
    p2pLogger = Context_1.logger.getLogger('p2p');
    p2pLogger.info('HELLO');
    // Init state
    reset();
    // Register routes
    for (const route of routes.external) {
        // [TODO] - Add Comms.registerExternalGet and Post that pass through to network.*
        //          so that we can always just use Comms.* instead of network.*
        Context_1.network._registerExternal(route.method, route.name, route.authHandler, route.handler);
    }
    for (const route of routes.internal) {
        Comms.registerInternal(route.name, route.handler);
    }
    for (const [name, handler] of Object.entries(routes.gossip)) {
        Comms.registerGossipHandler(name, handler);
    }
    Comms.registerInternalBinary(LostReportBinaryHandler.name, LostReportBinaryHandler.handler);
    Context_1.p2p.registerInternal('proxy', async (payload, respond, _sender, _tracker, msgSize) => {
        profiler_1.profilerInstance.scopedProfileSectionStart('proxy');
        let proxyRes = {
            success: false,
            response: null,
        };
        try {
            let targetNode = NodeList_1.nodes.get(payload.nodeId);
            if (targetNode == null) {
                error(`proxy handler targetNode is null`);
                await respond(proxyRes);
                return;
            }
            let res = null;
            if (payload.route === 'get_trie_hashes' // &&
            // this.p2p.useBinarySerializedEndpoints &&
            // this.p2p.getTrieHashesBinary
            ) {
                nestedCounters_1.nestedCountersInstance.countEvent('p2p', 'getTrieHashesBinary', 1);
                res = await Comms.askBinary(targetNode, InternalRouteEnum_1.InternalRouteEnum.binary_get_trie_hashes, payload.message, GetTrieHashesReq_1.serializeGetTrieHashesReq, GetTrieHashesResp_1.deserializeGetTrieHashesResp, {});
            }
            else {
                error(`proxy handler route is not get_trie_hashes: ${payload.route}`);
                await respond(proxyRes);
                return;
            }
            proxyRes = {
                success: true,
                response: res,
            };
            await respond(proxyRes);
        }
        catch (e) {
            error(`proxy handler error: ${e.message}`);
            await respond(proxyRes);
        }
        finally {
            profiler_1.profilerInstance.scopedProfileSectionEnd('proxy');
        }
    });
}
// This gets called before start of Q1
function reset() {
    const lostCacheCycles = Context_1.config.p2p.lostMapPruneCycles;
    for (let [key, lostRecordItems] of receivedLostRecordMap) {
        let shouldRemove = false;
        for (let [checker, report] of lostRecordItems) {
            // delete old lost reports
            if (report.cycle < CycleCreator_1.currentCycle - lostCacheCycles) {
                shouldRemove = true;
                break;
            }
            // delete once the target is removed from the node list
            if (NodeList_1.nodes.get(report.target) == null) {
                shouldRemove = true;
                break;
            }
        }
        if (shouldRemove) {
            lostReported.delete(key);
            checkedLostRecordMap.delete(key);
            receivedLostRecordMap.delete(key);
            upGossipMap.delete(key);
        }
    }
    appRemoved.clear();
    pruneIsDown(); // prune isUp and isDown status cache
    pruneStopReporting(); // prune stopReporting cache
}
// This gets called at the start of Q3
function getTxs() {
    /* prettier-ignore */ if (logger_1.logFlags.lost)
        console.log('getTxs: inside getTxs()');
    let lostTxs = [];
    let refutedTxs = [];
    let removedByAppTxs = [];
    // Check if the node in the lost list is in the apop list; remove it if there is one
    for (const [key, lostRecordItems] of receivedLostRecordMap) {
        if (lostRecordItems == null || lostRecordItems.size === 0)
            continue;
        let target;
        for (const [checker, record] of lostRecordItems) {
            if (record.target != null) {
                target = record.target;
                break;
            }
        }
        if (target && (0, Apoptosis_1.isApopMarkedNode)(target)) {
            receivedLostRecordMap.delete(key);
        }
    }
    // get winning item for each target from the array of lost records
    let seen = {}; // used to make sure we don't add the same node twice
    for (const [key, lostRecordItems] of receivedLostRecordMap) {
        if (lostRecordItems == null || lostRecordItems.size === 0)
            continue;
        let downMsgCount = 0;
        let upMsgCount = 0;
        let downRecord;
        let upRecord;
        for (const [checker, record] of lostRecordItems) {
            if (seen[record.target])
                continue;
            // if (record.cycle !== currentCycle) continue
            if (record.status === 'down') {
                downMsgCount++;
                downRecord = record;
            }
            else if (record.status === 'up') {
                upMsgCount++;
                upRecord = record;
            }
        }
        /* prettier-ignore */ if (logger_1.logFlags.lost)
            nestedCounters_1.nestedCountersInstance.countEvent('testingLost', `${CycleCreator_1.currentCycle}: down count: ${downMsgCount}, up count: ${upMsgCount}`);
        /* prettier-ignore */ if (logger_1.logFlags.lost)
            console.log(`getTxs: down count: ${downMsgCount}, up count: ${upMsgCount}`);
        if (upMsgCount >= Context_1.config.p2p.minChecksForUp) {
            seen[upRecord.target] = true;
            /* prettier-ignore */ if (logger_1.logFlags.lost)
                nestedCounters_1.nestedCountersInstance.countEvent('testingLost', `${CycleCreator_1.currentCycle}: Saw at least ${Context_1.config.p2p.minChecksForUp} up messages`);
            /* prettier-ignore */ if (logger_1.logFlags.lost)
                console.log(`getTxs: Saw at least ${Context_1.config.p2p.minChecksForUp} up messages: ${lib_types_1.Utils.safeStringify(upRecord)}`);
            if (logger_1.logFlags.verbose)
                info(`Saw at least ${Context_1.config.p2p.minChecksForUp} up messages: ${lib_types_1.Utils.safeStringify(upRecord)}`);
        }
        else if (downMsgCount >= Context_1.config.p2p.minChecksForDown) {
            lostTxs.push(downRecord.message);
            seen[downRecord.target] = true;
            /* prettier-ignore */ if (logger_1.logFlags.lost)
                nestedCounters_1.nestedCountersInstance.countEvent('testingLost', `${CycleCreator_1.currentCycle}: Adding lost record for ${downRecord.target} to lostTxs`);
            /* prettier-ignore */ if (logger_1.logFlags.lost)
                console.log(`getTxs: Adding lost record for ${downRecord.target} to lostTxs`);
            /* prettier-ignore */ if (logger_1.logFlags.lost)
                if (logger_1.logFlags.verbose)
                    info(`Adding lost record for ${downRecord.target} to lostTxs`);
        }
        else {
            /* prettier-ignore */ if (logger_1.logFlags.lost)
                info(`Not enough down messages to be considered lost: ${lib_types_1.Utils.safeStringify(downRecord)}`);
        }
    }
    // include up gossip messages that we received
    for (const [key, upGossipMsg] of upGossipMap) {
        let { target, cycle, status } = upGossipMsg;
        if (cycle == CycleCreator_1.currentCycle) {
            refutedTxs.push(upGossipMsg);
            if (logger_1.logFlags.verbose)
                info(`Adding up gossip message for ${target} to refutedTxs`);
        }
    }
    seen = {};
    for (const [key, obj] of appRemoved) {
        if (seen[obj.target])
            continue;
        removedByAppTxs.push(obj);
        seen[obj.target] = true;
    }
    return {
        lost: [...lostTxs],
        refuted: [...refutedTxs],
        removedByApp: [...removedByAppTxs],
    };
}
function validateRecordTypes(rec) {
    let err = (0, utils_1.validateTypes)(rec, { lost: 'a', refuted: 'a', appRemoved: 'a' });
    if (err)
        return err;
    for (const item of rec.lost) {
        if (typeof item !== 'string')
            return 'items of lost array must be strings';
    }
    for (const item of rec.refuted) {
        if (typeof item !== 'string')
            return 'items of refuted array must be strings';
    }
    for (const item of rec.appRemoved) {
        if (typeof item !== 'string')
            return 'items of appRemoved array must be strings';
    }
    return '';
}
// This gets called during Q3 after getTxs
function dropInvalidTxs(txs) {
    const validLost = txs.lost.filter((request) => checkDownMsg(request, CycleCreator_1.currentCycle)[0]);
    const validRefuted = txs.refuted.filter((request) => checkUpMsg(request, CycleCreator_1.currentCycle)[0]);
    const validRemovedByApp = txs.removedByApp.filter((request) => checkRemoveByAppMsg(request, CycleCreator_1.currentCycle)[0]);
    return { lost: validLost, refuted: validRefuted, removedByApp: validRemovedByApp };
}
/*
Given the txs and prev cycle record mutate the referenced record
*/
// This gets called during Q3 after dropInvalidTxs
function updateRecord(txs, record, prev) {
    const lostNodeIds = [];
    const lostSyncingNodeIds = [];
    const refutedNodeIds = [];
    const removedByAppNodeIds = [];
    let seen = {}; // used to make sure we don't add the same node twice
    for (const request of txs.lost) {
        if (seen[request.report.target])
            continue;
        lostNodeIds.push(request.report.target);
        seen[request.report.target] = true;
    }
    seen = {};
    for (const request of txs.refuted) {
        if (seen[request.target])
            continue;
        refutedNodeIds.push(request.target);
        seen[request.target] = true;
    }
    seen = {};
    for (const request of txs.removedByApp) {
        if (seen[request.target])
            continue;
        removedByAppNodeIds.push(request.target);
        seen[request.target] = true;
    }
    // remove activated nodes from syncing by id order
    /*
    we already remove from nodelist in updateNode, so this is redundant
    for (const nodeId of record.activated) {
      NodeList.removeSyncingNode(nodeId)
      NodeList.removeReadyNode(nodeId)
    }
    */
    if (Context_1.config.p2p.detectLostSyncing) {
        const syncingNodes = NodeList.syncingByIdOrder;
        const now = Math.floor((0, network_1.shardusGetTime)() / 1000);
        for (const syncingNode of syncingNodes) {
            const syncTime = now - syncingNode.syncingTimestamp;
            /* prettier-ignore */ if (logger_1.logFlags.p2pNonFatal)
                console.log('syncTime vs maxSyncTime', syncTime, record.maxSyncTime);
            if (record.maxSyncTime && syncTime > record.maxSyncTime) {
                if (logger_1.logFlags.lost) {
                    info(`Syncing time for node ${syncingNode.id}`, syncTime);
                    info(`Max sync time from record`, record.maxSyncTime);
                    info(`Sync time is longer than max sync time. Reporting as lost`);
                    info('adding node to lost syncing list', syncingNode.id, `${syncTime} > ${record.maxSyncTime}`);
                }
                //todo remove this later after we feel good about the system.. it wont really be that rare, so we dont want to swamp rare counters
                /* prettier-ignore */ nestedCounters_1.nestedCountersInstance.countRareEvent('lost', 'sync timeout ' + `${utils.stringifyReduce(syncingNode.id)} ${syncTime} > ${record.maxSyncTime}`);
                lostSyncingNodeIds.push(syncingNode.id);
            }
        }
    }
    record.lost = lostNodeIds.sort();
    record.lostSyncing = lostSyncingNodeIds.sort();
    record.refuted = refutedNodeIds.sort();
    record.appRemoved = removedByAppNodeIds.sort();
    if (prev) {
        let apop = prev.lost.filter((id) => NodeList_1.nodes.has(id)); // remove nodes that are no longer in the network
        apop = apop.filter((id) => !prev.appRemoved.includes(id));
        let apopSyncing = [];
        if (Context_1.config.p2p.detectLostSyncing) {
            apopSyncing = prev.lostSyncing.filter((id) => NodeList_1.nodes.has(id));
        }
        // neglect nodes that are refuted
        apop = apop.filter((id) => !refutedNodeIds.includes(id)); // remove nodes that refuted
        // filter adding nodes that are already in the apop record
        if (Context_1.config.p2p.uniqueRemovedIds) {
            apop = apop.filter((id) => !record.apoptosized.includes(id));
            apopSyncing = apopSyncing.filter((id) => !record.apoptosized.includes(id));
        }
        // If the apop nodes are in the removed record also, clear them from the removed record
        if (Context_1.config.p2p.uniqueRemovedIdsUpdate) {
            const nodesInRemoved = apop.filter((id) => record.removed.includes(id));
            record.removed = record.removed.filter((id) => !nodesInRemoved.includes(id));
        }
        // If the apop nodes are in the lost record also, clear them from the lost record
        if (Context_1.config.p2p.uniqueLostIdsUpdate) {
            const nodesInLost = apop.filter((id) => record.lost.includes(id));
            record.lost = record.lost.filter((id) => !nodesInLost.includes(id));
        }
        record.apoptosized = [...apop, ...apopSyncing, ...record.apoptosized].sort();
    }
}
// This gets called before Q1 when a new cycle is created or fetched
function parseRecord(record) {
    // If we see our node in the refute field clear flag to send an 'up' message at start of next cycle
    //   We ndded to do this check before checking the lost field for our node.
    for (const id of record.refuted) {
        const node = NodeList.nodes.get(id);
        if (node == null) {
            error(`Refuted node ${id} is not in the network`);
            continue;
        }
        const emitParams = {
            nodeId: node.id,
            reason: 'Node refuted',
            time: record.start,
            publicKey: node.publicKey,
            cycleNumber: record.counter,
        };
        Self.emitter.emit('node-refuted', emitParams);
        if (id === Self.id)
            sendRefute = -1;
    }
    // Once we see any node in the lost field of the cycle record, we should stop
    //   sending lost reports for it to reduce the amount of network messages caused by the lost node
    // If we see our node in the lost field set flag to send an 'up' message at start of next cycle
    for (const id of record.lost) {
        stopReporting[id] = record.counter;
        if (id === Self.id) {
            sendRefute = record.counter + 1;
            warn(`self-schedule refute currentC:${CycleCreator_1.currentCycle} inCycle:${record.counter} refuteat:${sendRefute}`);
            /* prettier-ignore */ nestedCounters_1.nestedCountersInstance.countEvent('p2p', `self-schedule refute currentC:${CycleCreator_1.currentCycle} inCycle:${record.counter}`, 1);
            (0, csvPerfEvents_1.logPerfEvents)('node_refute');
        }
    }
    // For every node in the lostSyncing list, emit a sync timeout event
    for (const id of record.lostSyncing) {
        const node = NodeList.nodes.get(id);
        if (node == null) {
            error(`Lost syncing node ${id} is not in the network`);
            continue;
        }
        // this event will be handled and eventually raise the 'node-sync-timeout' event to the dapp for handling
        NodeList.emitSyncTimeoutEvent(node, record);
        if (Context_1.config.p2p.removeLostSyncingNodeFromList)
            NodeList.removeSyncingNode(id);
    }
    if (record.lostSyncing.includes(Self.id)) {
        // This could happen if we take longer than maxSyncTime to sync
        error(`We got marked as lostSyncing. Being nice and leaving.`);
        Self.emitter.emit('invoke-exit', 'lostSyncing', (0, getCallstack_1.default)(), 'invoke-exit being called at parseRecord() => src/p2p/Lost.ts');
    }
    // Look at the app removed id's and make Self emit 'removed' if your own id is there
    if (record.appRemoved.includes(Self.id)) {
        /* prettier-ignore */
        nestedCounters_1.nestedCountersInstance.countEvent("p2p", `app-removed c:${CycleCreator_1.currentCycle}`, 1);
        Self.emitter.emit('app-removed', Self.id);
    }
    // We don't actually have to set removed because the Apoptosis module will do it.
    // Make sure the Lost module is listed after Apoptosis in the CycleCreator submodules list
    return {
        added: [],
        removed: [...record.appRemoved],
        updated: [],
    };
}
// This is called once per cycle at the start of Q1 by CycleCreator
function sendRequests() {
    if (Context_1.config.p2p.aggregateLostReportsTillQ1) {
        /* prettier-ignore */ if (logger_1.logFlags.lost)
            console.log('sendRequests: inside sendRequests()');
        /* prettier-ignore */ if (logger_1.logFlags.lost)
            console.log('sendRequests: scheduledForLostReport:', scheduledForLostReport);
        scheduledForLostReport.forEach((value, key) => {
            if (value.scheduledInCycle < CycleCreator_1.currentCycle - Context_1.config.p2p.delayLostReportByNumOfCycles) {
                /* prettier-ignore */ if (logger_1.logFlags.lost)
                    info(`Reporting lost: requestId: ${value.requestId}, scheduled in cycle: ${value.scheduledInCycle}, reporting in cycle ${CycleCreator_1.currentCycle}, originally reported at ${value.timestamp}`);
                /* prettier-ignore */ if (logger_1.logFlags.lost)
                    console.log('sendRequests: reporting lost node:', value.targetNode);
                reportLost(value.targetNode, value.reason, value.requestId);
                scheduledForLostReport.delete(key);
            }
        });
    }
    for (const [key, obj] of scheduledRemoveApp) {
        removeByApp(obj.target, obj.certificate);
        scheduledRemoveApp.delete(key);
    }
    // we will gossip the "down" lost records that we already checked for this cycle
    for (const [key, record] of checkedLostRecordMap) {
        if (record.status !== 'down')
            continue; // TEST
        if (record.message && record.checker && record.checker === Self.id) {
            if (record.gossiped)
                continue;
            if (record.status !== 'down')
                continue;
            if (stopReporting[record.message.target])
                continue; // TEST // this node already appeared in the lost field of the cycle record, we dont need to keep reporting
            let msg = { report: record.message, cycle: CycleCreator_1.currentCycle, status: 'down' };
            msg = Context_1.crypto.sign(msg);
            record.message = msg;
            record.gossiped = true;
            if (logger_1.logFlags.verbose)
                info(`Gossiping node down message for node: ${record.target} payload.cycle ${record.cycle}: ${lib_types_1.Utils.safeStringify(msg)}`);
            /* prettier-ignore */ nestedCounters_1.nestedCountersInstance.countEvent('p2p', 'send-lost-down', 1);
            //this next line is probably too spammy to leave in forever (but ok to comment out and keep)
            /* prettier-ignore */ nestedCounters_1.nestedCountersInstance.countEvent('p2p', `send-lost-down c:${CycleCreator_1.currentCycle}`, 1);
            Comms.sendGossip('lost-down', msg, '', null, NodeList_1.byIdOrder, true);
            // we add to our own map
            if (!receivedLostRecordMap.has(key)) {
                receivedLostRecordMap.set(key, new Map());
                receivedLostRecordMap.get(key).set(record.checker, record);
            }
        }
    }
    // We cannot count on the lost node seeing the gossip and refuting based on that.
    //   It has to be based on the lost node seeing it's node id in the lost field of the cycle record.
    //   Send refute is set to the cycle counter + 1 of the cycle record where we saw our id in the lost field
    //   We cannot create a message which has the down message since we may not have received that gossip
    if (sendRefute > 0) {
        warn(`pending sendRefute:${sendRefute} currentCycle:${CycleCreator_1.currentCycle}`);
    }
    if (sendRefute === CycleCreator_1.currentCycle) {
        let upGossipMsg = { target: Self.id, status: 'up', cycle: CycleCreator_1.currentCycle };
        warn(`Gossiping node up message: ${lib_types_1.Utils.safeStringify(upGossipMsg)}`);
        let signedUpGossipMsg = Context_1.crypto.sign(upGossipMsg);
        /* prettier-ignore */ nestedCounters_1.nestedCountersInstance.countEvent('p2p', 'self-refute', 1);
        //this next line is probably too spammy to leave in forever (but ok to comment out and keep)
        /* prettier-ignore */ nestedCounters_1.nestedCountersInstance.countEvent('p2p', `self-refute c:${CycleCreator_1.currentCycle}`, 1);
        (0, promises_1.fireAndForget)(() => Comms.sendGossip('lost-up', signedUpGossipMsg, '', null, NodeList_1.byIdOrder, true));
        upGossipMap.set(`${Self.id}-${CycleCreator_1.currentCycle}`, signedUpGossipMsg);
    }
}
/* Module functions */
async function killSelf(message) {
    error(`In killSelf`);
    Self.emitter.emit('invoke-exit', 'killSelf', (0, getCallstack_1.default)(), message);
    error(`I have been killed, will not restart.`);
}
async function killOther() {
    const requestId = (0, Utils_1.generateUUID)();
    if (logger_1.logFlags.verbose)
        info(`Explicitly injecting reportLost, requestId: ${requestId}`);
    let target = NodeList_1.activeByIdOrder[0];
    if (target.id === Self.id)
        target = NodeList_1.activeByIdOrder[1];
    scheduleLostReport(target, 'killother', requestId);
}
function scheduleLostReport(target, reason, requestId) {
    /* prettier-ignore */ if (logger_1.logFlags.lost)
        console.log('scheduleLostReport: inside scheduleLostReport()');
    if (!Context_1.config.p2p.aggregateLostReportsTillQ1)
        return reportLost(target, reason, requestId);
    if (requestId.length == 0)
        requestId = (0, Utils_1.generateUUID)();
    if (logger_1.logFlags.lost) {
        info(`Scheduling lost report for ${target.id}, requestId: ${requestId}.`);
        info(`Target node details for requestId: ${requestId}: ${(0, utils_1.logNode)(target)}`);
        info(`Scheduled lost report in ${CycleCreator_1.currentCycle} for requestId: ${requestId}.`);
    }
    const key = `${target.id}-${CycleCreator_1.currentCycle}`;
    if (scheduledForLostReport.has(key)) {
        const previousScheduleValue = scheduledForLostReport.get(key);
        if (logger_1.logFlags.verbose) {
            /* prettier-ignore */ info(`Target node ${target.id} already scheduled for lost report. requestId: ${previousScheduleValue.requestId}.`);
            /* prettier-ignore */ info(`Previous scheduled lost report details for ${target.id}: ${lib_types_1.Utils.safeStringify(previousScheduleValue)}`);
        }
    }
    scheduledForLostReport.set(key, {
        reason: reason,
        targetNode: target,
        timestamp: (0, network_1.shardusGetTime)(),
        scheduledInCycle: CycleCreator_1.currentCycle,
        requestId: requestId,
    });
}
/**
 * This gets called from Shardus when network module emits timeout or error
 *
 * This code will call 'lost-report' on another node which will help it do the actual down check
 *
 * currently this down check is a bit of a hack as it uses the 'apoptosize' route with a param
 * to indicate that it is not a normal apoptosize action
 *
 * @param target
 * @param reason
 * @param requestId
 * @returns
 */
function reportLost(target, reason, requestId) {
    /* prettier-ignore */ if (logger_1.logFlags.lost)
        console.log('reportLost: this is to send to checker nodes');
    try {
        /* prettier-ignore */ if (logger_1.logFlags.lost)
            info(`Reporting lost for ${target.id}, requestId: ${requestId}.`);
        /* prettier-ignore */ if (logger_1.logFlags.lost)
            info(`Target node details for requestId: ${requestId}: ${(0, utils_1.logNode)(target)}`);
        if (target.id === Self.id) {
            nestedCounters_1.nestedCountersInstance.countEvent('p2p', 'reportLost skip: self');
            return; // don't report self
        }
        if (stopReporting[target.id]) {
            nestedCounters_1.nestedCountersInstance.countEvent('p2p', 'reportLost skip: already stopped reporting');
            return; // this node already appeared in the lost field of the cycle record, we dont need to keep reporting
        }
        if (NodeList_1.nodes.get(target.id)?.status === 'syncing') {
            nestedCounters_1.nestedCountersInstance.countEvent('p2p', 'reportLost skip: node syncing');
            return; // don't report syncing nodes
        }
        if (NodeList_1.nodes.get(target.id)?.status === 'selected') {
            nestedCounters_1.nestedCountersInstance.countEvent('p2p', 'reportLost skip: node selected');
            return; // don't report selected nodes
        }
        // we set isDown cache to the cycle number here; to speed up deciding if a node is down
        exports.isDown[target.id] = CycleCreator_1.currentCycle;
        const key = `${target.id}-${CycleCreator_1.currentCycle}`;
        const lostRec = lostReported.get(key);
        if (lostRec) {
            /* prettier-ignore */ if (logger_1.logFlags.lost)
                console.log('reportLost: lostRec:', lostRec);
            return; // we have already seen this node for this cycle
        }
        let obj = { target: target.id, status: 'reported', cycle: CycleCreator_1.currentCycle };
        let lostCycle = CycleCreator_1.currentCycle;
        let checkerNodes = getMultipleCheckerNodes(target.id, lostCycle, Self.id);
        /* prettier-ignore */ if (logger_1.logFlags.lost)
            console.log('reportLost: checkerNodes:', checkerNodes);
        for (let checker of checkerNodes) {
            // const checker = getCheckerNode(target.id, currentCycle)
            if (checker.id === Self.id && NodeList_1.activeByIdOrder.length >= 3)
                return; // we cannot be reporter and checker if there is 3 or more nodes in the network
            let report = {
                target: target.id,
                checker: checker.id,
                reporter: Self.id,
                cycle: CycleCreator_1.currentCycle,
            };
            // [TODO] - remove the following line after testing killother
            if (reason === 'killother')
                report.killother = true;
            if (logger_1.logFlags.lost) {
                /* prettier-ignore */ info(`Sending investigate request. requestId: ${requestId}, reporter: ${Self.ip}:${Self.port} id: ${Self.id}`);
                /* prettier-ignore */ info(`Sending investigate request. requestId: ${requestId}, checker: ${checker.internalIp}:${checker.internalPort} node details: ${(0, utils_1.logNode)(checker)}`);
                /* prettier-ignore */ info(`Sending investigate request. requestId: ${requestId}, target: ${target.internalIp}:${target.internalPort} cycle: ${report.cycle} node details: ${(0, utils_1.logNode)(target)}`);
                /* prettier-ignore */ info(`Sending investigate request. requestId: ${requestId}, msg: ${lib_types_1.Utils.safeStringify(report)}`);
            }
            const msgCopy = lib_types_1.Utils.safeJsonParse(lib_types_1.Utils.safeStringify(report));
            msgCopy.timestamp = (0, network_1.shardusGetTime)();
            msgCopy.requestId = requestId;
            report = Context_1.crypto.sign(msgCopy);
            // if (config.p2p.useBinarySerializedEndpoints && config.p2p.lostReportBinary) {
            const request = report;
            Comms.tellBinary([checker], InternalRouteEnum_1.InternalRouteEnum.binary_lost_report, request, LostReportReq_1.serializeLostReportReq, {});
            // } else {
            //   Comms.tell([checker], 'lost-report', report)
            // }
            lostReported.set(key, report);
        }
        /* prettier-ignore */ if (logger_1.logFlags.lost)
            console.log('reportLost: lostReported:', lostReported);
        /* prettier-ignore */ if (logger_1.logFlags.lost)
            console.log('reportLost: sent lost-report without errors');
    }
    catch (ex) {
        nestedCounters_1.nestedCountersInstance.countEvent('p2p', `reportLost error ${(0, network_1.shardusGetTime)()}`);
        error('reportLost: ' + utils.formatErrorMessage(ex));
    }
}
function getMultipleCheckerNodes(target, lostCycle, reporter) {
    let checkerNodes = new Map();
    let obj = { target, cycle: lostCycle };
    let key = Context_1.crypto.hash(obj);
    const firstFourBytesOfMarker = key.slice(0, 8);
    const offset = parseInt(firstFourBytesOfMarker, 16);
    let pickedIndexes = utils.getIndexesPicked(NodeList_1.activeByIdOrder.length, Context_1.config.p2p.numCheckerNodes, offset);
    let attemptLimit = NodeList_1.activeByIdOrder.length; // Prevent infinite loops
    for (let i = 0; i < pickedIndexes.length; i++) {
        let pickedIndex = pickedIndexes[i];
        let currentNode = NodeList_1.activeByIdOrder[pickedIndex];
        let attempts = 0;
        // Ensure we do not select the reporter, target, or a duplicate, and handle loop limits
        while ((currentNode.id === reporter || currentNode.id === target || checkerNodes.has(currentNode.id)) &&
            attempts < attemptLimit) {
            pickedIndex = (pickedIndex + 1) % NodeList_1.activeByIdOrder.length;
            currentNode = NodeList_1.activeByIdOrder[pickedIndex];
            attempts++;
        }
        // Check if we exceeded our attempt limit and break if so, indicating an issue
        if (attempts >= attemptLimit) {
            error('Failed to find suitable nodes; most nodes are either reporters or targets.');
            return [];
        }
        // Add the valid node to the map
        checkerNodes.set(currentNode.id, currentNode);
    }
    let selectedNodes = [...checkerNodes.values()];
    /* prettier-ignore */ if (logger_1.logFlags.lost)
        info(`in getMultipleCheckerNodes checkerNodes for target: ${target}, reporter: ${reporter}, cycle: ${lostCycle}: ${lib_types_1.Utils.safeStringify(selectedNodes)}`);
    return selectedNodes;
}
function removeByApp(target, certificate) {
    /* prettier-ignore */ if (logger_1.logFlags.lost)
        info(`Gossip remove for ${target}`);
    if (target.id === Self.id) {
        nestedCounters_1.nestedCountersInstance.countEvent('p2p', 'removeByApp skip: self');
        return; // don't report self
    }
    // we set isDown cache to the cycle number here; to speed up deciding if a node is down
    exports.isDown[target.id] = CycleCreator_1.currentCycle;
    const removedRec = appRemoved.get(target.id);
    if (removedRec)
        return; // we have already removed this node for this cycle
    appRemoved.set(target.id, { certificate, target: target.id });
    Comms.sendGossip('remove-by-app', certificate, '', Self.id, NodeList_1.byIdOrder, true);
}
function getCheckerNode(id, cycle) {
    const obj = { id, cycle };
    const near = Context_1.crypto.hash(obj);
    function compareNodes(i, r) {
        return i > r.id ? 1 : i < r.id ? -1 : 0;
    }
    let idx = (0, utils_1.binarySearch)(NodeList_1.activeByIdOrder, near, compareNodes);
    const oidx = idx;
    if (idx < 0)
        idx = (-1 - idx) % NodeList_1.activeByIdOrder.length;
    const foundNode = NodeList_1.activeByIdOrder[idx];
    if (foundNode == null) {
        throw new Error(`activeByIdOrder idx:${idx} length: ${NodeList_1.activeByIdOrder.length}`);
    }
    if (foundNode.id === id)
        idx = (idx + 1) % NodeList_1.activeByIdOrder.length; // skip to next node if the selected node is target
    if (logger_1.logFlags.lost) {
        info(`in getCheckerNode oidx:${oidx} idx:${idx} near:${near}  cycle:${cycle}  id:${id}`);
        info(`${lib_types_1.Utils.safeStringify(NodeList_1.activeByIdOrder.map((n) => n.id))}`);
    }
    return NodeList_1.activeByIdOrder[idx];
}
// async function lostReportHandler(payload, response, sender) {
//   /* prettier-ignore */ if (logFlags.lost) console.log('lostReportHandler: this is for the checker node')
//   /* prettier-ignore */ if (logFlags.lost) console.log('lostReportHandler: target:', payload.target)
//   profilerInstance.scopedProfileSectionStart('lost-report')
//   try {
//     let requestId = generateUUID()
//     /* prettier-ignore */ if (logFlags.lost) info(`Got investigate request requestId: ${requestId}, req: ${Utils.safeStringify(payload)} from ${logNode(sender)}`)
//     let err = ''
//     // for request tracing
//     err = validateTypes(payload, { timestamp: 'n', requestId: 's' })
//     if (!err) {
//       /* prettier-ignore */ if (logFlags.lost) info(`Lost report tracing, requestId: ${payload.requestId}, timestamp: ${payload.timestamp}, sender: ${logNode(sender)}`)
//       requestId = payload.requestId
//     }
//     err = validateTypes(payload, { target: 's', reporter: 's', checker: 's', cycle: 'n', sign: 'o' })
//     if (err) {
//       warn(`requestId: ${requestId} bad input ${err}`)
//       return
//     }
//     err = validateTypes(payload.sign, { owner: 's', sig: 's' })
//     if (err) {
//       warn(`requestId: ${requestId} bad input ${err}`)
//       return
//     }
//     if (stopReporting[payload.target]) return // this node already appeared in the lost field of the cycle record, we dont need to keep reporting
//     const key = `${payload.target}-${payload.cycle}`
//     if (checkedLostRecordMap.get(key)) return // we have already seen this node for this cycle
//     const [valid, reason] = checkReport(payload, currentCycle + 1)
//     if (!valid) {
//       warn(`Got bad investigate request. requestId: ${requestId}, reason: ${reason}`)
//       return
//     }
//     if (sender !== payload.reporter) return // sender must be same as reporter
//     if (payload.checker !== Self.id) return // the checker should be our node id
//     let record: P2P.LostTypes.LostRecord = {
//       target: payload.target,
//       cycle: payload.cycle,
//       status: 'checking',
//       message: payload,
//       reporter: payload.reporter,
//       checker: payload.checker,
//     }
//     // check if we already know that this node is down
//     if (isDown[payload.target]) {
//       record.status = 'down'
//       return
//     }
//     let result = await isDownCache(nodes.get(payload.target), requestId)
//     /* prettier-ignore */ if (logFlags.lost) console.log('lostReportHandler: result:', result)
//     /* prettier-ignore */ if (logFlags.lost) info(`isDownCache for requestId: ${requestId}, result ${result}`)
//     if (payload.killother) result = 'down'
//     if (record.status === 'checking') record.status = result
//     /* prettier-ignore */ if (logFlags.lost) info(
//       `Status after checking for node ${payload.target} payload cycle: ${payload.cycle}, currentCycle: ${currentCycle} is ` +
//         record.status
//     )
//     if (!checkedLostRecordMap.has(key)) {
//       checkedLostRecordMap.set(key, record)
//     }
//     // At start of Q1 of the next cycle sendRequests() will start a gossip if the node was found to be down
//   } finally {
//     profilerInstance.scopedProfileSectionEnd('lost-report')
//   }
// }
const LostReportBinaryHandler = {
    name: InternalRouteEnum_1.InternalRouteEnum.binary_lost_report,
    handler: async (payload, respond, header, sign) => {
        const route = InternalRouteEnum_1.InternalRouteEnum.binary_lost_report;
        nestedCounters_1.nestedCountersInstance.countEvent('internal', route);
        profiler_1.profilerInstance.scopedProfileSectionStart(route);
        const errorHandler = (errorType, opts) => (0, Helpers_1.requestErrorHandler)(route, errorType, header, opts);
        try {
            const requestStream = (0, Helpers_1.getStreamWithTypeCheck)(payload, TypeIdentifierEnum_1.TypeIdentifierEnum.cLostReportReq);
            if (!requestStream) {
                return errorHandler(RequestErrorEnum_1.RequestErrorEnum.InvalidRequestType);
            }
            const req = (0, LostReportReq_1.deserializeLostReportReq)(requestStream);
            let requestId = (0, Utils_1.generateUUID)();
            const sender = NodeList.nodes.get(header.sender_id);
            /* prettier-ignore */ if (logger_1.logFlags.verbose)
                info(`Got investigate request requestId: ${req.requestId}, req: ${lib_types_1.Utils.safeStringify(req)} from ${(0, utils_1.logNode)(sender)}`);
            if (stopReporting[req.target])
                return; // this node already appeared in the lost field of the cycle record, we dont need to keep reporting
            const key = `${req.target}-${req.cycle}`;
            if (checkedLostRecordMap.get(key))
                return; // we have already seen this node for this cycle
            if (sign.owner !== sender.publicKey) {
                errorHandler(RequestErrorEnum_1.RequestErrorEnum.InvalidRequest, {
                    customCounterSuffix: 'bad_sign_owner',
                    customErrorLog: 'bad sign owner',
                });
                return;
            }
            const [valid, reason] = checkReport(req, CycleCreator_1.currentCycle + 1);
            if (!valid) {
                warn(`Got bad investigate request. requestId: ${requestId}, reason: ${reason}`);
                errorHandler(RequestErrorEnum_1.RequestErrorEnum.InvalidRequest);
                return;
            }
            if (header.sender_id !== req.reporter) {
                errorHandler(RequestErrorEnum_1.RequestErrorEnum.InvalidSender);
                return; // sender must be same as reporter
            }
            if (req.checker !== Self.id) {
                errorHandler(RequestErrorEnum_1.RequestErrorEnum.InvalidRequest, {
                    customCounterSuffix: 'bad_checker',
                    customErrorLog: 'the checker should be our node id',
                });
                return; // the checker should be our node id
            }
            let record = {
                target: req.target,
                cycle: req.cycle,
                status: 'checking',
                message: req,
                reporter: req.reporter,
                checker: req.checker,
            };
            // check if we already know that this node is down
            if (exports.isDown[req.target]) {
                record.status = 'down';
                return;
            }
            let result = await isDownCache(NodeList_1.nodes.get(req.target), requestId);
            /* prettier-ignore */ if (logger_1.logFlags.verbose)
                info(`isDownCache for requestId: ${requestId}, result ${result}`);
            if (req.killother)
                result = 'down';
            if (record.status === 'checking')
                record.status = result;
            /* prettier-ignore */ if (logger_1.logFlags.verbose)
                info(`Status after checking for node ${req.target} payload cycle: ${req.cycle}, currentCycle: ${CycleCreator_1.currentCycle} is ` + record.status);
            if (!checkedLostRecordMap.has(key)) {
                checkedLostRecordMap.set(key, record);
            }
            // At start of Q1 of the next cycle sendRequests() will start a gossip if the node was found to be down
        }
        catch (e) {
            nestedCounters_1.nestedCountersInstance.countEvent('internal', `${route}-exception`);
            /* prettier-ignore */ if (logger_1.logFlags.error)
                p2pLogger.error(`${route}: Exception executing request: ${utils.errorToStringFull(e)}`);
        }
        finally {
            profiler_1.profilerInstance.scopedProfileSectionEnd(route);
        }
    },
};
function checkReport(report, expectCycle) {
    if (!report || typeof report !== 'object')
        return [false, 'no report given'];
    if (!report.reporter || typeof report.reporter !== 'string')
        return [false, 'no reporter field'];
    if (!report.checker || typeof report.checker !== 'string')
        return [false, 'no checker field'];
    if (!report.target || typeof report.target !== 'string')
        return [false, 'no target field'];
    if (!report.cycle || typeof report.cycle !== 'number')
        return [false, 'no cycle field'];
    if (!report.sign || typeof report.sign !== 'object')
        return [false, 'no sign field'];
    if (report.target == Self.id)
        return [false, 'target is self']; // Don' accept if target is our node
    const cyclediff = expectCycle - report.cycle;
    if (cyclediff < 0)
        return [false, 'reporter cycle is not as expected; too new'];
    if (cyclediff >= 2)
        return [false, 'reporter cycle is not as expected; too old'];
    if (report.target === report.reporter)
        return [false, 'target cannot be reporter']; // the target should not be the reporter
    if (report.checker === report.target)
        return [false, 'target cannot be checker']; // the target should not be the checker
    if (report.checker === report.reporter) {
        if (NodeList_1.activeByIdOrder.length >= 3)
            return [false, 'checker cannot be reporter'];
    }
    if (!NodeList_1.nodes.has(report.target))
        return [false, 'target not in network'];
    if (!NodeList_1.nodes.has(report.reporter))
        return [false, 'reporter not in network'];
    if (!NodeList_1.nodes.has(report.checker))
        return [false, 'checker not in network'];
    try {
        let checkerNodes = getMultipleCheckerNodes(report.target, report.cycle, report.reporter);
        let checkNodeIds = checkerNodes.map((node) => node.id);
        if (!checkNodeIds.includes(report.checker)) {
            error(`checkReport: report.checker ${report.checker} is not one of the valid checkers ${checkNodeIds}`);
            return [
                false,
                `report.checker ${report.checker} is not part of eligible checkers: ${utils.stringifyReduce(checkNodeIds)}`,
            ];
        }
    }
    catch (ex) {
        error('checkReport: ' + utils.formatErrorMessage(ex));
        return [false, `checker node look up fail ${report.checker}`]; // we should be the checker based on our own calculations
    }
    if (!Context_1.crypto.verify(report, NodeList_1.nodes.get(report.reporter).publicKey))
        return [false, 'bad sign from reporter']; // the report should be properly signed
    return [true, ''];
}
/*
This cache uses two lookup tables: isUp and isDown
The tables map a node id to the cycle counter when the node up/down status was checked
When we check a node and find it to be up we set isUp[node_id] = current_cycle_counter
When we check a node and find it to be down we set isDown[node_id] = current_cycle_counter
At the start of each cycle we delete entries in the tables that are older than 5 cycles.
A conditional check for an entry that is not in the table has a result of false
and a conditional check for a nonzero entry has a result of true.
We export the isDown table so that other modules can easily check if a node is down.
However, if isDown returns false it does not mean that a node is not actually down.
But if it returns true it means that the node was found to be down recently.
Also if isUp returns false it does not mean that a node is actually not up, but if it
returns true it means that it was found to be up recently.
*/
async function isDownCache(node, requestId) {
    // First check the isUp isDown caches to see if we already checked this node before
    const id = node.id;
    if (Context_1.config.p2p.isDownCacheEnabled) {
        if (exports.isDown[id]) {
            /* prettier-ignore */ nestedCounters_1.nestedCountersInstance.countEvent('p2p', 'isDownCheck-skipped-down', 1);
            /* prettier-ignore */ if (logger_1.logFlags.lost)
                info(`node with id ${node.id} found in isDown for requestId: ${requestId}`);
            return 'down';
        }
        if (isUp[id]) {
            /* prettier-ignore */ nestedCounters_1.nestedCountersInstance.countEvent('p2p', 'isDownCheck-skipped-up', 1);
            /* prettier-ignore */ if (logger_1.logFlags.lost)
                info(`node with id ${node.id} found in isUp for requestId: ${requestId}`);
            return 'up';
        }
    }
    const status = await isDownCheck(node);
    /* prettier-ignore */ if (logger_1.logFlags.lost)
        info(`isDownCheck for requestId: ${requestId} on node with id ${node.id} is ${status}`);
    if (status === 'down') {
        exports.isDown[id] = CycleCreator_1.currentCycle;
    }
    else {
        isUp[id] = CycleCreator_1.currentCycle;
    }
    return status;
}
function setIsUpTs(nodeId) {
    let timestamp = (0, network_1.shardusGetTime)();
    isUpTs[nodeId] = timestamp;
}
function isNodeUpRecent(nodeId, maxAge) {
    let lastCheck = isUpTs[nodeId];
    let age = (0, network_1.shardusGetTime)() - lastCheck;
    if (isNaN(age)) {
        return { upRecent: false, state: 'noLastState', age };
    }
    if (age < maxAge)
        return { upRecent: true, state: 'up', age };
    return { upRecent: false, state: 'noLastState', age };
}
function isNodeDown(nodeId) {
    // First check the isUp isDown caches to see if we already checked this node before
    if (exports.isDown[nodeId])
        return { down: true, state: 'down' };
    if (isUp[nodeId])
        return { down: false, state: 'up' };
    return { down: false, state: 'noLastState' };
}
function isNodeLost(nodeId) {
    // First check the isUp isDown caches to see if we already checked this node before
    const key = `${nodeId}-${CycleCreator_1.currentCycle}`;
    const lostRec = receivedLostRecordMap.get(key);
    if (lostRec != null) {
        return true;
    }
    return false;
}
function removeNodeWithCertificiate(certificate) {
    const [success, message] = verifyRemoveCertificate(certificate, CycleCreator_1.currentCycle - 1);
    if (!success) {
        error(`Bad certificate. reason:${message}`);
        return;
    }
    const node = NodeList_1.byPubKey.get(certificate.nodePublicKey);
    if (node == null) {
        /* prettier-ignore */
        error(`Target remove node ${certificate.nodePublicKey} is not found in the activeList`);
        return;
    }
    if (scheduledRemoveApp.has(certificate.nodePublicKey)) {
        const previousScheduleValue = scheduledRemoveApp.get(certificate.nodePublicKey);
        if (logger_1.logFlags.verbose) {
            /* prettier-ignore */ info(`Target node ${certificate.nodePublicKey} already scheduled for removing.`);
            /* prettier-ignore */ info(`Previous scheduled lost report details for ${certificate.nodePublicKey}: ${lib_types_1.Utils.safeStringify(previousScheduleValue)}`);
        }
        return;
    }
    scheduledRemoveApp.set(certificate.nodePublicKey, {
        target: node,
        reason: 'remove-by-app',
        certificate,
        timestamp: (0, network_1.shardusGetTime)(),
    });
}
// This is called once per cycle by reset
function pruneIsDown() {
    const cachePruneAge = Context_1.config.p2p.isDownCachePruneCycles;
    for (const [key, value] of Object.entries(exports.isDown)) {
        if (typeof value === 'number' && value < CycleCreator_1.currentCycle - cachePruneAge)
            delete exports.isDown[key];
    }
    for (const [key, value] of Object.entries(isUp)) {
        if (typeof value === 'number' && value < CycleCreator_1.currentCycle - cachePruneAge)
            delete isUp[key];
    }
}
function pruneStopReporting() {
    const stopReportingPruneCycles = Context_1.config.p2p.stopReportingLostPruneCycles;
    for (const [key, value] of Object.entries(stopReporting)) {
        if (value < CycleCreator_1.currentCycle - stopReportingPruneCycles)
            delete stopReporting[key];
    }
}
// Make sure that both the external and internal ports are working
//   if either is not working then the node is considered down.
// If internal and external are both on the same IP then only need to check one.
// This function has some deep knowledge from Sync and Apoptosis APIs
//    and could break if they are changed.
// [TODO] - create our own APIs to test the internal and external connection.
//          Although this could allow a rouge node to more easily fool checks.
async function isDownCheck(node) {
    // Check the internal route
    // The timeout for this is controled by the network.timeout paramater in server.json
    /* prettier-ignore */ if (logger_1.logFlags.lost)
        info(`Checking internal connection for ${node.id}, cycle: ${CycleCreator_1.currentCycle}`);
    try {
        if (Context_1.config.p2p.useProxyForDownCheck) {
            //using the 'apoptosize' route to check if the node is up.
            let obj = { counter: CycleCreator_1.currentCycle, checker: Self.id, target: node.id, timestamp: (0, network_1.shardusGetTime)() };
            let hash = Context_1.crypto.hash(obj);
            let closestNodes = Context_1.stateManager.getClosestNodes(hash, 5, true);
            let proxyNode;
            for (let closetNode of closestNodes) {
                if (closetNode.id !== node.id) {
                    proxyNode = closetNode;
                    break;
                }
            }
            if (proxyNode == null) {
                throw new Error(`isDownCheck unable to get proxy node to check the target node`);
            }
            let hashTrieReq = {
                radixList: ['0'],
            };
            let proxyRequest = {
                nodeId: node.id,
                route: 'get_trie_hashes',
                message: hashTrieReq,
            };
            const res = await Comms.ask(proxyNode, 'proxy', proxyRequest, true, '', 3000);
            if (logger_1.logFlags.verbose)
                info(`lost check result for node ${node.id} cycle ${CycleCreator_1.currentCycle} is ${utils.stringifyReduce(res)}`);
            if (res == null || res.success === false || res.response == null || res.response.isResponse == null) {
                /* prettier-ignore */ nestedCounters_1.nestedCountersInstance.countEvent('p2p', 'isDownCheck-down-proxy-fail', 1);
                return 'down';
            }
            let nodeHashes = res.response.nodeHashes;
            if (nodeHashes == null || nodeHashes.length === 0) {
                /* prettier-ignore */ nestedCounters_1.nestedCountersInstance.countEvent('p2p', 'isDownCheck-down-via-proxy-empty-hashes', 1);
                return 'down';
            }
        }
        else {
            //using the 'apoptosize' route to check if the node is up.
            const res = await Comms.askBinary(node, 'apoptosize', {
                id: 'isDownCheck',
                when: 1,
            }, ApoptosisProposalReq_1.serializeApoptosisProposalReq, ApoptosisProposalResp_1.deserializeApoptosisProposalResp, {});
            if (res == null) {
                /* prettier-ignore */ nestedCounters_1.nestedCountersInstance.countEvent('p2p', 'isDownCheck-down-0', 1);
                return 'down';
            }
            if (typeof res.s !== 'string') {
                /* prettier-ignore */ nestedCounters_1.nestedCountersInstance.countEvent('p2p', 'isDownCheck-down-1', 1);
                return 'down';
            }
            //adding this check so that a node can repond that is is down. aka, realizes it is not funcitonal and wants to be removed from the network
            if (res.s === Apoptosis_1.nodeDownString) {
                /* prettier-ignore */ nestedCounters_1.nestedCountersInstance.countEvent('p2p', 'isDownCheck-down-self-reported-zombie', 1);
                return 'down';
            }
        }
    }
    catch (e) {
        /* prettier-ignore */ nestedCounters_1.nestedCountersInstance.countEvent('p2p', 'isDownCheck-down-2', 1);
        return 'down';
    }
    //Note 20230630:  the code below here has not likely had any coverage for a few years due to an upstream issue
    if (node.externalIp === node.internalIp)
        return 'up';
    /* prettier-ignore */ if (logger_1.logFlags.lost)
        info(`Checking external connection for ${node.id}`);
    // Check the external route if ip is different than internal
    const queryExt = async (node) => {
        const ip = node.ip ? node.ip : node.externalIp;
        const port = node.port ? node.port : node.externalPort;
        // the queryFunction must return null if the given node is our own
        // while syncing nodeList we dont have node.id, so use ip and port
        if (ip === Self.ip && port === Self.port)
            return null;
        const resp = await http.get(`${ip}:${port}/sync-newest-cycle`);
        return resp;
    };
    try {
        const resp = await queryExt(node); // if the node is down, reportLost() will set status to 'down'
        if (typeof resp.newestCycle.counter !== 'number')
            return 'down';
    }
    catch {
        /* prettier-ignore */ nestedCounters_1.nestedCountersInstance.countEvent('p2p', 'isDownCheck-down-3', 1);
        return 'down';
    }
    /* prettier-ignore */ nestedCounters_1.nestedCountersInstance.countEvent('p2p', 'isDownCheck-up-1', 1);
    return 'up';
}
function downGossipHandler(payload, sender, tracker) {
    /* prettier-ignore */ if (logger_1.logFlags.lost)
        console.log('downGossipHandler: this is to receive the gossip from checker nodes');
    /* prettier-ignore */ if (logger_1.logFlags.lost)
        console.log('downGossipHandler: target:', payload.report.target);
    /* prettier-ignore */ if (logger_1.logFlags.lost)
        info(`Got downGossip: ${lib_types_1.Utils.safeStringify(payload)}`);
    let err = '';
    err = (0, utils_1.validateTypes)(payload, { cycle: 'n', report: 'o', status: 's', sign: 'o' });
    if (err) {
        warn('bad input ' + err);
        return;
    }
    err = (0, utils_1.validateTypes)(payload.report, { target: 's', reporter: 's', checker: 's', cycle: 'n', sign: 'o' });
    if (err) {
        warn('bad input report ' + err);
        return;
    }
    err = (0, utils_1.validateTypes)(payload.report.sign, { owner: 's', sig: 's' });
    if (err) {
        warn('bad input report sign ' + err);
        return;
    }
    err = (0, utils_1.validateTypes)(payload.sign, { owner: 's', sig: 's' });
    if (err) {
        warn('bad input sign ' + err);
        return;
    }
    const key = `${payload.report.target}-${payload.report.cycle}`;
    const checkedRecord = checkedLostRecordMap.get(key);
    if (checkedRecord && ['up', 'down'].includes(checkedRecord.status)) {
        return; // we have already gossiped this node for this cycle
    }
    const alreadyProcessedLostRecord = receivedLostRecordMap.get(key)
        ? receivedLostRecordMap.get(key).get(payload.report.checker)
        : null;
    if (alreadyProcessedLostRecord && ['up', 'down'].includes(alreadyProcessedLostRecord.status)) {
        return; // we have already gossiped this node for this cycle
    }
    let [valid, reason] = checkQuarter(payload.report.checker, sender);
    if (!valid) {
        warn(`Bad downGossip message. reason:${reason} message:${lib_types_1.Utils.safeStringify(payload)}`);
        warn(`cycle:${CycleCreator_1.currentCycle} quarter:${CycleCreator_1.currentQuarter} sender:${sender}`);
        return;
    }
    ;
    [valid, reason] = checkDownMsg(payload, CycleCreator_1.currentCycle);
    if (!valid) {
        warn(`Bad downGossip message. reason:${reason}. message:${lib_types_1.Utils.safeStringify(payload)}`);
        warn(`cycle:${CycleCreator_1.currentCycle} quarter:${CycleCreator_1.currentQuarter} sender:${sender}`);
        return;
    }
    let receivedRecord = {
        target: payload.report.target,
        cycle: payload.report.cycle,
        status: 'down',
        message: payload,
        checker: payload.report.checker,
        reporter: payload.report.reporter,
    };
    if (receivedLostRecordMap.has(key) && receivedLostRecordMap.get(key).has(payload.report.checker)) {
        if (logger_1.logFlags.verbose)
            info(`downGossip already seen and processed. report ${lib_types_1.Utils.safeStringify(payload.report)}`);
        return;
    }
    if (receivedLostRecordMap.has(key)) {
        receivedLostRecordMap.get(key).set(payload.report.checker, receivedRecord);
    }
    else {
        receivedLostRecordMap.set(key, new Map());
        receivedLostRecordMap.get(key).set(payload.report.checker, receivedRecord);
    }
    if (logger_1.logFlags.verbose)
        info(`downGossip for target ${payload.report.target} at cycle ${payload.report.cycle} is processed. Total received: ${receivedLostRecordMap.get(key).size}`);
    /* prettier-ignore */ if (logger_1.logFlags.lost)
        console.log('downGossipHandler: sending gossip');
    Comms.sendGossip('lost-down', payload, tracker, Self.id, NodeList_1.byIdOrder, false);
    // After message has been gossiped in Q1 and Q2 we wait for getTxs() to be invoked in Q3
}
// Fn is used to limit gossip to Q1 and Q2 of the cycle and stop originator from gossiping in Q2
function checkQuarter(source, sender) {
    if (![1, 2].includes(CycleCreator_1.currentQuarter))
        return [false, 'not in Q1 or Q2'];
    if (sender === source && CycleCreator_1.currentQuarter === 2)
        return [false, 'originator cannot gossip in Q2'];
    return [true, ''];
}
function checkDownMsg(payload, expectedCycle) {
    if (payload.cycle !== expectedCycle)
        return [false, 'checker cycle is not as expected'];
    const [valid, reason] = checkReport(payload.report, expectedCycle - 1);
    if (!valid)
        return [valid, reason];
    if (!Context_1.crypto.verify(payload, NodeList_1.nodes.get(payload.report.checker).publicKey))
        return [false, `bad sign from checker.`];
    return [true, ''];
}
function upGossipHandler(payload, sender, tracker) {
    /* prettier-ignore */ if (logger_1.logFlags.lost)
        info(`Got upGossip: ${lib_types_1.Utils.safeStringify(payload)}`);
    let err = '';
    err = (0, utils_1.validateTypes)(payload, { cycle: 'n', target: 's', status: 's', sign: 'o' });
    if (err) {
        warn('bad input ' + err);
        return;
    }
    err = (0, utils_1.validateTypes)(payload.sign, { owner: 's', sig: 's' });
    if (err) {
        warn('bad input sign ' + err);
        return;
    }
    if (!stopReporting[payload.target]) {
        warn('Bad upGossip. We did not see this node in the lost field, but got a up msg from it; ignoring it');
        return;
    }
    let [valid, reason] = checkQuarter(payload.target, sender);
    if (!valid) {
        warn(`Bad upGossip message. reason:${reason} message:${lib_types_1.Utils.safeStringify(payload)}`);
        return;
    }
    const key = `${payload.target}-${payload.cycle}`;
    const rec = upGossipMap.get(key);
    if (rec && rec.status === 'up')
        return // we have already gossiped this node for this cycle
        ;
    [valid, reason] = checkUpMsg(payload, CycleCreator_1.currentCycle);
    if (!valid) {
        warn(`Bad upGossip message. reason:${reason} message:${lib_types_1.Utils.safeStringify(payload)}`);
        return;
    }
    upGossipMap.set(key, payload);
    (0, promises_1.fireAndForget)(() => Comms.sendGossip('lost-up', payload, tracker, Self.id, NodeList_1.byIdOrder, false));
    // the getTxs() function will loop through the lost object to make txs in Q3 and build the cycle record from them
}
function removeByAppHandler(payload, sender, tracker) {
    const [success, message] = verifyRemoveCertificate(payload, CycleCreator_1.currentCycle - 2);
    if (!success) {
        error(`Bad certificate. reason:${message}`);
        return;
    }
    const target = NodeList_1.byPubKey.get(payload.nodePublicKey).id;
    const rec = appRemoved.get(target);
    if (rec)
        return; // we have already gossiped this node for this cycle
    let [valid, reason] = checkQuarter(target, sender);
    if (!valid) {
        warn(`Bad downGossip message. reason:${reason} message:${lib_types_1.Utils.safeStringify(payload)}`);
        warn(`cycle:${CycleCreator_1.currentCycle} quarter:${CycleCreator_1.currentQuarter} sender:${sender}`);
        return;
    }
    appRemoved.set(target, { target: target, certificate: payload });
    Comms.sendGossip('remove-by-app', payload, tracker, Self.id, NodeList_1.byIdOrder, false);
}
function checkUpMsg(payload, expectedCycle) {
    if (!NodeList_1.nodes.has(payload.target))
        return [false, `target is not an active node  ${payload.target}  ${lib_types_1.Utils.safeStringify(NodeList_1.activeByIdOrder)}`];
    if (!Context_1.crypto.verify(payload, NodeList_1.nodes.get(payload.target).publicKey))
        return [false, 'bad sign from target'];
    return [true, ''];
}
function checkRemoveByAppMsg(payload, expectedCycle) {
    if (!NodeList_1.nodes.has(payload.target))
        return [false, `target is not an active node  ${payload.target}  ${lib_types_1.Utils.safeStringify(NodeList_1.activeByIdOrder)}`];
    // todo: verify signature
    // if (!crypto.verify(payload, nodes.get(payload.target).publicKey)) return [false, 'bad sign from target']
    return [true, ''];
}
function verifyRemoveCertificate(certificate, cycle) {
    if (!certificate)
        return [false, 'no certificate given'];
    if (!certificate.nodePublicKey || typeof certificate.nodePublicKey !== 'string')
        return [false, 'no nodePublicKey field'];
    const node = NodeList_1.byPubKey.get(certificate.nodePublicKey);
    if (!node)
        return [false, 'nodePublicKey not in network'];
    if (node.publicKey !== certificate.nodePublicKey)
        return [false, 'nodePublicKey does not match node'];
    // the cycle should be the previous cycle when processing
    if (certificate.cycle !== cycle)
        return [false, `cycle is not as expected. certificate.cycle: ${certificate.cycle} expected: ${cycle}`];
    const { success, reason } = Context_1.shardus.validateClosestActiveNodeSignatures(certificate, certificate.signs, 4, 5, 2);
    if (!success)
        return [false, reason];
    return [true, ''];
}
function info(...msg) {
    const entry = `Lost: ${msg.join(' ')}`;
    p2pLogger.info(entry);
}
function warn(...msg) {
    const entry = `Lost: ${msg.join(' ')}`;
    p2pLogger.warn(entry);
}
function error(...msg) {
    const entry = `Lost: ${msg.join(' ')}`;
    p2pLogger.error(entry);
}
//# sourceMappingURL=Lost.js.map