"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.HierarchicalNavigableSmallWorld = void 0;
const vector_ts_1 = require("./vector.js");
const msgpackr_1 = require("msgpackr");
const logger_ts_1 = require("../../utility/logging/logger.js");
const hdbError_ts_1 = require("../../utility/errors/hdbError.js");
const rocksdb_js_1 = require("@harperfast/rocksdb-js");
const extended_iterable_1 = require("@harperfast/extended-iterable");
const logger = (0, logger_ts_1.loggerWithTag)('HNSW');
// int8 scalar quantization of stored graph nodes is ON by default. Each node holds the
// vector as a compact int8 `bin` plus a per-vector `scale`, roughly a 5x size reduction
// over float32 and ~10x cheaper to decode (a single typed-array view instead of decoding
// 768 individually-tagged floats into a boxed Array). The full-precision vector still lives
// on the record, so only graph navigation is approximate:
//   - sort (nearest-neighbor) queries: reranked on exact distances after loading records (~0% recall loss)
//   - lt/le threshold queries: over-fetched and re-filtered on exact distances after loading records
// Opt out per-index with `@indexed(type: "HNSW", quantization: "none")`.
//
// Decode auto-detects the stored format (number[] = float, bin = int8), so indexes written
// before this default change (float nodes) continue to work transparently.
/** Symmetric int8 scalar-quantize a float vector. scale = max|component| / 127. */
function quantizeInt8(vector) {
    let max = 0;
    for (let i = 0; i < vector.length; i++) {
        const a = vector[i] < 0 ? -vector[i] : vector[i];
        if (a > max)
            max = a;
    }
    const scale = max / 127 || 1;
    const inv = 1 / scale;
    const q = new Int8Array(vector.length);
    // clamp guards against a float-rounding edge landing on 128 (which Int8Array would wrap to -128)
    for (let i = 0; i < vector.length; i++)
        q[i] = Math.max(-127, Math.min(127, Math.round(vector[i] * inv)));
    return { bytes: Buffer.from(q.buffer, q.byteOffset, q.byteLength), scale };
}
/** Reconstruct an approximate float array from an int8 vector + scale. */
function dequantizeInt8(q, scale) {
    const out = new Array(q.length);
    for (let i = 0; i < q.length; i++)
        out[i] = q[i] * scale;
    return out;
}
// Auto-scaled search ef, used only when an index does not explicitly configure efConstructionSearch
// and a query does not pass its own ef. A fixed ef makes recall decay as the graph grows (it explores
// a shrinking fraction of the graph), so ef grows with sqrt(node count), capped to bound search cost.
// Constants from a recall/latency-vs-N sweep (768-dim cosine, int8): ef≈400 holds ~0.8 recall@10 from
// 5K–30K, and the recall/latency tradeoff is steep (ef 800 at 30K ≈ 0.92 recall but ~2s p50), so the
// cap deliberately favors latency — apps wanting higher recall set efConstructionSearch or a per-query
// ef. Tune as graph build quality / larger-N data improves.
const AUTO_EF_BASE = 100;
const AUTO_EF_REF = 1000;
const AUTO_EF_MAX = 512;
function autoScaleEf(nodeCount) {
    const scaled = Math.round(AUTO_EF_BASE * Math.sqrt(Math.max(1, nodeCount / AUTO_EF_REF)));
    return Math.min(AUTO_EF_MAX, Math.max(AUTO_EF_BASE, scaled));
}
class MinHeap {
    data = [];
    get size() {
        return this.data.length;
    }
    push(item) {
        this.data.push(item);
        let i = this.data.length - 1;
        while (i > 0) {
            const p = (i - 1) >> 1;
            if (this.data[p].distance <= this.data[i].distance)
                break;
            const tmp = this.data[p];
            this.data[p] = this.data[i];
            this.data[i] = tmp;
            i = p;
        }
    }
    pop() {
        if (this.data.length === 0)
            return undefined;
        const top = this.data[0];
        const last = this.data.pop();
        if (this.data.length > 0) {
            this.data[0] = last;
            let i = 0;
            for (;;) {
                const l = 2 * i + 1, r = l + 1;
                let min = i;
                if (l < this.data.length && this.data[l].distance < this.data[min].distance)
                    min = l;
                if (r < this.data.length && this.data[r].distance < this.data[min].distance)
                    min = r;
                if (min === i)
                    break;
                const tmp = this.data[min];
                this.data[min] = this.data[i];
                this.data[i] = tmp;
                i = min;
            }
        }
        return top;
    }
}
function bisectInsert(arr, distance) {
    let lo = 0, hi = arr.length;
    while (lo < hi) {
        const mid = (lo + hi) >> 1;
        if (arr[mid].distance <= distance)
            lo = mid + 1;
        else
            hi = mid;
    }
    return lo;
}
/**
 * Implementation of a vector index for Harper, using hierarchical navigable small world graphs.
 */
const ENTRY_POINT = Symbol.for('entryPoint');
const KEY_PREFIX = Symbol.for('key');
const MAX_LEVEL = 10; // should give good high-level skip list performance up to trillions of nodes
/**
 * Represents a Hierarchical Navigable Small World (HNSW) index for approximate nearest neighbor search.
 * This implementation is based on hierarchical graph navigation to efficiently index and search high-dimensional vectors.
 * A HNSW is basically a multi-dimensional skip list. Each node has (potentially) higher levels that are used for quickly
 * traversing the graph get in the neighborhood of the node, and then lower levels are used to more accurately find the
 * closest neighbors.
 *
 * This implementation is based on the paper "Efficient and Robust Approximate Nearest Neighbor Search in High Dimensions"
 * (mostly influenced AI's contributions)
 */
class HierarchicalNavigableSmallWorld {
    static useObjectStore = true;
    // Index options that only affect search, not the stored graph — changing them must not trigger a
    // reindex (databases.ts persists the new value but skips rebuilding). efConstructionSearch is the
    // search-time candidate-list size; the build uses efConstruction/M/distance, which are structural.
    static searchOnlyOptions = ['efConstructionSearch'];
    indexStore;
    M = 16; // max number of connections per layer
    efConstruction = 100; // size of dynamic candidate list
    efConstructionSearch = 50; // size of dynamic candidate list for search
    mL = 1 / Math.log(this.M); // normalization factor for level generation
    // how aggressive do we avoid connections that have alternate indirect routes; a value of 0 never avoids connections,
    // a value of 1 is extremely aggressive.
    optimizeRouting = 0.5;
    nodesVisitedCount = 0;
    idIncrementer;
    distance;
    int8 = true; // store vectors as int8-quantized bins by default; opt out with `quantization: "none"`
    efSearchConfigured = false; // whether the schema set an explicit search ef; if not, search ef auto-scales with N
    // Caches the Int8Array-converted clone of a frozen (decoded-from-disk) int8 node, keyed by the
    // frozen node the object store hands back. WeakMap so entries are collected when the store evicts
    // the frozen node — without it, every cache hit on a frozen node would re-slice and re-clone.
    convertedNodes = new WeakMap();
    constructor(indexStore, options) {
        this.indexStore = indexStore;
        if (indexStore) {
            // use float32 representation of numbers as it is twice as space efficient as typical float64 and plenty accurate
            // (we would actually like to use float16 if it were available)
            this.indexStore.encoder.useFloat32 = msgpackr_1.FLOAT32_OPTIONS.ALWAYS;
        }
        this.int8 = options?.quantization !== 'none';
        // Respect an explicitly-configured search ef (or efConstruction, which seeds it); otherwise auto-scale.
        this.efSearchConfigured = options?.efConstructionSearch !== undefined || options?.efConstruction !== undefined;
        this.distance =
            options?.distance === 'euclidean'
                ? vector_ts_1.euclideanDistance
                : options?.distance === 'dotProduct'
                    ? vector_ts_1.dotProductDistance
                    : vector_ts_1.cosineDistance;
        if (options) {
            // allow all the HNSW parameters to be configured/tuned
            if (options.M !== undefined) {
                this.M = options.M;
                this.mL = 1 / Math.log(this.M); // recalculate
            }
            if (options.efConstruction !== undefined)
                this.efConstruction = this.efConstructionSearch = options.efConstruction;
            if (options.efConstructionSearch !== undefined)
                this.efConstructionSearch = options.efConstructionSearch;
            if (options.mL !== undefined)
                this.mL = options.mL;
            if (options.optimizeRouting !== undefined)
                this.optimizeRouting = options.optimizeRouting;
        }
    }
    index(primaryKey, vector, existingVector, options = {}) {
        // Reject non-finite components before touching the graph. NaN in particular poisons
        // bisectInsert (arr[mid].distance <= NaN is always false → returns 0, pinning the
        // candidate to rank 1 of every future search). Infinity causes analogous ordering
        // anomalies. embedHook.ts intentionally passes NaN through and expects HNSW to guard.
        if (vector) {
            for (let i = 0; i < vector.length; i++) {
                if (!Number.isFinite(vector[i])) {
                    throw new hdbError_ts_1.ClientError(`Vector for attribute "${String(primaryKey)}" contains non-finite component at index ${i}: ${vector[i]}. Ensure the embedding produces only finite values.`);
                }
            }
        }
        // first get the node id for the primary key; we use internal node ids for better efficiency,
        // but we must use a safe key that won't collide with the node ids
        const safeKey = typeof primaryKey === 'number' ? [KEY_PREFIX, primaryKey] : primaryKey;
        let nodeId = this.indexStore.getSync(safeKey, options);
        // if the node id is not found, create a new node (and store it in the index store)
        // (note that we don't need to check if the node id is already in the index store,
        // because we use internal node ids for better efficiency, and we use a safe key
        // that won't collide with the node ids, so we can't have a collision with internal
        if (!nodeId) {
            if (!vector)
                return; // didn't exist before, doesn't exist now, nothing to do
            if (!this.idIncrementer) {
                let largestNodeId = 0;
                for (const key of this.indexStore.getKeys({
                    reverse: true,
                    limit: 1,
                    start: Infinity,
                    end: 0,
                    transaction: options.transaction,
                })) {
                    if (typeof key === 'number')
                        largestNodeId = key;
                }
                this.idIncrementer = new BigInt64Array([BigInt(largestNodeId) + 1n]);
                this.idIncrementer = new BigInt64Array(this.indexStore.getUserSharedBuffer('next-id', this.idIncrementer.buffer));
            }
            nodeId = Number(Atomics.add(this.idIncrementer, 0, 1n));
            this.indexStore.put(safeKey, nodeId, options);
        }
        const updatedNodes = new Map();
        let oldNode;
        // If this is the first entry, create it as the entry point
        let entryPointId = this.indexStore.getSync(ENTRY_POINT, options);
        if (existingVector) {
            // If we are updating an existing entry, we need to update the entry point
            // if the new entry is closer to the entry point than the old one
            oldNode = { ...this.safeGetSync(nodeId, options) };
        }
        else {
            // If this key already has a graph node — which happens
            // when runIndexing re-feeds an already-indexed record after a crash/restart — load it
            // and treat the call as an update rather than a fresh insert. Without this, the absent
            // existingVector causes oldNode = {} → a new random level, connections overwritten,
            // and old-connection cleanup skipped, leaving dangling reverse edges and wrong levels.
            const storedNode = nodeId && vector ? this.safeGetSync(nodeId, options) : undefined;
            if (storedNode && storedNode.level !== undefined) {
                // Treat as an update: carry forward the stored graph state so cleanup and level
                // assignment below use the real existing connections instead of starting fresh.
                oldNode = { ...storedNode };
                // Reconstruct the existingVector from the stored node so distance computations
                // in the cleanup pass use the right baseline (dequantize if int8).
                if (!existingVector) {
                    existingVector =
                        storedNode.scale !== undefined
                            ? dequantizeInt8(storedNode.vector, storedNode.scale)
                            : storedNode.vector;
                }
            }
            else {
                oldNode = {};
            }
        }
        if (vector) {
            // Pre-compute 1/|vector| for cosine distance so searchLayer can skip sqrt per neighbor
            let invMag;
            if (this.distance === vector_ts_1.cosineDistance) {
                let magSq = 0;
                for (const v of vector)
                    magSq += v * v;
                invMag = 1 / (Math.sqrt(magSq) || 1);
            }
            // Quantized storage form. The float `vector` is still used as the query for every
            // searchLayer call below (asymmetric distance: float query x int8 stored); only what
            // we PUT to the store is quantized.
            const q = this.int8 ? quantizeInt8(vector) : undefined;
            const storedVector = q ? q.bytes : vector;
            const storedScale = q ? q.scale : undefined;
            let entryPoint = entryPointId && this.safeGetSync(entryPointId, options);
            if (entryPoint == null) {
                const level = Math.floor(-Math.log(Math.random()) * this.mL);
                const node = {
                    vector: storedVector,
                    scale: storedScale,
                    invMag,
                    level,
                    primaryKey,
                };
                for (let i = 0; i <= level; i++) {
                    node[i] = [];
                }
                this.indexStore.put(nodeId, node, options);
                if (typeof nodeId !== 'number') {
                    throw new Error('Invalid nodeId: ' + nodeId);
                }
                logger.debug?.('setting entry point to', nodeId);
                this.indexStore.put(ENTRY_POINT, nodeId, options);
                return;
            }
            // Generate random level for this new element
            const level = oldNode.level ?? Math.min(Math.floor(-Math.log(Math.random()) * this.mL), MAX_LEVEL);
            let currentLevel = entryPoint.level;
            if (level > currentLevel) {
                // if we are at a higher level, make this the new entry point
                if (typeof nodeId !== 'number') {
                    throw new Error('Invalid nodeId: ' + nodeId);
                }
                logger.debug?.('setting entry point to', nodeId);
                this.indexStore.put(ENTRY_POINT, nodeId, options);
            }
            // For each level from top to bottom
            while (currentLevel > level) {
                // Search for closest neighbors at current level
                const neighbors = this.searchLayer(vector, entryPointId, entryPoint, this.efConstruction, currentLevel, options);
                if (neighbors.length > 0) {
                    entryPointId = neighbors[0].id; // closest neighbor becomes new entry point
                    entryPoint = neighbors[0].node;
                }
                currentLevel--;
            }
            const connections = new Array(level + 1);
            for (let i = 0; i <= level; i++) {
                connections[i] = [];
            }
            // Connect the new element to neighbors at its level and below
            for (let l = Math.min(level, currentLevel); l >= 0; l--) {
                let neighbors = this.searchLayer(vector, entryPointId, entryPoint, this.efConstruction, l, options);
                neighbors = neighbors.slice(0, this.M << 1);
                if (neighbors.length === 0 && l === 0) {
                    logger.info?.('should not have zero connections for', entryPointId);
                }
                const connectionsAtLevel = connections[l];
                // Create bidirectional connections
                for (let i = 0; i < neighbors.length; i++) {
                    const { id, distance, node } = neighbors[i];
                    if (id === nodeId)
                        continue; // don't connect to self
                    const connectionsToBeReplaced = [];
                    if (this.optimizeRouting) {
                        // if we have existing connections through other nodes, we deprioritize new connections through them.
                        // I believe this yields better HNSW graphs, avoiding redundant paths, with better directed connectivity
                        // towards desired results
                        let skipping = false;
                        const neighborNeighbors = node[l];
                        const distanceThreshold = 1 + this.optimizeRouting * (1 + (0.5 * i) / this.M);
                        for (let i2 = 0; i2 < neighborNeighbors?.length; i2++) {
                            const { id: neighborId, distance: neighborDistance } = neighborNeighbors[i2];
                            const neighborDistanceThreshold = 1 + this.optimizeRouting * (1 + (0.5 * i2) / this.M);
                            for (let i3 = 0; i3 < connectionsAtLevel.length; i3++) {
                                const { id: addedId, distance: addedDistance } = connectionsAtLevel[i3];
                                if (addedId === neighborId) {
                                    if (distance * distanceThreshold > addedDistance + neighborDistance) {
                                        // if the new distance is relatively low compared to existing indirect connections,
                                        // we skip this neighbor since it is of less value
                                        skipping = true;
                                    }
                                    else if (neighborDistance * neighborDistanceThreshold > distance + addedDistance) {
                                        // potentially remove the neighbor's neighbor, because we are adding a better route (if we do add it)
                                        connectionsToBeReplaced.push({ fromId: addedId, toId: id });
                                        connectionsToBeReplaced.push({ fromId: id, toId: addedId });
                                    }
                                    break;
                                }
                            }
                            if (skipping)
                                break;
                        }
                        if (skipping)
                            continue;
                    }
                    else if (i >= (l > 0 ? this.M : this.M << 1)) {
                        // fallback to traditional HNSW level limiting; if we are at the maximum number of neighbors, we skip this one
                        continue;
                    }
                    // Add connection to the new element
                    connectionsAtLevel.push({ id, distance });
                    for (const { fromId, toId } of connectionsToBeReplaced) {
                        let from = updateNode(fromId);
                        if (!from)
                            from = updateNode(fromId, this.safeGetSync(fromId, options));
                        if (!from)
                            continue;
                        const fromAtLevel = from[l];
                        if (!fromAtLevel)
                            continue;
                        for (let i = 0; i < fromAtLevel.length; i++) {
                            if (from[l][i].id === toId) {
                                if (Object.isFrozen(from[l])) {
                                    from[l] = from[l].slice();
                                }
                                from[l].splice(i, 1);
                                break;
                            }
                        }
                    }
                    // Add reverse connection from neighbor to new element if it didn't exist before
                    // First check to see if we had an existing neighbor connection before. If we did we can
                    // just remove from the list of the connections to remove (don't remove, leave it in place)
                    let oldConnections = oldNode[l];
                    const oldConnection = oldConnections?.find(({ id: nid }) => nid === id);
                    if (oldConnection) {
                        const oldPosition = oldConnections?.indexOf(oldConnection);
                        if (!oldConnections.copied) {
                            // make a copy, it is likely frozen
                            oldConnections = [...oldConnections];
                            oldConnections.copied = true;
                            oldNode[l] = oldConnections;
                        }
                        oldConnections.splice(oldPosition, 1);
                        // update the distance in the reverse connection if the vector changed
                        if (oldConnection.distance !== distance) {
                            const neighborNode = updateNode(id, node);
                            if (neighborNode[l]) {
                                if (Object.isFrozen(neighborNode[l])) {
                                    neighborNode[l] = neighborNode[l].slice();
                                }
                                const reverseIdx = neighborNode[l].findIndex(({ id: nid }) => nid === nodeId);
                                if (reverseIdx >= 0) {
                                    neighborNode[l][reverseIdx] = { id: nodeId, distance };
                                }
                            }
                        }
                    }
                    else {
                        // add new connection since this is truly a new connection now
                        this.addConnection(id, updateNode(id, node), nodeId, l, distance, updateNode, options);
                    }
                }
            }
            // Store the new element
            this.indexStore.put(nodeId, {
                vector: storedVector,
                scale: storedScale,
                invMag,
                level,
                primaryKey,
                ...connections,
            }, options);
        }
        else {
            // removal of this node, but first make sure we have a valid entry point
            if (entryPointId === nodeId) {
                // if this is the entry point, find a new entry point
                const lastLevel = oldNode.level ?? 0;
                for (let l = lastLevel; l >= 0; l--) {
                    entryPointId = oldNode[l]?.[0]?.id;
                    if (entryPointId !== undefined)
                        break;
                }
                if (entryPointId === undefined) {
                    // Fallback scan: pass transaction so it sees the same write-set (not stale committed state),
                    // skip the node being deleted (it is typically the highest-level node and would be
                    // re-elected), and verify each candidate actually resolves before committing it.
                    let highestLevel = -1;
                    for (const { key, value } of this.indexStore.getRange({
                        start: 0,
                        end: Infinity,
                        transaction: options.transaction,
                    })) {
                        // skip the node being removed (safeKey mappings can't appear here: symbol-array
                        // and string keys sort outside the numeric 0..Infinity range)
                        if (key === nodeId)
                            continue;
                        if (!value || value.level === undefined)
                            continue;
                        if (value.level > highestLevel) {
                            entryPointId = key;
                            if (value.level === lastLevel)
                                break; // found a node at the same level as the old entry point
                            highestLevel = value.level;
                        }
                    }
                }
                if (entryPointId === undefined) {
                    // no nodes left in index
                    this.indexStore.remove(ENTRY_POINT, options);
                }
                else {
                    // set the new entry point
                    if (typeof entryPointId !== 'number') {
                        throw new Error('Invalid nodeId: ' + entryPointId);
                    }
                    logger.debug?.('setting entry point to', entryPointId);
                    this.indexStore.put(ENTRY_POINT, entryPointId, options);
                }
            }
            this.indexStore.remove(nodeId, options);
            // Remove the safeKey→nodeId mapping so the key count used by autoScaleEf stays accurate
            // and a re-insert of this primary key gets a fresh node rather than the deleted node's id.
            this.indexStore.remove(safeKey, options);
        }
        const needsReindexing = new Map();
        // remove connections to this node that are no longer valid
        if (oldNode.level !== undefined) {
            for (let l = 0; l <= oldNode.level; l++) {
                const oldConnections = oldNode[l];
                for (const { id: neighborId } of oldConnections) {
                    // get and copy the neighbor node so we can modify it
                    const neighborNode = updateNode(neighborId, this.safeGetSync(neighborId, options));
                    if (!neighborNode)
                        continue;
                    // On an UPDATE (vector != null), only remove the reverse edge at
                    // the exact level l where the old connection existed. Sweeping 0..l would destroy reverse
                    // edges at lower levels that were just re-added by addConnection or preserved by the
                    // splice logic above — causing asymmetry that accumulates with every re-embed.
                    // On DELETE (vector == null), the full 0..l sweep is correct: we want to remove every
                    // occurrence of nodeId from all levels of each neighbor.
                    const levelStart = vector ? l : 0;
                    for (let l2 = levelStart; l2 <= l; l2++) {
                        // remove the connection to this node from the neighbor node
                        neighborNode[l2] = neighborNode[l2]?.filter(({ id: nid }) => {
                            return nid !== nodeId;
                        });
                        if (neighborNode[l2]?.length === 0) {
                            logger.trace?.('node was left orphaned, will reindex', neighborId);
                            // reindex re-feeds this vector into index() as a float query, so dequantize int8 back to float
                            needsReindexing.set(neighborNode.primaryKey, neighborNode.scale !== undefined
                                ? dequantizeInt8(neighborNode.vector, neighborNode.scale)
                                : neighborNode.vector);
                        }
                    }
                }
            }
        }
        function updateNode(id, node) {
            // keep a record of all our changes, maintaining any changes that are queued to be written
            let updatedNode = updatedNodes.get(id);
            if (!updatedNode && node) {
                // copy the node so we can modify it
                updatedNode = { ...node };
                updatedNodes.set(id, updatedNode);
            }
            return updatedNode;
        }
        for (const [id, updatedNode] of updatedNodes) {
            this.indexStore.put(id, updatedNode, options);
        }
        for (const [key, orphanVector] of needsReindexing) {
            // If the orphan IS the current entry point, re-running
            // index() from it would find no other nodes to connect to (the entry point search returns
            // itself), leaving it permanently isolated. Elect a surviving neighbor as entry point first
            // so the orphan can reconnect to the live graph.
            const currentEP = this.indexStore.getSync(ENTRY_POINT, options);
            const orphanNodeId = this.indexStore.getSync(typeof key === 'number' ? [KEY_PREFIX, key] : key, options);
            if (currentEP !== undefined && currentEP === orphanNodeId) {
                // The orphan's own connection lists are empty, so look for a surviving node to elect:
                // first among the nodes updated in this pass, then a full scan as fallback.
                let replacementEP;
                for (const [candidateId, candidateNode] of updatedNodes) {
                    if (candidateId !== orphanNodeId && candidateNode[0]?.length > 0) {
                        replacementEP = candidateId;
                        break;
                    }
                }
                if (replacementEP === undefined) {
                    for (const { key: candidateKey, value: candidateNode } of this.indexStore.getRange({
                        start: 0,
                        end: Infinity,
                        transaction: options.transaction,
                    })) {
                        if (candidateKey === orphanNodeId)
                            continue;
                        if (candidateNode?.level !== undefined) {
                            replacementEP = candidateKey;
                            break;
                        }
                    }
                }
                if (replacementEP !== undefined) {
                    this.indexStore.put(ENTRY_POINT, replacementEP, options);
                }
            }
            this.index(key, orphanVector, orphanVector, options);
        }
        this.checkSymmetry(nodeId, this.safeGetSync(nodeId, options), options);
    }
    safeGetSync(key, options) {
        try {
            let node = this.indexStore.getSync(key, options);
            // A quantized vector decodes as a bin (Uint8Array/Buffer) that is a view into the
            // store's read buffer, which may be reused on the next getSync — so copy the bytes
            // into a retained Int8Array (raw two's-complement reinterpret). The Int8Array guard
            // skips re-conversion when the object store (useObjectStore) hands back an
            // already-converted cached node. Float nodes (vector is a number[]) pass through.
            if (node && node.vector && !Array.isArray(node.vector) && !(node.vector instanceof Int8Array)) {
                // A node decoded from disk (a cache miss, common once the table outgrows the object
                // cache) is frozen — the index store sets freezeData — so assigning node.vector would
                // throw and the catch below would silently drop the node, fragmenting the graph (#1161).
                // Clone the frozen node, memoizing the clone against the frozen node so repeated cache
                // hits skip re-slicing/re-cloning. Mutate in place only the writable just-written object.
                const cached = this.convertedNodes.get(node);
                if (cached)
                    return cached;
                const u8 = node.vector;
                const vector = new Int8Array(u8.buffer, u8.byteOffset, u8.byteLength).slice();
                if (Object.isFrozen(node)) {
                    const converted = { ...node, vector };
                    this.convertedNodes.set(node, converted);
                    node = converted;
                }
                else
                    node.vector = vector;
            }
            return node;
        }
        catch {
            logger.warn?.('Failed to decode HNSW node, skipping', key);
            return undefined;
        }
    }
    getEntryPoint(options = {}) {
        // Get entry point
        const entryPointId = this.indexStore.getSync(ENTRY_POINT, options);
        if (entryPointId === undefined)
            return;
        const node = this.safeGetSync(entryPointId, options);
        if (!node)
            return;
        return { id: entryPointId, ...node };
    }
    /**
     * Search one layer of the skip-list using HNSW algorithm for creating a candidate list and navigating the graph
     * TODO: This should be async, but we can't really do that with lmdb-js's transaction system right now. Should be
     * doable with RocksDB. We could also create an async version for searching.
     * @param queryVector
     * @param entryPointId
     * @param entryPoint
     * @param ef
     * @param level
     * @param distanceFunction
     * @param options
     * @private
     */
    searchLayer(queryVector, entryPointId, entryPoint, ef, level, options = {}, distanceFunction = this.distance) {
        // Pre-compute query magnitude for cosine; use cached invMag on stored nodes to skip sqrt per neighbor.
        // Asymmetric distance: the query stays full-precision float; a stored neighbor may be int8
        // (with per-vector `scaleB`) or float (`scaleB` undefined).
        let computeDistance;
        if (distanceFunction === vector_ts_1.cosineDistance) {
            let magASq = 0;
            for (const v of queryVector)
                magASq += v * v;
            const invMagA = 1 / (Math.sqrt(magASq) || 1);
            computeDistance = (b, invMagB, scaleB) => {
                let dot = 0;
                for (let i = 0; i < b.length; i++)
                    dot += queryVector[i] * b[i];
                if (scaleB !== undefined)
                    dot *= scaleB; // dequantize the int8 dot product
                if (invMagB !== undefined)
                    return 1 - dot * invMagA * invMagB;
                // Fallback when the stored node has no cached invMag (a non-cosine index queried as
                // cosine). Compute the stored magnitude and dequantize it by scaleB so it matches the
                // already-dequantized dot product.
                let magBSq = 0;
                for (let i = 0; i < b.length; i++)
                    magBSq += b[i] * b[i];
                let magB = Math.sqrt(magBSq) || 1;
                if (scaleB !== undefined)
                    magB *= scaleB;
                return 1 - (dot * invMagA) / magB;
            };
        }
        else if (distanceFunction === vector_ts_1.euclideanDistance) {
            // Asymmetric squared-euclidean, dequantizing each int8 component inline (no allocation).
            computeDistance = (b, _invMagB, scaleB) => {
                if (scaleB === undefined)
                    return distanceFunction(queryVector, b);
                let distanceSquared = 0;
                for (let i = 0; i < b.length; i++) {
                    const diff = queryVector[i] - b[i] * scaleB;
                    distanceSquared += diff * diff;
                }
                return distanceSquared;
            };
        }
        else {
            // Negated inner product, dequantizing the int8 dot product inline (no allocation).
            computeDistance = (b, _invMagB, scaleB) => {
                if (scaleB === undefined)
                    return distanceFunction(queryVector, b);
                let dot = 0;
                for (let i = 0; i < b.length; i++)
                    dot += queryVector[i] * b[i];
                return -(dot * scaleB);
            };
        }
        const visited = new Set([entryPointId]);
        const initialCandidate = {
            id: entryPointId,
            distance: computeDistance(entryPoint.vector, entryPoint.invMag, entryPoint.scale),
            node: entryPoint,
        };
        const candidates = new MinHeap();
        candidates.push(initialCandidate);
        const results = [initialCandidate];
        while (candidates.size > 0) {
            const current = candidates.pop();
            const furthestDistance = results[results.length - 1].distance;
            if (current.distance > furthestDistance)
                break;
            for (const { id: neighborId } of current.node[level] || []) {
                if (visited.has(neighborId) || neighborId === undefined)
                    continue;
                visited.add(neighborId);
                const neighbor = this.safeGetSync(neighborId, options);
                if (!neighbor)
                    continue;
                this.nodesVisitedCount++;
                const distance = computeDistance(neighbor.vector, neighbor.invMag, neighbor.scale);
                if (distance < furthestDistance || results.length < ef) {
                    const candidate = { id: neighborId, distance, node: neighbor };
                    candidates.push(candidate);
                    results.splice(bisectInsert(results, distance), 0, candidate);
                    if (results.length > ef)
                        results.pop();
                }
            }
        }
        results.visited = visited.size;
        return results;
    }
    /**
     * This the main entry from Harper's query functionality, where we actually search for an ordered list of nearest
     * neighbors, using the provided sort/order definition object and performing the multi-layer skip-list search.
     * This returns an iterable of the nearest neighbors to the provided target vector, with nearest ordered first.
     * @param target
     * @param value
     * @param descending
     * @param distance
     * @param comparator
     * @param context
     */
    search({ target, value, descending, distance, comparator, ef, }, context) {
        let limit; // only set for threshold comparators; 0 is a valid threshold (e.g. dotProduct)
        let limitInclusive = false; // true for `le`, false for `lt`
        switch (comparator) {
            case 'le':
                limitInclusive = true;
            // fallthrough
            case 'lt':
                limit = value;
            // fallthrough
            case 'sort':
                break;
            default:
                throw new hdbError_ts_1.ClientError(`Can not use "${comparator}" comparator with HNSW`);
        }
        // For quantized (int8) threshold queries, suppress the distance limit so the full candidate
        // set is returned; rescoreResults() re-filters on exact full-precision distances post-load.
        if (this.int8 && limit !== undefined)
            limit = undefined;
        if (descending)
            throw new hdbError_ts_1.ClientError(`Can not use descending sort order with HNSW`);
        let distanceFunction;
        if (distance === 'cosine')
            distanceFunction = vector_ts_1.cosineDistance;
        else if (distance === 'euclidean')
            distanceFunction = vector_ts_1.euclideanDistance;
        else if (distance === 'dotProduct')
            distanceFunction = vector_ts_1.dotProductDistance;
        else if (distance)
            throw new hdbError_ts_1.ClientError('Unknown distance function');
        else
            distanceFunction = this.distance;
        if (!target)
            throw new hdbError_ts_1.ClientError('A target vector must be provided for an HNSW query');
        if (!Array.isArray(target))
            throw new hdbError_ts_1.ClientError('The target vector must be an array');
        const options = context.transaction; // should have a nested RocksDB transaction
        // Resolve search ef: per-query ef wins; else an explicitly-configured efConstructionSearch;
        // else auto-scale with the graph size so recall holds as the table grows.
        let effectiveEf = this.efConstructionSearch;
        if (ef !== undefined && ef > 0)
            effectiveEf = ef;
        else if (!this.efSearchConfigured) {
            const nodeCount = this.indexStore.getKeysCount
                ? this.indexStore.getKeysCount()
                : (this.indexStore.getStats?.()?.entryCount ?? 0);
            effectiveEf = autoScaleEf(nodeCount);
        }
        let entryPoint = this.getEntryPoint(options);
        if (!entryPoint)
            return [];
        let entryPointId = entryPoint.id;
        let results = [];
        // For each level from top to bottom
        for (let l = entryPoint.level; l >= 0; l--) {
            // Search for closest neighbors at current level
            results = this.searchLayer(target, entryPointId, entryPoint, effectiveEf, l, options, distanceFunction);
            if (results.length > 0) {
                const neighbor = results[0]; // closest neighbor becomes new entry point
                entryPoint = neighbor.node;
                entryPointId = neighbor.id;
            }
        }
        if (limit !== undefined)
            results = results.filter((candidate) => limitInclusive ? candidate.distance <= limit : candidate.distance < limit);
        return results.map((candidate) => ({
            // we return the result as an entry so we can provide distance as metadata
            key: candidate.node.primaryKey, // return value
            distance: candidate.distance,
        }));
    }
    /**
     * Exact distance between a query and a record's FULL-precision vector, mirroring search()'s metric
     * selection. Used to rerank quantized (int8) results: graph traversal navigates on approximate
     * (quantized) distances, but the caller has the exact record vector and can restore exact ordering
     * and $distance.
     */
    exactDistance(searchCondition, recordVector) {
        if (recordVector == null)
            return Infinity; // missing vector sorts last
        // distance fns require a plain Array (they guard on Array.isArray); records normally store a
        // float[] vector, but convert defensively in case a typed array slips through.
        const vec = Array.isArray(recordVector) ? recordVector : Array.from(recordVector);
        const fn = searchCondition.distance === 'euclidean'
            ? vector_ts_1.euclideanDistance
            : searchCondition.distance === 'dotProduct'
                ? vector_ts_1.dotProductDistance
                : searchCondition.distance === 'cosine'
                    ? vector_ts_1.cosineDistance
                    : this.distance;
        return fn(searchCondition.target, vec);
    }
    /**
     * Post-load rescoring hook called by search.ts after full records have been loaded.
     * Handles two quantized (int8) cases:
     *   - 'sort': recompute exact distances and re-sort for correct nearest-neighbor ordering.
     *   - 'lt'/'le': recompute exact distances and re-filter by the threshold value (over-fetch
     *     was applied before the search to ensure enough candidates).
     * Returns null when rescoring doesn't apply so the caller uses the loaded iterable as-is.
     */
    rescoreResults(loaded, searchCondition, comparator, attributeName) {
        if (!this.int8 || !searchCondition.target || typeof attributeName !== 'string')
            return null;
        if (comparator === 'sort') {
            const rescored = loaded.filter((e) => e !== extended_iterable_1.SKIP && e && e.value);
            for (const e of rescored) {
                const d = this.exactDistance(searchCondition, e.value[attributeName]);
                // Non-finite exact distances (NaN from a corrupt record vector, Infinity from a
                // missing vector) sort last — consistent with the missing-vector sentinel in exactDistance.
                e.distance = Number.isFinite(d) ? d : Infinity;
            }
            // comparison-based (not subtraction) so Infinity sentinels for missing vectors
            // sort last without producing NaN (Infinity - Infinity).
            rescored.sort((a, b) => (a.distance === b.distance ? 0 : a.distance < b.distance ? -1 : 1));
            return rescored;
        }
        if (comparator === 'lt' || comparator === 'le') {
            const thresholdValue = searchCondition.value;
            const rescored = loaded.filter((e) => e !== extended_iterable_1.SKIP && e && e.value);
            for (const e of rescored)
                e.distance = this.exactDistance(searchCondition, e.value[attributeName]);
            return rescored.filter((e) => (comparator === 'le' ? e.distance <= thresholdValue : e.distance < thresholdValue));
        }
        return null;
    }
    checkSymmetry(id, node, options) {
        if (!node)
            return;
        let l = 0;
        let connections;
        while ((connections = node[l])) {
            // verify that the level is not empty, otherwise this means we have an orphaned node
            if (connections.length === 0)
                break;
            for (const { id: neighbor } of connections) {
                const neighborNode = this.safeGetSync(neighbor, options);
                if (!neighborNode) {
                    logger.info?.('could not find neighbor node', neighbor);
                    continue;
                }
                // verify that the connection is symmetrical
                const symmetrical = neighborNode[l]?.find(({ id: nid }) => nid == id);
                if (!symmetrical) {
                    logger.info?.('asymmetry detected', neighborNode[l], 'does not have', id);
                }
            }
            l++;
        }
    }
    addConnection(fromId, node, toId, level, distance, updateNode, options) {
        if (!node[level]) {
            node[level] = [];
        }
        let maxConnections = level === 0 ? this.M << 1 : this.M;
        if (this.optimizeRouting)
            maxConnections <<= 2; // bump up the max connections beyond traditional HNSW because we are naturally limiting
        // have we exceeded the max connections (with 25% grace period)
        if (node[level].length >= maxConnections + (maxConnections >> 2)) {
            logger.debug?.('maxConnections reached, removing some connections', maxConnections);
            // Get all connections with their similarities
            // Sort by distance but prioritize nodes that have reverse connections
            const connections = [...node[level]];
            connections.sort((a, b) => {
                return a.distance - b.distance;
            });
            // Keep the best connections
            const keptConnections = connections.slice(0, maxConnections);
            const removedConnections = connections.slice(maxConnections);
            // Update this node's connections
            node[level] = keptConnections;
            // For removed connections, ensure there's still a path to them
            for (const removed of removedConnections) {
                let removedNode = updateNode(removed.id) ?? this.safeGetSync(removed.id, options);
                if (removedNode) {
                    // Remove the reverse connection if it exists
                    if (removedNode[level]) {
                        const filtered = removedNode[level].filter(({ id }) => id !== fromId);
                        if (level === 0 && filtered.length === 0) {
                            // don't remove the last connection at level 0 — it would orphan this node
                            logger.info?.('skipping removal of last connection', fromId, toId);
                        }
                        else {
                            removedNode = updateNode(removed.id, removedNode);
                            removedNode[level] = filtered;
                        }
                    }
                }
            }
        }
        if (node[level].find(({ id }) => id === toId)) {
            logger.debug?.('already connected', fromId, toId);
        }
        else {
            node[level] = [...node[level], { id: toId, distance }]; // add
        }
        //this.indexStore.put(fromId, node, options);
        //this.checkSymmetry(fromId, node, options);
    }
    validateConnectivity(startLevel = 0) {
        const entryPoint = this.getEntryPoint();
        if (!entryPoint)
            return;
        const visited = new Set();
        // BFS from entry point to ensure all nodes are reachable. Asymmetric stale neighbor
        // references can survive deletes, so a referenced node may not actually exist anymore;
        // only count a node as visited once we confirm the underlying record is present.
        const queue = [entryPoint.id];
        const enqueued = new Set([entryPoint.id]);
        let connections = 0;
        while (queue.length > 0) {
            const currentId = queue.shift();
            const current = this.safeGetSync(currentId);
            if (!current)
                continue;
            visited.add(currentId);
            for (let level = startLevel; level <= current.level; level++) {
                for (const { id: neighborId } of current[level] || []) {
                    connections++;
                    if (!enqueued.has(neighborId)) {
                        enqueued.add(neighborId);
                        queue.push(neighborId);
                    }
                }
            }
        }
        // Check if all nodes are reachable
        // This would require maintaining a separate set/count of all nodes
        return {
            isFullyConnected: visited.size === this.totalNodes,
            averageConnections: connections / visited.size,
        };
    }
    get totalNodes() {
        return Array.from(this.indexStore.getKeys({ start: 0, end: Infinity })).length;
    }
    /**
     * This is used by the query planner to determine what order to apply conditions. It is our best guess at an estimated count.
     * This unit is typically the number of records that need to be accessed to satisfy the query. We know that we will visit
     * a minimum of efConstructionSearch nodes and a maximum of the total nodes (in absolute worst case).
     * The original paper described the complexity as polylogarithmic. From my testing, the
     * best and simplest guess at the number of nodes that need to be accessed is the geometric mean of the total number of nodes
     * and the efConstruction parameter (for search), which clearly constrains the estimate to the correct range and is
     * similar to polylogarithmic for realistic values.
     *
     * @returns
     */
    estimateCountAsSort() {
        const count = this.indexStore instanceof rocksdb_js_1.RocksDatabase ? this.indexStore.getKeysCount() : this.indexStore.getStats().entryCount;
        return Math.sqrt(count * this.efConstructionSearch);
    }
    /**
     * This is used to resolve the vector property, which should be resolved to the distance when used in a sort comparator
     * We also want to cache distance calculations so they can be accessed efficently later
     * @param vector
     * @param context
     * @param entry
     */
    propertyResolver(vector, context, entry) {
        const sortDefinition = context?.sort;
        if (sortDefinition) {
            // set up a cache for these so they can be accessed by $distance and not be recalculated during a sort
            let vectorDistances = sortDefinition.vectorDistances;
            if (vectorDistances) {
                const difference = vectorDistances.get(entry);
                if (difference)
                    return difference;
            }
            else
                vectorDistances = context.vectorDistances = sortDefinition.vectorDistances = new Map();
            let distanceFunction = this.distance;
            if (sortDefinition.type)
                distanceFunction =
                    sortDefinition.distance === 'euclidean'
                        ? vector_ts_1.euclideanDistance
                        : sortDefinition.distance === 'dotProduct'
                            ? vector_ts_1.dotProductDistance
                            : vector_ts_1.cosineDistance;
            const distance = distanceFunction(sortDefinition.target, vector);
            vectorDistances.set(entry, distance);
            return distance;
        }
        return vector;
    }
}
exports.HierarchicalNavigableSmallWorld = HierarchicalNavigableSmallWorld;
//# sourceMappingURL=HierarchicalNavigableSmallWorld.js.map