"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.COERCIBLE_OPERATORS = void 0;
exports.executeConditions = executeConditions;
exports.searchByIndex = searchByIndex;
exports.findAttribute = findAttribute;
exports.filterByType = filterByType;
exports.estimateCondition = estimateCondition;
exports.parseQuery = parseQuery;
exports.flattenKey = flattenKey;
exports.intersectionEstimate = intersectionEstimate;
const hdbError_js_1 = require("../utility/errors/hdbError.js");
const terms_js_1 = require("../utility/lmdb/terms.js");
const ordered_binary_1 = require("ordered-binary");
const extended_iterable_1 = require("@harperfast/extended-iterable");
const Table_ts_1 = require("./Table.js");
const RecordEncoder_ts_1 = require("./RecordEncoder.js");
const write_1 = require("./analytics/write");
// these are ratios/percentages of overall table size
const OPEN_RANGE_ESTIMATE = 0.3;
const BETWEEN_ESTIMATE = 0.1;
const STARTS_WITH_ESTIMATE = 0.05;
const SYMBOL_OPERATORS = {
    // these are coercing operators
    '<': 'lt',
    '<=': 'le',
    '>': 'gt',
    '>=': 'ge',
    '!=': 'ne',
    '==': 'eq',
    // these are strict operators:
    '===': 'equals',
    '!==': 'not_equal',
};
exports.COERCIBLE_OPERATORS = {
    lt: true,
    le: true,
    gt: true,
    ge: true,
    ne: true,
    eq: true,
};
function executeConditions(conditions, operator, table, txn, request, context, transformToEntries, filtered) {
    const firstSearch = conditions[0];
    // both AND and OR start by getting an iterator for the ids for first condition
    // and then things diverge...
    if (operator === 'or') {
        let results = executeCondition(firstSearch);
        //get the union of ids from all condition searches
        for (let i = 1; i < conditions.length; i++) {
            const condition = conditions[i];
            // might want to lazily execute this after getting to this point in the iteration
            const nextResults = executeCondition(condition);
            results = results.concat(nextResults);
        }
        const returnedIds = new Set();
        return results.filter((entry) => {
            const id = entry.key ?? entry;
            if (returnedIds.has(id))
                // skip duplicate ids
                return false;
            returnedIds.add(id);
            return true;
        });
    }
    else {
        // AND
        const results = executeCondition(firstSearch);
        // get the intersection of condition searches by using the indexed query for the first condition
        // and then filtering by all subsequent conditions.
        // now apply filters that require looking up records
        const filters = mapConditionsToFilters(conditions.slice(1), true, firstSearch.estimated_count);
        return filters.length > 0 ? transformToEntries(results, filters) : results;
    }
    function executeCondition(condition) {
        if (condition.conditions)
            return executeConditions(condition.conditions, condition.operator, table, txn, request, context, transformToEntries, filtered);
        return searchByIndex(condition, txn, condition.descending || request.reverse === true, table, request.allowFullScan, filtered, context);
    }
    function mapConditionsToFilters(conditions, intersection, estimatedIncomingCount) {
        return conditions
            .map((condition, index) => {
            if (condition.conditions) {
                // this is a group of conditions, we need to combine them
                const union = condition.operator === 'or';
                const filters = mapConditionsToFilters(condition.conditions, !union, estimatedIncomingCount);
                if (union)
                    return (record, entry) => filters.some((filter) => filter(record, entry));
                else
                    return (record, entry) => filters.every((filter) => filter(record, entry));
            }
            const isPrimaryKey = (condition.attribute || condition[0]) === table.primaryKey;
            const filter = filterByType(condition, table, context, filtered, isPrimaryKey, estimatedIncomingCount);
            if (intersection && index < conditions.length - 1 && estimatedIncomingCount) {
                estimatedIncomingCount = intersectionEstimate(table.primaryStore, condition.estimated_count, estimatedIncomingCount);
            }
            return filter;
        })
            .filter(Boolean);
    }
}
/**
 * Search for records or keys, based on the search condition, using an index if available
 * @param searchCondition
 * @param transaction
 * @param reverse
 * @param Table
 * @param allowFullScan
 * @param filtered
 */
function searchByIndex(searchCondition, transaction, reverse, Table, allowFullScan, filtered, context) {
    let attribute_name = searchCondition[0] ?? searchCondition.attribute;
    let value = searchCondition[1] ?? searchCondition.value;
    const comparator = searchCondition.comparator;
    if (value === undefined && comparator !== 'sort') {
        throw new hdbError_js_1.ClientError(`Search condition for ${attribute_name} must have a value`);
    }
    if (Array.isArray(attribute_name)) {
        const firstAttributeName = attribute_name[0];
        // get the potential relationship attribute
        const attribute = findAttribute(Table.attributes, firstAttributeName);
        if (attribute.relationship) {
            // it is a join/relational query
            if (attribute_name.length < 2)
                throw new hdbError_js_1.ClientError('Can not directly query a relational attribute, must query an attribute within the target table');
            const relatedTable = attribute.definition?.tableClass || attribute.elements?.definition?.tableClass;
            const joined = new Map();
            // search the related table
            let results = searchByIndex({
                attribute: attribute_name.length > 2 ? attribute_name.slice(1) : attribute_name[1],
                value,
                comparator,
            }, transaction, reverse, relatedTable, allowFullScan, joined);
            if (attribute.relationship.to) {
                // this is one-to-many or many-to-many, so we need to track the filtering of related entries that match
                filtered[attribute_name[0]] = joined;
                // Use the joinTo to join the results of the related table to the current table (can be one-to-many or many-to-many)
                const isManyToMany = Boolean(findAttribute(relatedTable.attributes, attribute.relationship.to)?.elements);
                results = joinTo(results, attribute, relatedTable.primaryStore, isManyToMany, joined);
            }
            if (attribute.relationship.from) {
                const searchEntry = (relatedEntry) => {
                    if (relatedEntry?.key !== undefined)
                        relatedEntry = relatedEntry.key;
                    return searchByIndex({ attribute: attribute.relationship.from, value: relatedEntry }, transaction, reverse, Table, allowFullScan, joined);
                };
                if (attribute.elements) {
                    filtered[attribute_name[0]] = joined;
                    // many-to-many relationship (forward), get all the ids first
                    results = joinFrom(results, attribute, relatedTable.primaryStore, joined, searchEntry);
                }
                else {
                    // many-to-one relationship, need to flatten the ids that point back to potentially many instances of this
                    results = results.flatMap(searchEntry);
                }
            }
            return results;
        }
        else if (attribute_name.length === 1) {
            attribute_name = attribute_name[0];
        }
        else {
            throw new hdbError_js_1.ClientError('Unable to query by attribute ' + JSON.stringify(attribute_name));
        }
    }
    const isPrimaryKey = attribute_name === Table.primaryKey || attribute_name == null;
    const index = isPrimaryKey ? Table.primaryStore : Table.indices[attribute_name];
    let start;
    let end, inclusiveEnd, exclusiveStart;
    if (value instanceof Date)
        value = value.getTime();
    let needFullScan;
    switch (ALTERNATE_COMPARATOR_NAMES[comparator] || comparator) {
        case 'lt':
            start = true;
            end = value;
            break;
        case 'le':
            start = true;
            end = value;
            inclusiveEnd = true;
            break;
        case 'gt':
            start = value;
            exclusiveStart = true;
            break;
        case 'ge':
            start = value;
            break;
        case 'prefix': // this is form finding multi-part keys that start with the provided prefix
            // this search needs to be of the form:
            // start: [prefix, null], end: [prefix, MAXIMUM_KEY]
            if (!Array.isArray(value))
                value = [value, null];
            else if (value[value.length - 1] != null)
                value = value.concat(null);
            start = value;
            end = value.slice(0);
            end[end.length - 1] = ordered_binary_1.MAXIMUM_KEY;
            break;
        case 'starts_with':
            start = value.toString();
            end = value + String.fromCharCode(0xffff);
            break;
        case 'between':
        case 'gele':
        case 'gelt':
        case 'gtlt':
        case 'gtle':
            start = value[0];
            if (start instanceof Date)
                start = start.getTime();
            end = value[1];
            if (end instanceof Date)
                end = end.getTime();
            inclusiveEnd = comparator === 'gele' || comparator === 'gtle' || comparator === 'between';
            exclusiveStart = comparator === 'gtlt' || comparator === 'gtle';
            break;
        case 'equals':
        case undefined:
            start = value;
            end = value;
            inclusiveEnd = true;
            break;
        case 'ne':
            if (value === null) {
                // since null is the lowest value in an index, we can treat anything higher as a non-null
                start = value;
                exclusiveStart = true;
                break;
            }
        case 'sort': // this is a special case for when we want to get all records for sorting
        case 'contains':
        case 'ends_with':
            // we have to revert to full table scan here
            start = true;
            needFullScan = true;
            break;
        default:
            throw new hdbError_js_1.ClientError(`Unknown query comparator "${comparator}"`);
    }
    let filter;
    if (typeof start === 'string' && start.length > terms_js_1.MAX_SEARCH_KEY_LENGTH) {
        // if the key is too long, we need to truncate it and filter the results
        start = start.slice(0, terms_js_1.MAX_SEARCH_KEY_LENGTH) + terms_js_1.OVERFLOW_MARKER;
        exclusiveStart = false;
        filter = filterByType(searchCondition, Table, null, filtered, isPrimaryKey);
    }
    if (typeof end === 'string' && end.length > terms_js_1.MAX_SEARCH_KEY_LENGTH) {
        // if the key is too long, we need to truncate it and filter the results
        end = end.slice(0, terms_js_1.MAX_SEARCH_KEY_LENGTH) + terms_js_1.OVERFLOW_MARKER;
        inclusiveEnd = true;
        filter = filter ?? filterByType(searchCondition, Table, null, filtered, isPrimaryKey);
    }
    if (reverse) {
        let newEnd = start;
        start = end;
        end = newEnd;
        newEnd = !exclusiveStart;
        exclusiveStart = !inclusiveEnd;
        inclusiveEnd = newEnd;
    }
    if (!index || index.isIndexing || needFullScan || (value === null && !index.indexNulls)) {
        // no indexed searching available, need a full scan
        if (allowFullScan === false && !index)
            throw new hdbError_js_1.ClientError(`"${attribute_name}" is not indexed, can not search for this attribute`, 404);
        if (allowFullScan === false && needFullScan)
            throw new hdbError_js_1.ClientError(`Can not use ${comparator || 'equal'} operator without combining with a condition that uses an index, can not search for attribute ${attribute_name}`, 403);
        if (index?.isIndexing)
            throw new hdbError_js_1.ServerError(`"${attribute_name}" is not indexed yet, can not search for this attribute`, 503);
        if (value === null && index && !index.indexNulls)
            throw new hdbError_js_1.ClientError(`"${attribute_name}" is not indexed for nulls, index needs to be rebuilt to search for nulls, can not search for this attribute`, 400);
        filter = filter ?? filterByType(searchCondition, Table, null, filtered, isPrimaryKey);
        if (!filter) {
            throw new hdbError_js_1.ClientError(`Unknown search operator ${searchCondition.comparator}`);
        }
    }
    const rangeOptions = {
        start,
        end,
        inclusiveEnd,
        exclusiveStart,
        values: true,
        versions: isPrimaryKey,
        transaction,
        reverse,
    };
    if (isPrimaryKey) {
        const results = index.getRange(rangeOptions).map(filter
            ? function ({ key, value }) {
                if (this?.isSync)
                    return value && filter(value) ? key : extended_iterable_1.SKIP;
                // for filter operations, we intentionally yield the event turn so that scanning queries
                // do not hog resources
                return new Promise((resolve, reject) => setImmediate(() => {
                    try {
                        resolve(value && filter(value) ? key : extended_iterable_1.SKIP);
                    }
                    catch (error) {
                        reject(error);
                    }
                }));
            }
            : function (entry) {
                let result;
                if (entry.value == null && !(entry.metadataFlags & (Table_ts_1.INVALIDATED | Table_ts_1.EVICTED)))
                    result = extended_iterable_1.SKIP;
                else {
                    Object.freeze(entry.value);
                    recordRead(entry);
                    result = entry;
                }
                if (this.isSync)
                    return result;
                return new Promise((resolve) => setImmediate(() => resolve(result)));
            });
        results.hasEntries = true;
        return results;
    }
    else if (index) {
        if (index.customIndex) {
            return index.customIndex.search(searchCondition, context).map((entry) => {
                // if the custom index returns an entry with metadata, merge it with the loaded entry
                if (typeof entry === 'object' && entry) {
                    const { key, ...otherProps } = entry;
                    const loadedEntry = Table.primaryStore.getEntry(key);
                    Object.freeze(loadedEntry?.value);
                    recordRead(loadedEntry);
                    return { ...otherProps, ...loadedEntry };
                }
                return entry;
            });
        }
        return index.getRange(rangeOptions).map(filter
            ? function ({ key, value }) {
                let recordMatcher;
                if (typeof key === 'string' && key.length > terms_js_1.MAX_SEARCH_KEY_LENGTH) {
                    // if it is an overflow string, need to get the actual value from the database
                    recordMatcher = Table.primaryStore.getSync(value);
                }
                else
                    recordMatcher = { [attribute_name]: key };
                if (this.isSync)
                    return filter(recordMatcher) ? value : extended_iterable_1.SKIP;
                // for filter operations, we intentionally yield the event turn so that scanning queries
                // do not hog resources
                return new Promise((resolve, reject) => setImmediate(() => {
                    try {
                        resolve(filter(recordMatcher) ? value : extended_iterable_1.SKIP);
                    }
                    catch (error) {
                        reject(error);
                    }
                }));
            }
            : ({ value }) => value);
    }
    else {
        return Table.primaryStore
            .getRange(reverse ? { end: true, transaction, reverse: true } : { start: true, transaction })
            .map(function (entry) {
            const { key, value } = entry;
            if (this.isSync) {
                recordRead(entry);
                return value && filter(value) ? key : extended_iterable_1.SKIP;
            }
            // for filter operations, we intentionally yield the event turn so that scanning queries
            // do not hog resources
            return new Promise((resolve, reject) => setImmediate(() => {
                try {
                    recordRead(entry);
                    resolve(value && filter(value) ? key : extended_iterable_1.SKIP);
                }
                catch (error) {
                    reject(error);
                }
            }));
        });
    }
    function recordRead(entry) {
        if ((Table.databaseName !== 'system' || Table.name === 'hdb_analytics') && entry?.value) {
            (0, write_1.recordAction)(entry.size ?? 1, 'db-read', Table.name, null);
        }
    }
}
function findAttribute(attributes, attribute_name) {
    if (Array.isArray(attribute_name)) {
        if (attribute_name.length > 1) {
            const firstAttribute = findAttribute(attributes, attribute_name[0]);
            const nextAttributes = (firstAttribute?.definition?.tableClass || firstAttribute?.elements?.definition?.tableClass)?.attributes ??
                firstAttribute?.properties;
            if (nextAttributes)
                return findAttribute(nextAttributes, attribute_name.slice(1));
            return;
        }
        else
            attribute_name = attribute_name.toString();
    }
    else if (typeof attribute_name !== 'string')
        attribute_name = attribute_name.toString();
    return attributes.find((attribute) => attribute.name === attribute_name);
}
/**
 * This is used to join the results of a query where the right side is a set of records with the foreign key that
 * points to the left side (from right to left)
 * @param rightIterable
 * @param attribute
 * @param store
 * @param isManyToMany
 * @param joined
 * @returns
 */
function joinTo(rightIterable, attribute, store, isManyToMany, joined) {
    return new rightIterable.constructor({
        [Symbol.iterator]() {
            let joinedIterator;
            joined.hasMappings = true;
            return {
                next() {
                    if (!joinedIterator) {
                        const rightProperty = attribute.relationship.to;
                        const addEntry = (key, entry) => {
                            let entriesForKey = joined.get(key);
                            if (entriesForKey)
                                entriesForKey.push(entry);
                            else
                                joined.set(key, (entriesForKey = [entry]));
                        };
                        //let i = 0;
                        // get all the ids of the related records
                        for (const entry of rightIterable) {
                            const record = entry.value ?? store.getSync(entry.key ?? entry);
                            const leftKey = record?.[rightProperty];
                            if (leftKey == null)
                                continue;
                            if (joined.filters?.some((filter) => !filter(record)))
                                continue;
                            if (isManyToMany) {
                                for (let i = 0; i < leftKey.length; i++) {
                                    addEntry(leftKey[i], entry);
                                }
                            }
                            else {
                                addEntry(leftKey, entry);
                            }
                            // TODO: Enable this with async iterator manually iterating so that we don't need to do an await on every iteration
                            /*
                            if (i++ > 100) {
                                // yield the event turn every 100 ids. See below for more explanation
                                await new Promise(setImmediate);
                                i = 0;
                            }*/
                        }
                        joinedIterator = joined.keys()[Symbol.iterator]();
                        return this.next();
                    }
                    const joinedEntry = joinedIterator.next();
                    if (joinedEntry.done)
                        return joinedEntry;
                    return {
                        // if necessary, get the original key from the entries array
                        value: joinedEntry.value,
                    };
                },
                return() {
                    if (joinedIterator?.return)
                        return joinedIterator.return();
                },
            };
        },
    });
}
/**
 * This is used to join the results of a query where the right side is a set of ids and the left side is a set of records
 * that have the foreign key (from left to right)
 * @param rightIterable
 * @param attribute
 * @param store
 * @param joined
 * @param searchEntry
 * @returns
 */
function joinFrom(rightIterable, attribute, store, joined, searchEntry) {
    return new rightIterable.constructor({
        [Symbol.iterator]() {
            let idIterator;
            let joinedIterator;
            const seenIds = new Set();
            return {
                next() {
                    let joinedEntry;
                    if (joinedIterator) {
                        while (true) {
                            joinedEntry = joinedIterator.next();
                            if (joinedEntry.done)
                                break; // and continue to find next
                            const id = joinedEntry.value;
                            if (seenIds.has(id))
                                continue;
                            seenIds.add(id);
                            return joinedEntry;
                        }
                    }
                    if (!idIterator) {
                        // get the ids of the related records as a Set so we can quickly check if it is in the set
                        // when are iterating through the results
                        const ids = new Set();
                        // Define the fromRecord function so that we can use it to filter the related records
                        // that are in the select(), to only those that are in this set of ids
                        joined.fromRecord = (record) => {
                            // TODO: Sort based on order ids
                            return record[attribute.relationship.from]?.filter?.((id) => ids.has(id));
                        };
                        //let i = 0;
                        // get all the ids of the related records
                        for (const id of rightIterable) {
                            if (joined.filters) {
                                // if additional filters are defined, we need to check them
                                const record = store.getSync(id);
                                if (joined.filters.some((filter) => !filter(record)))
                                    continue;
                            }
                            ids.add(id);
                            // TODO: Re-enable this when async iteration is used, and do so with manually iterating so that we don't need to do an await on every iteration
                            /*
                            if (i++ > 100) {
                                // yield the event turn every 100 ids. We don't want to monopolize the
                                // event loop, give others a chance to run. However, we are much more aggressive
                                // about running here than in simple filter operations, because we are
                                // executing a very minimal range iteration and because this is consuming
                                // memory (so we want to get it over with) and the user isn't getting any
                                // results until we finish
                                await new Promise(setImmediate);
                                i = 0;
                            }*/
                        }
                        // and now start iterating through the ids
                        idIterator = ids[Symbol.iterator]();
                        return this.next();
                    }
                    do {
                        const idEntry = idIterator.next();
                        if (idEntry.done)
                            return idEntry;
                        joinedIterator = searchEntry(idEntry.value)[Symbol.iterator]();
                        return this.next();
                    } while (true);
                },
                return() {
                    return joinedIterator?.return?.();
                },
                throw() {
                    return joinedIterator?.throw?.();
                },
            };
        },
    });
}
const ALTERNATE_COMPARATOR_NAMES = {
    'eq': 'equals',
    'greater_than': 'gt',
    'greaterThan': 'gt',
    'greater_than_equal': 'ge',
    'greaterThanEqual': 'ge',
    'less_than': 'lt',
    'lessThan': 'lt',
    'less_than_equal': 'le',
    'lessThanEqual': 'le',
    'not_equal': 'ne',
    'notEqual': 'ne',
    'equal': 'equals',
    'sw': 'starts_with',
    'startsWith': 'starts_with',
    'ew': 'ends_with',
    'endsWith': 'ends_with',
    'ct': 'contains',
    '>': 'gt',
    '>=': 'ge',
    '<': 'lt',
    '<=': 'le',
    '...': 'between',
};
/**
 * Create a filter based on the search condition that can be used to test each supplied record.
 * @param {SearchObject} searchCondition
 * @returns {({}) => boolean}
 */
function filterByType(searchCondition, Table, context, filtered, isPrimaryKey, estimatedIncomingCount) {
    const comparator = searchCondition.comparator;
    let attribute = searchCondition[0] ?? searchCondition.attribute;
    let value = searchCondition[1] ?? searchCondition.value;
    if (Array.isArray(attribute)) {
        if (attribute.length === 0)
            return () => true;
        if (attribute.length === 1)
            attribute = attribute[0];
        else if (attribute.length > 1) {
            const firstAttributeName = attribute[0];
            // get the relationship attribute
            const firstAttribute = findAttribute(Table.attributes, firstAttributeName);
            const relatedTable = firstAttribute.definition?.tableClass || firstAttribute.elements.definition?.tableClass;
            // TODO: If this is a relationship, we can potentially make this more efficient by using the index
            // and retrieving the set of matching ids first
            const filterMap = filtered?.[firstAttributeName];
            const nextFilter = filterByType({
                attribute: attribute.length > 2 ? attribute.slice(1) : attribute[1],
                value,
                comparator,
            }, relatedTable, context, filterMap?.[firstAttributeName]?.joined, attribute[1] === relatedTable.primaryKey, estimatedIncomingCount);
            if (!nextFilter)
                return;
            if (filterMap) {
                if (!filterMap.filters)
                    filterMap.filters = [];
                filterMap.filters.push(nextFilter);
                return;
            }
            const resolver = Table.propertyResolvers?.[firstAttributeName];
            if (resolver.to)
                nextFilter.to = resolver.to;
            let subIdFilter;
            const getSubObject = (record, entry) => {
                let subObject, subEntry;
                if (resolver) {
                    if (resolver.returnDirect) {
                        // indicates that the resolver will direct return the value instead of an entry
                        subObject = resolver(record, context, entry);
                        subEntry = RecordEncoder_ts_1.lastMetadata;
                    }
                    else {
                        subEntry = resolver(record, context, entry, true);
                        if (Array.isArray(subEntry)) {
                            // if any array, map the values
                            subObject = subEntry.map((subEntry) => subEntry.value);
                            subEntry = null;
                        }
                        else {
                            subObject = subEntry?.value;
                        }
                    }
                }
                else
                    subObject = record[firstAttributeName];
                return { subObject, subEntry };
            };
            const recordFilter = (record, entry) => {
                if (resolver) {
                    if (nextFilter.idFilter) {
                        // if we are filtering by id, we can use the idFilter to avoid loading the record
                        if (!subIdFilter) {
                            if (nextFilter.idFilter.idSet?.size === 1) {
                                // if there is a single id we are looking for, we can create a new search condition that the
                                // attribute comparator could eventually use to create a recursive id set
                                // TODO: Eventually we should be able to handle multiple ids by creating a union
                                for (const id of nextFilter.idFilter.idSet) {
                                    searchCondition = {
                                        attribute: resolver.from ?? Table.primaryKey, // if no from, we use our primary key
                                        value: id,
                                    };
                                }
                                // indicate that we can use an index for this. also we indicate that we allow object matching to allow array ids to directly tested
                                subIdFilter = attributeComparator(resolver.from ?? Table.primaryKey, nextFilter.idFilter, true, true);
                            }
                            else
                                subIdFilter = attributeComparator(resolver.from ?? Table.primaryKey, nextFilter.idFilter, false, true);
                        }
                        const matches = subIdFilter(record);
                        if (subIdFilter.idFilter)
                            recordFilter.idFilter = subIdFilter.idFilter;
                        return matches;
                    }
                }
                const { subObject, subEntry } = getSubObject(record, entry);
                if (!subObject)
                    return false;
                if (!Array.isArray(subObject))
                    return nextFilter(subObject, subEntry);
                const filterMap = filtered?.[firstAttributeName];
                if (!filterMap && filtered) {
                    // establish a filtering that can preserve this filter for the select() results of these sub objects
                    filtered[firstAttributeName] = {
                        fromRecord(record) {
                            // this is called when selecting the fields to include in results
                            const value = getSubObject(record).subObject;
                            if (Array.isArray(value))
                                return value.filter(nextFilter).map((value) => value[relatedTable.primaryKey]);
                            return value;
                        },
                    };
                }
                return subObject.some(nextFilter);
            };
            return recordFilter;
        }
    }
    if (value instanceof Date)
        value = value.getTime();
    switch (ALTERNATE_COMPARATOR_NAMES[comparator] || comparator) {
        case terms_js_1.SEARCH_TYPES.EQUALS:
        case undefined:
            return attributeComparator(attribute, (recordValue) => recordValue === value, true);
        case 'contains':
            return attributeComparator(attribute, (recordValue) => recordValue?.toString().includes(value));
        case 'ends_with':
            return attributeComparator(attribute, (recordValue) => recordValue?.toString().endsWith(value));
        case 'starts_with':
            return attributeComparator(attribute, (recordValue) => typeof recordValue === 'string' && recordValue.startsWith(value), true);
        case 'prefix':
            if (!Array.isArray(value))
                value = [value];
            else if (value[value.length - 1] == null)
                value = value.slice(0, -1);
            return attributeComparator(attribute, (recordValue) => {
                if (!Array.isArray(recordValue))
                    return false;
                for (let i = 0, l = value.length; i < l; i++) {
                    if (recordValue[i] !== value[i])
                        return false;
                }
                return true;
            }, true);
        case 'between':
            if (value[0] instanceof Date)
                value[0] = value[0].getTime();
            if (value[1] instanceof Date)
                value[1] = value[1].getTime();
            return attributeComparator(attribute, (recordValue) => {
                return (0, ordered_binary_1.compareKeys)(recordValue, value[0]) >= 0 && (0, ordered_binary_1.compareKeys)(recordValue, value[1]) <= 0;
            }, true);
        case 'gt':
            return attributeComparator(attribute, (recordValue) => (0, ordered_binary_1.compareKeys)(recordValue, value) > 0);
        case 'ge':
            return attributeComparator(attribute, (recordValue) => (0, ordered_binary_1.compareKeys)(recordValue, value) >= 0);
        case 'lt':
            return attributeComparator(attribute, (recordValue) => (0, ordered_binary_1.compareKeys)(recordValue, value) < 0);
        case 'le':
            return attributeComparator(attribute, (recordValue) => (0, ordered_binary_1.compareKeys)(recordValue, value) <= 0);
        case 'ne':
            return attributeComparator(attribute, (recordValue) => (0, ordered_binary_1.compareKeys)(recordValue, value) !== 0, false, true);
        case 'sort':
            return () => true;
        default:
            throw new hdbError_js_1.ClientError(`Unknown query comparator "${comparator}"`);
    }
    /** Create a comparison function that can take the record and check the attribute's value with the filter function */
    function attributeComparator(attribute, filter, canUseIndex, allowObjectMatching) {
        let thresholdRemainingMisses;
        canUseIndex =
            canUseIndex && // is it a comparator that makes sense to use index
                !isPrimaryKey && // no need to use index for primary keys, since we will be iterating over the primary keys
                Table?.indices[attribute] && // is there an index for this attribute
                estimatedIncomingCount > 3; // do we have a valid estimate of multiple incoming records (that is worth using an index for)
        if (canUseIndex) {
            if (searchCondition.estimated_count == undefined)
                estimateCondition(Table)(searchCondition);
            thresholdRemainingMisses = searchCondition.estimated_count >> 4;
            if (isNaN(thresholdRemainingMisses) || thresholdRemainingMisses >= estimatedIncomingCount)
                // invalid or can't be ever reached
                canUseIndex = false;
        }
        let misses = 0;
        let filteredSoFar = 3; // what we use to calculate miss rate; we give some buffer so we don't jump to indexed retrieval too quickly
        function recordFilter(record) {
            const value = record[attribute];
            let matches;
            if (typeof value !== 'object' || !value || allowObjectMatching)
                matches = filter(value);
            else if (Array.isArray(value))
                matches = value.some(filter);
            else if (value instanceof Date)
                matches = filter(value.getTime());
            //else matches = false;
            // As we are filtering, we can lazily/reactively switch to indexing if we are getting a low match rate, allowing use to load
            // a set of ids instead of loading each record. This can be a significant performance improvement for large queries with low match rates
            if (canUseIndex) {
                filteredSoFar++;
                if (!matches &&
                    !recordFilter.idFilter &&
                    // miss rate x estimated remaining to filter > 10% of estimated incoming
                    (++misses / filteredSoFar) * estimatedIncomingCount > thresholdRemainingMisses) {
                    // if we have missed too many times, we need to switch to indexed retrieval
                    const searchResults = searchByIndex(searchCondition, Table._readTxnForContext(context), false, Table);
                    let matchingIds;
                    if (recordFilter.to) {
                        // the values could be an array of keys, so we flatten the mapping
                        matchingIds = searchResults.flatMap((id) => Table.primaryStore.getSync(id)[recordFilter.to]);
                    }
                    else {
                        matchingIds = searchResults.map(flattenKey);
                    }
                    // now generate a hash set that we can efficiently check primary keys against
                    // TODO: Do this asynchronously
                    const idSet = new Set(matchingIds);
                    recordFilter.idFilter = (id) => idSet.has(flattenKey(id));
                    recordFilter.idFilter.idSet = idSet;
                }
            }
            return matches;
        }
        if (isPrimaryKey) {
            recordFilter.idFilter = filter;
        }
        return recordFilter;
    }
}
function estimateCondition(table) {
    function estimateConditionForTable(condition) {
        if (condition.estimated_count === undefined) {
            if (condition.conditions) {
                // for a group of conditions, we can estimate the count by combining the estimates of the sub-conditions
                let estimatedCount;
                if (condition.operator === 'or') {
                    // with a union, we can just add the estimated counts
                    estimatedCount = 0;
                    for (const subCondition of condition.conditions) {
                        estimateConditionForTable(subCondition);
                        estimatedCount += subCondition.estimated_count;
                    }
                }
                else {
                    // with an intersection, we have to use the rate of the sub-conditions to apply to estimate count of last condition
                    estimatedCount = Infinity;
                    for (const subCondition of condition.conditions) {
                        estimateConditionForTable(subCondition);
                        estimatedCount = isFinite(estimatedCount)
                            ? (estimatedCount * subCondition.estimated_count) / estimatedEntryCount(table.primaryStore)
                            : subCondition.estimated_count;
                    }
                }
                condition.estimated_count = estimatedCount;
                return condition.estimated_count;
            }
            // skip if it is cached
            let searchType = condition.comparator || condition.search_type;
            searchType = ALTERNATE_COMPARATOR_NAMES[searchType] || searchType;
            if (searchType === terms_js_1.SEARCH_TYPES.EQUALS || !searchType) {
                const attribute_name = condition[0] ?? condition.attribute;
                if (attribute_name == null || attribute_name === table.primaryKey)
                    condition.estimated_count = 1;
                else if (Array.isArray(attribute_name) && attribute_name.length > 1) {
                    const attribute = findAttribute(table.attributes, attribute_name[0]);
                    const relatedTable = attribute.definition?.tableClass || attribute.elements.definition?.tableClass;
                    const estimate = estimateCondition(relatedTable)({
                        value: condition.value,
                        attribute: attribute_name.length > 2 ? attribute_name.slice(1) : attribute_name[1],
                        comparator: 'equals',
                    });
                    const fromIndex = table.indices[attribute.relationship.from];
                    // the estimated count is sum of the estimate of the related table and the estimate of the index
                    condition.estimated_count =
                        estimate +
                            (fromIndex
                                ? (estimate * estimatedEntryCount(table.indices[attribute.relationship.from])) /
                                    (estimatedEntryCount(relatedTable.primaryStore) || 1)
                                : estimate);
                }
                else {
                    // we only attempt to estimate count on equals operator because that's really all that LMDB supports (some other key-value stores like libmdbx could be considered if we need to do estimated counts of ranges at some point)
                    const index = table.indices[attribute_name];
                    condition.estimated_count = index ? index.getValuesCount(condition[1] ?? condition.value) : Infinity;
                }
            }
            else if (searchType === 'contains' || searchType === 'ends_with' || searchType === 'ne') {
                const attribute_name = condition[0] ?? condition.attribute;
                const index = table.indices[attribute_name];
                if (condition.value === null && searchType === 'ne') {
                    condition.estimated_count =
                        estimatedEntryCount(table.primaryStore) - (index ? index.getValuesCount(null) : 0);
                }
                else
                    condition.estimated_count = Infinity;
                // for range queries (betweens, startsWith, greater, etc.), just arbitrarily guess
            }
            else if (searchType === 'starts_with' || searchType === 'prefix')
                condition.estimated_count = STARTS_WITH_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
            else if (searchType === 'between')
                condition.estimated_count = BETWEEN_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
            else if (searchType === 'sort') {
                const attribute_name = condition[0] ?? condition.attribute;
                const index = table.indices[attribute_name];
                if (index?.customIndex?.estimateCountAsSort)
                    // allow custom index to define its own estimation of counts
                    condition.estimated_count = index.customIndex.estimateCountAsSort(condition);
                else
                    condition.estimated_count = estimatedEntryCount(table.primaryStore) + 1; // only used by sort
            }
            else {
                // for the search types that use the broadest range, try do them last
                const attribute_name = condition[0] ?? condition.attribute;
                const index = table.indices[attribute_name];
                if (index?.customIndex?.estimateCount)
                    // allow custom index to define its own estimation of counts
                    condition.estimated_count = index.customIndex.estimateCount(condition.value);
                else
                    condition.estimated_count = OPEN_RANGE_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
            }
            // we give a condition significantly more weight/preference if we will be ordering by it
            if (typeof condition.descending === 'boolean')
                condition.estimated_count /= 2;
        }
        return condition.estimated_count; // use cached count
    }
    return estimateConditionForTable;
}
class SyntaxViolation extends hdbError_js_1.Violation {
}
const NEEDS_PARSER = /[()[\]|!<>.]|(=\w*=)/;
const QUERY_PARSER = /([^?&|=<>!([{}\]),]*)([([{}\])|,&]|[=<>!]*)/g;
const VALUE_PARSER = /([^&|=[\]{}]+)([[\]{}]|[&|=]*)/g;
let lastIndex;
let currentQuery;
let queryString;
/**
 * This is responsible for taking a query string (from a get()) and merging the parsed elements into a RequestTarget object.
 * @param queryString
 */
function parseQuery(queryToParse, query) {
    if (!queryToParse)
        return;
    queryString = queryToParse;
    // TODO: We can remove this if we are sure all exits points end with lastIndex as zero (reaching the end of parsing will do that)
    QUERY_PARSER.lastIndex = 0;
    if (NEEDS_PARSER.test(queryToParse)) {
        try {
            if (query)
                query.conditions = [];
            currentQuery = query ?? new Query();
            parseBlock(currentQuery, '');
            if (lastIndex !== queryString.length)
                recordError(`Unable to parse query, unexpected end of query`);
            if (currentQuery.parseErrorMessage) {
                currentQuery.parseError = new SyntaxViolation(query.parseErrorMessage);
                if (!query)
                    throw currentQuery.parseError;
            }
            return currentQuery;
        }
        catch (error) {
            error.statusCode = 400;
            error.message = `Unable to parse query, ${error.message} at position ${lastIndex} in '${queryString}'`;
            if (currentQuery.parseErrorMessage)
                error.message += ', ' + currentQuery.parseErrorMessage;
            if (query) {
                query.parseError = error;
            }
            else {
                throw error;
            }
        }
    }
    else {
        return query ?? new URLSearchParams(queryToParse);
    }
}
function recordError(message) {
    const errorMessage = `${message} at position ${lastIndex}`;
    currentQuery.parseErrorMessage = currentQuery.parseErrorMessage
        ? currentQuery.parseErrorMessage + ', ' + errorMessage
        : errorMessage;
}
function parseBlock(query, expectedEnd) {
    let parser = QUERY_PARSER;
    let match;
    let attribute, comparator, expectingDelimiter, expectingValue;
    let valueDecoder = decodeURIComponent;
    let lastBinaryOperator;
    while ((match = parser.exec(queryString))) {
        lastIndex = parser.lastIndex;
        const [, value, operator] = match;
        if (expectingDelimiter) {
            if (value)
                recordError(`expected operator, but encountered '${value}'`);
            expectingDelimiter = false;
            expectingValue = false;
        }
        else
            expectingValue = true;
        let entry;
        switch (operator) {
            case '=':
                if (attribute != undefined) {
                    // a FIQL operator like =gt= (and don't allow just any string)
                    if (value.length <= 2)
                        comparator = value;
                    else
                        recordError(`invalid FIQL operator ${value}`);
                    valueDecoder = typedDecoding; // use typed/auto-cast decoding for FIQL operators
                }
                else {
                    // standard equal comparison
                    valueDecoder = decodeURIComponent; // use strict decoding
                    comparator = 'equals'; // strict equals
                    if (!value)
                        recordError(`attribute must be specified before equality comparator`);
                    attribute = decodeProperty(value);
                }
                break;
            case '==':
            // TODO: Separate decoder to handle * operator here for startsWith, endsWith, and contains?
            // fall through
            case '!=':
            case '<':
            case '<=':
            case '>':
            case '>=':
            case '===':
            case '!==':
                comparator = SYMBOL_OPERATORS[operator];
                valueDecoder = exports.COERCIBLE_OPERATORS[comparator] ? typedDecoding : decodeURIComponent;
                if (!value)
                    recordError(`attribute must be specified before comparator ${operator}`);
                attribute = decodeProperty(value);
                break;
            case '&=': // for chaining conditions on to the same attribute
            case '|=':
            case '|':
            case '&':
            case '':
            case undefined:
                if (attribute == null) {
                    if (attribute === undefined) {
                        if (expectedEnd)
                            recordError(`expected '${expectedEnd}', but encountered ${operator[0] ? "'" + operator[0] + "'" : 'end of string'}}`);
                        recordError(`no comparison specified before ${operator ? "'" + operator + "'" : 'end of string'}`);
                    }
                }
                else {
                    if (!query.conditions)
                        recordError('conditions/comparisons are not allowed in a property list');
                    const condition = {
                        comparator,
                        attribute: attribute || null,
                        value: valueDecoder(value),
                    };
                    if (comparator === 'eq')
                        wildcardDecoding(condition, value);
                    if (attribute === '') {
                        // this is a nested condition
                        const lastCondition = query.conditions[query.conditions.length - 1];
                        lastCondition.chainedConditions = lastCondition.chainedConditions || [];
                        lastCondition.chainedConditions.push(condition);
                        lastCondition.operator = lastBinaryOperator;
                    }
                    else {
                        assignOperator(query, lastBinaryOperator);
                        query.conditions.push(condition);
                    }
                }
                if (operator === '&') {
                    lastBinaryOperator = 'and';
                    attribute = undefined;
                }
                else if (operator === '|') {
                    lastBinaryOperator = 'or';
                    attribute = undefined;
                }
                else if (operator === '&=') {
                    lastBinaryOperator = 'and';
                    attribute = '';
                }
                else if (operator === '|=') {
                    lastBinaryOperator = 'or';
                    attribute = '';
                }
                break;
            case ',':
                if (query.conditions) {
                    // TODO: Add support for a list of values
                    recordError('conditions/comparisons are not allowed in a property list');
                }
                else {
                    query.push(decodeProperty(value));
                }
                attribute = undefined;
                break;
            case '(':
                QUERY_PARSER.lastIndex = lastIndex;
                const args = parseBlock(value ? [] : new Query(), ')');
                switch (value) {
                    case '': // nested/grouped condition
                        assignOperator(query, lastBinaryOperator);
                        query.conditions.push(args);
                        break;
                    case 'limit':
                        switch (args.length) {
                            case 1:
                                query.limit = +args[0];
                                break;
                            case 2:
                                query.offset = +args[0];
                                query.limit = args[1] - query.offset;
                                break;
                            default:
                                recordError('limit must have 1 or 2 arguments');
                        }
                        break;
                    case 'select':
                        if (Array.isArray(args[0]) && args.length === 1 && !args[0].name) {
                            query.select = args[0];
                            query.select.asArray = true;
                        }
                        else if (args.length === 1)
                            query.select = args[0];
                        else if (args.length === 2 && args[1] === '')
                            query.select = args.slice(0, 1);
                        else
                            query.select = args;
                        break;
                    case 'group-by':
                        recordError('group by is not implemented yet');
                    case 'sort':
                        query.sort = toSortObject(args);
                        break;
                    default:
                        recordError(`unknown query function call ${value}`);
                }
                if (queryString[lastIndex] === ',') {
                    parser.lastIndex = ++lastIndex;
                }
                else
                    expectingDelimiter = true;
                attribute = null;
                break;
            case '{':
                if (query.conditions)
                    recordError('property sets are not allowed in a queries');
                if (!value)
                    recordError('property sets must have a defined parent property name');
                // this is interpreted as property{subProperty}
                QUERY_PARSER.lastIndex = lastIndex;
                entry = parseBlock([], '}');
                entry.name = value;
                query.push(entry);
                if (queryString[lastIndex] === ',') {
                    parser.lastIndex = ++lastIndex;
                }
                else
                    expectingDelimiter = true;
                break;
            case '[':
                QUERY_PARSER.lastIndex = lastIndex;
                if (value) {
                    // this is interpreted as propertyWithArray[name=value&anotherOtherConditions...]
                    entry = parseBlock(new Query(), ']');
                    entry.name = value;
                }
                else {
                    // this is interpreted a property list that can be used within other lists
                    entry = parseBlock(query.conditions ? new Query() : [], ']');
                }
                if (query.conditions) {
                    assignOperator(query, lastBinaryOperator);
                    if (queryString[lastIndex] === '=') {
                        // handle the case of a query parameter like property[]=value, using the standard equal behavior
                        valueDecoder = decodeURIComponent; // use strict decoding
                        comparator = 'equals'; // strict equals
                        attribute = decodeProperty(value);
                        parser.lastIndex = ++lastIndex;
                        break;
                    }
                    else {
                        query.conditions.push(entry);
                        attribute = null;
                    }
                }
                else
                    query.push(entry);
                if (queryString[lastIndex] === ',') {
                    parser.lastIndex = ++lastIndex;
                }
                else
                    expectingDelimiter = true;
                break;
            case ')':
            case ']':
            case '}':
                if (expectedEnd === operator[0]) {
                    // assert that it is expected
                    if (query.conditions) {
                        // finish condition
                        if (attribute) {
                            const condition = {
                                comparator: comparator || 'equals',
                                attribute,
                                value: valueDecoder(value),
                            };
                            if (comparator === 'eq')
                                wildcardDecoding(condition, value);
                            assignOperator(query, lastBinaryOperator);
                            query.conditions.push(condition);
                        }
                        else if (value) {
                            recordError('no attribute or comparison specified');
                        }
                    }
                    else if (value || (query.length > 0 && expectingValue)) {
                        query.push(decodeProperty(value));
                    }
                    return query;
                }
                else if (expectedEnd)
                    recordError(`expected '${expectedEnd}', but encountered '${operator[0]}'`);
                else
                    recordError(`unexpected token '${operator[0]}'`);
            default:
                recordError(`unexpected operator '${operator}'`);
        }
        if (expectedEnd !== ')') {
            parser = attribute ? VALUE_PARSER : QUERY_PARSER;
            parser.lastIndex = lastIndex;
        }
        if (lastIndex === queryString.length)
            return query;
    }
    if (expectedEnd)
        recordError(`expected '${expectedEnd}', but encountered end of string`);
}
function assignOperator(query, lastBinaryOperator) {
    if (query.conditions.length > 0) {
        if (query.operator) {
            if (query.operator !== lastBinaryOperator)
                recordError('Can not mix operators within a condition grouping');
        }
        else
            query.operator = lastBinaryOperator;
    }
}
function decodeProperty(name) {
    if (name.indexOf('.') > -1) {
        return name.split('.').map(decodeProperty);
    }
    return decodeURIComponent(name);
}
function typedDecoding(value) {
    // for non-strict operators, we allow for coercion of types
    if (value === 'null')
        return null;
    if (value.indexOf(':') > -1) {
        const [type, valueToCoerce] = value.split(':');
        if (type === 'number') {
            if (valueToCoerce[0] === '$')
                return parseInt(valueToCoerce.slice(1), 36);
            return +valueToCoerce;
        }
        else if (type === 'boolean')
            return valueToCoerce === 'true';
        else if (type === 'date')
            return new Date(isNaN(valueToCoerce) ? decodeURIComponent(valueToCoerce) : +valueToCoerce);
        else if (type === 'string')
            return decodeURIComponent(valueToCoerce);
        else
            throw new hdbError_js_1.ClientError(`Unknown type ${type}`);
    }
    return decodeURIComponent(value);
}
/**
 * Perform wildcard detection and conversion to correct comparator
 * @param condition
 * @param value
 */
function wildcardDecoding(condition, value) {
    if (value.indexOf('*') > -1) {
        if (value.endsWith('*')) {
            condition.comparator = 'starts_with';
            condition.value = decodeURIComponent(value.slice(0, -1));
        }
        else {
            throw new hdbError_js_1.ClientError('wildcard can only be used at the end of a string');
        }
    }
}
function toSortObject(sort) {
    const sortObject = toSortEntry(sort[0]);
    if (sort.length > 1) {
        sortObject.next = toSortObject(sort.slice(1));
    }
    return sortObject;
}
function toSortEntry(sort) {
    if (Array.isArray(sort)) {
        const sortObject = toSortEntry(sort[0]);
        sort[0] = sortObject.attribute;
        sortObject.attribute = sort;
        return sortObject;
    }
    if (typeof sort === 'string') {
        switch (sort[0]) {
            case '-':
                return { attribute: sort.slice(1), descending: true };
            case '+':
                return { attribute: sort.slice(1), descending: false };
            default:
                return { attribute: sort, descending: false };
        }
    }
    recordError(`Unknown sort type ${sort}`);
}
class Query {
    constructor() {
        this.conditions = [];
    }
    [Symbol.iterator]() {
        return this.conditions[Symbol.iterator]();
    }
    get(name) {
        for (let i = 0; i < this.conditions.length; i++) {
            const condition = this.conditions[i];
            if (condition.attribute === name)
                return condition.value;
        }
    }
    getAll() {
        const values = [];
        for (let i = 0, len = this.conditions.length; i < len; i++) {
            const condition = this.conditions[i];
            if (condition.attribute)
                values.push(condition.value);
        }
        return values;
    }
}
function flattenKey(key) {
    if (Array.isArray(key))
        return key.join('\x00');
    return key;
}
function estimatedEntryCount(store) {
    const now = Date.now();
    if ((store.estimatedEntryCountExpires || 0) < now) {
        // use getStats for LMDB because it is fast path, otherwise RocksDB can handle fast path on its own
        store.estimatedEntryCount = store.readerCheck ? store.getStats().entryCount : store.getKeysCount();
        store.estimatedEntryCountExpires = now + 10000;
    }
    return store.estimatedEntryCount;
}
function intersectionEstimate(store, left, right) {
    return (left * right) / estimatedEntryCount(store);
}
//# sourceMappingURL=search.js.map