"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
    var ownKeys = function(o) {
        ownKeys = Object.getOwnPropertyNames || function (o) {
            var ar = [];
            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
            return ar;
        };
        return ownKeys(o);
    };
    return function (mod) {
        if (mod && mod.__esModule) return mod;
        var result = {};
        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
        __setModuleDefault(result, mod);
        return result;
    };
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.databases = exports.tables = exports.databaseEventsEmitter = exports.NON_REPLICATING_SYSTEM_TABLES = void 0;
exports.getTables = getTables;
exports.getDatabases = getDatabases;
exports.readMetaDb = readMetaDb;
exports.resetDatabases = resetDatabases;
exports.database = database;
exports.dropDatabase = dropDatabase;
exports.table = table;
exports.dropTableMeta = dropTableMeta;
exports.onUpdatedTable = onUpdatedTable;
exports.onRemovedTable = onRemovedTable;
exports.onRemovedDB = onRemovedDB;
exports.getDefaultCompression = getDefaultCompression;
const node_events_1 = require("node:events");
const environmentManager_js_1 = require("../utility/environment/environmentManager.js");
const terms_js_1 = require("../utility/lmdb/terms.js");
const lmdb_1 = require("lmdb");
const path_1 = require("path");
const node_fs_1 = require("node:fs");
const promises_1 = require("node:fs/promises");
const initializePaths_js_1 = require("../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js");
const Table_ts_1 = require("./Table.js");
const OpenEnvironmentObject_js_1 = __importDefault(require("../utility/lmdb/OpenEnvironmentObject.js"));
const hdbTerms_ts_1 = require("../utility/hdbTerms.js");
const configUtils_js_1 = require("../config/configUtils.js");
const globals_js_1 = require("../globals.js");
const commonUtility_js_1 = require("../utility/lmdb/commonUtility.js");
const signalling = __importStar(require("../utility/signalling.js"));
const itc_js_1 = require("../server/threads/itc.js");
const worker_threads_1 = require("worker_threads");
const harper_logger_js_1 = __importDefault(require("../utility/logging/harper_logger.js"));
const { forComponent } = harper_logger_js_1.default;
const manageThreads = __importStar(require("../server/threads/manageThreads.js"));
const auditStore_ts_1 = require("./auditStore.js");
const RecordEncoder_ts_1 = require("./RecordEncoder.js");
const blob_ts_1 = require("./blob.js");
const customIndexes_ts_1 = require("./indexes/customIndexes.js");
const OpenDBIObject_js_1 = require("../utility/lmdb/OpenDBIObject.js");
const rocksdb_js_1 = require("@harperfast/rocksdb-js");
const replayLogs_ts_1 = require("./replayLogs.js");
const node_os_1 = require("node:os");
const RocksIndexStore_ts_1 = require("./RocksIndexStore.js");
const when_ts_1 = require("../utility/when.js");
const processManagement_js_1 = require("../utility/processManagement/processManagement.js");
function createOpenDBIObject(dupSort = false, isPrimary = false) {
    return new OpenDBIObject_js_1.OpenDBIObject(dupSort, isPrimary);
}
const logger = forComponent('storage');
const DEFAULT_DATABASE_NAME = 'data';
const DEFINED_TABLES = Symbol('defined-tables');
const DEFAULT_COMPRESSION_THRESHOLD = ((0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PAGESIZE) || 4096) - 60; // larger than this requires multiple pages
(0, environmentManager_js_1.initSync)();
// I don't know if this is the best place for this, but somewhere we need to specify which tables
// replicate by default:
exports.NON_REPLICATING_SYSTEM_TABLES = [
    'hdb_temp',
    'hdb_certificate',
    'hdb_raw_analytics',
    'hdb_session_will',
    'hdb_job',
    'hdb_info',
];
exports.databaseEventsEmitter = new node_events_1.EventEmitter();
exports.tables = Object.create(null);
exports.databases = Object.create(null);
const MEMORY_FOR_ROCKS_DB = Math.min(process.constrainedMemory?.() ?? Infinity, (0, node_os_1.totalmem)()) * 0.25; // 25% of available memory
function openRocksDatabase(path, options) {
    options.disableWAL ??= true;
    rocksdb_js_1.RocksDatabase.config({ blockCacheSize: MEMORY_FOR_ROCKS_DB });
    if (!(0, node_fs_1.existsSync)(path)) {
        (0, node_fs_1.mkdirSync)(path, { recursive: true });
    }
    let db;
    if (options.dupSort) {
        db = new RocksIndexStore_ts_1.RocksIndexStore(path, options).open();
    }
    else {
        db = rocksdb_js_1.RocksDatabase.open(path, options);
        // the RocksDB put and remove return promises, which masks thrown errors in non-awaiting calls to put/remove,
        // making them unsafe to replace LMDB methods, which will synchronously throw errors if there is a problem
        db.put = db.putSync;
        db.remove = db.removeSync;
        db.encoder.name = options.name;
    }
    db.env = {};
    return db;
}
const lmdbDatabaseEnvs = new Map();
const rocksdbDatabaseEnvs = new Map();
// set the following in both global and exports
(0, globals_js_1._assignPackageExport)('databases', exports.databases);
(0, globals_js_1._assignPackageExport)('tables', exports.tables);
const NEXT_TABLE_ID = Symbol.for('next-table-id');
let loadedDatabases; // indicates if we have loaded databases from the file system yet
// This is used to track all the databases that are found when iterating through the file system so that anything that is missing
// can be removed:
let definedDatabases;
/**
 * This gets the set of tables from the default database ("data").
 */
function getTables() {
    if (!loadedDatabases) {
        getDatabases();
    }
    return exports.tables || {};
}
/**
 * This provides the main entry point for getting the set of all Harper tables (organized by schemas/databases).
 * This proactively scans the known
 * databases/schemas directories and finds any databases and opens them. This done proactively so that there is a fast
 * object available to all consumers that doesn't require runtime checks for database open states.
 * This also attaches the audit store associated with table. Note that legacy tables had a single audit table per db table
 * but in newer multi-table databases, there is one consistent, integrated audit table for the database since transactions
 * can span any tables in the database.
 */
function getDatabases() {
    if (loadedDatabases) {
        return exports.databases;
    }
    loadedDatabases = true;
    definedDatabases = new Map();
    const hdbBasePath = (0, environmentManager_js_1.getHdbBasePath)();
    let databasePath = hdbBasePath && (0, path_1.join)(hdbBasePath, hdbTerms_ts_1.DATABASES_DIR_NAME);
    const schemaConfigs = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.DATABASES) || {};
    // not sure why this doesn't work with the environmemt manager
    if (process.env.SCHEMAS_DATA_PATH)
        schemaConfigs.data = { path: process.env.SCHEMAS_DATA_PATH };
    databasePath =
        process.env.STORAGE_PATH ||
            (0, configUtils_js_1.getConfigPath)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PATH) ||
            (databasePath && ((0, node_fs_1.existsSync)(databasePath) ? databasePath : (0, path_1.join)((0, environmentManager_js_1.getHdbBasePath)(), hdbTerms_ts_1.LEGACY_DATABASES_DIR_NAME)));
    if (!databasePath)
        return;
    if ((0, node_fs_1.existsSync)(databasePath)) {
        // First load all the databases from our main database folder
        // TODO: Load any databases defined with explicit storage paths from the config
        for (const databaseEntry of (0, node_fs_1.readdirSync)(databasePath, { withFileTypes: true })) {
            const dbName = (0, path_1.basename)(databaseEntry.name, '.mdb');
            const dbPath = (0, path_1.join)(databasePath, databaseEntry.name);
            if (databaseEntry.isFile() &&
                (0, path_1.extname)(databaseEntry.name).toLowerCase() === '.mdb' &&
                !schemaConfigs[dbName]?.path) {
                logger.trace(`loading lmdb database: ${dbPath}`);
                readMetaDb(dbPath, null, dbName);
                continue;
            }
            try {
                const files = (0, node_fs_1.readdirSync)(dbPath, { withFileTypes: true });
                if (files.find((file) => file.name === 'CURRENT')?.isFile() &&
                    files.some((file) => file.name.startsWith('MANIFEST-')) &&
                    !schemaConfigs[dbName]?.path) {
                    readRocksMetaDb(dbPath, null, dbName);
                    continue;
                }
            }
            catch (err) {
                if (!('code' in err && (err.code === 'ENOENT' || err.code === 'ENOTDIR'))) {
                    throw err;
                }
            }
        }
    }
    // now we load databases from the legacy "schema" directory folder structure
    const baseSchemaPath = (0, initializePaths_js_1.getBaseSchemaPath)();
    if ((0, node_fs_1.existsSync)(baseSchemaPath)) {
        for (const schemaEntry of (0, node_fs_1.readdirSync)(baseSchemaPath, { withFileTypes: true })) {
            if (!schemaEntry.isFile()) {
                const schemaPath = (0, path_1.join)(baseSchemaPath, schemaEntry.name);
                const schemaAuditPath = (0, path_1.join)((0, initializePaths_js_1.getTransactionAuditStoreBasePath)(), schemaEntry.name);
                for (const tableEntry of (0, node_fs_1.readdirSync)(schemaPath, { withFileTypes: true })) {
                    if (tableEntry.isFile() && (0, path_1.extname)(tableEntry.name).toLowerCase() === '.mdb') {
                        const auditPath = (0, path_1.join)(schemaAuditPath, tableEntry.name);
                        readMetaDb((0, path_1.join)(schemaPath, tableEntry.name), (0, path_1.basename)(tableEntry.name, '.mdb'), schemaEntry.name, auditPath, true);
                    }
                }
            }
        }
    }
    if (schemaConfigs) {
        for (const dbName in schemaConfigs) {
            const schemaConfig = schemaConfigs[dbName];
            const databasePath = schemaConfig.path;
            if ((0, node_fs_1.existsSync)(databasePath)) {
                for (const databaseEntry of (0, node_fs_1.readdirSync)(databasePath, { withFileTypes: true })) {
                    if (databaseEntry.isFile() && (0, path_1.extname)(databaseEntry.name).toLowerCase() === '.mdb') {
                        readMetaDb((0, path_1.join)(databasePath, databaseEntry.name), (0, path_1.basename)(databaseEntry.name, '.mdb'), dbName);
                    }
                    else {
                        try {
                            const dbPath = (0, path_1.join)(databasePath, databaseEntry.name);
                            const files = (0, node_fs_1.readdirSync)(dbPath, { withFileTypes: true });
                            if (files.find((file) => file.name === 'CURRENT')?.isFile() &&
                                files.some((file) => file.name.startsWith('MANIFEST-'))) {
                                readRocksMetaDb(dbPath, null, dbName);
                                continue;
                            }
                        }
                        catch (err) {
                            if (!('code' in err && (err.code === 'ENOENT' || err.code === 'ENOTDIR'))) {
                                throw err;
                            }
                        }
                    }
                }
            }
            const tableConfigs = schemaConfig.tables;
            if (tableConfigs) {
                for (const tableName in tableConfigs) {
                    const tableConfig = tableConfigs[tableName];
                    const tablePath = (0, path_1.join)(tableConfig.path, (0, path_1.basename)(tableName + '.mdb'));
                    if ((0, node_fs_1.existsSync)(tablePath)) {
                        readMetaDb(tablePath, tableName, dbName, null, true);
                    }
                }
            }
            //TODO: Iterate configured table paths
        }
    }
    // now remove any databases or tables that have been removed
    for (const dbName in exports.databases) {
        const definedTables = definedDatabases.get(dbName);
        if (definedTables) {
            const tables = exports.databases[dbName];
            if (dbName.includes('delete'))
                logger.trace(`defined tables ${Array.from(definedTables.keys())}`);
            for (const tableName in tables) {
                if (!definedTables.has(tableName)) {
                    logger.trace(`delete table class ${tableName}`);
                    delete tables[tableName];
                }
            }
        }
        else {
            delete exports.databases[dbName];
            if (dbName === 'data') {
                for (const tableName in exports.tables) {
                    delete exports.tables[tableName];
                }
                delete exports.tables[DEFINED_TABLES];
            }
        }
    }
    if ((0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.ANALYTICS_REPLICATE) === false) {
        if (!exports.NON_REPLICATING_SYSTEM_TABLES.includes('hdb_analytics'))
            exports.NON_REPLICATING_SYSTEM_TABLES.push('hdb_analytics');
    }
    else {
        // auditing must be enabled for replication
        exports.databases.system?.hdb_analytics?.enableAuditing();
        exports.databases.system?.hdb_analytics_hostname?.enableAuditing();
    }
    if (exports.databases.system) {
        for (const tableName of exports.NON_REPLICATING_SYSTEM_TABLES) {
            if (exports.databases.system[tableName]) {
                exports.databases.system[tableName].replicate = false;
            }
        }
    }
    return exports.databases;
}
/**
 * This is responsible for reading the internal dbi of a single database file to get a list of all the tables and
 * their indexed or registered attributes
 * @param path
 * @param defaultTable
 * @param databaseName
 */
function readMetaDb(path, defaultTable, databaseName = DEFAULT_DATABASE_NAME, auditPath, isLegacy) {
    const envInit = new OpenEnvironmentObject_js_1.default(path, false);
    try {
        let rootStore = lmdbDatabaseEnvs.get(path);
        if (rootStore) {
            rootStore.needsDeletion = false;
        }
        else {
            rootStore = (0, lmdb_1.open)(envInit);
            lmdbDatabaseEnvs.set(path, rootStore);
        }
        return initStores(path, rootStore, databaseName, defaultTable, auditPath, isLegacy);
    }
    catch (error) {
        error.message += ` opening database ${path}`;
        throw error;
    }
}
function readRocksMetaDb(path, defaultTable, databaseName = DEFAULT_DATABASE_NAME) {
    try {
        logger.trace(`loading rocksdb database: ${path}`);
        if (process.env.HARPER_PARENT_PROCESS_PID) {
            const parentProcessPid = parseInt(process.env.HARPER_PARENT_PROCESS_PID);
            if ((0, processManagement_js_1.isProcessRunning)(parentProcessPid)) {
                logger.info(`Parent process ${parentProcessPid} is still running!`);
            }
        }
        let rootStore = rocksdbDatabaseEnvs.get(path);
        if (rootStore) {
            initStores(path, rootStore, databaseName, defaultTable);
        }
        else {
            rootStore = openRocksDatabase(path, { disableWAL: false });
            rocksdbDatabaseEnvs.set(path, rootStore);
            initStores(path, rootStore, databaseName, defaultTable);
            (0, replayLogs_ts_1.replayLogs)(rootStore, exports.databases[databaseName]);
        }
        return rootStore;
    }
    catch (error) {
        error.message += ` opening database ${path}`;
        throw error;
    }
}
function initStores(path, rootStore, databaseName, defaultTable, auditPath, isLegacy) {
    const envInit = new OpenEnvironmentObject_js_1.default(path, false);
    const internalDbiInit = createOpenDBIObject(false);
    let attributesDbi = rootStore.dbisDb;
    if (!attributesDbi) {
        if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
            attributesDbi = openRocksDatabase(rootStore.path, {
                ...internalDbiInit,
                disableWAL: false,
                name: terms_js_1.INTERNAL_DBIS_NAME,
            });
        }
        else {
            attributesDbi = rootStore.openDB(terms_js_1.INTERNAL_DBIS_NAME, internalDbiInit);
        }
        rootStore.dbisDb = attributesDbi;
    }
    let auditStore = rootStore.auditStore;
    if (!auditStore) {
        if (auditPath) {
            if ((0, node_fs_1.existsSync)(auditPath)) {
                envInit.path = auditPath;
                if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
                    auditStore = (0, auditStore_ts_1.openAuditStore)(rootStore);
                }
                else {
                    auditStore = (0, lmdb_1.open)({
                        ...envInit,
                        encoder: {
                            encode: (auditRecord) => (0, auditStore_ts_1.createAuditEntry)(auditRecord),
                            decode: (encoding) => (0, auditStore_ts_1.readAuditEntry)(encoding),
                        },
                    });
                }
                auditStore.isLegacy = true;
            }
        }
        else {
            auditStore = (0, auditStore_ts_1.openAuditStore)(rootStore);
        }
    }
    const tables = ensureDB(databaseName);
    const definedTables = tables[DEFINED_TABLES];
    definedTables.rootStore = rootStore;
    const tablesToLoad = new Map();
    for (const result of attributesDbi.getRange({ start: false })) {
        const { key, value } = result;
        let [tableName, attribute_name] = key.toString().split('/');
        if (attribute_name === '') {
            // primary key
            attribute_name = value.name;
        }
        else if (!attribute_name) {
            attribute_name = tableName;
            tableName = defaultTable;
            if (!value.name) {
                // legacy attribute
                value.name = attribute_name;
                value.indexed = !value.isPrimaryKey;
            }
        }
        definedTables?.add(tableName);
        let tableDef = tablesToLoad.get(tableName);
        if (!tableDef)
            tablesToLoad.set(tableName, (tableDef = { attributes: [] }));
        if (attribute_name == null || value.isPrimaryKey)
            tableDef.primary = value;
        if (attribute_name != null)
            tableDef.attributes.push(value);
        Object.defineProperty(value, 'key', { value: key, configurable: true });
    }
    for (const [tableName, tableDef] of tablesToLoad) {
        let { attributes, primary: primaryAttribute } = tableDef;
        if (!primaryAttribute) {
            // this isn't defined, find it in the attributes
            for (const attribute of attributes) {
                if (attribute.isPrimaryKey) {
                    primaryAttribute = attribute;
                    break;
                }
            }
            if (!primaryAttribute) {
                logger.warn(`Unable to find a primary key attribute on table ${tableName}, with attributes: ${JSON.stringify(attributes)}`);
                continue;
            }
        }
        // if the table has already been defined, use that class, don't create a new one
        let table = tables[tableName];
        let indices = {}, existingAttributes = [];
        let tableId;
        let primaryStore;
        const audit = typeof primaryAttribute.audit === 'boolean' ? primaryAttribute.audit : (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.LOGGING_AUDITLOG);
        const trackDeletes = primaryAttribute.trackDeletes;
        const expiration = primaryAttribute.expiration;
        const eviction = primaryAttribute.eviction;
        const sealed = primaryAttribute.sealed;
        const splitSegments = primaryAttribute.splitSegments;
        const replicate = primaryAttribute.replicate;
        if (table) {
            indices = table.indices;
            existingAttributes = table.attributes;
            table.schemaVersion++;
        }
        else {
            tableId = primaryAttribute.tableId;
            if (tableId) {
                if (tableId >= (attributesDbi.getSync(NEXT_TABLE_ID) || 0)) {
                    attributesDbi.putSync(NEXT_TABLE_ID, tableId + 1);
                    logger.info(`Updating next table id (it was out of sync) to ${tableId + 1} for ${tableName}`);
                }
            }
            else {
                primaryAttribute.tableId = tableId = attributesDbi.getSync(NEXT_TABLE_ID);
                if (!tableId)
                    tableId = 1;
                logger.debug(`Table {tableName} missing an id, assigning {tableId}`);
                attributesDbi.putSync(NEXT_TABLE_ID, tableId + 1);
                attributesDbi.putSync(primaryAttribute.key, primaryAttribute);
            }
            const dbiInit = createOpenDBIObject(!primaryAttribute.isPrimaryKey, primaryAttribute.isPrimaryKey);
            dbiInit.compression = primaryAttribute.compression;
            if (dbiInit.compression) {
                const compressionThreshold = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION_THRESHOLD) || DEFAULT_COMPRESSION_THRESHOLD; // this is the only thing that can change;
                dbiInit.compression.threshold = compressionThreshold;
            }
            if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
                primaryStore = (0, RecordEncoder_ts_1.handleLocalTimeForGets)(openRocksDatabase(rootStore.path, { ...dbiInit, name: primaryAttribute.key }), rootStore);
            }
            else {
                primaryStore = (0, RecordEncoder_ts_1.handleLocalTimeForGets)(rootStore.openDB(primaryAttribute.key, dbiInit), rootStore);
            }
            rootStore.databaseName = databaseName;
            primaryStore.tableId = tableId;
        }
        let attributesUpdated;
        for (const attribute of attributes) {
            attribute.attribute = attribute.name;
            try {
                // now load the non-primary keys, opening the dbs as necessary for indices
                if (!attribute.isPrimaryKey && (attribute.indexed || (attribute.attribute && !attribute.name))) {
                    if (!indices[attribute.name]) {
                        const dbi = openIndex(attribute.key, rootStore, attribute);
                        indices[attribute.name] = dbi;
                        indices[attribute.name].indexNulls = attribute.indexNulls;
                    }
                    const existingAttribute = existingAttributes.find((existingAttribute) => existingAttribute.name === attribute.name);
                    if (existingAttribute)
                        existingAttributes.splice(existingAttributes.indexOf(existingAttribute), 1, attribute);
                    else
                        existingAttributes.push(attribute);
                    attributesUpdated = true;
                }
            }
            catch (error) {
                logger.error(`Error trying to update attribute`, attribute, existingAttributes, indices, error);
            }
        }
        for (const existingAttribute of existingAttributes) {
            const attribute = attributes.find((attribute) => attribute.name === existingAttribute.name);
            if (!attribute) {
                if (existingAttribute.isPrimaryKey) {
                    logger.error(new Error('Unable to remove existing primary key attribute'), existingAttribute, 'from attributes', existingAttributes, 'in', tableName, 'requesting new attribute list', attributes, 'full metadata list', Array.from(attributesDbi.getRange({ start: false })));
                    continue;
                }
                if (existingAttribute.indexed) {
                    // we only remove attributes if they were indexed, in order to support dropAttribute that removes dynamic indexed attributes
                    existingAttributes.splice(existingAttributes.indexOf(existingAttribute), 1);
                    attributesUpdated = true;
                }
            }
        }
        if (table) {
            if (attributesUpdated) {
                table.schemaVersion++;
                table.updatedAttributes();
            }
        }
        else {
            table = setTable(tables, tableName, (0, Table_ts_1.makeTable)({
                primaryStore,
                auditStore,
                audit,
                sealed,
                splitSegments,
                replicate,
                expirationMS: expiration && expiration * 1000,
                evictionMS: eviction && eviction * 1000,
                trackDeletes,
                tableName,
                tableId,
                primaryKey: primaryAttribute.name,
                databasePath: isLegacy ? `${databaseName}/${tableName}` : databaseName,
                databaseName,
                indices,
                attributes,
                schemaDefined: primaryAttribute.schemaDefined,
                dbisDB: attributesDbi,
            }));
            table.schemaVersion = 1;
            exports.databaseEventsEmitter.emit('updateTable', table);
        }
    }
    return rootStore;
}
function resetDatabases() {
    loadedDatabases = false;
    for (const store of Object.values(lmdbDatabaseEnvs)) {
        store.needsDeletion = true;
    }
    getDatabases();
    for (const [path, store] of lmdbDatabaseEnvs) {
        if (store.needsDeletion && !path.endsWith('system.mdb')) {
            store.close();
            lmdbDatabaseEnvs.delete(path);
            const db = exports.databases[store.databaseName];
            for (const tableName in db) {
                const table = db[tableName];
                if (table.primaryStore.path === path) {
                    delete exports.databases[store.databaseName];
                    exports.databaseEventsEmitter.emit('dropDatabase', store.databaseName);
                    break;
                }
            }
        }
    }
    return exports.databases;
}
/**
 * Ensure that we have this database object (that holds a set of tables) set up
 * @param databaseName
 * @returns
 */
function ensureDB(databaseName) {
    let dbTables = exports.databases[databaseName];
    if (!dbTables) {
        if (databaseName === 'data')
            // preserve the data tables objet
            dbTables = exports.databases[databaseName] = exports.tables;
        else if (databaseName === 'system')
            // make system non-enumerable
            Object.defineProperty(exports.databases, 'system', {
                value: (dbTables = Object.create(null)),
                configurable: true, // no enum
            });
        else {
            dbTables = exports.databases[databaseName] = Object.create(null);
        }
    }
    if (definedDatabases && !definedDatabases.has(databaseName)) {
        const definedTables = new Set(); // we create this so we can determine what was found in a reset and remove any removed dbs/tables
        dbTables[DEFINED_TABLES] = definedTables;
        definedDatabases.set(databaseName, definedTables);
    }
    return dbTables;
}
/**
 * Set the table class into the database's tables object
 * @param tables
 * @param tableName
 * @param Table
 * @returns
 */
function setTable(tables, tableName, Table) {
    tables[tableName] = Table;
    return Table;
}
/**
 * Get root store for a database
 * @param options
 * @returns
 */
function database({ database: databaseName, table: tableName }) {
    if (!databaseName)
        databaseName = DEFAULT_DATABASE_NAME;
    getDatabases();
    ensureDB(databaseName);
    const definedDatabase = definedDatabases.get(databaseName);
    if (definedDatabase?.rootStore) {
        return definedDatabase.rootStore;
    }
    const databaseConfig = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.DATABASES) || {};
    if (process.env.SCHEMAS_DATA_PATH) {
        databaseConfig.data = { path: process.env.SCHEMAS_DATA_PATH };
    }
    const tablePath = tableName && databaseConfig[databaseName]?.tables?.[tableName]?.path;
    const hdbBasePath = (0, environmentManager_js_1.getHdbBasePath)();
    const databasePath = tablePath ||
        databaseConfig[databaseName]?.path ||
        process.env.STORAGE_PATH ||
        (0, configUtils_js_1.getConfigPath)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PATH) ||
        ((0, node_fs_1.existsSync)((0, path_1.join)(hdbBasePath, hdbTerms_ts_1.DATABASES_DIR_NAME))
            ? (0, path_1.join)(hdbBasePath, hdbTerms_ts_1.DATABASES_DIR_NAME)
            : (0, path_1.join)(hdbBasePath, hdbTerms_ts_1.LEGACY_DATABASES_DIR_NAME));
    let rootStore;
    const useRocksdb = (process.env.HARPER_STORAGE_ENGINE || (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_ENGINE)) !== 'lmdb';
    if (useRocksdb) {
        const path = (0, path_1.join)(databasePath, tablePath ? tableName : databaseName);
        rootStore = rocksdbDatabaseEnvs.get(path);
        if (!rootStore || rootStore.status === 'closed') {
            rootStore = openRocksDatabase(path, {
                disableWAL: false,
            });
            rocksdbDatabaseEnvs.set(path, rootStore);
        }
    }
    else {
        const path = (0, path_1.join)(databasePath, `${tablePath ? tableName : databaseName}.mdb`);
        rootStore = lmdbDatabaseEnvs.get(path);
        if (!rootStore || rootStore.status === 'closed') {
            // TODO: validate database name
            const envInit = new OpenEnvironmentObject_js_1.default(path, false);
            rootStore = (0, lmdb_1.open)(envInit);
            lmdbDatabaseEnvs.set(path, rootStore);
        }
    }
    if (!rootStore.auditStore) {
        rootStore.auditStore = (0, auditStore_ts_1.openAuditStore)(rootStore);
    }
    if (definedDatabase)
        definedDatabase.rootStore = rootStore;
    return rootStore;
}
/**
 * Delete the database
 * @param databaseName
 */
async function dropDatabase(databaseName) {
    if (!exports.databases[databaseName])
        throw new Error('Database does not exist');
    const dbTables = exports.databases[databaseName];
    let rootStore;
    for (const tableName in dbTables) {
        const table = dbTables[tableName];
        rootStore = table.primaryStore.rootStore;
        lmdbDatabaseEnvs.delete(rootStore.path);
        rocksdbDatabaseEnvs.delete(rootStore.path);
    }
    for (const tableName in dbTables) {
        exports.databaseEventsEmitter.emit('dropTable', tableName, databaseName);
    }
    if (databaseName === 'data') {
        for (const tableName in exports.tables) {
            delete exports.tables[tableName];
        }
        delete exports.tables[DEFINED_TABLES];
    }
    delete exports.databases[databaseName];
    exports.databaseEventsEmitter.emit('dropDatabase', databaseName);
    if (rootStore) {
        if (rootStore.status === 'open') {
            if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
                rootStore.close();
                rootStore.destroy();
            }
            else {
                await rootStore.close();
                await (0, promises_1.unlink)(rootStore.path);
            }
        }
    }
    else {
        rootStore = database({ database: databaseName, table: null });
        if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
            rootStore.close();
            rootStore.destroy();
        }
        else if (rootStore.status === 'open') {
            await rootStore.close();
            await (0, promises_1.unlink)(rootStore.path);
        }
    }
    await (0, blob_ts_1.deleteRootBlobPathsForDB)(rootStore);
}
// opens an index, consulting with custom indexes that may use alternate store configuration
function openIndex(dbiKey, rootStore, attribute) {
    const objectStorage = attribute.isPrimaryKey || (attribute.indexed.type && customIndexes_ts_1.CUSTOM_INDEXES[attribute.indexed.type]?.useObjectStore);
    const dbiInit = createOpenDBIObject(!objectStorage, objectStorage);
    let dbi;
    if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
        dbi = openRocksDatabase(rootStore.path, { ...dbiInit, name: dbiKey });
        dbi.rootStore = rootStore;
    }
    else {
        dbi = rootStore.openDB(dbiKey, dbiInit);
    }
    if (attribute.indexed.type) {
        const CustomIndex = customIndexes_ts_1.CUSTOM_INDEXES[attribute.indexed.type];
        if (CustomIndex) {
            dbi.customIndex = new CustomIndex(dbi, attribute.indexed);
        }
        else {
            logger.error(`The indexing type '${attribute.indexed.type}' is unknown`);
        }
    }
    return dbi;
}
/**
 * This can be called to ensure that the specified table exists and if it does not exist, it should be created.
 * @param tableName
 * @param databaseName
 * @param customPath
 * @param expiration
 * @param eviction
 * @param scanInterval
 * @param attributes
 * @param audit
 * @param sealed
 * @param splitSegments
 * @param replicate
 */
function table(tableDefinition) {
    let { table: tableName, database: databaseName, expiration, eviction, scanInterval, attributes, audit, sealed, splitSegments, replicate, trackDeletes, schemaDefined, origin, } = tableDefinition;
    if (!databaseName)
        databaseName = DEFAULT_DATABASE_NAME;
    const rootStore = database({ database: databaseName, table: tableName });
    const tables = exports.databases[databaseName];
    logger.trace(`Defining ${tableName} in ${databaseName}`);
    let Table = tables?.[tableName];
    if (rootStore.status === 'closed') {
        throw new Error(`Can not use a closed data store for ${tableName}`);
    }
    let primaryKey;
    let primaryKeyAttribute;
    let attributesDbi;
    if (schemaDefined == undefined)
        schemaDefined = true;
    const internalDbiInit = createOpenDBIObject(false);
    for (const attribute of attributes) {
        if (attribute.attribute && !attribute.name) {
            // there is some legacy code that calls the attribute's name the attribute's attribute
            attribute.name = attribute.attribute;
            attribute.indexed = true;
        }
        else
            attribute.attribute = attribute.name;
        if (attribute.expiresAt)
            attribute.indexed = true;
    }
    let hasChanges;
    let releaseExclusiveLock;
    if (Table) {
        primaryKey = Table.primaryKey;
        if (Table.primaryStore.rootStore.status === 'closed') {
            throw new Error(`Can not use a closed data store from ${tableName} class`);
        }
        // it table already exists, get the split segments setting
        if (splitSegments == undefined)
            splitSegments = Table.splitSegments;
        Table.attributes.splice(0, Table.attributes.length, ...attributes);
    }
    else {
        const auditStore = rootStore.auditStore;
        primaryKeyAttribute = attributes.find((attribute) => attribute.isPrimaryKey) || {};
        primaryKey = primaryKeyAttribute.name;
        primaryKeyAttribute.isPrimaryKey = true;
        primaryKeyAttribute.schemaDefined = schemaDefined;
        // can't change compression after the fact (except threshold), so save only when we create the table
        primaryKeyAttribute.compression = getDefaultCompression();
        if (trackDeletes)
            primaryKeyAttribute.trackDeletes = true;
        audit = primaryKeyAttribute.audit = typeof audit === 'boolean' ? audit : (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.LOGGING_AUDITLOG);
        if (expiration)
            primaryKeyAttribute.expiration = expiration;
        if (eviction)
            primaryKeyAttribute.eviction = eviction;
        splitSegments ??= false;
        primaryKeyAttribute.splitSegments = splitSegments; // always default to not splitting segments going forward
        if (typeof sealed === 'boolean')
            primaryKeyAttribute.sealed = sealed;
        if (typeof replicate === 'boolean')
            primaryKeyAttribute.replicate = replicate;
        if (origin) {
            if (!primaryKeyAttribute.origins)
                primaryKeyAttribute.origins = [origin];
            else if (!primaryKeyAttribute.origins.includes(origin))
                primaryKeyAttribute.origins.push(origin);
        }
        logger.trace(`${tableName} table loading, opening primary store`);
        const dbiInit = createOpenDBIObject(false, true);
        dbiInit.compression = primaryKeyAttribute.compression;
        const dbiName = tableName + '/';
        if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
            attributesDbi = rootStore.dbisDb = openRocksDatabase(rootStore.path, {
                ...internalDbiInit,
                disableWAL: false,
                name: terms_js_1.INTERNAL_DBIS_NAME,
            });
        }
        else {
            attributesDbi = rootStore.dbisDb = rootStore.openDB(terms_js_1.INTERNAL_DBIS_NAME, internalDbiInit);
        }
        exclusiveLock(); // get an exclusive lock on the database so we can verify that we are the only thread creating the table (and assigning the table id)
        if (attributesDbi.getSync(dbiName)) {
            // table was created while we were setting up
            if (releaseExclusiveLock)
                releaseExclusiveLock();
            resetDatabases();
            return table(tableDefinition);
        }
        let primaryStore;
        if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
            primaryStore = openRocksDatabase(rootStore.path, { ...dbiInit, name: dbiName });
        }
        else {
            primaryStore = rootStore.openDB(dbiName, dbiInit);
        }
        primaryStore = (0, RecordEncoder_ts_1.handleLocalTimeForGets)(primaryStore, rootStore);
        rootStore.databaseName = databaseName;
        primaryStore.tableId = attributesDbi.getSync(NEXT_TABLE_ID);
        logger.trace(`Assigning new table id ${primaryStore.tableId} for ${tableName}`);
        if (!primaryStore.tableId)
            primaryStore.tableId = 1;
        attributesDbi.put(NEXT_TABLE_ID, primaryStore.tableId + 1);
        primaryKeyAttribute.tableId = primaryStore.tableId;
        Table = setTable(tables, tableName, (0, Table_ts_1.makeTable)({
            primaryStore,
            auditStore,
            audit,
            sealed,
            splitSegments,
            replicate,
            trackDeletes,
            expirationMS: expiration && expiration * 1000,
            evictionMS: eviction && eviction * 1000,
            primaryKey,
            tableName,
            tableId: primaryStore.tableId,
            databasePath: databaseName,
            databaseName,
            indices: {},
            attributes,
            schemaDefined,
            dbisDB: attributesDbi,
        }));
        Table.schemaVersion = 1;
        hasChanges = true;
        attributesDbi.put(dbiName, primaryKeyAttribute);
    }
    const indices = Table.indices;
    if (!attributesDbi) {
        if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
            rootStore.dbisDb = openRocksDatabase(rootStore.path, {
                ...internalDbiInit,
                disableWAL: false,
                name: terms_js_1.INTERNAL_DBIS_NAME,
            });
        }
        else {
            rootStore.dbisDb = rootStore.openDB(terms_js_1.INTERNAL_DBIS_NAME, internalDbiInit);
        }
        attributesDbi = rootStore.dbisDb;
    }
    Table.dbisDB = attributesDbi;
    const indicesToRemove = [];
    for (const { key, value } of attributesDbi.getRange({ start: true })) {
        let [attributeTableName, attribute_name] = key.toString().split('/');
        if (attribute_name === '')
            attribute_name = value.name; // primary key
        if (attribute_name) {
            if (attributeTableName !== tableName)
                continue;
        }
        else {
            // table attribute for a table with no primary key, we don't want to remove this, so continue on
            continue;
        }
        const attribute = attributes.find((attribute) => attribute.name === attribute_name);
        const removeIndex = !attribute?.indexed && value.indexed && !value.isPrimaryKey;
        if (!attribute || removeIndex) {
            exclusiveLock();
            hasChanges = true;
            if (!attribute)
                attributesDbi.remove(key);
            if (removeIndex) {
                const indexDbi = Table.indices[attributeTableName];
                if (indexDbi)
                    indicesToRemove.push(indexDbi);
            }
        }
    }
    const attributesToIndex = [];
    try {
        // TODO: If we have attributes and the schemaDefined flag is not set, turn it on
        // iterate through the attributes to ensure that we have all the dbis created and indexed
        for (const attribute of attributes || []) {
            if (attribute.relationship || attribute.computed) {
                hasChanges = true; // need to update the table so the computed properties are translated to property resolvers
                if (attribute.relationship)
                    continue;
            }
            let dbiKey = tableName + '/' + (attribute.name || '');
            Object.defineProperty(attribute, 'key', { value: dbiKey, configurable: true });
            let attributeDescriptor = attributesDbi.getSync(dbiKey);
            if (attribute.isPrimaryKey) {
                attributeDescriptor = attributeDescriptor || attributesDbi.getSync((dbiKey = tableName + '/')) || {};
                // primary key can't change indexing, but settings can change
                if ((audit !== undefined && audit !== Table.audit) ||
                    (sealed !== undefined && sealed !== Table.sealed) ||
                    (replicate !== undefined && replicate !== Table.replicate) ||
                    (+expiration || undefined) !== (+attributeDescriptor.expiration || undefined) ||
                    (+eviction || undefined) !== (+attributeDescriptor.eviction || undefined) ||
                    attribute.type !== attributeDescriptor.type) {
                    const updatedPrimaryAttribute = { ...attributeDescriptor };
                    if (typeof audit === 'boolean') {
                        if (audit)
                            Table.enableAuditing(audit);
                        updatedPrimaryAttribute.audit = audit;
                    }
                    if (expiration)
                        updatedPrimaryAttribute.expiration = +expiration;
                    if (eviction)
                        updatedPrimaryAttribute.eviction = +eviction;
                    if (sealed !== undefined)
                        updatedPrimaryAttribute.sealed = sealed;
                    if (replicate !== undefined)
                        updatedPrimaryAttribute.replicate = replicate;
                    if (attribute.type)
                        updatedPrimaryAttribute.type = attribute.type;
                    hasChanges = true; // send out notification of the change
                    exclusiveLock();
                    attributesDbi.put(dbiKey, updatedPrimaryAttribute);
                }
                continue;
            }
            // note that non-indexed attributes do not need a dbi
            if (attributeDescriptor?.attribute && !attributeDescriptor.name)
                attributeDescriptor.indexed = true; // legacy descriptor
            const changed = !attributeDescriptor ||
                attributeDescriptor.type !== attribute.type ||
                JSON.stringify(attributeDescriptor.indexed) !== JSON.stringify(attribute.indexed) ||
                attributeDescriptor.nullable !== attribute.nullable ||
                attributeDescriptor.version !== attribute.version ||
                attributeDescriptor.enumerable !== attribute.enumerable ||
                JSON.stringify(attributeDescriptor.properties) !== JSON.stringify(attribute.properties) ||
                JSON.stringify(attributeDescriptor.elements) !== JSON.stringify(attribute.elements);
            if (attribute.indexed) {
                const dbi = openIndex(dbiKey, rootStore, attribute);
                if (changed ||
                    (attributeDescriptor.indexingPID && attributeDescriptor.indexingPID !== process.pid) ||
                    attributeDescriptor.restartNumber < worker_threads_1.workerData?.restartNumber) {
                    hasChanges = true;
                    exclusiveLock();
                    attributeDescriptor = attributesDbi.getSync(dbiKey);
                    if (changed ||
                        (attributeDescriptor.indexingPID && attributeDescriptor.indexingPID !== process.pid) ||
                        attributeDescriptor.restartNumber < worker_threads_1.workerData?.restartNumber) {
                        hasChanges = true;
                        if (attribute.indexNulls === undefined)
                            attribute.indexNulls = true;
                        let hasExistingData = false;
                        for (let _entry of Table.primaryStore.getRange({ start: true })) {
                            hasExistingData = true;
                            break;
                        }
                        if (hasExistingData) {
                            attribute.lastIndexedKey = attributeDescriptor?.lastIndexedKey ?? undefined;
                            attribute.indexingPID = process.pid;
                            dbi.isIndexing = true;
                            Object.defineProperty(attribute, 'dbi', { value: dbi });
                            // we only set indexing nulls to true if new or reindexing, we can't have partial indexing of null
                            attributesToIndex.push(attribute);
                        }
                    }
                    attributesDbi.put(dbiKey, attribute);
                }
                if (attributeDescriptor?.indexNulls && attribute.indexNulls === undefined)
                    attribute.indexNulls = true;
                dbi.indexNulls = attribute.indexNulls;
                indices[attribute.name] = dbi;
            }
            else if (changed) {
                hasChanges = true;
                exclusiveLock();
                attributesDbi.put(dbiKey, attribute);
            }
        }
    }
    finally {
        if (releaseExclusiveLock)
            releaseExclusiveLock();
    }
    if (hasChanges) {
        Table.schemaVersion++;
        Table.updatedAttributes();
    }
    logger.trace(`${tableName} table loading, running index`);
    if (attributesToIndex.length > 0 || indicesToRemove.length > 0) {
        Table.indexingOperation = runIndexing(Table, attributesToIndex, indicesToRemove);
    }
    else if (hasChanges)
        signalling.signalSchemaChange(new itc_js_1.SchemaEventMsg(process.pid, 'schema-change', Table.databaseName, Table.tableName));
    Table.origin = origin;
    if (hasChanges) {
        exports.databaseEventsEmitter.emit('updateTable', Table, origin !== 'cluster');
    }
    if (expiration || eviction || scanInterval)
        Table.setTTLExpiration({
            expiration,
            eviction,
            scanInterval,
        });
    logger.trace(`${tableName} table loaded`);
    return Table;
    // Acquire an exclusive lock for attribute updates
    function exclusiveLock() {
        if (releaseExclusiveLock)
            return;
        if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
            while (!rootStore.tryLock('update-attributes')) { } // use a spin lock, we really need an synchronous exclusive lock here
            releaseExclusiveLock = () => {
                rootStore.unlock('update-attributes');
            };
        }
        else {
            // we only need an exclusive transaction lock in lmdb
            rootStore.transactionSync(() => {
                return {
                    then(callback) {
                        releaseExclusiveLock = callback;
                    },
                };
            });
        }
    }
}
const MAX_OUTSTANDING_INDEXING = 1000;
const MIN_OUTSTANDING_INDEXING = 10;
async function runIndexing(Table, attributes, indicesToRemove) {
    try {
        logger.info(`Indexing ${Table.tableName} attributes`, attributes);
        await signalling.signalSchemaChange(new itc_js_1.SchemaEventMsg(process.pid, 'schema-change', Table.databaseName, Table.tableName));
        let lastResolution;
        for (const index of indicesToRemove) {
            lastResolution = index.drop();
        }
        let interrupted;
        const attributeErrorReported = {};
        let indexed = 0;
        const attributesLength = attributes.length;
        await new Promise((resolve) => setImmediate(resolve)); // yield event turn, indexing should consistently take at least one event turn
        if (attributesLength > 0) {
            let start;
            for (const attribute of attributes) {
                // if we are resuming, we need to start from the last key we indexed by all attributes
                if ((0, lmdb_1.compareKeys)(attribute.lastIndexedKey, start) < 0)
                    start = attribute.lastIndexedKey;
                if (attribute.lastIndexedKey == undefined) {
                    // if we are starting from the beginning, clear out any previous index entries since we are rewriting
                    if (attribute.dbi.clearAsync) {
                        // LMDB, note that we don't need to wait for this to complete, just gets enqueued in front of the other writes
                        attribute.dbi.clearAsync();
                    }
                    else {
                        await attribute.dbi.clear();
                    }
                }
            }
            let outstanding = 0;
            // this means that a new attribute has been introduced that needs to be indexed
            for (const { key, value: record } of Table.primaryStore.getRange({
                start,
                lazy: attributesLength < 4,
                versions: true,
                snapshot: false, // don't hold a read transaction this whole time
            })) {
                if (!record)
                    continue; // deletion entry
                // TODO: Do we ever need to interrupt due to a schema change that was not a restart?
                //if (Table.schemaVersion !== schemaVersion) return; // break out if there are any schema changes and let someone else pick it up
                outstanding++;
                // every index operation needs to be guarded by the version still be the same. If it has already changed before
                // we index, that's fine because indexing is idempotent, we can just put the same values again. If it changes
                // during the indexing, the indexing here will fail. This is also fine because it means the other thread will have
                // performed indexing and we don't need to do anything further
                for (let i = 0; i < attributesLength; i++) {
                    const attribute = attributes[i];
                    const property = attribute.name;
                    const index = attribute.dbi;
                    try {
                        const resolver = attribute.resolve;
                        const value = record && (resolver ? resolver(record) : record[property]);
                        if (index.customIndex) {
                            index.customIndex.index(key, value);
                            continue;
                        }
                        const values = (0, commonUtility_js_1.getIndexedValues)(value, index.indexNulls);
                        if (values) {
                            for (let i = 0, l = values.length; i < l; i++) {
                                lastResolution = index.put(values[i], key);
                            }
                        }
                    }
                    catch (error) {
                        if (!attributeErrorReported[property]) {
                            // just report an indexing error once per attribute so we don't spam the logs
                            attributeErrorReported[property] = true;
                            logger.error(`Error indexing attribute ${property}`, error);
                        }
                    }
                }
                (0, when_ts_1.when)(lastResolution, () => outstanding--, (error) => {
                    outstanding--;
                    logger.error(error);
                });
                if (worker_threads_1.workerData && worker_threads_1.workerData.restartNumber !== manageThreads.restartNumber) {
                    interrupted = true;
                }
                if (++indexed % 100 === 0 || interrupted) {
                    // occasionally update our progress so if we crash, we can resume
                    for (const attribute of attributes) {
                        attribute.lastIndexedKey = key;
                        Table.dbisDB.put(attribute.key, attribute);
                    }
                    if (interrupted)
                        return;
                }
                if (outstanding > MAX_OUTSTANDING_INDEXING)
                    await lastResolution;
                else if (outstanding > MIN_OUTSTANDING_INDEXING)
                    await new Promise((resolve) => setImmediate(resolve)); // yield event turn, don't want to use all computation
            }
            // update the attributes to indicate that we are finished
            for (const attribute of attributes) {
                delete attribute.lastIndexedKey;
                delete attribute.indexingPID;
                attribute.dbi.isIndexing = false;
                lastResolution = Table.dbisDB.put(attribute.key, attribute);
            }
        }
        await lastResolution;
        // now notify all the threads that we are done and the index is ready to use
        await signalling.signalSchemaChange(new itc_js_1.SchemaEventMsg(process.pid, 'indexing-finished', Table.databaseName, Table.tableName));
        logger.info(`Finished indexing ${Table.tableName} attributes`, attributes);
    }
    catch (error) {
        logger.error('Error in indexing', error);
    }
}
function dropTableMeta({ table: tableName, database: databaseName }) {
    const rootStore = database({ database: databaseName, table: tableName });
    const removals = [];
    const dbisDb = rootStore.dbisDb;
    for (const key of dbisDb.getKeys({ start: tableName + '/', end: tableName + '0' })) {
        removals.push(dbisDb.remove(key));
    }
    exports.databaseEventsEmitter.emit('dropTable', tableName, databaseName);
    return Promise.all(removals);
}
function onUpdatedTable(listener) {
    exports.databaseEventsEmitter.on('updateTable', listener);
    return {
        remove() {
            exports.databaseEventsEmitter.off('updateTable', listener);
        },
    };
}
function onRemovedTable(listener) {
    exports.databaseEventsEmitter.on('dropTable', listener);
    return {
        remove() {
            exports.databaseEventsEmitter.off('dropTable', listener);
        },
    };
}
function onRemovedDB(listener) {
    exports.databaseEventsEmitter.on('dropDatabase', listener);
    return {
        remove() {
            exports.databaseEventsEmitter.off('dropDatabase', listener);
        },
    };
}
function getDefaultCompression() {
    const LMDB_COMPRESSION = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION);
    const STORAGE_COMPRESSION_DICTIONARY = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION_DICTIONARY);
    const STORAGE_COMPRESSION_THRESHOLD = (0, environmentManager_js_1.get)(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_COMPRESSION_THRESHOLD) || DEFAULT_COMPRESSION_THRESHOLD;
    const LMDB_COMPRESSION_OPTS = { startingOffset: 32 };
    if (STORAGE_COMPRESSION_DICTIONARY)
        LMDB_COMPRESSION_OPTS['dictionary'] = (0, node_fs_1.readFileSync)(STORAGE_COMPRESSION_DICTIONARY);
    if (STORAGE_COMPRESSION_THRESHOLD)
        LMDB_COMPRESSION_OPTS['threshold'] = STORAGE_COMPRESSION_THRESHOLD;
    return LMDB_COMPRESSION && LMDB_COMPRESSION_OPTS;
}
//# sourceMappingURL=databases.js.map