import { assert, unreachable } from "../../../../shared/src/asserts.js";
import { deepEqual } from "../../../../shared/src/json.js";
import { must } from "../../../../shared/src/must.js";
import { skipYields } from "../../../../zql/src/ivm/operator.js";
import { MeasurePushOperator } from "../../../../zql/src/query/measure-push-operator.js";
import { buildPipeline } from "../../../../zql/src/builder/builder.js";
import { TableSource } from "../../../../zqlite/src/table-source.js";
import { Debug, runtimeDebugFlags } from "../../../../zql/src/builder/debug-delegate.js";
import "../../types/row-key.js";
import { ZERO_VERSION_COLUMN_NAME } from "../replicator/schema/constants.js";
import { getSubscriptionState } from "../replicator/schema/replication-state.js";
import { computeZqlSpecs, mustGetTableSpec } from "../../db/lite-tables.js";
import { resolveSimpleScalarSubqueries } from "../../../../zqlite/src/resolve-scalar-subqueries.js";
import { createSQLiteCostModel } from "../../../../zqlite/src/sqlite-cost-model.js";
import { reloadPermissionsIfChanged } from "../../auth/load-permissions.js";
import { getOrCreateCounter, getOrCreateHistogram } from "../../observability/metrics.js";
import { checkClientSchema } from "./client-schema.js";
import { ResetPipelinesSignal } from "./snapshotter.js";
//#region ../zero-cache/src/services/view-syncer/pipeline-driver.ts
/**
* No matter how fast hydration is, advancement is given at least this long to
* complete before doing a pipeline reset.
*/
var MIN_ADVANCEMENT_TIME_LIMIT_MS = 50;
/**
* Manages the state of IVM pipelines for a given ViewSyncer (i.e. client group).
*/
var PipelineDriver = class {
	#tables = /* @__PURE__ */ new Map();
	#pipelines = /* @__PURE__ */ new Map();
	#lc;
	#snapshotter;
	#storage;
	#shardID;
	#logConfig;
	#config;
	#tableSpecs = /* @__PURE__ */ new Map();
	#allTableNames = /* @__PURE__ */ new Set();
	#costModels;
	#yieldThresholdMs;
	#streamer = null;
	#hydrateContext = null;
	#advanceContext = null;
	#replicaVersion = null;
	#primaryKeys = null;
	#permissions = null;
	#advanceTime = getOrCreateHistogram("sync", "ivm.advance-time", {
		description: "Time to advance all queries for a given client group for in response to a single change.",
		unit: "s"
	});
	#conflictRowsDeleted = getOrCreateCounter("sync", "ivm.conflict-rows-deleted", "Number of rows deleted because they conflicted with added row");
	#inspectorDelegate;
	constructor(lc, logConfig, snapshotter, shardID, storage, clientGroupID, inspectorDelegate, yieldThresholdMs, enablePlanner, config) {
		this.#lc = lc.withContext("clientGroupID", clientGroupID);
		this.#snapshotter = snapshotter;
		this.#storage = storage;
		this.#shardID = shardID;
		this.#logConfig = logConfig;
		this.#config = config;
		this.#inspectorDelegate = inspectorDelegate;
		this.#costModels = enablePlanner ? /* @__PURE__ */ new WeakMap() : void 0;
		this.#yieldThresholdMs = yieldThresholdMs;
	}
	/**
	* Initializes the PipelineDriver to the current head of the database.
	* Queries can then be added (i.e. hydrated) with {@link addQuery()}.
	*
	* Must only be called once.
	*/
	init(clientSchema) {
		assert(!this.#snapshotter.initialized(), "Already initialized");
		this.#snapshotter.init();
		this.#initAndResetCommon(clientSchema);
	}
	/**
	* @returns Whether the PipelineDriver has been initialized.
	*/
	initialized() {
		return this.#snapshotter.initialized();
	}
	/**
	* Clears the current pipelines and TableSources, returning the PipelineDriver
	* to its initial state. This should be called in response to a schema change,
	* as TableSources need to be recomputed.
	*/
	reset(clientSchema) {
		for (const pipeline of this.#pipelines.values()) {
			pipeline.input.destroy();
			for (const companion of pipeline.companions) companion.input.destroy();
		}
		this.#pipelines.clear();
		this.#tables.clear();
		this.#allTableNames.clear();
		this.#initAndResetCommon(clientSchema);
	}
	#initAndResetCommon(clientSchema) {
		const { db } = this.#snapshotter.current();
		const fullTables = /* @__PURE__ */ new Map();
		computeZqlSpecs(this.#lc, db.db, { includeBackfillingColumns: false }, this.#tableSpecs, fullTables);
		checkClientSchema(this.#shardID, clientSchema, this.#tableSpecs, fullTables);
		this.#allTableNames.clear();
		for (const table of fullTables.keys()) this.#allTableNames.add(table);
		const primaryKeys = this.#primaryKeys ?? /* @__PURE__ */ new Map();
		this.#primaryKeys = primaryKeys;
		primaryKeys.clear();
		for (const [table, spec] of this.#tableSpecs.entries()) primaryKeys.set(table, spec.tableSpec.primaryKey);
		buildPrimaryKeys(clientSchema, primaryKeys);
		const { replicaVersion } = getSubscriptionState(db);
		this.#replicaVersion = replicaVersion;
	}
	/** @returns The replica version. The PipelineDriver must have been initialized. */
	get replicaVersion() {
		return must(this.#replicaVersion, "Not yet initialized");
	}
	/**
	* Returns the current version of the database. This will reflect the
	* latest version change when calling {@link advance()} once the
	* iteration has begun.
	*/
	currentVersion() {
		assert(this.initialized(), "Not yet initialized");
		return this.#snapshotter.current().version;
	}
	/**
	* Returns the current upstream {app}.permissions, or `null` if none are defined.
	*/
	currentPermissions() {
		assert(this.initialized(), "Not yet initialized");
		const res = reloadPermissionsIfChanged(this.#lc, this.#snapshotter.current().db, this.#shardID.appID, this.#permissions, this.#config);
		if (res.changed) {
			this.#permissions = res.permissions;
			this.#lc.debug?.("Reloaded permissions", JSON.stringify(this.#permissions));
		}
		return this.#permissions;
	}
	advanceWithoutDiff() {
		const { db, version } = this.#snapshotter.advanceWithoutDiff().curr;
		for (const table of this.#tables.values()) table.setDB(db.db);
		return version;
	}
	#ensureCostModelExistsIfEnabled(db) {
		let existing = this.#costModels?.get(db);
		if (existing) return existing;
		if (this.#costModels) {
			const costModel = createSQLiteCostModel(db, this.#tableSpecs);
			this.#costModels.set(db, costModel);
			return costModel;
		}
	}
	/**
	* Clears storage used for the pipelines. Call this when the
	* PipelineDriver will no longer be used.
	*/
	destroy() {
		this.#storage.destroy();
		this.#snapshotter.destroy();
	}
	/** @return Map from query ID to PipelineInfo for all added queries. */
	queries() {
		return this.#pipelines;
	}
	totalHydrationTimeMs() {
		let total = 0;
		for (const pipeline of this.#pipelines.values()) total += pipeline.hydrationTimeMs;
		return total;
	}
	#resolveScalarSubqueries(ast) {
		const companionRows = [];
		const companionInputs = [];
		const executor = (subqueryAST, childField) => {
			const input = buildPipeline(subqueryAST, {
				getSource: (name) => this.#getSource(name),
				createStorage: () => this.#createStorage(),
				decorateSourceInput: (input) => input,
				decorateInput: (input) => input,
				addEdge() {},
				decorateFilterInput: (input) => input
			}, "scalar-subquery");
			let node;
			for (const n of skipYields(input.fetch({}))) node ??= n;
			if (!node) {
				companionInputs.push(input);
				return;
			}
			companionRows.push({
				table: subqueryAST.table,
				row: node.row
			});
			companionInputs.push(input);
			return node.row[childField] ?? null;
		};
		const { ast: resolved, companions } = resolveSimpleScalarSubqueries(ast, this.#tableSpecs, executor);
		return {
			ast: resolved,
			companionRows,
			companions,
			companionInputs
		};
	}
	/**
	* Adds a pipeline for the query. The method will hydrate the query using the
	* driver's current snapshot of the database and return a stream of results.
	* Henceforth, updates to the query will be returned when the driver is
	* {@link advance}d. The query and its pipeline can be removed with
	* {@link removeQuery()}.
	*
	* If a query with the same queryID is already added, the existing pipeline
	* will be removed and destroyed before adding the new pipeline.
	*
	* @param timer The caller-controlled {@link Timer} used to determine the
	*        final hydration time. (The caller may pause and resume the timer
	*        when yielding the thread for time-slicing).
	* @return The rows from the initial hydration of the query.
	*/
	*addQuery(transformationHash, queryID, query, timer) {
		assert(this.initialized(), "Pipeline driver must be initialized before adding queries");
		this.removeQuery(queryID);
		const debugDelegate = runtimeDebugFlags.trackRowsVended ? new Debug() : void 0;
		const costModel = this.#ensureCostModelExistsIfEnabled(this.#snapshotter.current().db.db);
		assert(this.#advanceContext === null, "Cannot hydrate while advance is in progress");
		this.#hydrateContext = { timer };
		try {
			const { ast: resolvedQuery, companionRows, companions: companionMeta, companionInputs } = this.#resolveScalarSubqueries(query);
			const input = buildPipeline(resolvedQuery, {
				debug: debugDelegate,
				enableNotExists: true,
				getSource: (name) => this.#getSource(name),
				createStorage: () => this.#createStorage(),
				decorateSourceInput: (input, _queryID) => new MeasurePushOperator(input, queryID, this.#inspectorDelegate, "query-update-server"),
				decorateInput: (input) => input,
				addEdge() {},
				decorateFilterInput: (input) => input
			}, queryID, costModel);
			const schema = input.getSchema();
			input.setOutput({ push: (change) => {
				const streamer = this.#streamer;
				assert(streamer, "must #startAccumulating() before pushing changes");
				streamer.accumulate(queryID, schema, [change]);
				return [];
			} });
			yield* hydrateInternal(input, queryID, must(this.#primaryKeys), this.#tableSpecs);
			for (const { table, row } of companionRows) yield {
				type: "add",
				queryID,
				table,
				rowKey: getRowKey(mustGetPrimaryKey(this.#primaryKeys, table), row),
				row
			};
			const hydrationTimeMs = timer.totalElapsed();
			if (runtimeDebugFlags.trackRowCountsVended) {
				if (hydrationTimeMs > this.#logConfig.slowHydrateThreshold) {
					let totalRowsConsidered = 0;
					const lc = this.#lc.withContext("queryID", queryID).withContext("hydrationTimeMs", hydrationTimeMs);
					for (const tableName of this.#tables.keys()) {
						const entries = Object.entries(debugDelegate?.getVendedRowCounts()[tableName] ?? {});
						totalRowsConsidered += entries.reduce((acc, entry) => acc + entry[1], 0);
						lc.info?.(tableName + " VENDED: ", entries);
					}
					lc.info?.(`Total rows considered: ${totalRowsConsidered}`);
				}
			}
			debugDelegate?.reset();
			const liveCompanions = [];
			for (let i = 0; i < companionMeta.length; i++) {
				const meta = companionMeta[i];
				const companionInput = companionInputs[i];
				const companionSchema = companionInput.getSchema();
				const { childField, resolvedValue } = meta;
				companionInput.setOutput({ push: (change) => {
					let newValue;
					switch (change.type) {
						case "add":
						case "edit":
							newValue = change.node.row[childField] ?? null;
							break;
						case "remove":
							newValue = void 0;
							break;
						case "child": return [];
					}
					if (!scalarValuesEqual(newValue, resolvedValue)) throw new ResetPipelinesSignal(`Scalar subquery value changed for ${meta.ast.table}: ${String(resolvedValue)} -> ${String(newValue)}`, "scalar-subquery");
					const streamer = this.#streamer;
					assert(streamer, "must #startAccumulating() before pushing changes");
					streamer.accumulate(queryID, companionSchema, [change]);
					return [];
				} });
				liveCompanions.push({
					input: companionInput,
					childField,
					resolvedValue
				});
			}
			this.#pipelines.set(queryID, {
				input,
				hydrationTimeMs,
				transformedAst: resolvedQuery,
				transformationHash,
				companions: liveCompanions
			});
		} finally {
			this.#hydrateContext = null;
		}
	}
	/**
	* Removes the pipeline for the query. This is a no-op if the query
	* was not added.
	*/
	removeQuery(queryID) {
		const pipeline = this.#pipelines.get(queryID);
		if (pipeline) {
			this.#pipelines.delete(queryID);
			pipeline.input.destroy();
			for (const companion of pipeline.companions) companion.input.destroy();
		}
	}
	/**
	* Returns the value of the row with the given primary key `pk`,
	* or `undefined` if there is no such row. The pipeline must have been
	* initialized.
	*/
	getRow(table, pk) {
		assert(this.initialized(), "Not yet initialized");
		return must(this.#tables.get(table)).getRow(pk);
	}
	/**
	* Advances to the new head of the database.
	*
	* @param timer The caller-controlled {@link Timer} that will be used to
	*        measure the progress of the advancement and abort with a
	*        {@link ResetPipelinesSignal} if it is estimated to take longer
	*        than a hydration.
	* @return The resulting row changes for all added queries. Note that the
	*         `changes` must be iterated over in their entirety in order to
	*         advance the database snapshot.
	*/
	advance(timer) {
		assert(this.initialized(), "Pipeline driver must be initialized before advancing");
		const diff = this.#snapshotter.advance(this.#tableSpecs, this.#allTableNames);
		const { prev, curr, changes } = diff;
		this.#lc.debug?.(`advance ${prev.version} => ${curr.version}: ${changes} changes`);
		return {
			version: curr.version,
			numChanges: changes,
			changes: this.#advance(diff, timer, changes)
		};
	}
	*#advance(diff, timer, numChanges) {
		assert(this.#hydrateContext === null, "Cannot advance while hydration is in progress");
		const totalHydrationTimeMs = this.totalHydrationTimeMs();
		this.#advanceContext = {
			timer,
			totalHydrationTimeMs,
			numChanges,
			pos: 0
		};
		this.#lc.info?.(`starting pipeline advancement of ${numChanges} changes with an advancement time limited based on total hydration time of ${totalHydrationTimeMs} ms.`);
		try {
			for (const { table, prevValues, nextValue } of diff) {
				if (this.#shouldAdvanceYieldMaybeAbortAdvance()) yield "yield";
				const start = timer.totalElapsed();
				let type;
				try {
					const tableSource = this.#tables.get(table);
					if (!tableSource) continue;
					const primaryKey = mustGetPrimaryKey(this.#primaryKeys, table);
					let editOldRow = void 0;
					for (const prevValue of prevValues) if (nextValue && deepEqual(getRowKey(primaryKey, prevValue), getRowKey(primaryKey, nextValue))) editOldRow = prevValue;
					else {
						if (nextValue) this.#conflictRowsDeleted.add(1);
						yield* this.#push(tableSource, {
							type: "remove",
							row: prevValue
						});
					}
					if (nextValue) if (editOldRow) yield* this.#push(tableSource, {
						type: "edit",
						row: nextValue,
						oldRow: editOldRow
					});
					else yield* this.#push(tableSource, {
						type: "add",
						row: nextValue
					});
				} finally {
					this.#advanceContext.pos++;
				}
				const elapsed = timer.totalElapsed() - start;
				this.#advanceTime.record(elapsed / 1e3, {
					table,
					type
				});
			}
			const { curr } = diff;
			for (const table of this.#tables.values()) table.setDB(curr.db.db);
			this.#ensureCostModelExistsIfEnabled(curr.db.db);
			this.#lc.debug?.(`Advanced to ${curr.version}`);
		} finally {
			this.#advanceContext = null;
		}
	}
	/** Implements `BuilderDelegate.getSource()` */
	#getSource(tableName) {
		let source = this.#tables.get(tableName);
		if (source) return source;
		const tableSpec = mustGetTableSpec(this.#tableSpecs, tableName);
		const primaryKey = mustGetPrimaryKey(this.#primaryKeys, tableName);
		const { db } = this.#snapshotter.current();
		source = new TableSource(this.#lc, this.#logConfig, db.db, tableName, tableSpec.zqlSpec, primaryKey, () => this.#shouldYield());
		this.#tables.set(tableName, source);
		this.#lc.debug?.(`created TableSource for ${tableName}`);
		return source;
	}
	#shouldYield() {
		if (this.#hydrateContext) return this.#hydrateContext.timer.elapsedLap() > this.#yieldThresholdMs();
		if (this.#advanceContext) return this.#shouldAdvanceYieldMaybeAbortAdvance();
		throw new Error("shouldYield called outside of hydration or advancement");
	}
	/**
	* Cancel the advancement processing, by throwing a ResetPipelinesSignal, if
	* it has taken longer than half the total hydration time to make it through
	* half of the advancement, or if processing time exceeds total hydration
	* time.  This serves as both a circuit breaker for very large transactions,
	* as well as a bound on the amount of time the previous connection locks
	* the inactive WAL file (as the lock prevents WAL2 from switching to the
	* free WAL when the current one is over the size limit, which can make
	* the WAL grow continuously and compound slowness).
	* This is checked:
	* 1. before starting to process each change in an advancement is processed
	* 2. whenever a row is fetched from a TableSource during push processing
	*/
	#shouldAdvanceYieldMaybeAbortAdvance() {
		const { pos, numChanges, timer: advanceTimer, totalHydrationTimeMs } = must(this.#advanceContext);
		const elapsed = advanceTimer.totalElapsed();
		if (elapsed > MIN_ADVANCEMENT_TIME_LIMIT_MS && (elapsed > totalHydrationTimeMs || elapsed > totalHydrationTimeMs / 2 && pos <= numChanges / 2)) throw new ResetPipelinesSignal(`Advancement exceeded timeout at ${pos} of ${numChanges} changes after ${elapsed} ms. Advancement time limited based on total hydration time of ${totalHydrationTimeMs} ms.`, "advancement-timeout");
		return advanceTimer.elapsedLap() > this.#yieldThresholdMs();
	}
	/** Implements `BuilderDelegate.createStorage()` */
	#createStorage() {
		return this.#storage.createStorage();
	}
	*#push(source, change) {
		this.#startAccumulating();
		try {
			for (const val of source.genPush(change)) {
				if (val === "yield") yield "yield";
				for (const changeOrYield of this.#stopAccumulating().stream()) yield changeOrYield;
				this.#startAccumulating();
			}
		} finally {
			if (this.#streamer !== null) this.#stopAccumulating();
		}
	}
	#startAccumulating() {
		assert(this.#streamer === null, "Streamer already started");
		this.#streamer = new Streamer(must(this.#primaryKeys), this.#tableSpecs);
	}
	#stopAccumulating() {
		const streamer = this.#streamer;
		assert(streamer, "Streamer not started");
		this.#streamer = null;
		return streamer;
	}
};
var Streamer = class {
	#primaryKeys;
	#tableSpecs;
	constructor(primaryKeys, tableSpecs) {
		this.#primaryKeys = primaryKeys;
		this.#tableSpecs = tableSpecs;
	}
	#changes = [];
	accumulate(queryID, schema, changes) {
		this.#changes.push([
			queryID,
			schema,
			changes
		]);
		return this;
	}
	*stream() {
		for (const [queryID, schema, changes] of this.#changes) yield* this.#streamChanges(queryID, schema, changes);
	}
	*#streamChanges(queryID, schema, changes) {
		if (schema.system === "permissions") return;
		for (const change of changes) {
			if (change === "yield") {
				yield change;
				continue;
			}
			const { type } = change;
			switch (type) {
				case "add":
				case "remove":
					yield* this.#streamNodes(queryID, schema, type, () => [change.node]);
					break;
				case "child": {
					const { child } = change;
					const childSchema = must(schema.relationships[child.relationshipName]);
					yield* this.#streamChanges(queryID, childSchema, [child.change]);
					break;
				}
				case "edit":
					yield* this.#streamNodes(queryID, schema, type, () => [{
						row: change.node.row,
						relationships: {}
					}]);
					break;
				default: unreachable(type);
			}
		}
	}
	*#streamNodes(queryID, schema, op, nodes) {
		const { tableName: table, system } = schema;
		const primaryKey = must(this.#primaryKeys.get(table));
		const spec = must(this.#tableSpecs.get(table)).tableSpec;
		if (system === "permissions") return;
		for (const node of nodes()) {
			if (node === "yield") {
				yield node;
				continue;
			}
			const { relationships } = node;
			let { row } = node;
			const rowKey = getRowKey(primaryKey, row);
			if (op !== "remove") {
				const rowVersion = row[ZERO_VERSION_COLUMN_NAME];
				if (typeof rowVersion === "string" && rowVersion < (spec.minRowVersion ?? "00")) row = {
					...row,
					[ZERO_VERSION_COLUMN_NAME]: spec.minRowVersion
				};
			}
			yield {
				type: op,
				queryID,
				table,
				rowKey,
				row: op === "remove" ? void 0 : row
			};
			for (const [relationship, children] of Object.entries(relationships)) {
				const childSchema = must(schema.relationships[relationship]);
				yield* this.#streamNodes(queryID, childSchema, op, children);
			}
		}
	}
};
function* toAdds(nodes) {
	for (const node of nodes) {
		if (node === "yield") {
			yield node;
			continue;
		}
		yield {
			type: "add",
			node
		};
	}
}
function getRowKey(cols, row) {
	return Object.fromEntries(cols.map((col) => [col, must(row[col])]));
}
/**
* Core hydration logic used by {@link PipelineDriver#addQuery}, extracted to a
* function for reuse by bin-analyze so that bin-analyze's hydration logic
* is as close as possible to zero-cache's real hydration logic.
*/
function* hydrate(input, hash, clientSchema, tableSpecs) {
	const res = input.fetch({});
	yield* new Streamer(buildPrimaryKeys(clientSchema), tableSpecs).accumulate(hash, input.getSchema(), toAdds(res)).stream();
}
function* hydrateInternal(input, hash, primaryKeys, tableSpecs) {
	const res = input.fetch({});
	yield* new Streamer(primaryKeys, tableSpecs).accumulate(hash, input.getSchema(), toAdds(res)).stream();
}
function buildPrimaryKeys(clientSchema, primaryKeys = /* @__PURE__ */ new Map()) {
	for (const [tableName, { primaryKey }] of Object.entries(clientSchema.tables)) primaryKeys.set(tableName, primaryKey);
	return primaryKeys;
}
function mustGetPrimaryKey(primaryKeys, table) {
	const pKeys = must(primaryKeys, "primaryKey map must be non-null");
	const rv = pKeys.get(table);
	assert(rv, () => `table '${table}' is not one of: ${[...pKeys.keys()].sort()}. Check the spelling and ensure that the table has a primary key.`);
	return rv;
}
/**
* Compares two scalar subquery resolved values for equality.
* Unlike `valuesEqual` in data.ts (which treats null != null for join
* semantics), this uses identity semantics: undefined === undefined
* (no row matched), null === null (row matched but field was NULL).
*/
function scalarValuesEqual(a, b) {
	return a === b;
}
//#endregion
export { PipelineDriver, hydrate };

//# sourceMappingURL=pipeline-driver.js.map