import { assert } from "../../../../../shared/src/asserts.js";
import { must } from "../../../../../shared/src/must.js";
import { stringify } from "../../../../../shared/src/bigint-json.js";
import { randInt } from "../../../../../shared/src/rand.js";
import { stateVersionFromString, stateVersionToString } from "../../../types/state-version.js";
import "../../../types/lite.js";
import { CustomKeyMap } from "../../../../../shared/src/custom-key-map.js";
import { resolver } from "@rocicorp/resolver";
//#region ../zero-cache/src/services/change-source/common/backfill-manager.ts
function tableKey({ schema, name }) {
	return `${schema}.${name}`;
}
var MIN_BACKOFF_INTERVAL_MS = 2e3;
var MAX_BACKOFF_INTERVAL_MS = 6e4;
/**
* The BackfillManager initiates backfills for BackfillRequests from the
* change-streamer (i.e. unfinished backfills from previous sessions)
* or for new backfills signaled by `create-table` or `add-column` messages
* from the change-source.
*
* The BackfillManager registers itself as a change stream listener in order
* to track necessary backfills, and potentially invalidate the in-progress
* backfill (e.g. due to a schema change) so that it can be retried at a
* new snapshot.
*
* The manager also handles low priority streaming of the backfill messages
* using the {@link ChangeStreamMultiplexer}, implementing a policy of always
* releasing its reservation if another producer (i.e. the main change stream)
* has messages to stream.
*/
var BackfillManager = class {
	#lc;
	/**
	* Tracks the metadata of required backfills based on schema changes
	* and initial backfill requests.
	*/
	#requiredBackfills = new CustomKeyMap(tableKey);
	#changeStreamer;
	#backfillStreamer;
	#jsonFormat;
	/**
	* The current running backfill. The backfill request is always also in
	* `#requiredBackfills` (technically, it can be a subset of what's in
	* `#requiredBackfills`); the request is removed from `#requiredBackfills`
	* upon completion.
	*/
	#runningBackfill = null;
	/** The last seen watermark in the change stream. */
	#lastStatusWatermark = null;
	#awaitingStatusWatermarks = [];
	/** The watermark of the current transaction in the change stream. */
	#currentTxWatermark = null;
	constructor(lc, changeStreamer, backfillStreamer, jsonFormat = "s", minBackoffMs = MIN_BACKOFF_INTERVAL_MS, maxBackoffMs = MAX_BACKOFF_INTERVAL_MS) {
		this.#lc = lc.withContext("component", "backfill-manager");
		this.#changeStreamer = changeStreamer;
		this.#backfillStreamer = backfillStreamer;
		this.#jsonFormat = jsonFormat;
		this.#minBackoffMs = minBackoffMs;
		this.#maxBackoffMs = maxBackoffMs;
		this.#retryDelayMs = minBackoffMs;
	}
	run(lastWatermark, initialRequests) {
		this.#lc.info?.(`starting backfill manager with ${initialRequests.length} initial requests`, { requests: initialRequests });
		this.#lastStatusWatermark = lastWatermark;
		initialRequests.forEach((req) => this.#setRequiredBackfill("initial-request", req));
		this.#checkAndStartBackfill();
	}
	#setLastStatusWatermark({ watermark }) {
		if ((this.#lastStatusWatermark ?? "") < watermark) {
			this.#lastStatusWatermark = watermark;
			for (let i = this.#awaitingStatusWatermarks.length - 1; i >= 0; i--) {
				const awaiting = this.#awaitingStatusWatermarks[i];
				if (watermark >= awaiting.watermark) {
					awaiting.reached();
					this.#awaitingStatusWatermarks.splice(i, 1);
				}
			}
		}
	}
	#changeStreamReached(lc, watermark) {
		if ((this.#lastStatusWatermark ?? "") < watermark) {
			const { promise, resolve: reached } = resolver();
			this.#awaitingStatusWatermarks.push({
				watermark,
				reached
			});
			lc.info?.(`waiting for change stream (at ${this.#lastStatusWatermark}) to reach ${watermark}`);
			return promise;
		}
		return null;
	}
	#minBackoffMs;
	#maxBackoffMs;
	#retryDelayMs;
	#backfillRetryTimer;
	#checkAndStartBackfill() {
		if (!this.#backfillRetryTimer && !this.#runningBackfill && this.#requiredBackfills.size) {
			const candidates = [...this.#requiredBackfills.values()];
			const request = candidates[randInt(0, candidates.length - 1)];
			const state = {
				request,
				minWatermark: ""
			};
			const lc = this.#lc.withContext("table", request.table.name);
			this.#runningBackfill = state;
			this.#runBackfill(lc, state).then(() => {
				this.#stopRunningBackfill("backfill exited", state);
				this.#retryDelayMs = this.#minBackoffMs;
			}).catch((e) => {
				this.#stopRunningBackfill(String(e), state);
				this.#retryBackfillWithBackoff(e);
			});
		}
	}
	#retryBackfillWithBackoff(e) {
		const log = this.#retryDelayMs === this.#maxBackoffMs ? "error" : "warn";
		this.#lc[log]?.(`Error running backfill. Retrying in ${this.#retryDelayMs} ms`, e);
		this.#backfillRetryTimer = setTimeout(() => {
			this.#backfillRetryTimer = void 0;
			this.#checkAndStartBackfill();
		}, this.#retryDelayMs);
		this.#retryDelayMs = Math.min(this.#retryDelayMs * 2, this.#maxBackoffMs);
	}
	async #runBackfill(lc, state) {
		const changeStream = this.#changeStreamer;
		let backfillTx = null;
		/**
		* @returns the new tx watermark, or null if backfill was cancelled
		*/
		const beginTxFor = async (msg) => {
			assert(backfillTx === null, "Expected no active backfill transaction");
			const lastWatermark = await changeStream.reserve("backfill");
			if (state.canceledReason || msg.tag === "backfill" && msg.watermark < state.minWatermark) {
				if (state.canceledReason === void 0) {
					assert(msg.tag === "backfill", "Expected backfill message tag");
					this.#stopRunningBackfill(`row key change at ${state.minWatermark} postdates backfill watermark at ${msg.watermark}`, state);
				}
				changeStream.release(lastWatermark);
				return null;
			}
			const { major, minor = 0n } = stateVersionFromString(lastWatermark);
			let tx = stateVersionToString({
				major,
				minor: BigInt(minor) + 1n
			});
			if (msg.tag === "backfill-completed" && tx < msg.watermark) tx = msg.watermark;
			changeStream.push([
				"begin",
				{
					tag: "begin",
					json: this.#jsonFormat,
					skipAck: true
				},
				{ commitWatermark: tx }
			]);
			return backfillTx = tx;
		};
		const commitTx = () => {
			if (backfillTx) {
				changeStream.push([
					"commit",
					{ tag: "commit" },
					{ watermark: backfillTx }
				]);
				changeStream.release(backfillTx);
			}
			backfillTx = null;
		};
		for await (const msg of this.#backfillStreamer(state.request)) {
			const mustWaitBeforeFlush = msg.tag === "backfill-completed" && (this.#changeStreamReached(lc, msg.watermark) || backfillTx !== null && backfillTx < msg.watermark);
			if (backfillTx && (changeStream.waiterDelay() > 0 || mustWaitBeforeFlush)) commitTx();
			mustWaitBeforeFlush && await mustWaitBeforeFlush;
			if (msg.tag === "backfill" && msg.rowValues.length > 0 && msg.relation.rowKey.columns.length === 0) throw new MissingRowKeyError(state.request);
			if ((backfillTx ??= await beginTxFor(msg)) === null) {
				lc.info?.(`backfill stream canceled: ${state.canceledReason}`, state.request);
				this.#checkAndStartBackfill();
				return;
			}
			await changeStream.push(["data", msg]);
		}
		backfillTx && commitTx();
		lc.debug?.(`backfill stream exited`, state.canceledReason ?? "");
	}
	#backfillRunningFor(table) {
		const state = this.#runningBackfill;
		return state?.request.table.schema === table.schema && state.request.table.name === table.name ? state : null;
	}
	/**
	* Stops the running backfill for the specified `reason`. If `instance` is
	* specified, the running backfill is stopped only if it is that instance.
	* This allows the running backfill itself to clear backfill state without
	* accidentally stopping a different (e.g. subsequent) backfill.
	*/
	#stopRunningBackfill(reason, instance) {
		const backfill = this.#runningBackfill;
		if (backfill && backfill === (instance ?? backfill)) {
			backfill.canceledReason = reason;
			this.#runningBackfill = null;
			reason && this.#lc.info?.(`canceling backfill:`, reason);
		}
	}
	#setRequiredBackfill(source, req) {
		const action = this.#requiredBackfills.has(req.table) ? "updated" : "added";
		this.#lc.info?.(`Backfill ${action}: ${source}`, { backfill: req });
		this.#requiredBackfills.set(req.table, req);
	}
	#deleteRequiredBackfill(source, id) {
		const req = this.#requiredBackfills.get(id);
		if (req) {
			const action = source === "backfill-completed" ? "completed" : "dropped";
			this.#lc.info?.(`Backfill ${action}: ${source}`, { backfill: req });
			this.#requiredBackfills.delete(id);
		}
	}
	/**
	* Implements {@link Listener.onChange()}, invoked by the
	* {@link ChangeStreamMultiplexer}.
	*/
	onChange(message) {
		if (message[0] === "begin") {
			this.#currentTxWatermark = message[2].commitWatermark;
			return;
		}
		if (message[0] === "commit") {
			this.#currentTxWatermark = null;
			this.#setLastStatusWatermark(message[2]);
			this.#checkAndStartBackfill();
			return;
		}
		if (message[0] === "status") {
			this.#setLastStatusWatermark(message[2]);
			return;
		}
		if (message[0] !== "data") return;
		const change = message[1];
		const { tag } = change;
		switch (tag) {
			case "update-table-metadata": {
				const { table, new: metadata } = change;
				const backfillRequest = this.#requiredBackfills.get(table);
				if (backfillRequest) {
					this.#setRequiredBackfill(tag, {
						...backfillRequest,
						table: {
							...backfillRequest.table,
							metadata
						}
					});
					if (this.#backfillRunningFor(table)) this.#stopRunningBackfill(`TableMetadata updated`);
				}
				break;
			}
			case "create-table": {
				const { spec: { schema, name }, metadata = null, backfill } = change;
				if (backfill) this.#setRequiredBackfill(tag, {
					table: {
						schema,
						name,
						metadata
					},
					columns: backfill
				});
				break;
			}
			case "rename-table": {
				const { old, new: newTable } = change;
				const backfillRequest = this.#requiredBackfills.get(old);
				if (backfillRequest) {
					const { schema, name } = newTable;
					this.#deleteRequiredBackfill(tag, old);
					this.#setRequiredBackfill(tag, {
						...backfillRequest,
						table: {
							...backfillRequest.table,
							schema,
							name
						}
					});
					if (this.#backfillRunningFor(old)) this.#stopRunningBackfill(`table renamed`);
				}
				break;
			}
			case "drop-table": {
				const { id } = change;
				if (this.#requiredBackfills.get(id)) {
					this.#deleteRequiredBackfill(tag, id);
					if (this.#backfillRunningFor(id)) this.#stopRunningBackfill(`table dropped`);
				}
				break;
			}
			case "add-column": {
				const { table, tableMetadata: metadata = null, column, backfill } = change;
				if (backfill) {
					const backfillRequest = this.#requiredBackfills.get(table);
					if (!backfillRequest) this.#setRequiredBackfill(tag, {
						table: {
							...table,
							metadata
						},
						columns: { [column.name]: backfill }
					});
					else this.#setRequiredBackfill(tag, {
						...backfillRequest,
						table: {
							...backfillRequest.table,
							metadata
						},
						columns: {
							...backfillRequest.columns,
							[column.name]: backfill
						}
					});
				}
				break;
			}
			case "update-column": {
				const { table, old: { name: oldName }, new: { name: newName } } = change;
				if (oldName !== newName) {
					const backfillRequest = this.#requiredBackfills.get(table);
					if (backfillRequest && oldName in backfillRequest.columns) {
						const { [oldName]: colSpec, ...otherCols } = backfillRequest.columns;
						this.#setRequiredBackfill(tag, {
							...backfillRequest,
							columns: {
								...otherCols,
								[newName]: colSpec
							}
						});
						const backfill = this.#backfillRunningFor(table);
						if (backfill && oldName in backfill.request.columns) this.#stopRunningBackfill(`column renamed`);
					}
				}
				break;
			}
			case "drop-column": {
				const { table, column } = change;
				const backfillRequest = this.#requiredBackfills.get(table);
				if (backfillRequest && column in backfillRequest.columns) {
					const { [column]: _excluded, ...remaining } = backfillRequest.columns;
					if (Object.keys(remaining).length === 0) this.#deleteRequiredBackfill(tag, table);
					else this.#setRequiredBackfill(tag, {
						...backfillRequest,
						columns: remaining
					});
					const backfill = this.#backfillRunningFor(table);
					if (backfill && column in backfill.request.columns) this.#stopRunningBackfill(`column dropped`);
				}
				break;
			}
			case "update": {
				const { relation, key, new: row } = change;
				const backfill = this.#backfillRunningFor(relation);
				const txWatermark = must(this.#currentTxWatermark, `not in a tx`);
				if (backfill?.request.table.metadata && key !== null) {
					for (const col of Object.keys(backfill.request.table.metadata.rowKey)) if (key[col] !== row[col]) {
						backfill.minWatermark = txWatermark;
						this.#lc.info?.(`key for row as changed (col: ${col}). backfill data must not predate ${backfill.minWatermark}`);
						break;
					}
				}
				break;
			}
			case "backfill-completed": {
				const { relation, columns } = change;
				const backfillRequest = this.#requiredBackfills.get(relation);
				assert(backfillRequest, () => `No BackfillRequest completed backfill ${stringify(change)}`);
				const remaining = Object.entries(backfillRequest.columns).filter(([col]) => !(columns.includes(col) || relation.rowKey.columns.includes(col)));
				if (remaining.length === 0) this.#deleteRequiredBackfill(tag, relation);
				else this.#setRequiredBackfill(tag, {
					...backfillRequest,
					columns: Object.fromEntries(remaining)
				});
				this.#stopRunningBackfill();
				break;
			}
		}
	}
	cancel() {
		this.#stopRunningBackfill(`change stream canceled`);
		clearTimeout(this.#backfillRetryTimer);
	}
};
var BackfillStreamError = class extends Error {
	constructor(bf, msg, cause) {
		super(`Cannot backfill ${bf.table.schema}.${bf.table.name}[${Object.keys(bf.columns).join(",")}]: ${msg}`, { cause });
	}
};
/**
* Background: The zero-cache supports replication of tables without a
* PRIMARY KEY to facilitate the onboarding process. These rows can be
* INSERT'ed, but postgres will rightfully prohibit UPDATEs and DELETEs
* on such tables because the rows cannot be identified by a key. Supporting
* this mode of replication allows the user to "fix" the setup by adding the
* primary key, after which the table can be published downstream without
* requiring a resync of the data.
*
* In terms of backfill, however, non-empty tables without a row key **cannot**
* be backfilled, because backfill retries would result in writing duplicating
* rows. (Empty tables, on the other hand, are fine because there is no data
* to be deduped.)
*
* The MissingRowKeyError is used to signal that the table cannot be backfilled
* in its current state. For simplicity, it is handled like runtime errors and
* retried with backoff, with which it can eventually succeed if (1) a primary
* key is added or (2) the table is emptied, e.g. via a TRUNCATE.
*/
var MissingRowKeyError = class extends BackfillStreamError {
	name = "MissingRowKeyError";
	constructor(bf, cause) {
		super(bf, `"${bf.table.name}" is missing a PRIMARY KEY`, cause);
	}
};
/**
* Error type for backfill stream implementations to throw indicating that
* the backfill request failed due to a schema incompatibility error. This
* type of error does not need exponential backoff, as the retry happens
* naturally once the invalidating schema change is processed and committed.
*/
var SchemaIncompatibilityError = class extends BackfillStreamError {
	name = "SchemaIncompatibilityError";
	constructor(bf, msg, cause) {
		super(bf, msg, cause);
	}
};
//#endregion
export { BackfillManager, SchemaIncompatibilityError };

//# sourceMappingURL=backfill-manager.js.map