import { promiseVoid } from "../../../../shared/src/resolved-promises.js";
import { Database } from "../../../../zqlite/src/db.js";
import { RunningState } from "../running-state.js";
import { getOrCreateGauge } from "../../observability/metrics.js";
import { Subscription } from "../../types/subscription.js";
import parsePrometheusTextFormat from "parse-prometheus-text-format";
//#region ../zero-cache/src/services/change-streamer/backup-monitor.ts
var CHECK_INTERVAL_MS = 6e4;
var MIN_CLEANUP_DELAY_MS = 3e4;
/**
* The BackupMonitor polls the litestream "/metrics" endpoint to track the
* watermark (label) value of the `litestream_replica_progress` gauge and
* schedules cleanup of change log entries that can be purged as a result.
*
* See: https://github.com/rocicorp/litestream/pull/3
*
* Note that change log entries cannot simply be purged as soon as they
* have been applied and backed up by litestream. Consider the case in which
* litestream backs up new wal segments every minute, but it takes 5 minutes
* to restore a replica: if a zero-cache starts restoring a replica at
* minute 0, and new watermarks are replicated at minutes 1, 2, 3, 4, and 5,
* purging changelog records as soon as those watermarks are replicated would
* result in the zero-cache not being able to catch up from minute 0 once it
* has finished restoring the replica.
*
* The `/snapshot` reservation protocol is used to prevent premature change
* log cleanup:
* - Clients restoring a snapshot initiate a `/snapshot` request and hold that
*   request open while it restores its snapshot, prepares it, and
*   starts its subscription to the change stream. During this time, no
*   cleanups are scheduled.
* - When the subscription is started, the interval since the beginning of
*   of the reservation is tracked to increase the background cleanup delay
*   interval if needed. The reservation is ended (and request closed), and
*   cleanup scheduling is resumed with the current delay interval.
*
* Note that the reservation request is the primary mechanism by which
* premature change log cleanup is prevented. The cleanup delay interval is
* a secondary safeguard.
*/
var BackupMonitor = class {
	id = "backup-monitor";
	#lc;
	#replicaFile;
	#backupURL;
	#metricsEndpoint;
	#changeStreamer;
	#state = new RunningState(this.id);
	#reservations = /* @__PURE__ */ new Map();
	#watermarks = /* @__PURE__ */ new Map();
	#lastWatermark = "";
	#latestBackupTime = null;
	#cleanupDelayMs;
	#checkMetricsTimer;
	constructor(lc, replicaFile, backupURL, metricsEndpoint, changeStreamer, initialCleanupDelayMs) {
		this.#lc = lc.withContext("component", this.id);
		this.#replicaFile = replicaFile;
		this.#backupURL = backupURL;
		this.#metricsEndpoint = metricsEndpoint;
		this.#changeStreamer = changeStreamer;
		this.#cleanupDelayMs = Math.max(initialCleanupDelayMs, MIN_CLEANUP_DELAY_MS);
		this.#lc.info?.(`backup monitor started ${initialCleanupDelayMs} ms after snapshot restore`);
	}
	run() {
		this.#lc.info?.(`monitoring backups at ${this.#metricsEndpoint} with ${this.#cleanupDelayMs} ms cleanup delay`);
		this.#checkMetricsTimer = setInterval(this.checkWatermarksAndScheduleCleanup, CHECK_INTERVAL_MS);
		this.#initBackupLagMetric();
		return this.#state.stopped();
	}
	startSnapshotReservation(taskID) {
		this.#lc.info?.(`pausing change-log cleanup while ${taskID} snapshots`);
		this.#reservations.get(taskID)?.sub.cancel();
		const sub = Subscription.create({ cleanup: () => this.endReservation(taskID, false) });
		this.#reservations.set(taskID, {
			start: /* @__PURE__ */ new Date(),
			sub
		});
		this.#changeStreamer.getChangeLogState().then((changeLogState) => {
			sub.push(["status", {
				tag: "status",
				backupURL: this.#backupURL,
				...changeLogState
			}]);
		}).catch((e) => {
			this.#lc.warn?.(`failing snapshot reservation`, e);
			sub.fail(e);
		});
		return sub;
	}
	endReservation(taskID, updateCleanupDelay = true) {
		const res = this.#reservations.get(taskID);
		if (res === void 0) return;
		this.#reservations.delete(taskID);
		const { start, sub } = res;
		sub.cancel();
		if (updateCleanupDelay) {
			const duration = Date.now() - start.getTime();
			this.#lc.info?.(`snapshot initialized by ${taskID} in ${duration} ms`);
			if (duration > this.#cleanupDelayMs) {
				this.#cleanupDelayMs = duration;
				this.#lc.info?.(`increased cleanup delay to ${duration} ms`);
			}
		}
	}
	checkWatermarksAndScheduleCleanup = async () => {
		try {
			await this.#checkWatermarks();
		} catch (e) {
			this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);
		}
		try {
			this.#scheduleCleanup();
		} catch (e) {
			this.#lc.warn?.(`error scheduling cleanup`, e);
		}
	};
	async *#fetchWatermarks() {
		const metricsEndpoint = this.#metricsEndpoint;
		const signal = this.#state.signal;
		let resp;
		try {
			resp = await fetch(metricsEndpoint, { signal });
		} catch (e) {
			if (signal.aborted) return;
			this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);
			return;
		}
		if (!resp.ok) {
			this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`);
			return;
		}
		const families = parsePrometheusTextFormat(await resp.text());
		for (const family of families) if (family.type === "GAUGE" && family.name === "litestream_replica_progress") for (const metric of family.metrics) {
			const watermark = metric.labels?.watermark;
			const name = metric.labels?.name;
			const time = /* @__PURE__ */ new Date(parseFloat(metric.value) * 1e3);
			if (watermark) yield {
				watermark,
				time,
				name
			};
		}
	}
	async #checkWatermarks() {
		for await (const { watermark, name, time } of this.#fetchWatermarks()) if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {
			this.#lc.info?.(`replicated watermark=${watermark} to ${name} at ${time.toISOString()}.`);
			this.#watermarks.set(watermark, time);
			this.#latestBackupTime = time;
		}
		return this.#latestBackupTime;
	}
	#scheduleCleanup() {
		if (this.#reservations.size > 0) {
			this.#lc.info?.(`watermark cleanup paused for snapshot(s): ${[...this.#reservations.keys()]}`);
			return;
		}
		const latestCleanupTime = Date.now() - this.#cleanupDelayMs;
		let maxWatermark = "";
		for (const [watermark, backupTime] of this.#watermarks.entries()) if (backupTime.getTime() <= latestCleanupTime && watermark > maxWatermark) maxWatermark = watermark;
		if (maxWatermark.length) {
			this.#changeStreamer.scheduleCleanup(maxWatermark);
			for (const watermark of this.#watermarks.keys()) if (watermark <= maxWatermark) this.#watermarks.delete(watermark);
			this.#lastWatermark = maxWatermark;
		}
	}
	stop() {
		clearInterval(this.#checkMetricsTimer);
		for (const { sub } of this.#reservations.values()) sub.cancel();
		this.#state.stop(this.#lc);
		return promiseVoid;
	}
	#initBackupLagMetric() {
		getOrCreateGauge("replica", "backup_lag", {
			description: "Latency from when a change is written to the replica to when it is backed up to litestream. It is expected to create a saw pattern from 0 to the configured ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES.",
			unit: "millisecond"
		}).addCallback(async (o) => {
			const latestBackup = await this.#checkWatermarks();
			if (!latestBackup) {
				this.#lc.warn?.(`no backed up watermarks. unable to report replica.backup_lag`);
				return;
			}
			const db = new Database(this.#lc, this.#replicaFile, { readonly: true });
			try {
				const { writeTimeMs } = db.prepare(`SELECT writeTimeMs FROM "_zero.replicationState"`).get();
				const backupLag = Math.max(0, writeTimeMs - latestBackup.getTime());
				o.observe(backupLag);
			} catch (e) {
				this.#lc.warn?.(`error measuring replica.backup_lag metric`, e);
			} finally {
				db.close();
			}
		});
	}
};
//#endregion
export { BackupMonitor };

//# sourceMappingURL=backup-monitor.js.map