import { __addDisposableResource, __disposeResources } from "tslib";
import { assertNever } from '../utils/never.js';
import { asleep } from '../utils/asleep.js';
import { Errors } from './common.js';
import { MethodLogger, ManageStatusUpdates, ConvertStringToRequestID } from './internal.js';
import { AsyncDisposableStack } from '../utils/defer.js';
const keetaAnchorPipeableQueueStatuses = ['completed', 'failed_permanently'];
/**
 * An in-memory implementation of the KeetaAnchorQueueStorageDriver
 */
export class KeetaAnchorQueueStorageDriverMemory {
    queueStorage = {};
    logger;
    partitionCounter = 0;
    destroyed = false;
    name = 'KeetaAnchorQueueStorageDriverMemory';
    id;
    path = [];
    constructor(options) {
        this.id = options?.id ?? crypto.randomUUID();
        this.logger = options?.logger;
        this.path.push(...(options?.path ?? []));
        Object.freeze(this.path);
        this.methodLogger('new')?.debug('Created new in-memory queue storage driver');
    }
    clone(options) {
        const cloned = new KeetaAnchorQueueStorageDriverMemory({
            logger: this.logger,
            id: `${this.id}::${this.partitionCounter++}`,
            path: [...this.path],
            ...options
        });
        cloned.queueStorage = this.queueStorage;
        return (cloned);
    }
    get queue() {
        const pathKey = ['root', ...this.path].join('.');
        let retval = this.queueStorage[pathKey];
        if (retval === undefined) {
            retval = this.queueStorage[pathKey] = [];
        }
        return (retval);
    }
    methodLogger(method) {
        return (MethodLogger(this.logger, {
            class: 'KeetaAnchorQueueStorageDriverMemory',
            file: 'src/lib/queue/index.ts',
            method: method,
            instanceID: this.id
        }));
    }
    checkDestroyed() {
        if (this.destroyed) {
            throw (new Error('Queue has been destroyed'));
        }
    }
    async add(request, info) {
        this.checkDestroyed();
        const logger = this.methodLogger('add');
        let id = ConvertStringToRequestID(info?.id);
        if (id) {
            const duplicateID = this.queue.some(function (checkEntry) {
                return (checkEntry.id === id);
            });
            if (duplicateID) {
                logger?.debug(`Request with id ${String(id)} already exists, ignoring`);
                return (id);
            }
        }
        const idempotentIDs = info?.idempotentKeys;
        if (idempotentIDs) {
            const matchingIdempotentEntries = new Set();
            for (const idempotentID of idempotentIDs) {
                const idempotentEntryExists = this.queue.some(function (checkEntry) {
                    return (checkEntry.idempotentKeys?.has(idempotentID) ?? false);
                });
                if (idempotentEntryExists) {
                    matchingIdempotentEntries.add(idempotentID);
                }
            }
            if (matchingIdempotentEntries.size !== 0) {
                throw (new Errors.IdempotentExistsError('One or more idempotent entries already exist in the queue', matchingIdempotentEntries));
            }
        }
        /**
         * The status to use for the new entry
         */
        const status = info?.status ?? 'pending';
        /*
         * The ID is a branded string, so we must convert the generated UUID
         */
        id ??= ConvertStringToRequestID(crypto.randomUUID());
        logger?.debug(`Enqueuing request with id ${String(id)}`);
        this.queue.push({
            id: id,
            request: request,
            output: null,
            lastError: null,
            status: status,
            failures: 0,
            created: new Date(),
            updated: new Date(),
            worker: null,
            idempotentKeys: idempotentIDs ? new Set(idempotentIDs) : undefined
        });
        return (id);
    }
    async setStatus(id, status, ancillary) {
        this.checkDestroyed();
        const logger = this.methodLogger('setStatus');
        const entry = this.queue.find(function (checkEntry) {
            return (checkEntry.id === id);
        });
        if (!entry) {
            throw (new Error(`Request with ID ${String(id)} not found`));
        }
        const changedFields = ManageStatusUpdates(id, entry, status, ancillary, logger);
        Object.assign(entry, changedFields);
    }
    async get(id) {
        this.checkDestroyed();
        const entry = this.queue.find(function (checkEntry) {
            return (checkEntry.id === id);
        });
        if (!entry) {
            return (null);
        }
        return (structuredClone(entry));
    }
    async query(filter) {
        this.checkDestroyed();
        const logger = this.methodLogger('query');
        const queueDuplicate = structuredClone(this.queue);
        logger?.debug(`Querying queue with id ${this.id} with filter:`, filter);
        const allEntriesInStatus = (function () {
            const filterStatus = filter?.status;
            const filterLastUpdateBefore = filter?.updatedBefore;
            if (filterStatus || filterLastUpdateBefore) {
                return (queueDuplicate.filter(function (entry) {
                    if (filterStatus) {
                        if (entry.status !== filterStatus) {
                            return (false);
                        }
                    }
                    if (filterLastUpdateBefore) {
                        if (entry.updated >= filterLastUpdateBefore) {
                            return (false);
                        }
                    }
                    return (true);
                }));
            }
            else {
                return (queueDuplicate);
            }
        })();
        let retval = allEntriesInStatus;
        if (filter?.limit !== undefined) {
            retval = allEntriesInStatus.slice(0, filter.limit);
        }
        logger?.debug(`Queried queue with id ${this.id} with filter:`, filter, '-- found', retval.length, 'entries');
        return (retval);
    }
    async partition(path) {
        this.checkDestroyed();
        const logger = this.methodLogger('partition');
        logger?.debug(`Creating partitioned queue storage driver for path "${path}"`);
        const partitioned = this.clone({
            path: [...this.path, path]
        });
        return (partitioned);
    }
    async destroy() {
        this.destroyed = true;
        this.methodLogger('destroy')?.debug('Destroying in-memory queue');
    }
    async [Symbol.asyncDispose]() {
        return (await this.destroy());
    }
}
/**
 * A Queue Runner and Request Translator for processing entries in a queue
 *
 * The queue runner is responsible for pulling entries from the queue,
 * processing them, and updating their status in the queue.  As well
 * as moving jobs between queues by piping the output of one runner
 * to another.  Additionally, maintenance tasks such as re-queuing
 * failed jobs and marking stuck jobs are also handled by the runner.
 *
 * This is an abstract base class that must be extended to provide
 * the actual processing logic as well as the encoding and decoding
 * for requests and responses.
 */
export class KeetaAnchorQueueRunner {
    /**
     * The queue this runner is responsible for running
     */
    queue;
    /**
     * The logger we should use for logging anything
     */
    logger;
    /**
     * Worker configuration
     */
    workers;
    workerID;
    /**
     * Pipes to other runners we have registered
     */
    pipes = [];
    /**
     * Initialization promise
     */
    initializePromise;
    /**
     * Configuration for this queue
     */
    /**
     * The maximum number of times to retry a failed job before giving up
     * and marking it as permanently failed
     */
    maxRetries = 5;
    /**
     * The amount of time to allow a job to run before considering it
     * timed out and marking it as aborted, the `aborted` processor
     * (if provided) will be responsible for determining what to do with
     * the job at that point (e.g. retry it, mark it as failed, etc)
     * because some work from the job may have been completed
     * before the timeout was reached -- it could even still be on-going
     * after the timeout is reached, so the `aborted` processor should be
     * prepared to handle that situation as well.
     */
    processTimeout = 300_000; /* 5 minutes */
    /**
     * When piping a batch of jobs to another runner, this is the number
     * of jobs to include in each batch (max).
     */
    batchSize = 100;
    /**
     * The amount of time to wait before retrying a failed job -- this should
     * be long enough to allow any transient issues to be resolved (e.g. a downstream
     * service to come back up, etc) but not so long that it causes excessive
     * delays in processing.
     *
     * Keep in mind that after the {@link maxRetries} is reached, the job will
     * be marked as permanently failed, so if the retry delay is too low then
     * it could cause jobs to be marked as permanently failed before transient
     * issues have a chance to be resolved.
     */
    get retryDelay() {
        if (this.internalRetryDelay !== undefined) {
            return (this.internalRetryDelay);
        }
        return (this.processTimeout * 10);
    }
    set retryDelay(value) {
        this.internalRetryDelay = value;
    }
    internalRetryDelay;
    /**
     * The number of {@link processTimeout} intervals to wait before
     * considering a job to be stuck -- this is for jobs that are
     * still in the 'processing' state but have not updated their
     * timestamp in a long time, which likely indicates that the worker
     * processing the job has died or is otherwise no longer making
     * progress on the job.
     *
     * Like the `aborted` status, the `stuck` status means the job is in
     * an indeterminate state where it may have done some of the work of
     * the processor but we don't know how much (if any) of it was
     * completed, so the `stuck` processor should be prepared to handle
     * that situation.  It is unlikely that the job is still being actively
     * processed by a worker at this point, but it is possible, so the
     * `stuck` processor should be prepared to handle that as well.
     */
    get stuckMultiplier() {
        if (this.internalStuckMultiplier !== undefined) {
            return (this.internalStuckMultiplier);
        }
        return (10);
    }
    set stuckMultiplier(value) {
        this.internalStuckMultiplier = value;
    }
    internalStuckMultiplier;
    /**
     * How many runners can process this queue in parallel
     */
    maxRunners;
    runnerLockKey;
    /**
     * The ID of this runner for diagnostic purposes
     */
    id;
    constructor(config) {
        this.queue = config.queue;
        this.logger = config.logger;
        this.workers = config.workers ?? {
            count: 1,
            id: 0
        };
        if (this.workers.id < 0) {
            throw (new Error('Worker ID cannot be negative'));
        }
        if (this.maxRunners) {
            if (this.workers.id > this.maxRunners - 1 || this.workers.count > this.maxRunners) {
                throw (new Error('Worker ID other than 0 or worker count other than 1 is not supported yet'));
            }
        }
        /*
         * The worker ID is just a branded version of the worker number
         */
        // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
        this.workerID = this.workers.id;
        /*
         * The runner lock key, a unique key used to ensure only
         * one instance of a given runner is running at a time
         */
        this.runnerLockKey = ConvertStringToRequestID(`@runner-lock:9ba756f0-7aa2-41c7-a1ea-b010dc752ae8.worker.${this.workerID}`);
        /**
         * Instance ID
         */
        this.id = config.id ?? crypto.randomUUID();
        this.methodLogger('new')?.debug('Created new queue runner attached to queue', this.queue.id);
    }
    async initialize() {
        if (this.initializePromise) {
            return (await this.initializePromise);
        }
        /* Ensure the sequential lock entry exists */
        this.initializePromise = (async () => {
            /*
             * We store `null` as the request value because we
             * don't have anything better to store -- it's not
             * always going to be compatible with the type
             * QueueRequest but we know that we will never actually
             * use the value.
             */
            // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
            await this.queue.add(null, {
                id: this.runnerLockKey,
                status: '@internal'
            });
        })();
        return (await this.initializePromise);
    }
    setConfiguration(parameters) {
        const parameterNames = ['batchSize', 'maxRetries', 'processTimeout', 'retryDelay', 'stuckMultiplier'];
        for (const parameterName of parameterNames) {
            const value = parameters[parameterName];
            if (value === undefined) {
                continue;
            }
            this[parameterName] = value;
        }
    }
    methodLogger(method) {
        return (MethodLogger(this.logger, {
            class: 'KeetaAnchorQueueRunner',
            file: 'src/lib/queue/index.ts',
            method: method,
            instanceID: this.id
        }));
    }
    /** @internal */
    _Testing(key) {
        if (key !== 'bc81abf8-e43b-490b-b486-744fb49a5082') {
            throw (new Error('This is a testing only method'));
        }
        return ({
            setParams: (params) => {
                const { maxRunners, ...configParams } = params;
                this.setConfiguration(configParams);
                if (maxRunners !== undefined) {
                    this.maxRunners = maxRunners;
                }
            },
            queue: () => {
                return (this.queue);
            },
            markWorkerAsProcessing: async () => {
                await this.queue.setStatus(this.runnerLockKey, 'processing', {
                    oldStatus: '@internal',
                    by: this.workerID
                });
            }
        });
    }
    decodeEntry(entry) {
        return ({
            ...entry,
            request: this.decodeRequest(entry.request),
            output: this.decodeResponse(entry.output)
        });
    }
    /**
     * Enqueue an item to be processed by the queue
     */
    async add(request, info) {
        await this.initialize();
        const encodedRequest = this.encodeRequest(request);
        const newID = await this.queue.add(encodedRequest, info);
        return (newID);
    }
    /**
     * Get a single entry from storage by ID
     */
    async get(id) {
        await this.initialize();
        const entry = await this.queue.get(id);
        if (!entry) {
            return (null);
        }
        return (this.decodeEntry(entry));
    }
    /**
     * Get entries from storage with an optional filter
     */
    async query(filter) {
        await this.initialize();
        const entries = await this.queue.query(filter);
        return (entries.map((entry) => {
            return (this.decodeEntry(entry));
        }));
    }
    /**
     * Set the status of an entry in the queue
     */
    async setStatus(id, status, ancillary) {
        await this.initialize();
        let encodedOutput = undefined;
        if (ancillary?.output !== undefined) {
            encodedOutput = this.encodeResponse(ancillary.output);
        }
        return (await this.queue.setStatus(id, status, {
            ...ancillary,
            output: encodedOutput
        }));
    }
    /**
     * Checks to see if the queue is runnable
     */
    async runnable() {
        await this.initialize();
        const pendingEntries = await this.queue.query({ status: 'pending', limit: 1 });
        if (pendingEntries.length > 0) {
            return (true);
        }
        for (const pipe of this.pipes) {
            const pipeRunnable = await pipe.target.runnable();
            if (pipeRunnable) {
                return (true);
            }
        }
        return (false);
    }
    async updateRunnerLockTimestamp() {
        const logger = this.methodLogger('updateRunnerLockTimestamp');
        try {
            logger?.debug('Updating sequential processing lock timestamp for worker ID', this.workerID);
            await this.queue.setStatus(this.runnerLockKey, 'processing', {
                oldStatus: 'processing',
                by: this.workerID
            });
            logger?.debug('Updated sequential processing lock timestamp for worker ID', this.workerID);
        }
        catch (error) {
            logger?.error('Failed to update sequential processing lock timestamp for worker ID', this.workerID, ':', error);
            throw (error);
        }
    }
    async getRunnerLock(cleanup) {
        const logger = this.methodLogger('getRunnerLock');
        try {
            logger?.debug('Acquiring sequential processing lock for worker ID', this.workerID);
            await this.queue.setStatus(this.runnerLockKey, 'processing', {
                oldStatus: '@internal',
                by: this.workerID
            });
            logger?.debug('Acquired sequential processing lock for worker ID', this.workerID);
        }
        catch (error) {
            if (Errors.IncorrectStateAssertedError.isInstance(error)) {
                return (false);
            }
            throw (error);
        }
        cleanup.defer(async () => {
            for (let retry = 0; retry < 10; retry++) {
                logger?.debug(`Releasing sequential processing lock try #${retry + 1} for worker ID`, this.workerID);
                try {
                    await this.queue.setStatus(this.runnerLockKey, '@internal', {
                        oldStatus: 'processing',
                        by: undefined
                    });
                }
                catch {
                    await asleep(1000);
                    continue;
                }
                break;
            }
        });
        return (true);
    }
    async maintainRunnerLock() {
        const env_1 = { stack: [], error: void 0, hasError: false };
        try {
            const logger = this.methodLogger('maintainRunnerLock');
            const moment = new Date();
            const cleanup = __addDisposableResource(env_1, new AsyncDisposableStack(), true);
            const obtained = await this.getRunnerLock(cleanup);
            if (obtained) {
                return;
            }
            /**
             * Check to see if the lock is stale
             */
            const lockEntry = await this.queue.get(this.runnerLockKey);
            if (!lockEntry) {
                return;
            }
            const lockAge = moment.getTime() - lockEntry.updated.getTime();
            if (lockAge > this.processTimeout * this.stuckMultiplier) {
                logger?.warn('Processing lock is stale, taking over lock for worker ID', this.workerID);
                await this.queue.setStatus(this.runnerLockKey, '@internal', {
                    oldStatus: 'processing',
                    by: this.workerID
                });
            }
        }
        catch (e_1) {
            env_1.error = e_1;
            env_1.hasError = true;
        }
        finally {
            const result_1 = __disposeResources(env_1);
            if (result_1)
                await result_1;
        }
    }
    /**
     * Run the queue processor
     *
     * Processes up to `batchSize` entries from the queue and returns
     * true if there may be more work to do, or false if the queue
     * is empty.
     *
     * @param options Optional run options
     */
    async run(options) {
        const env_2 = { stack: [], error: void 0, hasError: false };
        try {
            const timeout = options?.timeoutMs;
            await this.initialize();
            const logger = this.methodLogger('run');
            const batchSize = this.batchSize;
            const processTimeout = this.processTimeout;
            const cleanup = __addDisposableResource(env_2, new AsyncDisposableStack(), true);
            let retval = true;
            const startTime = Date.now();
            const locked = await this.getRunnerLock(cleanup);
            if (!locked) {
                logger?.debug('Another worker is already processing the queue, skipping run');
                return (true);
            }
            const processJobOk = Symbol('processJobOk');
            const processJobTimeout = Symbol('processJobTimeout');
            const processJob = async (index, entry, startingStatus, processor) => {
                if (timeout !== undefined) {
                    const elapsed = Date.now() - startTime;
                    if (elapsed >= timeout) {
                        logger?.debug(`Timeout of ${timeout}ms reached after processing ${index} entries (${startingStatus} phase; elapsed ${elapsed}ms)`);
                        return (processJobTimeout);
                    }
                }
                await this.updateRunnerLockTimestamp();
                let setEntryStatus = { status: 'failed_temporarily', output: null };
                logger?.debug(`Processing entry request with id ${String(entry.id)}`);
                try {
                    /*
                     * Get a lock by setting it to 'processing'
                     */
                    await this.queue.setStatus(entry.id, 'processing', { oldStatus: startingStatus, by: this.workerID });
                    /*
                     * Process the entry with a timeout, if the timeout is reached
                     * we should mark the process as aborted because we no longer
                     * know what state the work is in and someone will need to
                     * inspect the job and determine through some other means if
                     * it is completed or failed.
                     */
                    let timeoutTimer = null;
                    setEntryStatus = await Promise.race([
                        new Promise(function (resolve) {
                            timeoutTimer = setTimeout(function () {
                                resolve({ status: 'aborted', output: null });
                            }, processTimeout);
                        }),
                        (async () => {
                            try {
                                return (await processor(this.decodeEntry(entry)));
                            }
                            finally {
                                if (timeoutTimer) {
                                    clearTimeout(timeoutTimer);
                                }
                            }
                        })()
                    ]);
                }
                catch (error) {
                    if (Errors.IncorrectStateAssertedError.isInstance(error)) {
                        logger?.info(`Skipping request with id ${String(entry.id)} because it is no longer in the expected state "${startingStatus}"`, error);
                        return (processJobOk);
                    }
                    logger?.error(`Failed to process request with id ${String(entry.id)}, setting state to "${setEntryStatus.status}":`, error);
                    setEntryStatus.status = 'failed_temporarily';
                    setEntryStatus.error = String(error);
                }
                if (setEntryStatus.status === 'processing') {
                    logger?.error(`Processor for request with id ${String(entry.id)} returned invalid status "processing"`);
                    setEntryStatus.status = 'failed_temporarily';
                    setEntryStatus.error = 'Processor returned invalid status "processing"';
                }
                let by = this.workerID;
                if (setEntryStatus.status === 'pending') {
                    by = undefined;
                }
                await this.queue.setStatus(entry.id, setEntryStatus.status, { oldStatus: 'processing', by: by, output: this.encodeResponse(setEntryStatus.output), error: setEntryStatus.error });
                return (processJobOk);
            };
            /*
             * Process pending jobs first
             */
            for (let index = 0; index < batchSize; index++) {
                const entries = await this.queue.query({ status: 'pending', limit: 1 });
                const entry = entries[0];
                if (entry === undefined) {
                    retval = false;
                    break;
                }
                const result = await processJob(index, entry, 'pending', this.processor.bind(this));
                if (result === processJobTimeout) {
                    break;
                }
            }
            /*
             * Next process any pipes to other runners
             */
            const pipes = [...this.pipes];
            for (const pipe of pipes) {
                if (pipe.exclusiveTarget === false) {
                    continue;
                }
                let remainingTime = undefined;
                if (timeout !== undefined) {
                    const elapsed = Date.now() - startTime;
                    remainingTime = timeout - elapsed;
                    if (remainingTime <= 0) {
                        remainingTime = -1;
                    }
                }
                const pipeHasMoreWork = await pipe.target.run({
                    ...options,
                    timeoutMs: remainingTime
                });
                if (pipeHasMoreWork) {
                    retval = true;
                }
            }
            /**
             * Process stuck or aborted jobs (if possible)
             */
            const conditions = [{
                    status: 'aborted',
                    processor: this.processorAborted?.bind(this)
                }, {
                    status: 'stuck',
                    processor: this.processorStuck?.bind(this)
                }];
            let timeoutReached = false;
            for (const condition of conditions) {
                if (condition.processor === undefined) {
                    continue;
                }
                for (let index = 0; index < batchSize; index++) {
                    const entries = await this.queue.query({ status: condition.status, limit: 1 });
                    const entry = entries[0];
                    if (entry === undefined) {
                        break;
                    }
                    const result = await processJob(index, entry, condition.status, condition.processor);
                    if (result === processJobTimeout) {
                        timeoutReached = true;
                        break;
                    }
                }
                if (timeoutReached) {
                    break;
                }
            }
            return (retval);
        }
        catch (e_2) {
            env_2.error = e_2;
            env_2.hasError = true;
        }
        finally {
            const result_2 = __disposeResources(env_2);
            if (result_2)
                await result_2;
        }
    }
    async markStuckRequestsAsStuck() {
        const stuckThreshold = this.processTimeout * this.stuckMultiplier;
        const logger = this.methodLogger('markStuckRequestsAsStuck');
        const now = Date.now();
        const requests = await this.queue.query({ status: 'processing', limit: 100, updatedBefore: new Date(now - stuckThreshold) });
        for (const request of requests) {
            /*
             * Skip the runner lock entries, they are managed separately
             */
            if (request.id.toString().startsWith('@runner-lock:9ba756f0-7aa2-41c7-a1ea-b010dc752ae8.worker.')) {
                continue;
            }
            try {
                logger?.warn(`Marking request with id ${String(request.id)} as stuck`);
                await this.queue.setStatus(request.id, 'stuck', { oldStatus: 'processing', by: this.workerID });
            }
            catch (error) {
                logger?.error(`Failed to mark request with id ${String(request.id)} as stuck:`, error);
            }
        }
    }
    async requeueFailedRequests() {
        const retryDelay = this.retryDelay;
        const maxRetries = this.maxRetries;
        const logger = this.methodLogger('requeueFailedRequests');
        const now = Date.now();
        const requests = await this.queue.query({ status: 'failed_temporarily', limit: 100, updatedBefore: new Date(now - retryDelay) });
        for (const request of requests) {
            try {
                if (request.failures >= maxRetries) {
                    logger?.info(`Request with id ${String(request.id)} has exceeded maximum retries, not requeuing -- moving to failed_permanently`);
                    await this.queue.setStatus(request.id, 'failed_permanently', { oldStatus: 'failed_temporarily', by: this.workerID });
                    continue;
                }
                logger?.debug(`Requeuing failed request with id ${String(request.id)}`);
                await this.queue.setStatus(request.id, 'pending', { oldStatus: 'failed_temporarily', by: this.workerID });
            }
            catch (error) {
                logger?.error(`Failed to requeue request with id ${String(request.id)}:`, error);
            }
        }
    }
    async moveCompletedToNextStage(statusTarget) {
        const logger = this.methodLogger('moveCompletedToNextStage');
        const pipes = [...this.pipes].filter(function (pipe) {
            return (pipe.acceptStatus === statusTarget);
        });
        if (pipes.length === 0) {
            return;
        }
        const allRequests = await this.queue.query({ status: statusTarget, limit: 100 });
        let requests = allRequests;
        const RequestSentToPipes = new Map();
        function IncrRequestSentToPipes(requestID) {
            const sentCount = RequestSentToPipes.get(requestID) ?? 0;
            RequestSentToPipes.set(requestID, sentCount + 1);
        }
        const getNextPipeRequestInput = (entry) => {
            if (statusTarget === 'completed') {
                return (this.decodeResponse(entry.output));
            }
            else if (statusTarget === 'failed_permanently') {
                return (this.decodeRequest(entry.request));
            }
            assertNever(statusTarget);
        };
        for (const pipe of pipes) {
            logger?.debug('Processing pipe to target', pipe.target.id, pipe.isBatchPipe ? '(batch pipe)' : '(single item pipe)');
            if (pipe.isBatchPipe) {
                /**
                 * Keep track of all the requests we successfully
                 * sent to the target stage
                 */
                const allTargetSeenRequestIDs = new Set();
                /**
                 * During each iteration of the batch processing, we keep track
                 * of the IDs we have already seen by the target and processed
                 * so we don't try to reprocess them again
                 */
                const iterationTargetSeenRequestIDs = new Set();
                /**
                 * If we get a batch that cannot be added to the target pipe,
                 * we just skip over them for retrying at a later date
                 */
                const skipRequestIDs = new Set();
                /**
                 * Compute a durable ID for this batch and target
                 */
                let batchID = ConvertStringToRequestID(crypto.randomUUID());
                /**
                 * Keep track of sequential failures to find enough entries
                 * and stop processing if we can't find enough after a few tries
                 * in a row
                 */
                let sequentialFailureCount = 0;
                for (; requests.length >= pipe.minBatchSize; 
                /*
                 * Remove any entries we have already seen during
                 * the last iteration of the loop
                 */
                requests = requests.filter(function (entry) {
                    return (!iterationTargetSeenRequestIDs.has(entry.id) && !skipRequestIDs.has(entry.id));
                })) {
                    iterationTargetSeenRequestIDs.clear();
                    logger?.debug(`Preparing to move ${statusTarget} requests to next stage ${pipe.target.id} (min=${pipe.minBatchSize}, max=${pipe.maxBatchSize}), have ${requests.length} ${statusTarget} requests available`);
                    /**
                     * Compute a batch of entries to send to the next stage,
                     * constrained to the max batch size of the pipe and
                     * the entries which have non-null outputs
                     */
                    const batchRaw = requests.map((entry) => {
                        return ({ output: getNextPipeRequestInput(entry), id: entry.id });
                    }).filter(function (entry) {
                        if (entry === null) {
                            return (false);
                        }
                        return (true);
                    }).slice(0, pipe.maxBatchSize);
                    /*
                     * If we don't have enough entries to meet the minimum
                     * batch size, skip this iteration
                     */
                    if (batchRaw.length < pipe.minBatchSize) {
                        sequentialFailureCount++;
                        if (sequentialFailureCount >= 3) {
                            logger?.debug(`Not enough ${statusTarget} requests to move to next stage ${pipe.target.id}, stopping batch processing`);
                            break;
                        }
                        logger?.debug(`Not moving ${statusTarget} requests to next stage ${pipe.target.id} because batch size ${batchRaw.length} is less than minimum size ${pipe.minBatchSize}`);
                        continue;
                    }
                    sequentialFailureCount = 0;
                    /**
                     * The IDs for the entries we are sending to the next stage
                     * target -- this may get reduced if we find there are already
                     * jobs in the next stage that have the idempotentIDs of one of
                     * these jobs
                     */
                    const batchLocalIDs = new Set(batchRaw.map(function (entry) {
                        return (entry.id);
                    }));
                    /**
                     * The outputs for the batch we are sending to the next stage
                     */
                    const batchOutput = batchRaw.map(function (entry) {
                        return (entry.output);
                    });
                    logger?.debug(`Moving batch of ${batchOutput.length} ${statusTarget} requests to next pipe`, pipe.target.id, '(input entry IDs:', Array.from(batchLocalIDs), '->', `${pipe.target.id}:${String(batchID)})`);
                    try {
                        // There is no way for typescript to cleanly infer the type here, so we assert it.
                        // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
                        await pipe.target.add(batchOutput, {
                            id: batchID,
                            /* Use the set of IDs as the idempotent IDs for the batch */
                            idempotentKeys: batchLocalIDs
                        });
                        batchID = ConvertStringToRequestID(crypto.randomUUID());
                    }
                    catch (error) {
                        if (Errors.IdempotentExistsError.isInstance(error) && error.idempotentIDsFound) {
                            logger?.debug('Some of the jobs have already been added to the target queue, skipping those:', error.idempotentIDsFound.values());
                            for (const requestID of error.idempotentIDsFound) {
                                iterationTargetSeenRequestIDs.add(requestID);
                                allTargetSeenRequestIDs.add(requestID);
                            }
                        }
                        else {
                            /*
                             * If we got some kind of other error adding these
                             * items to the target queue runner, just skip them
                             * and we will retry them on the next iteration
                             */
                            logger?.error(`Failed to move completed batch to next stage ${pipe.target.id}, will try to create another batch without them:`, error);
                            for (const requestID of batchLocalIDs) {
                                skipRequestIDs.add(requestID);
                            }
                        }
                        continue;
                    }
                    for (const requestID of batchLocalIDs) {
                        iterationTargetSeenRequestIDs.add(requestID);
                        allTargetSeenRequestIDs.add(requestID);
                    }
                }
                /*
                 * For every request we know the target has definitely seen, mark it
                 * as moved for this pipe
                 */
                for (const requestID of allTargetSeenRequestIDs) {
                    IncrRequestSentToPipes(requestID);
                }
            }
            else {
                for (const request of requests) {
                    let shouldMarkAsMoved = true;
                    try {
                        const output = getNextPipeRequestInput(request);
                        if (output === null) {
                            logger?.debug(`Completed request with id ${String(request.id)} has no output -- next stage will not be run`);
                        }
                        else {
                            logger?.debug(`Moving ${statusTarget} request with id ${String(request.id)} to next pipe`, pipe.target.id);
                            // There is no way for typescript to cleanly infer the type here, so we assert it.
                            // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
                            await pipe.target.add(output, { id: request.id });
                        }
                    }
                    catch (error) {
                        logger?.error(`Failed to move completed request with id ${String(request.id)} to next stage:`, error);
                        shouldMarkAsMoved = false;
                    }
                    if (shouldMarkAsMoved) {
                        IncrRequestSentToPipes(request.id);
                    }
                }
            }
        }
        const TotalPipes = pipes.length;
        for (const request of allRequests) {
            const sentCount = RequestSentToPipes.get(request.id) ?? 0;
            if (sentCount !== TotalPipes) {
                logger?.debug(`${statusTarget} request with id ${String(request.id)} was only moved to ${sentCount} out of ${TotalPipes} pipes -- not marking as moved`);
                continue;
            }
            logger?.debug(`Marking ${statusTarget} request with id ${String(request.id)} as moved`);
            await this.queue.setStatus(request.id, 'moved', { oldStatus: statusTarget, by: this.workerID });
        }
    }
    async maintain() {
        const logger = this.methodLogger('maintain');
        await this.initialize();
        /*
         * Each worker should maintain its own lock
         */
        try {
            await this.maintainRunnerLock();
        }
        catch (error) {
            logger?.debug('Failed to maintain runner lock:', error);
        }
        if (this.workers.id !== 0) {
            return;
        }
        /*
         * Only the worker with ID 0 should perform maintenance tasks on requests
         */
        try {
            await this.markStuckRequestsAsStuck();
        }
        catch (error) {
            logger?.debug('Failed to mark stuck requests as stuck:', error);
        }
        try {
            await this.requeueFailedRequests();
        }
        catch (error) {
            logger?.debug('Failed to requeue failed requests:', error);
        }
        for (const pipeStatus of keetaAnchorPipeableQueueStatuses) {
            try {
                await this.moveCompletedToNextStage(pipeStatus);
            }
            catch (error) {
                logger?.debug(`Failed to move ${pipeStatus} requests to next stage:`, error);
            }
        }
        for (const pipe of this.pipes) {
            try {
                if (pipe.exclusiveTarget === false) {
                    continue;
                }
                await pipe.target.maintain();
            }
            catch (error) {
                logger?.debug(`Failed to maintain piped runner with ID ${pipe.target.id}:`, error);
            }
        }
        if (this.queue.maintain) {
            try {
                await this.queue.maintain();
            }
            catch (error) {
                logger?.debug(`Failed to maintain queue storage driver with ID ${this.queue.id}`, error);
            }
        }
    }
    /**
     * Pipe the the completed entries of this runner to another runner
     */
    pipe(target, options) {
        this.pipes.push({
            ...options,
            isBatchPipe: false,
            target: target,
            acceptStatus: 'completed'
        });
        return (target);
    }
    pipeFailed(target, options) {
        this.pipes.push({
            ...options,
            isBatchPipe: false,
            target: target,
            acceptStatus: 'failed_permanently'
        });
        return (target);
    }
    /**
     * Pipe batches of completed entries from this runner to another runner
     */
    pipeBatch(target, maxBatchSize = 100, minBatchSize = 1, options) {
        this.pipes.push({
            ...options,
            isBatchPipe: true,
            target: target,
            minBatchSize: minBatchSize,
            maxBatchSize: maxBatchSize,
            acceptStatus: 'completed'
        });
        return (target);
    }
    pipeBatchFailed(target, maxBatchSize = 100, minBatchSize = 1, options) {
        this.pipes.push({
            ...options,
            isBatchPipe: true,
            target: target,
            minBatchSize: minBatchSize,
            maxBatchSize: maxBatchSize,
            acceptStatus: 'failed_permanently'
        });
        return (target);
    }
    async destroy() {
        this.methodLogger('destroy')?.debug('Destroying queue runner attached to queue', this.queue.id);
    }
    async [Symbol.asyncDispose]() {
        await this.destroy();
    }
}
/**
 * A KeetaAnchorQueueRunner for use when you want to process already
 * JSON-serializable data without any encoding/decoding needed
 */
export class KeetaAnchorQueueRunnerJSON extends KeetaAnchorQueueRunner {
    decodeRequest(request) {
        // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
        return request;
    }
    decodeResponse(response) {
        // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
        return response;
    }
    encodeRequest(request) {
        return (request);
    }
    encodeResponse(response) {
        return (response);
    }
}
/**
 * A KeetaAnchorQueueRunnerJSON that takes a processor function
 * in the constructor -- this is mainly useful for testing
 */
export class KeetaAnchorQueueRunnerJSONConfigProc extends KeetaAnchorQueueRunnerJSON {
    processor;
    constructor(config) {
        super(config);
        const { processor, processorStuck, processorAborted, ...parameters } = config;
        this.processor = processor;
        if (processorStuck) {
            this.processorStuck = processorStuck;
        }
        if (processorAborted) {
            this.processorAborted = processorAborted;
        }
        this.setConfiguration(parameters);
    }
}
//# sourceMappingURL=index.js.map