import { JobJson } from './job-json'; import { KeysMap } from '../classes/queue-keys'; import { DependenciesOpts, MinimalJob, MoveToDelayedOpts, MoveToWaitingChildrenOpts, RetryJobOpts } from './minimal-job'; import { ParentKeyOpts } from './parent'; import { QueueBaseOptions } from './queue-options'; import { RepeatableOptions } from './repeatable-options'; import { RetryOptions } from './retry-options'; import { StreamReadRaw } from './redis-streams'; import { FinishedStatus, JobProgress, JobsOptions, JobState, JobType, KeepJobs } from '../types'; type FinishedState = FinishedStatus; /** * IQueueBackend * * Database-agnostic contract describing every *high-level* operation that the * {@link Queue}, {@link Worker} and {@link Job} classes need in order to * function. The goal of this interface is to express the queue semantics * ("move job to active", "extend lock", "promote job", …) **independently of * the underlying datastore**. * * Built-in implementations currently include the Redis adapter * ({@link RedisQueueBackend}) and the PostgreSQL adapter. Both fulfil the same * operations over different datastores without requiring any change to * `Queue`, `Worker` or `Job`. * * The method names and signatures intentionally mirror the existing * `RedisQueueBackend` class so that the Redis adapter is a near * drop-in implementation. * Operations that used to be performed via direct datastore * commands scattered across the three classes (queue metadata, job getters, * the blocking "wait for next job" primitive, …) have been promoted into * this interface so that the three classes never need to talk to the * datastore directly. * * @remarks * Low-level, Redis-specific helpers (Lua KEYS/ARGV builders, error-code * mapping, `runCommand`, …) are deliberately **not** part of this contract. * They remain private implementation details of the Redis adapter. * * The interface intentionally exposes **no connection or transaction type**: a * concrete adapter owns its connection(s). For example, the Redis adapter is * built from a context that provides an {@link IRedisClient} (plus a dedicated * blocking client for {@link IQueueBackend.waitForJob}), so callers never * thread a connection or transaction through an operation. */ export interface IQueueBackend { /** * Resolves once the backend's underlying connection(s) are ready to accept * operations. */ waitUntilReady(): Promise; /** * Closes the backend and its underlying connection(s), waiting for any * in-flight work to settle. * * @param force - When `true`, forcibly tears down the connection(s) without * waiting for in-flight (e.g. blocking) commands to finish. */ close(force?: boolean): Promise; /** * Truthy once {@link IQueueBackend.close} has begun (resolves when the close * completes). Used by the worker to decide whether it is still safe to issue * datastore operations (e.g. completing the current job) while the * higher-level instance is shutting down. */ readonly closing: Promise | undefined; /** * Forcibly disconnects the backend's underlying connection(s). */ disconnect(): Promise; /** * Sets a human-readable name on the underlying connection (for * observability). No-op for backends that have no such concept. */ setName(name: string): Promise; /** * Smallest meaningful block timeout (in seconds) supported by the backend's * blocking primitive. Used by workers to bound `waitForJob`. */ readonly minimumBlockTimeout: number; /** * Subscribes to normalized backend lifecycle events (`'ready'`, `'error'`, * `'close'`), derived from the underlying connection(s). */ on(event: 'ready' | 'error' | 'close', listener: (...args: any[]) => void): this; once(event: 'ready' | 'error' | 'close', listener: (...args: any[]) => void): this; removeListener(event: string, listener: (...args: any[]) => void): this; /** * Returns a sibling backend bound to a different queue (by name) that shares * this backend's underlying connection(s). * * This is used by {@link FlowProducer}, which spans multiple queues over a * single connection: every node in a flow needs datastore operations scoped * to its own queue, but they must all reuse the same connection. The * returned backend has an independent identity (its operations target the * given queue) but does not own the connection, so closing it is a no-op on * the shared connection. * * @param queueName - The queue the sibling backend should operate on. * @param prefix - Optional key prefix for the target queue. Flows may span * queues under different prefixes, so when omitted the backend's own prefix * is used. */ forQueue(queueName: string, prefix?: string): IQueueBackend; /** * The queue's fully-qualified name (the cross-backend logical identifier used * e.g. as a flow parent reference). Redis: `":"`. */ readonly qualifiedName: string; /** * The map of named sub-keys/identifiers for the queue. For Redis these are * the concrete Redis keys; backends that don't address jobs by key may return * an empty map. */ readonly keys: KeysMap; /** * Builds a namespaced sub-key/identifier of the given `type` for this queue * (e.g. a job's `"::dependencies"` key). */ toKey(type: string): string; /** * Parses a flow child/dependency node key (`":"`) back * into the components needed to locate the job: its queue keyspace `prefix` * (empty for backends without a prefix), `queueName` and `id`. Inverse of the * backend's key format; used when walking a flow tree. */ parseNodeKey(key: string): { prefix: string; queueName: string; id: string; }; /** * Builds the connection client name (used for `setName` and worker/queue * discovery). Redis: `":"`. Backends without a * client-name concept may return any stable string. */ clientName(suffix?: string): string; /** * Adds a single job to the queue, routing it to the correct initial state * (wait / delayed / prioritized / waiting-children) based on its options. * * The backend uses its own connection — callers never pass one in. */ addJob(job: JobJson, jobId: string, parentKeyOpts?: ParentKeyOpts): Promise; /** * Adds many jobs to the queue in a single, efficient operation. * * How the insert is batched (a Redis pipeline, a single multi-row SQL * `INSERT`, a transaction, …) is entirely an implementation detail of the * backend; the contract only requires that all jobs are added and their ids * returned in order. * * @returns The generated ids, in the same order as `entries`. */ addJobs(entries: { job: JobJson; jobId: string; parentKeyOpts?: ParentKeyOpts; }[]): Promise; /** * Atomically inserts a flow (tree) of jobs that may span multiple queues, * returning one `[error, idOrCode]` tuple per entry, in the same order they * were provided. Each entry is self-describing (it carries its own queue * `prefix`/`queueName`), so the operation is not bound to a single queue. * * For the Redis adapter this is a single `MULTI`; a SQL backend would use a * single transaction. */ addFlow(entries: { jobData: JobJson; jobId: string; parentKeyOpts: ParentKeyOpts; prefix: string; queueName: string; }[]): Promise<[Error | null, string | number][]>; /** * Registers a job scheduler and enqueues its next delayed iteration. * * Two job-option bags are involved, with deliberately different roles: * - `templateOpts` — the scheduler's *template* options, stored once and * reused as the basis for every future iteration produced by the scheduler. * - `delayedJobOpts` — the fully-resolved options for the *single* delayed * job created right now: the template plus this iteration's `jobId`, * `delay`, `repeat.offset`/`count`, etc. * * @returns A tuple of `[jobId, delay]` for the next iteration. */ addJobScheduler(jobSchedulerId: string, nextMillis: number, templateData: string, templateOpts: JobsOptions, opts: RepeatableOptions, delayedJobOpts: JobsOptions, producerId?: string): Promise<[string, number]>; /** * Atomically moves the next eligible job from wait/prioritized to active, * returning its data (or the delay/rate-limit signals when none is ready). */ moveToActive(token: string, name?: string): Promise; /** * Moves an active job to the completed state and, optionally, fetches the * next job to process. * @returns The next job data tuple when `fetchNext` is set, plus the * `finishedOn` timestamp that was recorded. */ moveToCompleted(job: MinimalJob, returnValue: R, removeOnComplete: boolean | number | KeepJobs, token: string, fetchNext: boolean): Promise<{ result: void | any[]; finishedOn: number; }>; /** * Moves an active job to the failed state and, optionally, fetches the next * job to process. * @returns The next job data tuple when `fetchNext` is set, plus the * `finishedOn` timestamp that was recorded. */ moveToFailed(job: MinimalJob, failedReason: string, removeOnFail: boolean | number | KeepJobs, token: string, fetchNext: boolean, fieldsToUpdate?: Record): Promise<{ result: void | any[]; finishedOn: number; }>; /** * Moves a job to the delayed state, scheduling it to run after `delay` ms. */ moveToDelayed(jobId: string, timestamp: number, delay: number, token?: string, opts?: MoveToDelayedOpts): Promise; /** * Moves a parent job to the waiting-children state. * @returns `true` if moved, `false` if there are pending dependencies. */ moveToWaitingChildren(jobId: string, token: string, opts?: MoveToWaitingChildrenOpts): Promise; /** * Moves a (manually rate-limited) job from active back to wait. */ moveJobFromActiveToWait(jobId: string, token?: string): Promise; /** * Retries a failed/active job immediately by pushing it back to wait. */ retryJob(jobId: string, lifo: boolean, token?: string, opts?: RetryJobOpts): Promise; /** * Reprocesses a finished (failed/completed) job, moving it back to wait. */ retryFinishedJob(job: MinimalJob, state: 'failed' | 'completed', opts?: RetryOptions): Promise; /** * Promotes a single delayed job so it can be processed as soon as possible. */ promote(jobId: string): Promise; /** * Recovers stalled jobs (active jobs whose lock expired) back to wait. * @returns The ids of the jobs that were moved. */ moveStalledJobsToWait(): Promise; /** * Moves up to `count` finished jobs of the given `state` back to wait. * @returns A cursor; `0` when there are no more jobs to move. */ retryFinishedJobs(state?: FinishedState, count?: number, timestamp?: number): Promise; /** * Promotes up to `count` delayed jobs back to wait. * @returns A cursor; `0` when there are no more jobs to promote. */ promoteJobs(count?: number): Promise; /** * Pauses or resumes the whole queue. */ pause(pause: boolean): Promise; /** * Removes waiting (and optionally delayed) jobs from the queue. */ drain(delayed: boolean): Promise; /** * Removes jobs in a given state that are older than `timestamp`. * @returns The ids of the removed jobs. */ cleanJobsByState(state: string, timestamp: number, limit?: number): Promise; /** * Irreversibly destroys the queue and all of its contents. * @returns A cursor; `0` when obliteration is complete. */ obliterate(opts: { force: boolean; count: number; }): Promise; /** * Removes orphaned job keys that exist in the datastore but are not * referenced by any queue state set. * @returns The total number of orphaned jobs removed. */ removeOrphanedJobs(count?: number, limit?: number): Promise; /** * Extends the lock of a single active job. */ extendLock(jobId: string, token: string, duration: number): Promise; /** * Extends the lock of several active jobs at once. * @returns The ids of the jobs whose lock could not be extended. */ extendLocks(jobIds: string[], tokens: string[], duration: number): Promise; /** * Replaces a job's data payload. */ updateData(job: MinimalJob, data: T): Promise; /** * Updates a job's progress and emits the corresponding event. */ updateProgress(jobId: string, progress: JobProgress): Promise; /** * Appends a row to a job's log, optionally trimming old entries. * @returns The total number of log entries. */ addLog(jobId: string, logRow: string, keepLogs?: number): Promise; /** * Clears a job's logs, optionally keeping the most recent `keepLogs` rows. */ clearLogs(jobId: string, keepLogs?: number): Promise; /** * Changes the delay of a delayed job. */ changeDelay(jobId: string, delay: number): Promise; /** * Changes the priority (and optionally lifo) of a waiting job. */ changePriority(jobId: string, priority?: number, lifo?: boolean): Promise; /** * Removes a job and (optionally) its children. * @returns `1` if removed, `0` if it (or a dependency) was locked. */ remove(jobId: string, removeChildren: boolean): Promise; /** * Removes all unprocessed children of a job. */ removeUnprocessedChildren(jobId: string): Promise; /** * Removes the child→parent dependency for a not-yet-finished child. * @returns `true` if the dependency existed and was removed. */ removeChildDependency(jobId: string, parentKey: string): Promise; /** * Removes a deduplication key if it still maps to the given job. * @returns `1` if removed, `0` otherwise. */ removeDeduplicationKey(deduplicationId: string, jobId: string): Promise; /** * Unconditionally deletes a deduplication key. * @returns The number of keys removed. */ deleteDeduplicationKey(deduplicationId: string): Promise; updateJobSchedulerNextMillis(jobSchedulerId: string, nextMillis: number, templateData: string, delayedJobOpts: JobsOptions, producerId?: string): Promise; removeJobScheduler(jobSchedulerId: string): Promise; getJobScheduler(id: string): Promise<[any, string | null]>; /** * Returns whether an id corresponds to a registered job scheduler. */ isJobScheduler(id: string): Promise; /** * Returns the raw stored metadata hash for a job scheduler. */ getJobSchedulerData(key: string): Promise>; /** * Returns a range of scheduler keys with their next-run scores, flattened as * `[key, score, key, score, …]`. */ getJobSchedulersRange(start: number, end: number, asc: boolean): Promise; /** * Returns the number of registered job schedulers. */ getJobSchedulersCount(): Promise; /** * Returns the current state of a job. */ getState(jobId: string): Promise; /** * Returns whether a job has finished and (optionally) its result. */ isFinished(jobId: string, returnValue?: boolean): Promise; /** * Returns whether the queue has reached its concurrency limit. */ isMaxed(): Promise; /** * Returns whether a job id is present in the given state. */ isJobInState(state: string, jobId: string): Promise; /** * Returns the stored data for a job, or `undefined` if it is missing. */ getJobData(jobId: string): Promise; /** * Returns the job id currently holding the given deduplication key, if any. */ getDeduplicationJobId(deduplicationId: string): Promise; /** * Returns a page of a job's logs together with the total log count. */ getJobLogs(jobId: string, start: number, end: number, asc: boolean): Promise<{ logs: string[]; count: number; }>; /** * Returns the ttl (ms) of the current rate-limit window. */ getRateLimitTtl(maxJobs?: number): Promise; getCounts(types: JobType[]): Promise; getCountsPerPriority(priorities: number[]): Promise; getRanges(types: JobType[], start?: number, end?: number, asc?: boolean): Promise<[string][]>; getDependencyCounts(jobId: string, types: string[]): Promise; /** * Returns a job's children dependencies (processed/unprocessed/ignored/failed). */ getDependencies(jobId: string, opts: DependenciesOpts): Promise<{ nextFailedCursor?: number; failed?: string[]; nextIgnoredCursor?: number; ignored?: Record; nextProcessedCursor?: number; processed?: Record; nextUnprocessedCursor?: number; unprocessed?: string[]; }>; /** * Returns the raw processed-children map (child key → serialized value). */ getProcessedChildrenValues(jobId: string): Promise>; /** * Returns the raw ignored-children failures map (child key → reason). */ getIgnoredChildrenFailures(jobId: string): Promise>; getMetrics(type: 'completed' | 'failed', start?: number, end?: number): Promise<[string[], string[], number]>; /** * Returns the raw worker/client list(s) for the queue's datastore. For the * Redis adapter this is `CLIENT LIST` (one string per cluster node, or a * single string otherwise). Backends with no notion of connected clients * may return an empty array. */ getClientList(): Promise; /** * Paginates a datastore set or hash, optionally fetching the jobs themselves. */ paginate(key: string, opts: { start: number; end: number; fetchJobs?: boolean; }): Promise<{ cursor: string; items: { id: string; v?: any; err?: string; }[]; total: number; jobs?: JobJson[]; }>; /** * Sets one or more queue metadata fields. */ setQueueMeta(values: Record): Promise; /** * Reads a single queue metadata field. */ getQueueMetaField(field: string): Promise; /** * Reads several queue metadata fields at once, in order. */ getQueueMetaFields(fields: string[]): Promise<(string | null)[]>; /** * Reads the entire queue metadata hash. */ getQueueMeta(): Promise>; /** * Removes one or more queue metadata fields. */ removeQueueMetaFields(fields: string[]): Promise; /** * Returns whether a queue metadata field exists. */ hasQueueMetaField(field: string): Promise; /** * Sets the global rate-limit window for the next jobs. */ setRateLimit(expireTimeMs: number): Promise; /** * Removes the rate-limit key. * @returns The number of keys removed. */ removeRateLimitKey(): Promise; /** * Removes the deprecated priority helper key. * @returns The number of keys removed. */ removeDeprecatedPriorityKey(): Promise; /** * Trims the event stream to an approximate maximum length. * @returns The number of entries removed. */ trimEvents(maxLength: number): Promise; /** * Publishes a custom event to the queue's event stream. * @returns The id of the appended event entry. */ publishEvent(fields: Record, maxEvents: number): Promise; /** * Blocks (up to `blockTimeout` ms) reading the queue's event stream for * entries newer than `id`, returning the raw stream entries (or a falsy value * on timeout). For the Redis adapter this is an `XREAD ... BLOCK`. */ readEvents(id: string, blockTimeout: number): Promise; /** * Blocks (up to `blockTimeout` seconds) until the queue signals that a new * job may be available, returning the next "block-until" timestamp. * * For the Redis adapter this is a `BZPOPMIN` on the marker sorted set using * the adapter's own dedicated blocking connection; other adapters may * implement it via `LISTEN`/`NOTIFY`, change-data-capture or polling. * * @returns The marker member/score on success, or `null` on timeout. */ waitForJob(blockTimeout: number): Promise<{ member: string; score: number; } | null>; /** * Interrupts the backend's in-flight blocking wait (so a worker can stop or * recover). No-op for backends without a dedicated blocking connection. */ disconnectBlocking(wait?: boolean): Promise; /** * Re-establishes the backend's blocking connection after an interrupt. */ reconnectBlocking(): Promise; } /** * Factory that builds an {@link IQueueBackend} for a given queue. Injected into * the queue classes so they depend only on the abstraction, never on a concrete * datastore/connection. The default factory is the Redis one * (`createRedisBackend`). * * The factory is generic over the concrete backend type `B` it produces, so a * caller (or class) parameterized on `B` keeps the concrete typing end-to-end * (e.g. `getBackend()` returning the concrete adapter instead of the bare * interface). */ export type BackendFactory = (name: string, opts: QueueBaseOptions, options?: { /** The backend's main connection is itself blocking (e.g. QueueEvents). */ blocking?: boolean; /** Provision a dedicated blocking connection (workers). */ withBlockingConnection?: boolean; }) => B; export {};