From d42c5038e8e68f8e4f98dfb93037887df44d6618 Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Sat, 29 Nov 2025 15:05:32 -0300 Subject: [PATCH 1/3] feat: add job polling interval configuration to the dispatcher --- packages/engine/src/engine.ts | 11 +++++++++-- packages/engine/src/execution/dispatcher.test.ts | 4 +++- packages/engine/src/execution/dispatcher.ts | 9 ++++----- packages/engine/src/workers/main.ts | 1 + 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/packages/engine/src/engine.ts b/packages/engine/src/engine.ts index f344dd84..dfdddaa7 100644 --- a/packages/engine/src/engine.ts +++ b/packages/engine/src/engine.ts @@ -61,7 +61,6 @@ export interface EngineConfig { * @see {@link QueueDefaults} for more details */ queueDefaults?: QueueDefaults; - /** * If true, job scripts will NOT be automatically resolved by the engine. * In this case, you need to create a `sidequest.jobs.js` file at the root of your project @@ -74,7 +73,6 @@ export interface EngineConfig { * Defaults to `false`. */ manualJobResolution?: boolean; - /** * Optional path to the `sidequest.jobs.js` file when using manual job resolution. * If not provided, the engine will search for `sidequest.jobs.js` starting from the current working directory @@ -92,6 +90,14 @@ export interface EngineConfig { * If manualJobResolution === false, this option is ignored. */ jobsFilePath?: string; + /** + * Interval in milliseconds for polling new jobs in the dispatcher loop. Every polling cycle, + * the dispatcher will check for new jobs in the DB to process. + * Increase this number to reduce DB load at the cost of job start latency. + * + * Defaults to 100 ms. + */ + jobPollingInterval?: number; } /** @@ -180,6 +186,7 @@ export class Engine { }, manualJobResolution: config?.manualJobResolution ?? false, jobsFilePath: config?.jobsFilePath?.trim() ?? "", + jobPollingInterval: config?.jobPollingInterval ?? 100, }; this.validateConfig(); diff --git a/packages/engine/src/execution/dispatcher.test.ts b/packages/engine/src/execution/dispatcher.test.ts index 6b3992aa..c13276dd 100644 --- a/packages/engine/src/execution/dispatcher.test.ts +++ b/packages/engine/src/execution/dispatcher.test.ts @@ -56,6 +56,7 @@ describe("Dispatcher", () => { backend, new QueueManager(backend, config.queues!), new ExecutorManager(backend, config as NonNullableEngineConfig), + 100, ); dispatcher.start(); @@ -81,7 +82,7 @@ describe("Dispatcher", () => { const mockClaim = vi.spyOn(backend, "claimPendingJob"); - const dispatcher = new Dispatcher(backend, new QueueManager(backend, config.queues!), executorManager); + const dispatcher = new Dispatcher(backend, new QueueManager(backend, config.queues!), executorManager, 100); dispatcher.start(); expect(mockClaim).not.toBeCalled(); @@ -99,6 +100,7 @@ describe("Dispatcher", () => { backend, new QueueManager(backend, config.queues!), new ExecutorManager(backend, config as NonNullableEngineConfig), + 100, ); dispatcher.start(); diff --git a/packages/engine/src/execution/dispatcher.ts b/packages/engine/src/execution/dispatcher.ts index 24456870..f364c371 100644 --- a/packages/engine/src/execution/dispatcher.ts +++ b/packages/engine/src/execution/dispatcher.ts @@ -3,8 +3,6 @@ import { JobData, logger } from "@sidequest/core"; import { ExecutorManager } from "./executor-manager"; import { QueueManager } from "./queue-manager"; -const sleepDelay = 100; - /** * Dispatcher for managing job execution and queue polling. */ @@ -22,6 +20,7 @@ export class Dispatcher { private backend: Backend, private queueManager: QueueManager, private executorManager: ExecutorManager, + private sleepDelay: number, ) {} /** @@ -38,14 +37,14 @@ export class Dispatcher { const availableSlots = this.executorManager.availableSlotsByQueue(queue); if (availableSlots <= 0) { logger("Dispatcher").debug(`Queue ${queue.name} limit reached!`); - await this.sleep(sleepDelay); + await this.sleep(this.sleepDelay); continue; } const globalSlots = this.executorManager.availableSlotsGlobal(); if (globalSlots <= 0) { logger("Dispatcher").debug(`Global concurrency limit reached!`); - await this.sleep(sleepDelay); + await this.sleep(this.sleepDelay); continue; } @@ -63,7 +62,7 @@ export class Dispatcher { } if (shouldSleep) { - await this.sleep(sleepDelay); + await this.sleep(this.sleepDelay); } } } diff --git a/packages/engine/src/workers/main.ts b/packages/engine/src/workers/main.ts index 1f33dcbc..7c006796 100644 --- a/packages/engine/src/workers/main.ts +++ b/packages/engine/src/workers/main.ts @@ -29,6 +29,7 @@ export class MainWorker { this.backend, new QueueManager(this.backend, nonNullConfig.queues, nonNullConfig.queueDefaults), new ExecutorManager(this.backend, nonNullConfig), + nonNullConfig.jobPollingInterval, ); this.dispatcher.start(); From bb012c593fcb5de81698506006dee890b1639d85 Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Sat, 29 Nov 2025 15:10:19 -0300 Subject: [PATCH 2/3] feat: update configuration documentation to include job polling interval and manual job resolution options --- packages/docs/engine/configuration.md | 59 +++++++++++++++------------ packages/docs/engine/index.md | 4 +- 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/packages/docs/engine/configuration.md b/packages/docs/engine/configuration.md index 47927c15..c9c86873 100644 --- a/packages/docs/engine/configuration.md +++ b/packages/docs/engine/configuration.md @@ -167,36 +167,44 @@ await Sidequest.start({ password: "secret", }, }, + + // 11. Job resolution + manualJobResolution: false, + jobsFilePath: "sidequest.jobs.js", + + // 12. Job polling interval + jobPollingInterval: 100, // 100 milliseconds }); ``` ### Configuration Options -| Option | Description | Default | -| -------------------------------- | ------------------------------------------------------------------------------------------------------------------ | --------------------------- | -| `backend.driver` | Backend driver package name (SQLite, Postgres, MySQL, MongoDB) | `@sidequest/sqlite-backend` | -| `backend.config` | Backend-specific connection string or [Knex configuration object](https://knexjs.org/guide/#configuration-options) | `./sidequest.sqlite` | -| `dashboard.enabled` | Whether to enable the dashboard web interface | `true` | -| `dashboard.port` | Port for the dashboard web interface | `8678` | -| `dashboard.auth` | Basic auth configuration with `user` and `password`. If omitted, no auth is required. | `undefined` | -| `queues` | Array of queue configurations with name, concurrency, priority, and state | `[]` | -| `maxConcurrentJobs` | Maximum number of jobs processed simultaneously across all queues | `10` | -| `minThreads` | Minimum number of worker threads to use | Number of CPU cores | -| `maxThreads` | Maximum number of worker threads to use | `minThreads * 2` | -| `idleWorkerTimeout` | Timeout (milliseconds) for idle workers before they are terminated | `10000` (10 seconds) | -| `skipMigration` | Whether to skip database migration on startup | `false` | -| `releaseStaleJobsIntervalMin` | Frequency (minutes) for releasing stale jobs. Set to `false` to disable | `60` | -| `releaseStaleJobsMaxStaleMs` | Maximum age (milliseconds) for a running job to be considered stale | `600000` (10 minutes) | -| `releaseStaleJobsMaxClaimedMs` | Maximum age (milliseconds) for a claimed job to be considered stale | `60000` (1 minute) | -| `cleanupFinishedJobsIntervalMin` | Frequency (minutes) for cleaning up finished jobs. Set to `false` to disable | `60` | -| `cleanupFinishedJobsOlderThan` | Age (milliseconds) after which finished jobs are deleted | `2592000000` (30 days) | -| `logger.level` | Minimum log level (`debug`, `info`, `warn`, `error`) | `info` | -| `logger.json` | Whether to output logs in JSON format | `false` | -| `gracefulShutdown` | Whether to enable graceful shutdown handling | `true` | -| `jobDefaults` | Default values for new jobs. Used while enqueueing | `undefined` | -| `queueDefaults` | Default values for auto-created queues | `undefined` | -| `manualJobResolution` | Whether to manually resolve job classes. See [Manual Job Resolution](/jobs/manual-resolution.md) | `false` | -| `jobsFilePath` | Optional path to the file where job classes are exported. Ignored if `manualJobResolution` is `false`. | `undefined` | +| Option | Description | Default | +| -------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | --------------------------- | +| `backend.driver` | Backend driver package name (SQLite, Postgres, MySQL, MongoDB) | `@sidequest/sqlite-backend` | +| `backend.config` | Backend-specific connection string or [Knex configuration object](https://knexjs.org/guide/#configuration-options) | `./sidequest.sqlite` | +| `dashboard.enabled` | Whether to enable the dashboard web interface | `true` | +| `dashboard.port` | Port for the dashboard web interface | `8678` | +| `dashboard.auth` | Basic auth configuration with `user` and `password`. If omitted, no auth is required. | `undefined` | +| `queues` | Array of queue configurations with name, concurrency, priority, and state | `[]` | +| `maxConcurrentJobs` | Maximum number of jobs processed simultaneously across all queues | `10` | +| `minThreads` | Minimum number of worker threads to use | Number of CPU cores | +| `maxThreads` | Maximum number of worker threads to use | `minThreads * 2` | +| `idleWorkerTimeout` | Timeout (milliseconds) for idle workers before they are terminated | `10000` (10 seconds) | +| `skipMigration` | Whether to skip database migration on startup | `false` | +| `releaseStaleJobsIntervalMin` | Frequency (minutes) for releasing stale jobs. Set to `false` to disable | `60` | +| `releaseStaleJobsMaxStaleMs` | Maximum age (milliseconds) for a running job to be considered stale | `600000` (10 minutes) | +| `releaseStaleJobsMaxClaimedMs` | Maximum age (milliseconds) for a claimed job to be considered stale | `60000` (1 minute) | +| `cleanupFinishedJobsIntervalMin` | Frequency (minutes) for cleaning up finished jobs. Set to `false` to disable | `60` | +| `cleanupFinishedJobsOlderThan` | Age (milliseconds) after which finished jobs are deleted | `2592000000` (30 days) | +| `logger.level` | Minimum log level (`debug`, `info`, `warn`, `error`) | `info` | +| `logger.json` | Whether to output logs in JSON format | `false` | +| `gracefulShutdown` | Whether to enable graceful shutdown handling | `true` | +| `jobDefaults` | Default values for new jobs. Used while enqueueing | `undefined` | +| `queueDefaults` | Default values for auto-created queues | `undefined` | +| `manualJobResolution` | Whether to manually resolve job classes. See [Manual Job Resolution](/jobs/manual-resolution.md) | `false` | +| `jobsFilePath` | Optional path to the file where job classes are exported. Ignored if `manualJobResolution` is `false`. | `undefined` | +| `jobPollingInterval` | Interval (milliseconds) for polling new jobs to process. Increase this number to reduce DB load at the cost of job start latency. | `100` (100 milliseconds) | ::: danger If `auth` is not configured and `dashboard: true` is enabled in production, the dashboard will be publicly accessible. This is a security risk and **not recommended**. @@ -299,6 +307,7 @@ await Sidequest.start({ queueDefaults: { concurrency: 20, // Higher default concurrency }, + jobPollingInterval: 50, // Faster job polling - every 50ms }); ``` diff --git a/packages/docs/engine/index.md b/packages/docs/engine/index.md index cb57a01b..6a0cd083 100644 --- a/packages/docs/engine/index.md +++ b/packages/docs/engine/index.md @@ -55,6 +55,8 @@ await Sidequest.configure({ **Configuration Options:** +Here are a few of the most common configuration options you can provide: + - `backend` - Database backend configuration (defaults to SQLite) - `queues` - Initial queue configurations - `logger` - Logging configuration @@ -63,7 +65,7 @@ await Sidequest.configure({ - `queueDefaults` - Default queue settings - `gracefulShutdown` - Enable graceful shutdown handling -More information about configuration options can be found in the [Configuration Guide](/engine/configuration). +More information about configuration options and a full list can be found in the [Configuration Guide](/engine/configuration). ### `Sidequest.start` From 442e88455e7a17e579ad4ef66e5bcd286e12b047 Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Sat, 29 Nov 2025 15:15:33 -0300 Subject: [PATCH 3/3] feat: improve documentation for job polling interval in EngineConfig --- packages/engine/src/engine.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/engine/src/engine.ts b/packages/engine/src/engine.ts index dfdddaa7..6338e099 100644 --- a/packages/engine/src/engine.ts +++ b/packages/engine/src/engine.ts @@ -91,9 +91,11 @@ export interface EngineConfig { */ jobsFilePath?: string; /** - * Interval in milliseconds for polling new jobs in the dispatcher loop. Every polling cycle, - * the dispatcher will check for new jobs in the DB to process. + * Interval in milliseconds for polling new jobs in the dispatcher loop. + * The dispatcher will check for new jobs in the DB to process at every polling cycle. + * * Increase this number to reduce DB load at the cost of job start latency. + * Decrease this number if you want to have lower latency at the cost of higher DB load. * * Defaults to 100 ms. */