diff --git a/packages/docs/engine/configuration.md b/packages/docs/engine/configuration.md index 47927c1..c9c8687 100644 --- a/packages/docs/engine/configuration.md +++ b/packages/docs/engine/configuration.md @@ -167,36 +167,44 @@ await Sidequest.start({ password: "secret", }, }, + + // 11. Job resolution + manualJobResolution: false, + jobsFilePath: "sidequest.jobs.js", + + // 12. Job polling interval + jobPollingInterval: 100, // 100 milliseconds }); ``` ### Configuration Options -| Option | Description | Default | -| -------------------------------- | ------------------------------------------------------------------------------------------------------------------ | --------------------------- | -| `backend.driver` | Backend driver package name (SQLite, Postgres, MySQL, MongoDB) | `@sidequest/sqlite-backend` | -| `backend.config` | Backend-specific connection string or [Knex configuration object](https://knexjs.org/guide/#configuration-options) | `./sidequest.sqlite` | -| `dashboard.enabled` | Whether to enable the dashboard web interface | `true` | -| `dashboard.port` | Port for the dashboard web interface | `8678` | -| `dashboard.auth` | Basic auth configuration with `user` and `password`. If omitted, no auth is required. | `undefined` | -| `queues` | Array of queue configurations with name, concurrency, priority, and state | `[]` | -| `maxConcurrentJobs` | Maximum number of jobs processed simultaneously across all queues | `10` | -| `minThreads` | Minimum number of worker threads to use | Number of CPU cores | -| `maxThreads` | Maximum number of worker threads to use | `minThreads * 2` | -| `idleWorkerTimeout` | Timeout (milliseconds) for idle workers before they are terminated | `10000` (10 seconds) | -| `skipMigration` | Whether to skip database migration on startup | `false` | -| `releaseStaleJobsIntervalMin` | Frequency (minutes) for releasing stale jobs. Set to `false` to disable | `60` | -| `releaseStaleJobsMaxStaleMs` | Maximum age (milliseconds) for a running job to be considered stale | `600000` (10 minutes) | -| `releaseStaleJobsMaxClaimedMs` | Maximum age (milliseconds) for a claimed job to be considered stale | `60000` (1 minute) | -| `cleanupFinishedJobsIntervalMin` | Frequency (minutes) for cleaning up finished jobs. Set to `false` to disable | `60` | -| `cleanupFinishedJobsOlderThan` | Age (milliseconds) after which finished jobs are deleted | `2592000000` (30 days) | -| `logger.level` | Minimum log level (`debug`, `info`, `warn`, `error`) | `info` | -| `logger.json` | Whether to output logs in JSON format | `false` | -| `gracefulShutdown` | Whether to enable graceful shutdown handling | `true` | -| `jobDefaults` | Default values for new jobs. Used while enqueueing | `undefined` | -| `queueDefaults` | Default values for auto-created queues | `undefined` | -| `manualJobResolution` | Whether to manually resolve job classes. See [Manual Job Resolution](/jobs/manual-resolution.md) | `false` | -| `jobsFilePath` | Optional path to the file where job classes are exported. Ignored if `manualJobResolution` is `false`. | `undefined` | +| Option | Description | Default | +| -------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | --------------------------- | +| `backend.driver` | Backend driver package name (SQLite, Postgres, MySQL, MongoDB) | `@sidequest/sqlite-backend` | +| `backend.config` | Backend-specific connection string or [Knex configuration object](https://knexjs.org/guide/#configuration-options) | `./sidequest.sqlite` | +| `dashboard.enabled` | Whether to enable the dashboard web interface | `true` | +| `dashboard.port` | Port for the dashboard web interface | `8678` | +| `dashboard.auth` | Basic auth configuration with `user` and `password`. If omitted, no auth is required. | `undefined` | +| `queues` | Array of queue configurations with name, concurrency, priority, and state | `[]` | +| `maxConcurrentJobs` | Maximum number of jobs processed simultaneously across all queues | `10` | +| `minThreads` | Minimum number of worker threads to use | Number of CPU cores | +| `maxThreads` | Maximum number of worker threads to use | `minThreads * 2` | +| `idleWorkerTimeout` | Timeout (milliseconds) for idle workers before they are terminated | `10000` (10 seconds) | +| `skipMigration` | Whether to skip database migration on startup | `false` | +| `releaseStaleJobsIntervalMin` | Frequency (minutes) for releasing stale jobs. Set to `false` to disable | `60` | +| `releaseStaleJobsMaxStaleMs` | Maximum age (milliseconds) for a running job to be considered stale | `600000` (10 minutes) | +| `releaseStaleJobsMaxClaimedMs` | Maximum age (milliseconds) for a claimed job to be considered stale | `60000` (1 minute) | +| `cleanupFinishedJobsIntervalMin` | Frequency (minutes) for cleaning up finished jobs. Set to `false` to disable | `60` | +| `cleanupFinishedJobsOlderThan` | Age (milliseconds) after which finished jobs are deleted | `2592000000` (30 days) | +| `logger.level` | Minimum log level (`debug`, `info`, `warn`, `error`) | `info` | +| `logger.json` | Whether to output logs in JSON format | `false` | +| `gracefulShutdown` | Whether to enable graceful shutdown handling | `true` | +| `jobDefaults` | Default values for new jobs. Used while enqueueing | `undefined` | +| `queueDefaults` | Default values for auto-created queues | `undefined` | +| `manualJobResolution` | Whether to manually resolve job classes. See [Manual Job Resolution](/jobs/manual-resolution.md) | `false` | +| `jobsFilePath` | Optional path to the file where job classes are exported. Ignored if `manualJobResolution` is `false`. | `undefined` | +| `jobPollingInterval` | Interval (milliseconds) for polling new jobs to process. Increase this number to reduce DB load at the cost of job start latency. | `100` (100 milliseconds) | ::: danger If `auth` is not configured and `dashboard: true` is enabled in production, the dashboard will be publicly accessible. This is a security risk and **not recommended**. @@ -299,6 +307,7 @@ await Sidequest.start({ queueDefaults: { concurrency: 20, // Higher default concurrency }, + jobPollingInterval: 50, // Faster job polling - every 50ms }); ``` diff --git a/packages/docs/engine/index.md b/packages/docs/engine/index.md index cb57a01..6a0cd08 100644 --- a/packages/docs/engine/index.md +++ b/packages/docs/engine/index.md @@ -55,6 +55,8 @@ await Sidequest.configure({ **Configuration Options:** +Here are a few of the most common configuration options you can provide: + - `backend` - Database backend configuration (defaults to SQLite) - `queues` - Initial queue configurations - `logger` - Logging configuration @@ -63,7 +65,7 @@ await Sidequest.configure({ - `queueDefaults` - Default queue settings - `gracefulShutdown` - Enable graceful shutdown handling -More information about configuration options can be found in the [Configuration Guide](/engine/configuration). +More information about configuration options and a full list can be found in the [Configuration Guide](/engine/configuration). ### `Sidequest.start` diff --git a/packages/engine/src/engine.ts b/packages/engine/src/engine.ts index f344dd8..6338e09 100644 --- a/packages/engine/src/engine.ts +++ b/packages/engine/src/engine.ts @@ -61,7 +61,6 @@ export interface EngineConfig { * @see {@link QueueDefaults} for more details */ queueDefaults?: QueueDefaults; - /** * If true, job scripts will NOT be automatically resolved by the engine. * In this case, you need to create a `sidequest.jobs.js` file at the root of your project @@ -74,7 +73,6 @@ export interface EngineConfig { * Defaults to `false`. */ manualJobResolution?: boolean; - /** * Optional path to the `sidequest.jobs.js` file when using manual job resolution. * If not provided, the engine will search for `sidequest.jobs.js` starting from the current working directory @@ -92,6 +90,16 @@ export interface EngineConfig { * If manualJobResolution === false, this option is ignored. */ jobsFilePath?: string; + /** + * Interval in milliseconds for polling new jobs in the dispatcher loop. + * The dispatcher will check for new jobs in the DB to process at every polling cycle. + * + * Increase this number to reduce DB load at the cost of job start latency. + * Decrease this number if you want to have lower latency at the cost of higher DB load. + * + * Defaults to 100 ms. + */ + jobPollingInterval?: number; } /** @@ -180,6 +188,7 @@ export class Engine { }, manualJobResolution: config?.manualJobResolution ?? false, jobsFilePath: config?.jobsFilePath?.trim() ?? "", + jobPollingInterval: config?.jobPollingInterval ?? 100, }; this.validateConfig(); diff --git a/packages/engine/src/execution/dispatcher.test.ts b/packages/engine/src/execution/dispatcher.test.ts index 6b3992a..c13276d 100644 --- a/packages/engine/src/execution/dispatcher.test.ts +++ b/packages/engine/src/execution/dispatcher.test.ts @@ -56,6 +56,7 @@ describe("Dispatcher", () => { backend, new QueueManager(backend, config.queues!), new ExecutorManager(backend, config as NonNullableEngineConfig), + 100, ); dispatcher.start(); @@ -81,7 +82,7 @@ describe("Dispatcher", () => { const mockClaim = vi.spyOn(backend, "claimPendingJob"); - const dispatcher = new Dispatcher(backend, new QueueManager(backend, config.queues!), executorManager); + const dispatcher = new Dispatcher(backend, new QueueManager(backend, config.queues!), executorManager, 100); dispatcher.start(); expect(mockClaim).not.toBeCalled(); @@ -99,6 +100,7 @@ describe("Dispatcher", () => { backend, new QueueManager(backend, config.queues!), new ExecutorManager(backend, config as NonNullableEngineConfig), + 100, ); dispatcher.start(); diff --git a/packages/engine/src/execution/dispatcher.ts b/packages/engine/src/execution/dispatcher.ts index 2445687..f364c37 100644 --- a/packages/engine/src/execution/dispatcher.ts +++ b/packages/engine/src/execution/dispatcher.ts @@ -3,8 +3,6 @@ import { JobData, logger } from "@sidequest/core"; import { ExecutorManager } from "./executor-manager"; import { QueueManager } from "./queue-manager"; -const sleepDelay = 100; - /** * Dispatcher for managing job execution and queue polling. */ @@ -22,6 +20,7 @@ export class Dispatcher { private backend: Backend, private queueManager: QueueManager, private executorManager: ExecutorManager, + private sleepDelay: number, ) {} /** @@ -38,14 +37,14 @@ export class Dispatcher { const availableSlots = this.executorManager.availableSlotsByQueue(queue); if (availableSlots <= 0) { logger("Dispatcher").debug(`Queue ${queue.name} limit reached!`); - await this.sleep(sleepDelay); + await this.sleep(this.sleepDelay); continue; } const globalSlots = this.executorManager.availableSlotsGlobal(); if (globalSlots <= 0) { logger("Dispatcher").debug(`Global concurrency limit reached!`); - await this.sleep(sleepDelay); + await this.sleep(this.sleepDelay); continue; } @@ -63,7 +62,7 @@ export class Dispatcher { } if (shouldSleep) { - await this.sleep(sleepDelay); + await this.sleep(this.sleepDelay); } } } diff --git a/packages/engine/src/workers/main.ts b/packages/engine/src/workers/main.ts index 1f33dcb..7c00679 100644 --- a/packages/engine/src/workers/main.ts +++ b/packages/engine/src/workers/main.ts @@ -29,6 +29,7 @@ export class MainWorker { this.backend, new QueueManager(this.backend, nonNullConfig.queues, nonNullConfig.queueDefaults), new ExecutorManager(this.backend, nonNullConfig), + nonNullConfig.jobPollingInterval, ); this.dispatcher.start();