From c8789bc98cd30cf194dc6ba8f343602f68635434 Mon Sep 17 00:00:00 2001
From: Giovani Guizzo <giovaniguizzo@gmail.com>
Date: Wed, 8 Oct 2025 09:04:30 -0300
Subject: [PATCH] feat: enable WAL mode for SQLite backend to improve
 concurrency and performance

---
 .gitignore                                    |  5 +++-
 packages/backends/sqlite/README.md            | 28 +++++++++++++++++++
 .../sqlite/migrations/3_enable_wal_mode.cjs   | 24 ++++++++++++++++
 3 files changed, 56 insertions(+), 1 deletion(-)
 create mode 100644 packages/backends/sqlite/migrations/3_enable_wal_mode.cjs

diff --git a/.gitignore b/.gitignore
index f3859fa0..e3996009 100644
--- a/.gitignore
+++ b/.gitignore
@@ -315,7 +315,10 @@ $RECYCLE.BIN/
 # End of https://www.toptal.com/developers/gitignore/api/node,macos,osx,windows,linux,visualstudiocode,vim,emacs
 
 *.sqlite
+*.sqlite-wal
+*.sqlite-shm
+*.sqlite-journal
 
 .sidequest.config.json
 .turbo
-.vscode
\ No newline at end of file
+.vscode
diff --git a/packages/backends/sqlite/README.md b/packages/backends/sqlite/README.md
index d912e396..43cfc3f3 100644
--- a/packages/backends/sqlite/README.md
+++ b/packages/backends/sqlite/README.md
@@ -75,10 +75,38 @@ backend: {
 - **Zero Configuration** - No database server setup required, just specify a file path
 - **File-Based Storage** - Self-contained database in a single file for easy deployment
 - **ACID Transactions** - Full transaction support for data integrity and safe job claiming
+- **WAL Mode** - Write-Ahead Logging enabled by default for better concurrent access
 - **Lightweight** - Minimal resource footprint, perfect for development and small deployments
 - **Migration Support** - Automatic database schema management with Knex.js migrations
 - **Portable** - Database files can be easily backed up, moved, or shared
 
+## WAL Mode and Concurrency
+
+The SQLite backend automatically enables **WAL (Write-Ahead Logging) mode** for improved concurrent access. This provides several benefits:
+
+- **Better Concurrency**: Allows multiple readers and one writer simultaneously
+- **Reduced Lock Contention**: Minimizes `SQLITE_BUSY` errors during concurrent job processing
+- **Improved Performance**: Faster writes and better throughput for job queue operations
+- **Safer Transactions**: More reliable atomic operations for job claiming
+
+### WAL Mode Files
+
+When WAL mode is enabled, SQLite creates additional files alongside your main database:
+
+- `sidequest.sqlite` - Main database file
+- `sidequest.sqlite-wal` - Write-ahead log file
+- `sidequest.sqlite-shm` - Shared memory index file
+
+These files are managed automatically by SQLite and should not be manually edited or deleted.
+
+### Limitations
+
+While WAL mode significantly improves concurrency, SQLite is still not recommended for high-concurrency production deployments. For production use with multiple workers or distributed systems, consider using:
+
+- **PostgreSQL** (`@sidequest/postgres-backend`) - Best for production
+- **MySQL** (`@sidequest/mysql-backend`) - Good for production
+- **MongoDB** (`@sidequest/mongo-backend`) - Alternative for production
+
 ## License
 
 LGPL-3.0-or-later
diff --git a/packages/backends/sqlite/migrations/3_enable_wal_mode.cjs b/packages/backends/sqlite/migrations/3_enable_wal_mode.cjs
new file mode 100644
index 00000000..e18724bf
--- /dev/null
+++ b/packages/backends/sqlite/migrations/3_enable_wal_mode.cjs
@@ -0,0 +1,24 @@
+/**
+ * Migration to enable WAL (Write-Ahead Logging) mode for SQLite
+ * 
+ * WAL mode provides better concurrency for job processing:
+ * - Allows simultaneous readers and one writer
+ * - Reduces SQLITE_BUSY errors
+ * - Better performance for write-heavy workloads
+ * - More predictable behavior under concurrent load
+ * 
+ * Learn more: https://www.sqlite.org/wal.html
+ */
+
+exports.up = async function(knex) {
+  await knex.schema.raw('PRAGMA journal_mode = WAL;');
+  await knex.schema.raw('PRAGMA busy_timeout = 5000;');
+  await knex.schema.raw('PRAGMA cache_size = -20000;');
+  await knex.schema.raw('PRAGMA temp_store = memory;');
+};
+
+exports.down = function(knex) {
+  // Revert to default rollback journal mode
+  return knex.schema.raw('PRAGMA journal_mode = DELETE;')
+    .then(() => knex.schema.raw('PRAGMA synchronous = FULL;'));
+};