diff --git a/EntityFrameworkCore.Sqlite.Concurrency/EFCore.Sqlite.Concurrency.csproj b/EntityFrameworkCore.Sqlite.Concurrency/EFCore.Sqlite.Concurrency.csproj index 13d1a16..31407e1 100644 --- a/EntityFrameworkCore.Sqlite.Concurrency/EFCore.Sqlite.Concurrency.csproj +++ b/EntityFrameworkCore.Sqlite.Concurrency/EFCore.Sqlite.Concurrency.csproj @@ -11,7 +11,7 @@ EntityFrameworkCore.Sqlite.Concurrency - 10.0.2 + 10.0.3 Mike Gotfryd @@ -34,29 +34,39 @@ β€” registers IDbContextFactory with all concurrency + settings. Use this for Task.WhenAll, background services, Channel consumers, and any + workload that creates concurrent database operations. DbContext is not thread-safe; the factory + pattern gives each concurrent flow its own independent instance. +β€’ Structured logging: pass ILoggerFactory (or let DI resolve it) to get Warning logs for + SQLITE_BUSY/SQLITE_BUSY_SNAPSHOT events, Error logs for SQLITE_LOCKED, and Debug logs for + BEGIN IMMEDIATE upgrades β€” all through your existing logging pipeline. +β€’ GetWalCheckpointStatusAsync β€” runs PRAGMA wal_checkpoint(PASSIVE) and returns a typed + WalCheckpointStatus with IsBusy, TotalWalFrames, CheckpointedFrames, and CheckpointProgress. + Call periodically to detect long-running readers blocking WAL reclamation before it degrades + read performance. +β€’ TryReleaseMigrationLockAsync β€” detects and optionally clears a stale __EFMigrationsLock + row left behind by a crashed migration process. Prevents indefinite blocking on Database.Migrate() + in multi-instance deployments. +β€’ SynchronousMode option β€” configures PRAGMA synchronous (Off / Normal / Full / Extra). + Default remains Normal (recommended for WAL: safe after app crash, fast writes). +β€’ UpgradeTransactionsToImmediate option β€” opt out of the BEGIN β†’ BEGIN IMMEDIATE rewrite + if you manage write transactions explicitly yourself. Default remains true. + +NO BREAKING CHANGES β€” all existing call sites compile and behave correctly without modification. ]]> @@ -93,9 +103,9 @@ Get started in one line. Stop compromising on SQLite reliability and speed. - + - + diff --git a/EntityFrameworkCore.Sqlite.Concurrency/doc/QUICKSTART.md b/EntityFrameworkCore.Sqlite.Concurrency/doc/QUICKSTART.md index 84b83db..7fa9285 100644 --- a/EntityFrameworkCore.Sqlite.Concurrency/doc/QUICKSTART.md +++ b/EntityFrameworkCore.Sqlite.Concurrency/doc/QUICKSTART.md @@ -34,30 +34,62 @@ public class BlogDbContext : DbContext } ``` -### 2. Configure with One Line of Code +### 2. Configure in Program.cs -In your `Program.cs` or startup configuration: +#### Single-threaded / request-scoped use (ASP.NET Core controllers, Razor Pages, Blazor Server) + +One context is created per HTTP request through the DI scope. ASP.NET Core processes requests one thread at a time per scope, so sharing a context here is safe. ```csharp -// Simple configuration -builder.Services.AddDbContext(options => - options.UseSqliteWithConcurrency("Data Source=blog.db")); +builder.Services.AddConcurrentSqliteDbContext("Data Source=blog.db"); ``` Or with custom options: ```csharp -builder.Services.AddDbContext(options => - options.UseSqliteWithConcurrency( - "Data Source=blog.db", - sqliteOptions => +builder.Services.AddConcurrentSqliteDbContext( + "Data Source=blog.db", + options => + { + options.BusyTimeout = TimeSpan.FromSeconds(30); + options.MaxRetryAttempts = 5; + }); +``` + +#### Concurrent use (background workers, Task.WhenAll, channels, hosted services) + +A `DbContext` is **not thread-safe** β€” it must not be shared across concurrent operations. Use `IDbContextFactory` instead. Each concurrent flow calls `CreateDbContext()` to get its own independent instance. + +```csharp +builder.Services.AddConcurrentSqliteDbContextFactory("Data Source=blog.db"); +``` + +Then inject and use the factory: + +```csharp +public class PostImportService +{ + private readonly IDbContextFactory _factory; + + public PostImportService(IDbContextFactory factory) + => _factory = factory; + + public async Task ImportPostsAsync(IEnumerable posts, CancellationToken ct) + { + var tasks = posts.Select(async post => { - sqliteOptions.UseWriteQueue = true; // Enable write serialization - sqliteOptions.BusyTimeout = TimeSpan.FromSeconds(30); - sqliteOptions.MaxRetryAttempts = 5; - })); + await using var db = _factory.CreateDbContext(); + db.Posts.Add(post); + await db.SaveChangesAsync(ct); + }); + + await Task.WhenAll(tasks); // βœ… Each task has its own context β€” no EF thread-safety violation + } +} ``` +> **Note:** `Cache=Shared` in the connection string is incompatible with WAL mode and will throw an `ArgumentException` at startup. Use the default connection string format (`Data Source=blog.db`) β€” connection pooling is enabled automatically. + ## Basic Usage Examples ### Writing Data (Automatically Thread-Safe) @@ -168,45 +200,60 @@ public class ImportService Imagine a scenario where multiple background workers are processing tasks: ```csharp -// WITHOUT ThreadSafeEFCore.SQLite - This would fail with "database is locked" +// ❌ WRONG β€” sharing one DbContext across concurrent tasks +// EF Core will throw InvalidOperationException about concurrent usage, +// and SQLite returns "database is locked" for simultaneous writers. public class TaskProcessor { + private readonly AppDbContext _context; // shared β€” unsafe for concurrent use + public async Task ProcessTasksConcurrently() { var tasks = Enumerable.Range(1, 10) .Select(i => ProcessSingleTaskAsync(i)); - - await Task.WhenAll(tasks); // πŸ’₯ Database locked errors! + + await Task.WhenAll(tasks); // πŸ’₯ EF thread-safety violation + database locked + } + + private async Task ProcessSingleTaskAsync(int taskId) + { + _context.TaskResults.Add(new TaskResult { TaskId = taskId }); + await _context.SaveChangesAsync(); // πŸ’₯ concurrent SaveChanges on one context } } -// WITH ThreadSafeEFCore.SQLite - Just works! +// βœ… CORRECT β€” one context per concurrent flow via IDbContextFactory +// Register with: builder.Services.AddConcurrentSqliteDbContextFactory("Data Source=app.db"); public class TaskProcessor { - private readonly AppDbContext _context; - + private readonly IDbContextFactory _factory; + + public TaskProcessor(IDbContextFactory factory) + => _factory = factory; + public async Task ProcessTasksConcurrently() { var tasks = Enumerable.Range(1, 10) .Select(i => ProcessSingleTaskAsync(i)); - + await Task.WhenAll(tasks); // βœ… All tasks complete successfully } - + private async Task ProcessSingleTaskAsync(int taskId) { - // Each task writes to the database var result = await PerformWorkAsync(taskId); - - // The package automatically queues these writes - _context.TaskResults.Add(new TaskResult + + // Each concurrent flow creates and disposes its own context. + // ThreadSafeEFCore.SQLite serializes the actual writes at the SQLite level. + await using var db = _factory.CreateDbContext(); + db.TaskResults.Add(new TaskResult { TaskId = taskId, Result = result, CompletedAt = DateTime.UtcNow }); - - await _context.SaveChangesAsync(); + + await db.SaveChangesAsync(); // βœ… Thread-safe β€” no shared context, writes queued automatically } } ``` @@ -216,8 +263,7 @@ public class TaskProcessor ```csharp // Create contexts manually when needed var dbContext = ThreadSafeFactory.CreateContext( - "Data Source=blog.db", - options => options.UseWriteQueue = true); + "Data Source=blog.db"); // Use it await dbContext.Posts.AddAsync(new Post { Title = "Hello World" }); @@ -254,19 +300,47 @@ public async Task UpdatePostWithRetryAsync(int postId, string newContent) | Option | Default | Description | |--------|---------|-------------| -| `UseWriteQueue` | `true` | Automatically queue write operations | -| `BusyTimeout` | 30 seconds | How long to wait if database is busy | -| `MaxRetryAttempts` | 3 | Number of retries for busy errors | -| `CommandTimeout` | 300 seconds | SQL command timeout | -| `EnableWalCheckpointManagement` | `true` | Automatically manage WAL checkpoints | +| `BusyTimeout` | 30 seconds | Per-connection `PRAGMA busy_timeout`. First layer of busy handling; SQLite retries lock acquisition internally for up to this duration. | +| `MaxRetryAttempts` | 3 | Application-level retry attempts for `SQLITE_BUSY*` errors, with exponential backoff and jitter. | +| `CommandTimeout` | 300 seconds | EF Core SQL command timeout in seconds. | +| `WalAutoCheckpoint` | 1000 pages | WAL auto-checkpoint interval (`PRAGMA wal_autocheckpoint`). Each page is 4 096 bytes by default (~4 MB). Set to `0` to disable. | +| `SynchronousMode` | `Normal` | Durability vs. performance trade-off (`PRAGMA synchronous`). `Normal` is recommended for WAL mode: safe against application crashes; a power loss or OS crash may roll back the last commit(s) not yet checkpointed. Use `Full` or `Extra` for stronger durability guarantees. | +| `UpgradeTransactionsToImmediate` | `true` | Rewrites `BEGIN`/`BEGIN TRANSACTION` to `BEGIN IMMEDIATE` to prevent `SQLITE_BUSY_SNAPSHOT` mid-transaction. Disable only if you manage write transactions explicitly yourself. | + +## Multi-Instance Deployments and Migration Locks + +EF Core uses a `__EFMigrationsLock` table to serialize concurrent migrations. If a migration process crashes after acquiring the lock but before releasing it, subsequent calls to `Database.Migrate()` will block indefinitely. + +**Recommended approach:** run migrations once as a controlled startup step rather than calling `Database.Migrate()` from every app instance simultaneously. + +If a stale lock does occur, use the built-in helper to detect and clear it: + +```csharp +// In your startup or migration runner: +using var db = factory.CreateDbContext(); +var connection = db.Database.GetDbConnection(); +await connection.OpenAsync(); + +var wasStale = await SqliteConnectionEnhancer.TryReleaseMigrationLockAsync(connection); +if (wasStale) + logger.LogWarning("Stale EF migration lock found and released. Proceeding with migration."); + +await db.Database.MigrateAsync(); +``` + +Pass `release: false` to check for a stale lock without removing it (useful for diagnostics). + +> **Network filesystem warning:** SQLite WAL mode requires all connections to be on the **same physical host**. Do not point the database at an NFS, SMB, or other network-mounted path. If your app runs across multiple machines or containers, use a client/server database instead. ## Best Practices -1. **Use Dependency Injection** when possible for automatic context management -2. **Keep write transactions short** - queue your data and write quickly -3. **Use `BulkInsertOptimizedAsync`** for importing large amounts of data -4. **Enable WAL mode** (already done by default) for better concurrency -5. **Monitor performance** with the built-in diagnostics when needed +1. **Use `IDbContextFactory` for concurrent workloads** β€” inject the factory and call `CreateDbContext()` per concurrent operation; never share a single `DbContext` instance across concurrent tasks +2. **Use `AddConcurrentSqliteDbContext` for request-scoped workloads** β€” standard ASP.NET Core controllers and Razor Pages where one request = one thread = one context +3. **Keep write transactions short** β€” acquire the write slot, write, commit; long-held write transactions block all other writers +4. **Use `BulkInsertOptimizedAsync`** for importing large amounts of data +5. **WAL mode is enabled automatically** β€” do not add `Cache=Shared` to the connection string; it is incompatible with WAL +6. **Run migrations from a single process** β€” avoid calling `Database.Migrate()` concurrently from multiple instances; use `TryReleaseMigrationLockAsync` if a stale lock occurs +7. **Stay on local disk** β€” WAL mode does not work over network filesystems (NFS, SMB); use a client/server database for multi-host deployments ## What Makes It Different diff --git a/EntityFrameworkCore.Sqlite.Concurrency/doc/v10_0_0.md b/EntityFrameworkCore.Sqlite.Concurrency/doc/v10_0_0.md index d1bbd66..2453d8e 100644 --- a/EntityFrameworkCore.Sqlite.Concurrency/doc/v10_0_0.md +++ b/EntityFrameworkCore.Sqlite.Concurrency/doc/v10_0_0.md @@ -19,7 +19,7 @@ options.UseSqlite("Data Source=app.db"); // With this: options.UseSqliteWithConcurrency("Data Source=app.db"); ``` -Guaranteed 100% write reliability and up to 10x faster bulk operations. +Eliminates write contention errors and provides up to 10x faster bulk operations. --- @@ -70,7 +70,7 @@ Next, explore high-performance bulk inserts or fine-tune the configuration. | **Mixed Read/Write Workload** | ~15.3 seconds | ~3.8 seconds | **4.0x faster** | | **Memory Usage (100k operations)** | ~425 MB | ~285 MB | **33% less memory** | -*Benchmark environment: .NET 10, Windows 11, Intel i7-13700K, 32GB RAM* +**Benchmark environment:** .NET 10, Windows 11, Intel i7-13700K, 32GB RAM* --- @@ -82,13 +82,7 @@ public async Task PerformDataMigrationAsync(List legacyRecords) { var modernRecords = legacyRecords.Select(ConvertToModernFormat); - await _context.BulkInsertSafeAsync(modernRecords, new BulkConfig - { - BatchSize = 5000, - PreserveInsertOrder = true, - EnableStreaming = true, - UseOptimalTransactionSize = true - }); + await _context.BulkInsertSafeAsyncmodernRecords); } ``` @@ -114,8 +108,7 @@ public async Task ExecuteHighPerformanceOperationAsync( Func> operation) { using var context = ThreadSafeFactory.CreateContext( - "Data Source=app.db", - options => options.EnablePerformanceOptimizations = true); + "Data Source=app.db"); return await context.ExecuteWithRetryAsync(operation, maxRetries: 2); } @@ -136,7 +129,6 @@ services.AddDbContext(options => concurrencyOptions.BusyTimeout = TimeSpan.FromSeconds(30); concurrencyOptions.MaxRetryAttempts = 3; // Performance-focused retry logic concurrencyOptions.CommandTimeout = 180; // 3-minute timeout for large operations - concurrencyOptions.EnablePerformanceOptimizations = true; // Additional speed boosts })); ``` diff --git a/EntityFrameworkCore.Sqlite.Concurrency/doc/v10_0_3.md b/EntityFrameworkCore.Sqlite.Concurrency/doc/v10_0_3.md new file mode 100644 index 0000000..d85460a --- /dev/null +++ b/EntityFrameworkCore.Sqlite.Concurrency/doc/v10_0_3.md @@ -0,0 +1,226 @@ +# EntityFrameworkCore.Sqlite.Concurrency β€” v10.0.3 Release Notes + +## Fix `SQLITE_BUSY`, `SQLITE_BUSY_SNAPSHOT`, and EF Core Thread-Safety in One Update + +This release closes the remaining correctness gaps in how the library handles SQLite locking errors, adds the `IDbContextFactory` registration pattern that EF Core recommends for concurrent workloads, and introduces structured diagnostics for production observability. + +If you have hit any of the following in your .NET / EF Core application, this update directly addresses them: + +- `Microsoft.Data.Sqlite.SqliteException: SQLite Error 5: 'database is locked'` +- `SQLITE_BUSY_SNAPSHOT` causing unexpected failures mid-transaction +- `InvalidOperationException: A second operation was started on this context` when using `Task.WhenAll` +- Retry storms where all threads wake simultaneously and re-contend for the write lock +- Silent misconfiguration (invalid `MaxRetryAttempts`, `Cache=Shared` + WAL conflicts) +- No visibility into why busy errors occur in production + +--- + +## Bug Fixes + +### SQLITE_BUSY_SNAPSHOT now correctly restarts the operation (not just retries) + +**Error extended code:** `517` (`SQLITE_BUSY | (2 << 8)`) + +This was the most impactful correctness bug. All `SQLITE_BUSY` variants were previously retried the same way β€” waiting a backoff delay and calling the same statement again. `SQLITE_BUSY_SNAPSHOT` has fundamentally different semantics: the connection's WAL read snapshot became stale after another writer committed. Re-executing the same statement produces the same error. The only correct fix is to roll back the entire transaction and **restart the operation from scratch** so that any data read inside it is re-queried against the current snapshot. + +`ExecuteWriteAsync` and `ExecuteWithRetryAsync` now correctly distinguish this case using `SqliteException.SqliteExtendedErrorCode` and restart the full operation lambda. + +### Full jitter added to exponential backoff (thundering herd prevention) + +Pure exponential backoff (`100ms Γ— 2^n`) causes every contending thread to wake at approximately the same time and immediately re-contend for the write lock β€” a classic thundering herd. Retry delays are now randomized in `[baseDelay, 2Γ—baseDelay]` so threads spread out naturally without coordination. + +### `Cache=Shared` incompatibility with WAL detected at startup + +`Cache=Shared` in a SQLite connection string enables a shared page cache across connections. This conflicts with WAL mode's snapshot isolation model, where each connection must independently track read snapshot boundaries. The combination silently corrupts WAL semantics and was previously accepted without warning. It now throws a descriptive `ArgumentException` at startup: + +``` +Cache=Shared is incompatible with WAL mode and cannot be used with ThreadSafeEFCore.SQLite. +Remove 'Cache=Shared' from your connection string. Connection pooling (Pooling=true) is +enabled automatically and provides efficient connection reuse without the WAL incompatibility. +``` + +### Startup validation for `SqliteConcurrencyOptions` + +Invalid option values (e.g. `MaxRetryAttempts = 0`, negative `BusyTimeout`) previously produced silent incorrect behavior. `Validate()` is now called during `UseSqliteWithConcurrency` and throws `ArgumentOutOfRangeException` with a descriptive message at startup. + +--- + +## New Features + +### `AddConcurrentSqliteDbContextFactory` β€” correct EF Core pattern for concurrent workloads + +A `DbContext` instance is not thread-safe and must not be shared across concurrent operations. EF Core's recommended pattern for concurrent workloads β€” background services, `Task.WhenAll`, `Parallel.ForEachAsync`, `Channel` consumers β€” is `IDbContextFactory`, which creates an independent context per concurrent flow. + +The new registration method wires this up with all concurrency settings and auto-injects `ILoggerFactory`: + +```csharp +// Program.cs β€” for concurrent workloads (hosted services, background queues, Task.WhenAll) +builder.Services.AddConcurrentSqliteDbContextFactory("Data Source=app.db"); + +// The existing method remains for request-scoped use (controllers, Razor Pages, Blazor Server) +builder.Services.AddConcurrentSqliteDbContext("Data Source=app.db"); +``` + +Inject `IDbContextFactory` and call `CreateDbContext()` per concurrent operation: + +```csharp +public class ReportGenerationService +{ + private readonly IDbContextFactory _factory; + + public ReportGenerationService(IDbContextFactory factory) + => _factory = factory; + + public async Task GenerateAllReportsAsync(IEnumerable reportIds, CancellationToken ct) + { + // Each task gets its own context β€” no EF thread-safety violation, + // and ThreadSafeEFCore.SQLite serializes the writes at the SQLite level. + var tasks = reportIds.Select(async id => + { + await using var db = _factory.CreateDbContext(); + var data = await db.ReportData.Where(r => r.ReportId == id).ToListAsync(ct); + var result = ComputeReport(data); + db.Reports.Add(result); + await db.SaveChangesAsync(ct); + }); + + await Task.WhenAll(tasks); + } +} +``` + +### Structured logging for `SQLITE_BUSY*` events + +Pass an `ILoggerFactory` (or let DI resolve it automatically through `AddConcurrentSqliteDbContext`/`AddConcurrentSqliteDbContextFactory`) and the interceptor emits structured log entries: + +| Event | Log Level | Message | +|---|---|---| +| `SQLITE_BUSY` / `SQLITE_BUSY_RECOVERY` | `Warning` | Includes command text and retry attempt number | +| `SQLITE_BUSY_SNAPSHOT` | `Warning` | Identifies stale snapshot, includes command text | +| `SQLITE_LOCKED` | `Error` | Same-connection conflict β€” indicates an application bug | +| `BEGIN IMMEDIATE` upgrade | `Debug` | Logged when a deferred `BEGIN` is rewritten | + +Manual wiring (non-DI scenario): + +```csharp +options.UseSqliteWithConcurrency("Data Source=app.db", o => +{ + o.LoggerFactory = loggerFactory; +}); +``` + +### WAL checkpoint health monitoring (`GetWalCheckpointStatusAsync`) + +Long-running read transactions block WAL checkpoint completion, causing the WAL file to grow unboundedly and degrade read performance. Call this periodically to detect pressure before it becomes a problem: + +```csharp +var status = await SqliteConnectionEnhancer.GetWalCheckpointStatusAsync(connection); + +if (status.IsBusy && status.TotalWalFrames > 5000) + logger.LogWarning( + "WAL checkpoint blocked. {Total} frames, {Checkpointed} checkpointed ({Progress:F1}%). " + + "A long-running read transaction may be preventing WAL reclamation.", + status.TotalWalFrames, status.CheckpointedFrames, status.CheckpointProgress); +``` + +### Migration lock recovery (`TryReleaseMigrationLockAsync`) + +EF Core uses a `__EFMigrationsLock` table to serialize concurrent migrations. If a migration process crashes after acquiring the lock, subsequent calls to `Database.Migrate()` block indefinitely. The new helper detects and clears stale locks: + +```csharp +// Run once at startup before Database.MigrateAsync() +var connection = db.Database.GetDbConnection(); +await connection.OpenAsync(); + +var wasStale = await SqliteConnectionEnhancer.TryReleaseMigrationLockAsync(connection); +if (wasStale) + logger.LogWarning("Stale EF migration lock detected and cleared. Proceeding with migration."); + +await db.Database.MigrateAsync(); +``` + +Pass `release: false` to check without modifying the database (diagnostics only). + +### Configurable `SynchronousMode` option + +`PRAGMA synchronous` is now configurable instead of hardcoded. Controls the durability vs. write-speed trade-off: + +| Mode | Durability | Use case | +|---|---|---| +| `Off` | Lowest β€” data loss on OS crash | Bulk import scratch databases | +| `Normal` | **Default** β€” safe after app crash; last commit may be lost on power loss | Most production apps with WAL | +| `Full` | Safe after power loss β€” extra `fsync` on every commit | Financial records, audit logs | +| `Extra` | Strongest β€” guards against certain filesystem clock skew bugs | High-compliance environments | + +```csharp +options.UseSqliteWithConcurrency("Data Source=app.db", o => +{ + o.SynchronousMode = SqliteSynchronousMode.Full; // power-loss safe +}); +``` + +### `UpgradeTransactionsToImmediate` opt-out + +The interceptor rewrites deferred `BEGIN` to `BEGIN IMMEDIATE` by default, which prevents `SQLITE_BUSY_SNAPSHOT` mid-transaction. Power users who manage write transactions explicitly can disable this: + +```csharp +options.UseSqliteWithConcurrency("Data Source=app.db", o => +{ + o.UpgradeTransactionsToImmediate = false; // you manage BEGIN IMMEDIATE yourself +}); +``` + +--- + +## Breaking Changes + +None. This release is fully backwards-compatible: + +- All existing `UseSqliteWithConcurrency`, `AddConcurrentSqliteDbContext`, and `ExecuteWithRetryAsync` call sites compile and behave correctly without modification. +- `AddConcurrentSqliteDbContextFactory` is additive. +- `Cache=Shared` rejection is technically a new startup error, but `Cache=Shared` + WAL was already producing incorrect behavior silently β€” this makes the failure explicit and actionable. +- `SynchronousMode` defaults to `Normal`, which was the hardcoded value in prior versions. +- `UpgradeTransactionsToImmediate` defaults to `true`, preserving the prior behavior. + +--- + +## Configuration Reference + +| Option | Default | Description | +|--------|---------|-------------| +| `BusyTimeout` | 30 s | `PRAGMA busy_timeout` β€” SQLite retries lock acquisition internally for this duration before surfacing `SQLITE_BUSY` to the application. | +| `MaxRetryAttempts` | 3 | Application-level retry attempts after `SQLITE_BUSY*`, with exponential backoff and full jitter. | +| `CommandTimeout` | 300 s | EF Core SQL command timeout. | +| `WalAutoCheckpoint` | 1000 pages | `PRAGMA wal_autocheckpoint` β€” triggers an automatic passive checkpoint after this many WAL frames (~4 MB at the default 4 096-byte page size). Set to `0` to disable. | +| `SynchronousMode` | `Normal` | `PRAGMA synchronous` β€” durability vs. write-speed trade-off. `Normal` is recommended for WAL mode. | +| `UpgradeTransactionsToImmediate` | `true` | Rewrites `BEGIN`/`BEGIN TRANSACTION` to `BEGIN IMMEDIATE` to prevent `SQLITE_BUSY_SNAPSHOT` mid-transaction. | +| `LoggerFactory` | `null` | Resolved automatically from DI when using `AddConcurrentSqliteDbContext` / `AddConcurrentSqliteDbContextFactory`. | + +--- + +## Upgrade Guide + +```csharp +// 1. For request-scoped use (controllers, Razor Pages) β€” no change needed: +builder.Services.AddConcurrentSqliteDbContext("Data Source=app.db"); + +// 2. For concurrent workloads β€” switch to the factory: +// Before: +builder.Services.AddConcurrentSqliteDbContext("Data Source=app.db"); +// After: +builder.Services.AddConcurrentSqliteDbContextFactory("Data Source=app.db"); +// Then inject IDbContextFactory and call CreateDbContext() per task. + +// 3. Remove Cache=Shared from any connection string that has it. + +// 4. That's it β€” no other changes required. +``` + +--- + +## System Requirements + +- .NET 10.0+ +- Entity Framework Core 10.0+ +- Microsoft.Data.Sqlite 10.0+ +- SQLite 3.35.0+ (WAL2 and `SQLITE_BUSY_SNAPSHOT` require 3.37.0+) diff --git a/EntityFrameworkCore.Sqlite.Concurrency/packages.lock.json b/EntityFrameworkCore.Sqlite.Concurrency/packages.lock.json index bfd6c23..c9da38b 100644 --- a/EntityFrameworkCore.Sqlite.Concurrency/packages.lock.json +++ b/EntityFrameworkCore.Sqlite.Concurrency/packages.lock.json @@ -30,11 +30,11 @@ }, "Microsoft.Extensions.DependencyInjection": { "type": "Direct", - "requested": "[10.0.2, )", - "resolved": "10.0.2", - "contentHash": "J/Zmp6fY93JbaiZ11ckWvcyxMPjD6XVwIHQXBjryTBgn7O6O20HYg9uVLFcZlNfgH78MnreE/7EH+hjfzn7VyA==", + "requested": "[10.0.5, )", + "resolved": "10.0.5", + "contentHash": "v1SVsowG6YE1YnHVGmLWz57YTRCQRx9pH5ebIESXfm5isI9gA3QaMyg/oMTzPpXYZwSAVDzYItGJKfmV+pqXkQ==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.2" + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.5" } }, "Microsoft.Extensions.Logging.Abstractions": { @@ -48,18 +48,22 @@ }, "Microsoft.SourceLink.GitHub": { "type": "Direct", - "requested": "[8.0.0, )", - "resolved": "8.0.0", - "contentHash": "G5q7OqtwIyGTkeIOAc3u2ZuV/kicQaec5EaRnc0pIeSnh9LUjj+PYQrJYBURvDt7twGl2PKA7nSN0kz1Zw5bnQ==", + "requested": "[10.0.201, )", + "resolved": "10.0.201", + "contentHash": "qxYAmO4ktzd9L+HMdnqWucxpu7bI9undPyACXOMqPyhaiMtbpbYL/n0ACyWIJlbyEJrXFwxiOaBOSasLtDvsCg==", "dependencies": { - "Microsoft.Build.Tasks.Git": "8.0.0", - "Microsoft.SourceLink.Common": "8.0.0" + "Microsoft.Build.Tasks.Git": "10.0.201", + "Microsoft.SourceLink.Common": "10.0.201", + "System.IO.Hashing": "10.0.5" } }, "Microsoft.Build.Tasks.Git": { "type": "Transitive", - "resolved": "8.0.0", - "contentHash": "bZKfSIKJRXLTuSzLudMFte/8CempWjVamNUR5eHJizsy+iuOuO/k2gnh7W0dHJmYY0tBf+gUErfluCv5mySAOQ==" + "resolved": "10.0.201", + "contentHash": "DMYBnrFZvLnBKn14VavEuuIr31CY6YY2i2L9P8DorS/Qp6ifRR8ZPLdJCFLFfjikNq8DykbYyLd/RP6lSqHcWw==", + "dependencies": { + "System.IO.Hashing": "10.0.5" + } }, "Microsoft.Data.Sqlite.Core": { "type": "Transitive", @@ -145,8 +149,8 @@ }, "Microsoft.Extensions.DependencyInjection.Abstractions": { "type": "Transitive", - "resolved": "10.0.2", - "contentHash": "zOIurr59+kUf9vNcsUkCvKWZv+fPosUZXURZesYkJCvl0EzTc9F7maAO4Cd2WEV7ZJJ0AZrFQvuH6Npph9wdBw==" + "resolved": "10.0.5", + "contentHash": "iVMtq9eRvzyhx8949EGT0OCYJfXi737SbRVzWXE5GrOgGj5AaZ9eUuxA/BSUfmOMALKn/g8KfFaNQw0eiB3lyA==" }, "Microsoft.Extensions.DependencyModel": { "type": "Transitive", @@ -179,8 +183,8 @@ }, "Microsoft.SourceLink.Common": { "type": "Transitive", - "resolved": "8.0.0", - "contentHash": "dk9JPxTCIevS75HyEQ0E4OVAFhB2N+V9ShCXf8Q6FkUQZDkgLI12y679Nym1YqsiSysuQskT7Z+6nUf3yab6Vw==" + "resolved": "10.0.201", + "contentHash": "QbBYhkjgL6rCnBfDbzsAJLlsad13TlBHqYCFDIw56OO2g6ix+9RsmY8uxiQGdWwFKbZXaXyAA6jDCzFYVGCZDw==" }, "SQLitePCLRaw.bundle_e_sqlite3": { "type": "Transitive", @@ -208,6 +212,11 @@ "dependencies": { "SQLitePCLRaw.core": "2.1.11" } + }, + "System.IO.Hashing": { + "type": "Transitive", + "resolved": "10.0.5", + "contentHash": "8IBJWcCT9+e4Bmevm4T7+fQEiAh133KGiz4oiVTgJckd3Q76OFdR1falgn9lpz7+C4HJvogCDJeAa2QmvbeVtg==" } } } diff --git a/EntityFrameworkCore.Sqlite.Concurrency/src/ExtensionMethods/SqliteConcurrencyServiceCollectionExtensions.cs b/EntityFrameworkCore.Sqlite.Concurrency/src/ExtensionMethods/SqliteConcurrencyServiceCollectionExtensions.cs index 6b78250..b95c4a9 100644 --- a/EntityFrameworkCore.Sqlite.Concurrency/src/ExtensionMethods/SqliteConcurrencyServiceCollectionExtensions.cs +++ b/EntityFrameworkCore.Sqlite.Concurrency/src/ExtensionMethods/SqliteConcurrencyServiceCollectionExtensions.cs @@ -1,6 +1,7 @@ using EntityFrameworkCore.Sqlite.Concurrency.Models; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; namespace EntityFrameworkCore.Sqlite.Concurrency.ExtensionMethods; @@ -18,6 +19,22 @@ public static class SqliteConcurrencyServiceCollectionExtensions /// An optional action to configure concurrency options. /// The lifetime of the DbContext. /// The service collection. + /// + /// + /// This overload automatically resolves from the DI + /// container and injects it into the concurrency options so that SQLITE_BUSY* + /// events and BEGIN IMMEDIATE upgrades are logged through the application's + /// normal logging pipeline. + /// + /// + /// A instance is not thread-safe. For workloads that + /// create concurrent database operations (e.g. Task.WhenAll, background queues, + /// hosted services), use + /// + /// instead and inject + /// to create a separate context per concurrent operation. + /// + /// public static IServiceCollection AddConcurrentSqliteDbContext( this IServiceCollection services, string connectionString, @@ -27,9 +44,87 @@ public static IServiceCollection AddConcurrentSqliteDbContext( { services.AddDbContext((provider, options) => { - options.UseSqliteWithConcurrency(connectionString, configure); + options.UseSqliteWithConcurrency(connectionString, o => + { + configure?.Invoke(o); + + // Inject the singleton ILoggerFactory so the interceptor can emit + // structured logs without the caller having to wire it up manually. + if (o.LoggerFactory is null) + o.LoggerFactory = provider.GetService(); + }); }, contextLifetime); - + + return services; + } + + /// + /// Adds an + /// configured with optimized SQLite concurrency and performance settings. + /// + /// The type of the DbContext. + /// The service collection. + /// The SQLite connection string. + /// An optional action to configure concurrency options. + /// + /// The lifetime of the factory. Defaults to + /// because the factory itself holds no per-request state and is safe to share. + /// + /// The service collection. + /// + /// + /// Prefer this overload whenever operations execute concurrently β€” for example inside + /// Task.WhenAll, Channel<T> consumers, + /// IHostedService workers, or + /// Parallel.ForEachAsync. Each concurrent flow should call + /// + /// to obtain its own independent context instance, which eliminates EF Core's + /// object-level thread-safety restriction entirely. + /// + /// + /// This overload automatically resolves from the DI + /// container so that SQLITE_BUSY* events and BEGIN IMMEDIATE upgrades + /// are logged through the application's normal logging pipeline. + /// + /// + /// + /// // Registration + /// builder.Services.AddConcurrentSqliteDbContextFactory<AppDbContext>( + /// "Data Source=app.db"); + /// + /// // Concurrent use β€” each task gets its own context + /// public async Task ProcessAllAsync(IEnumerable<int> ids, CancellationToken ct) + /// { + /// var tasks = ids.Select(async id => + /// { + /// await using var db = _factory.CreateDbContext(); + /// // ... read and write with db + /// }); + /// await Task.WhenAll(tasks); + /// } + /// + /// + /// + public static IServiceCollection AddConcurrentSqliteDbContextFactory( + this IServiceCollection services, + string connectionString, + Action? configure = null, + ServiceLifetime factoryLifetime = ServiceLifetime.Singleton) + where TContext : DbContext + { + services.AddDbContextFactory((provider, options) => + { + options.UseSqliteWithConcurrency(connectionString, o => + { + configure?.Invoke(o); + + // Inject the singleton ILoggerFactory so the interceptor can emit + // structured logs without the caller having to wire it up manually. + if (o.LoggerFactory is null) + o.LoggerFactory = provider.GetService(); + }); + }, factoryLifetime); + return services; } -} \ No newline at end of file +} diff --git a/EntityFrameworkCore.Sqlite.Concurrency/src/Models/SqliteConcurrencyOptions.cs b/EntityFrameworkCore.Sqlite.Concurrency/src/Models/SqliteConcurrencyOptions.cs index cfbd241..dc5fb54 100644 --- a/EntityFrameworkCore.Sqlite.Concurrency/src/Models/SqliteConcurrencyOptions.cs +++ b/EntityFrameworkCore.Sqlite.Concurrency/src/Models/SqliteConcurrencyOptions.cs @@ -1,3 +1,5 @@ +using Microsoft.Extensions.Logging; + namespace EntityFrameworkCore.Sqlite.Concurrency.Models; /// @@ -6,34 +8,161 @@ namespace EntityFrameworkCore.Sqlite.Concurrency.Models; public class SqliteConcurrencyOptions : IEquatable { /// - /// The maximum number of retry attempts for SQLITE_BUSY errors. + /// The maximum number of retry attempts when an operation fails with + /// SQLITE_BUSY or SQLITE_BUSY_SNAPSHOT. /// + /// + /// Each retry uses exponential backoff with jitter starting at 100 ms. + /// Must be greater than zero. Default is 3. + /// public int MaxRetryAttempts { get; set; } = 3; /// - /// The busy timeout for SQLite connections. + /// Per-connection busy timeout passed to PRAGMA busy_timeout. + /// SQLite will automatically retry lock acquisition for up to this duration + /// before returning SQLITE_BUSY. /// + /// + /// + /// This is the first layer of busy handling. The library also adds + /// application-level retry with jitter (the second layer) because SQLite may + /// bypass the busy handler to avoid deadlocks β€” for example, when a read + /// transaction attempts to upgrade to a write transaction. + /// + /// + /// Must be non-negative. A value of disables the + /// built-in handler and relies entirely on the application-level retry. + /// Default is 30 seconds. + /// + /// public TimeSpan BusyTimeout { get; set; } = TimeSpan.FromSeconds(30); /// - /// The command timeout for SQLite commands. + /// EF Core command timeout (seconds) applied via + /// SqliteDbContextOptionsBuilder.CommandTimeout. /// - public int CommandTimeout { get; set; } = 300; // 5 minutes + /// + /// Must be non-negative. Default is 300 (5 minutes). + /// + public int CommandTimeout { get; set; } = 300; /// - /// The number of pages for WAL auto-checkpoint. + /// Number of WAL frames after which SQLite automatically runs a passive + /// checkpoint (PRAGMA wal_autocheckpoint). /// + /// + /// + /// A lower value keeps the WAL file small (reducing read overhead) but adds more + /// checkpoint I/O. A higher value reduces checkpoint frequency at the cost of a + /// larger WAL file and potential read slowdown as readers must scan more frames. + /// + /// + /// Must be non-negative. Set to 0 to disable automatic checkpointing + /// (manual checkpoints only via PRAGMA wal_checkpoint). Default is + /// 1000 pages (~4 MB at the default 4 096-byte page size). + /// + /// public int WalAutoCheckpoint { get; set; } = 1000; + /// + /// Controls how aggressively SQLite flushes commits to stable storage + /// (PRAGMA synchronous). + /// + /// + /// + /// Default: β€” recommended for WAL + /// mode. The database is always consistent after an application crash. A power + /// failure or OS crash may roll back the last committed transaction(s) that had not + /// yet been checkpointed. + /// + /// + /// Use or + /// when power-loss durability is critical. + /// Be aware that these settings significantly increase write latency due to additional + /// fsync calls on every commit. + /// + /// + public SqliteSynchronousMode SynchronousMode { get; set; } = SqliteSynchronousMode.Normal; + + /// + /// When (the default), the interceptor rewrites plain + /// BEGIN and BEGIN TRANSACTION commands to BEGIN IMMEDIATE. + /// + /// + /// + /// BEGIN IMMEDIATE acquires a reserved write lock at transaction start, + /// guaranteeing that no later statement in the transaction will fail with + /// SQLITE_BUSY before commit. This eliminates the common failure mode where + /// a deferred transaction (plain BEGIN) successfully reads data but then + /// fails with SQLITE_BUSY_SNAPSHOT on the first write because another + /// writer committed in the meantime. + /// + /// + /// Trade-off: Upgrading read-only transactions to IMMEDIATE acquires + /// an unnecessary write lock, which can reduce concurrency slightly when many + /// read-only operations run simultaneously. If your application exclusively uses + /// explicit read-only transactions and manages write transactions manually, set this + /// to and start write transactions with + /// BEGIN IMMEDIATE yourself. + /// + /// + /// Commands that already contain IMMEDIATE or EXCLUSIVE are never + /// modified. + /// + /// + public bool UpgradeTransactionsToImmediate { get; set; } = true; + + /// + /// Optional logger factory used to create the interceptor's + /// . Not included in equality comparison. + /// + /// + /// When provided, the library logs SQLITE_BUSY* events (Warning), + /// checkpoint-blocked detections (Warning), and BEGIN IMMEDIATE upgrades + /// (Debug). In a DI scenario, prefer + /// + /// which resolves the factory automatically. + /// + public ILoggerFactory? LoggerFactory { get; set; } + + /// + /// Validates that all option values are within acceptable ranges. + /// + /// + /// Thrown when any option value is outside its valid range. + /// + public void Validate() + { + if (MaxRetryAttempts <= 0) + throw new ArgumentOutOfRangeException(nameof(MaxRetryAttempts), + MaxRetryAttempts, "MaxRetryAttempts must be greater than zero."); + + if (BusyTimeout < TimeSpan.Zero) + throw new ArgumentOutOfRangeException(nameof(BusyTimeout), + BusyTimeout, "BusyTimeout must be non-negative."); + + if (CommandTimeout < 0) + throw new ArgumentOutOfRangeException(nameof(CommandTimeout), + CommandTimeout, "CommandTimeout must be non-negative."); + + if (WalAutoCheckpoint < 0) + throw new ArgumentOutOfRangeException(nameof(WalAutoCheckpoint), + WalAutoCheckpoint, "WalAutoCheckpoint must be non-negative (0 disables auto-checkpoint)."); + } + /// public bool Equals(SqliteConcurrencyOptions? other) { if (other is null) return false; if (ReferenceEquals(this, other)) return true; - return MaxRetryAttempts == other.MaxRetryAttempts && - BusyTimeout.Equals(other.BusyTimeout) && - CommandTimeout == other.CommandTimeout && - WalAutoCheckpoint == other.WalAutoCheckpoint; + return MaxRetryAttempts == other.MaxRetryAttempts + && BusyTimeout.Equals(other.BusyTimeout) + && CommandTimeout == other.CommandTimeout + && WalAutoCheckpoint == other.WalAutoCheckpoint + && SynchronousMode == other.SynchronousMode + && UpgradeTransactionsToImmediate == other.UpgradeTransactionsToImmediate; + // LoggerFactory is intentionally excluded from equality: it is infrastructure + // metadata and does not affect SQLite behaviour. } /// @@ -47,7 +176,11 @@ public override bool Equals(object? obj) /// public override int GetHashCode() - { - return HashCode.Combine(MaxRetryAttempts, BusyTimeout, CommandTimeout, WalAutoCheckpoint); - } -} \ No newline at end of file + => HashCode.Combine( + MaxRetryAttempts, + BusyTimeout, + CommandTimeout, + WalAutoCheckpoint, + SynchronousMode, + UpgradeTransactionsToImmediate); +} diff --git a/EntityFrameworkCore.Sqlite.Concurrency/src/Models/SqliteSynchronousMode.cs b/EntityFrameworkCore.Sqlite.Concurrency/src/Models/SqliteSynchronousMode.cs new file mode 100644 index 0000000..25f9fea --- /dev/null +++ b/EntityFrameworkCore.Sqlite.Concurrency/src/Models/SqliteSynchronousMode.cs @@ -0,0 +1,80 @@ +namespace EntityFrameworkCore.Sqlite.Concurrency.Models; + +/// +/// Controls how aggressively SQLite flushes write operations to stable storage +/// (PRAGMA synchronous). +/// +/// +/// +/// The right choice depends on your durability requirements and workload: +/// +/// +/// +/// For WAL mode (the default), is almost always correct. +/// Commits are durable against application crashes; the small risk is losing the +/// last transaction or two after an unexpected power loss or OS crash. +/// +/// +/// For rollback journal mode, or provides +/// stronger guarantees at the cost of more fsync calls per commit. +/// +/// +/// trades all durability for maximum speed and is only +/// appropriate for ephemeral or fully-reproducible databases. +/// +/// +/// +/// See the SQLite +/// documentation for the complete specification. +/// +/// +public enum SqliteSynchronousMode +{ + /// + /// PRAGMA synchronous = OFF β€” SQLite hands off writes to the OS and + /// continues without waiting for any flush. Maximum performance. + /// + /// Risk: A power failure or OS crash at any point can corrupt the database. + /// Not recommended for production data. + /// + /// + Off = 0, + + /// + /// PRAGMA synchronous = NORMAL β€” SQLite syncs at WAL checkpoint boundaries + /// rather than on every commit. + /// + /// The database is always consistent after an application crash. A power failure or + /// OS crash may roll back the last one or two transactions that had been committed + /// by the application but not yet checkpointed. + /// + /// + /// This is the recommended setting for WAL mode and the library default. It + /// delivers significantly higher write throughput than while + /// providing acceptable durability for most workloads. + /// + /// + Normal = 1, + + /// + /// PRAGMA synchronous = FULL β€” SQLite issues an fsync (or platform + /// equivalent) on every commit, ensuring committed data survives a power failure + /// or OS crash. + /// + /// Significantly slower than for write-heavy workloads because + /// every commit blocks until the OS confirms the data is on stable storage. + /// + /// + Full = 2, + + /// + /// PRAGMA synchronous = EXTRA β€” Like , but also syncs + /// the directory containing the database file after deleting or truncating the + /// rollback journal (DELETE/TRUNCATE mode). Provides the strongest durability + /// guarantee against filesystems that do not persist directory entries on crash. + /// + /// Has no additional benefit beyond in WAL mode. + /// + /// + Extra = 3 +} diff --git a/EntityFrameworkCore.Sqlite.Concurrency/src/Models/WalCheckpointStatus.cs b/EntityFrameworkCore.Sqlite.Concurrency/src/Models/WalCheckpointStatus.cs new file mode 100644 index 0000000..6b2a4d8 --- /dev/null +++ b/EntityFrameworkCore.Sqlite.Concurrency/src/Models/WalCheckpointStatus.cs @@ -0,0 +1,42 @@ +namespace EntityFrameworkCore.Sqlite.Concurrency.Models; + +/// +/// Represents the WAL checkpoint state returned by +/// PRAGMA wal_checkpoint(PASSIVE). +/// +/// +/// if one or more active readers held a WAL read lock on a +/// frame the checkpoint attempted to overwrite, preventing the checkpoint from +/// completing fully. +/// +/// A persistently value means long-running read transactions are +/// blocking WAL reclamation. As the WAL grows, read performance degrades because readers +/// must scan the growing WAL for every page lookup. Investigate and shorten the longest +/// running read transactions. +/// +/// +/// +/// The total number of frames currently in the WAL file. Each frame corresponds to one +/// database page (default 4 096 bytes). Multiply by the page size to estimate WAL file +/// size on disk. +/// +/// +/// The number of WAL frames successfully transferred back to the main database file +/// during the last checkpoint pass. When greatly exceeds +/// this value, readers are blocking the checkpoint. +/// +public record WalCheckpointStatus(bool IsBusy, int TotalWalFrames, int CheckpointedFrames) +{ + /// + /// Returns when there are WAL frames that have not yet been + /// checkpointed back into the main database, indicating WAL growth pressure. + /// + public bool HasUncheckpointedFrames => TotalWalFrames > CheckpointedFrames; + + /// + /// The percentage of WAL frames that have been successfully checkpointed. + /// Returns 100.0 when is zero. + /// + public double CheckpointProgress => + TotalWalFrames == 0 ? 100.0 : (double)CheckpointedFrames / TotalWalFrames * 100.0; +} diff --git a/EntityFrameworkCore.Sqlite.Concurrency/src/SqliteConcurrencyExtensions.cs b/EntityFrameworkCore.Sqlite.Concurrency/src/SqliteConcurrencyExtensions.cs index e704a74..41add9d 100644 --- a/EntityFrameworkCore.Sqlite.Concurrency/src/SqliteConcurrencyExtensions.cs +++ b/EntityFrameworkCore.Sqlite.Concurrency/src/SqliteConcurrencyExtensions.cs @@ -18,6 +18,9 @@ public static class SqliteConcurrencyExtensions /// The SQLite connection string. /// An optional action to configure concurrency options. /// The options builder. + /// + /// Thrown when any option value is outside its valid range. + /// public static DbContextOptionsBuilder UseSqliteWithConcurrency( this DbContextOptionsBuilder optionsBuilder, string connectionString, @@ -25,36 +28,60 @@ public static DbContextOptionsBuilder UseSqliteWithConcurrency( { var options = new SqliteConcurrencyOptions(); configure?.Invoke(options); + options.Validate(); // Get the enhanced connection string var enhancedConnectionString = SqliteConnectionEnhancer .GetOptimizedConnectionString(connectionString); - + // Use the connection string with EF Core to allow proper pooling optionsBuilder.UseSqlite(enhancedConnectionString, sqliteOptions => { - // Configure command timeout sqliteOptions.CommandTimeout(options.CommandTimeout); }); - + // Add interceptors for PRAGMAs, performance, and concurrency var interceptor = SqliteConnectionEnhancer.GetInterceptor(enhancedConnectionString, options); optionsBuilder.AddInterceptors(interceptor); - + return optionsBuilder; } - - + /// - /// Executes an operation with automatic retry on SQLITE_BUSY errors. + /// Executes an operation with automatic retry on SQLITE_BUSY and + /// SQLITE_BUSY_SNAPSHOT errors. /// /// The result type. /// The database context. /// The operation to execute. - /// The maximum number of retries. + /// + /// The maximum number of retry attempts. Each retry waits using exponential backoff + /// with jitter starting at 100 ms. + /// /// The cancellation token. /// The result of the operation. + /// + /// + /// Two classes of busy error are handled: + /// + /// + /// + /// SQLITE_BUSY / SQLITE_BUSY_RECOVERY / SQLITE_BUSY_TIMEOUT β€” another + /// connection holds a lock. The operation is retried after a backoff delay. + /// + /// + /// SQLITE_BUSY_SNAPSHOT β€” the connection's read snapshot became stale + /// after another writer committed. The entire operation is restarted so that it + /// can acquire a fresh snapshot. Any data read in the failed attempt must be + /// re-queried inside the operation lambda. + /// + /// + /// + /// SQLITE_LOCKED (same-connection conflict) is not retried and propagates + /// immediately, as it indicates an application-level bug. + /// + /// public static async Task ExecuteWithRetryAsync( this DbContext context, Func> operation, @@ -68,10 +95,25 @@ public static async Task ExecuteWithRetryAsync( { return await operation(context); } - catch (SqliteException ex) when (ex.SqliteErrorCode == 5 && attempt < maxRetries) + catch (SqliteException ex) when (SqliteErrorCodes.IsAnyBusy(ex) && attempt < maxRetries) { + // SQLITE_LOCKED (code 6) is not caught here β€” it is an app-level bug and + // should not be silently retried. + attempt++; - await Task.Delay(TimeSpan.FromMilliseconds(100 * Math.Pow(2, attempt)), cancellationToken); + var isSnapshot = SqliteErrorCodes.IsBusySnapshot(ex); + + // Exponential backoff with full jitter: delay is uniformly distributed in + // [baseDelay, 2Γ—baseDelay] to prevent synchronized retry storms ("thundering + // herd") when multiple threads hit contention simultaneously. + var baseDelay = 100 * Math.Pow(2, attempt); + var jitter = Random.Shared.NextDouble() * baseDelay; + await Task.Delay(TimeSpan.FromMilliseconds(baseDelay + jitter), cancellationToken); + + // For SQLITE_BUSY_SNAPSHOT the operation lambda will restart on the next + // loop iteration, which is correct: it allows the caller to re-query any + // data that may now be stale. + _ = isSnapshot; // consumed for documentation clarity } } } @@ -107,7 +149,6 @@ public static async Task BulkInsertOptimizedAsync( { await using var transaction = await context.Database.BeginTransactionAsync(cancellationToken); - // Batch inserts var batchSize = 1000; var batches = entities.Chunk(batchSize); @@ -126,4 +167,4 @@ public static async Task BulkInsertOptimizedAsync( writeLock.Release(); } } -} \ No newline at end of file +} diff --git a/EntityFrameworkCore.Sqlite.Concurrency/src/SqliteConcurrencyInterceptor.cs b/EntityFrameworkCore.Sqlite.Concurrency/src/SqliteConcurrencyInterceptor.cs index 09035f3..5b6d853 100644 --- a/EntityFrameworkCore.Sqlite.Concurrency/src/SqliteConcurrencyInterceptor.cs +++ b/EntityFrameworkCore.Sqlite.Concurrency/src/SqliteConcurrencyInterceptor.cs @@ -1,6 +1,7 @@ using System.Data.Common; using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Logging; using EntityFrameworkCore.Sqlite.Concurrency.Models; namespace EntityFrameworkCore.Sqlite.Concurrency; @@ -13,6 +14,7 @@ public class SqliteConcurrencyInterceptor : DbCommandInterceptor, IDbConnectionI private readonly SqliteConcurrencyOptions _options; private readonly SemaphoreSlim _writeLock; private readonly string _connectionString; + private readonly ILogger? _logger; /// /// Gets the concurrency options configured for this interceptor. @@ -29,6 +31,7 @@ public SqliteConcurrencyInterceptor(SqliteConcurrencyOptions options, string con _options = options; _connectionString = connectionString; _writeLock = SqliteConnectionEnhancer.GetWriteLock(connectionString); + _logger = options.LoggerFactory?.CreateLogger(); } // --- Connection Management --- @@ -96,21 +99,82 @@ public override ValueTask> ScalarExecutingAsync( return base.ScalarExecutingAsync(command, eventData, result, cancellationToken); } - private static void UpgradeToBeginImmediate(DbCommand command) + // --- Command Failure Logging --- + + /// + public override void CommandFailed(DbCommand command, CommandErrorEventData eventData) + { + LogCommandFailure(eventData.Exception, command.CommandText); + base.CommandFailed(command, eventData); + } + + /// + public override Task CommandFailedAsync(DbCommand command, CommandErrorEventData eventData, CancellationToken cancellationToken = default) { + LogCommandFailure(eventData.Exception, command.CommandText); + return base.CommandFailedAsync(command, eventData, cancellationToken); + } + + private void LogCommandFailure(Exception? exception, string commandText) + { + if (_logger is null || exception is not SqliteException sqlEx) return; + + if (SqliteErrorCodes.IsBusySnapshot(sqlEx)) + { + _logger.LogWarning( + sqlEx, + "SQLITE_BUSY_SNAPSHOT on command [{Command}]: the connection's read snapshot is stale β€” " + + "another writer committed after this transaction began. The transaction will be rolled back " + + "and retried from scratch. Extended error code: {ExtendedCode}.", + TruncateCommand(commandText), + sqlEx.SqliteExtendedErrorCode); + } + else if (SqliteErrorCodes.IsRetryableBusy(sqlEx)) + { + _logger.LogWarning( + sqlEx, + "SQLITE_BUSY on command [{Command}]: the database is locked by another connection. " + + "Will retry with backoff. Extended error code: {ExtendedCode}.", + TruncateCommand(commandText), + sqlEx.SqliteExtendedErrorCode); + } + else if (SqliteErrorCodes.IsLocked(sqlEx)) + { + _logger.LogError( + sqlEx, + "SQLITE_LOCKED on command [{Command}]: conflict within the same connection. " + + "This typically indicates a statement is still open on the same connection while " + + "a write is attempted. Extended error code: {ExtendedCode}.", + TruncateCommand(commandText), + sqlEx.SqliteExtendedErrorCode); + } + } + + private void UpgradeToBeginImmediate(DbCommand command) + { + if (!_options.UpgradeTransactionsToImmediate) return; + var text = command.CommandText.Trim(); - if (text.StartsWith("BEGIN", StringComparison.OrdinalIgnoreCase) && - !text.Contains("IMMEDIATE", StringComparison.OrdinalIgnoreCase) && - !text.Contains("EXCLUSIVE", StringComparison.OrdinalIgnoreCase)) + if (!text.StartsWith("BEGIN", StringComparison.OrdinalIgnoreCase)) return; + if (text.Contains("IMMEDIATE", StringComparison.OrdinalIgnoreCase)) return; + if (text.Contains("EXCLUSIVE", StringComparison.OrdinalIgnoreCase)) return; + + if (text.Equals("BEGIN", StringComparison.OrdinalIgnoreCase) || + text.Equals("BEGIN TRANSACTION", StringComparison.OrdinalIgnoreCase) || + text.Equals("BEGIN DEFERRED", StringComparison.OrdinalIgnoreCase) || + text.Equals("BEGIN DEFERRED TRANSACTION", StringComparison.OrdinalIgnoreCase)) { - if (text.Equals("BEGIN", StringComparison.OrdinalIgnoreCase) || - text.Equals("BEGIN TRANSACTION", StringComparison.OrdinalIgnoreCase)) - { - command.CommandText = "BEGIN IMMEDIATE"; - } + command.CommandText = "BEGIN IMMEDIATE"; + _logger?.LogDebug( + "Upgraded [{Original}] to BEGIN IMMEDIATE to prevent SQLITE_BUSY_SNAPSHOT " + + "mid-transaction. Set UpgradeTransactionsToImmediate = false to disable.", + text); } } + private static string TruncateCommand(string commandText) + => commandText.Length <= 120 ? commandText : commandText[..120] + "…"; + // --- Transaction Interception --- /// @@ -185,4 +249,4 @@ public void ConnectionClosing(DbConnection connection, ConnectionEventData event public void ConnectionFailed(DbConnection connection, ConnectionErrorEventData eventData) { } /// public Task ConnectionFailedAsync(DbConnection connection, ConnectionErrorEventData eventData) => Task.CompletedTask; -} \ No newline at end of file +} diff --git a/EntityFrameworkCore.Sqlite.Concurrency/src/SqliteConnectionEnhancer.cs b/EntityFrameworkCore.Sqlite.Concurrency/src/SqliteConnectionEnhancer.cs index fa68986..a186bd2 100644 --- a/EntityFrameworkCore.Sqlite.Concurrency/src/SqliteConnectionEnhancer.cs +++ b/EntityFrameworkCore.Sqlite.Concurrency/src/SqliteConnectionEnhancer.cs @@ -14,10 +14,10 @@ public static class SqliteConnectionEnhancer { // Cache optimized connection strings to avoid repeated parsing private static readonly ConcurrentDictionary _connectionStringCache = new(); - + // Shared locks per connection string to ensure serialization across multiple DbContext instances private static readonly ConcurrentDictionary _writeLocks = new(); - + // Shared interceptors per connection string to avoid leaking background tasks private static readonly ConcurrentDictionary _interceptors = new(); @@ -37,7 +37,6 @@ public static class SqliteConnectionEnhancer /// An optimized connection string. public static string GetOptimizedConnectionString(string originalConnectionString) { - // Cache hit - return pre-computed optimized string return _connectionStringCache.GetOrAdd(originalConnectionString, ComputeOptimizedConnectionString); } @@ -57,9 +56,14 @@ public static SemaphoreSlim GetWriteLock(string connectionString) /// The connection string. /// The concurrency options. /// A instance. - /// Thrown when the provided do not match the options of an existing interceptor for the same . + /// + /// Thrown when the provided do not match the options of an + /// existing interceptor for the same . + /// /// - /// Callers must use consistent options for the same connection string, as interceptors are cached and shared. + /// Callers must use consistent options for the same connection string, as interceptors + /// are cached and shared. is + /// excluded from this check. /// public static SqliteConcurrencyInterceptor GetInterceptor(string connectionString, SqliteConcurrencyOptions options) { @@ -85,18 +89,34 @@ private static string FormatOptions(SqliteConcurrencyOptions options) return $"[MaxRetryAttempts={options.MaxRetryAttempts}, " + $"BusyTimeout={options.BusyTimeout}, " + $"CommandTimeout={options.CommandTimeout}, " + - $"WalAutoCheckpoint={options.WalAutoCheckpoint}]"; + $"WalAutoCheckpoint={options.WalAutoCheckpoint}, " + + $"SynchronousMode={options.SynchronousMode}, " + + $"UpgradeTransactionsToImmediate={options.UpgradeTransactionsToImmediate}]"; } private static string ComputeOptimizedConnectionString(string originalConnectionString) { - var builder = new SqliteConnectionStringBuilder(originalConnectionString) - { - Pooling = true, - ForeignKeys = true, - RecursiveTriggers = true, - Mode = SqliteOpenMode.ReadWriteCreate - }; + var builder = new SqliteConnectionStringBuilder(originalConnectionString); + + // Cache=Shared uses a single shared page cache across connections, which conflicts + // with WAL mode's snapshot isolation model. In WAL mode each connection must track + // its own read snapshot independently; shared-cache connections share internal + // pager state in a way that can produce inconsistent snapshot visibility and + // violates WAL's reader/writer non-blocking guarantee. + // Connection pooling (Pooling=true, set below) achieves efficient reuse without + // this incompatibility. See https://www.sqlite.org/wal.html for details. + if (builder.Cache == SqliteCacheMode.Shared) + throw new ArgumentException( + "Cache=Shared is incompatible with WAL mode and cannot be used with " + + "ThreadSafeEFCore.SQLite. Remove 'Cache=Shared' from your connection string. " + + "Connection pooling (Pooling=true) is enabled automatically and provides " + + "efficient connection reuse without the WAL incompatibility.", + nameof(originalConnectionString)); + + builder.Pooling = true; + builder.ForeignKeys = true; + builder.RecursiveTriggers = true; + builder.Mode = SqliteOpenMode.ReadWriteCreate; return builder.ToString(); } @@ -117,13 +137,15 @@ public static void ApplyRuntimePragmas(DbConnection connection) /// The concurrency options. public static void ApplyRuntimePragmas(DbConnection connection, SqliteConcurrencyOptions options) { - if (connection is not SqliteConnection sqliteConnection) + if (connection is not SqliteConnection sqliteConnection) return; var builder = new SqliteConnectionStringBuilder(sqliteConnection.ConnectionString); var dataSource = builder.DataSource; - // 1. Database-scoped Pragmas - Run once per process + // 1. Database-scoped PRAGMAs β€” executed once per process per database file. + // These settings are persistent (stored in the database header) and affect all + // connections to the same file. if (!_initializedDatabases.ContainsKey(dataSource)) { var lockObj = _pragmaLocks.GetOrAdd(dataSource, _ => new object()); @@ -135,14 +157,33 @@ public static void ApplyRuntimePragmas(DbConnection connection, SqliteConcurrenc { using var initCommand = sqliteConnection.CreateCommand(); initCommand.CommandText = $@" + -- WAL mode: readers and writers can proceed concurrently (readers never block + -- writers and writers never block readers). The WAL file must remain on the + -- same machine as the database β€” do not use WAL on network filesystems. PRAGMA journal_mode = WAL; + + -- 4 096 bytes aligns with modern OS page sizes (ext4, NTFS, APFS) and is the + -- SQLite recommended default. Changing page_size after data exists has no effect + -- without a VACUUM, so this is a no-op on pre-existing databases. PRAGMA page_size = 4096; + + -- INCREMENTAL auto-vacuum reclaims free pages on demand (PRAGMA incremental_vacuum) + -- without the heavy full-database rewrite that FULL auto-vacuum performs on every + -- commit. NONE means free pages are never returned to the OS. PRAGMA auto_vacuum = INCREMENTAL; + + -- Caps the on-disk size of the rollback journal / WAL after a checkpoint or commit. + -- 128 MB is a reasonable upper bound; without this the WAL can grow unbounded when + -- long-running readers prevent checkpoint completion. PRAGMA journal_size_limit = 134217728; + + -- Trigger an automatic passive checkpoint after this many WAL frames are written. + -- 1 000 frames Γ— 4 096 bytes β‰ˆ 4 MB. Smaller values keep the WAL compact (faster + -- reads) at the cost of more checkpoint I/O. Set to 0 to disable auto-checkpoint. PRAGMA wal_autocheckpoint = {options.WalAutoCheckpoint}; "; initCommand.ExecuteNonQuery(); - + _initializedDatabases.TryAdd(dataSource, true); } catch @@ -155,17 +196,169 @@ public static void ApplyRuntimePragmas(DbConnection connection, SqliteConcurrenc } } - // 2. Connection-scoped Pragmas - Run on every open + // 2. Connection-scoped PRAGMAs β€” applied on every connection open. + // These are per-connection settings that are not stored in the database file. using var command = sqliteConnection.CreateCommand(); command.CommandText = $@" + -- How long (ms) this connection will spin waiting for a lock before returning + -- SQLITE_BUSY. This is the first layer of busy handling; the library adds a + -- second layer (application-level retry with jitter) because SQLite bypasses + -- this handler when it detects a potential deadlock. PRAGMA busy_timeout = {(int)options.BusyTimeout.TotalMilliseconds}; + + -- Memory-mapped I/O size (256 MB). Allows the OS virtual-memory subsystem to + -- serve reads directly from the mapped region, bypassing read() syscalls for + -- hot pages. Adjust down on memory-constrained hosts. PRAGMA mmap_size = 268435456; + + -- Store internal temporary tables and indices in RAM instead of a temp file. + -- Eliminates temp-file I/O for sort and aggregation operations. PRAGMA temp_store = MEMORY; + + -- Negative value = kibibytes. -20 000 β‰ˆ 20 MB page cache per connection. + -- Each connection maintains its own cache; size accordingly for your process. PRAGMA cache_size = -20000; - PRAGMA synchronous = NORMAL; + + -- Durability vs. performance trade-off. See SqliteSynchronousMode for full + -- documentation. NORMAL is the recommended setting for WAL mode: the database + -- is always consistent after an application crash; a power loss or OS crash + -- may roll back the last one or two commits that had not yet been checkpointed. + PRAGMA synchronous = {options.SynchronousMode.ToString().ToUpperInvariant()}; + + -- NORMAL (default): connections release file locks between transactions, + -- allowing other processes to access the database. EXCLUSIVE holds locks + -- permanently and can improve single-process throughput but prevents any + -- other process from opening the file. PRAGMA locking_mode = NORMAL; + + -- OFF: deleted content is overwritten with zeros on VACUUM only, not on every + -- DELETE. Improves write performance. Enable if the database stores sensitive + -- data that must not be recoverable from free pages after deletion. PRAGMA secure_delete = OFF; "; command.ExecuteNonQuery(); } -} \ No newline at end of file + + /// + /// Runs a passive WAL checkpoint and returns its status, which indicates whether + /// the WAL is growing and whether long-running readers are blocking reclamation. + /// + /// An open SQLite connection to the target database. + /// A cancellation token. + /// + /// A describing the current WAL state. + /// Returns a zeroed status when the database is not in WAL mode. + /// + /// + /// + /// A passive checkpoint transfers already-committed WAL frames to the main + /// database file without blocking readers or writers. It is the safest checkpoint + /// mode for health monitoring. + /// + /// + /// Call this periodically (e.g., every few minutes) to detect WAL growth pressure. + /// A persistently result combined with a + /// large means long-running read + /// transactions are preventing WAL reclamation and will eventually degrade read + /// performance as readers must scan an ever-larger WAL on every page lookup. + /// + /// + public static async Task GetWalCheckpointStatusAsync( + DbConnection connection, + CancellationToken cancellationToken = default) + { + if (connection is not SqliteConnection) + return new WalCheckpointStatus(false, 0, 0); + + await using var command = connection.CreateCommand(); + // PRAGMA wal_checkpoint(PASSIVE) returns a single row: (busy, log, checkpointed) + // busy β€” 1 if blocked by an active reader, 0 otherwise + // log β€” total WAL frames + // checkpointed β€” frames successfully written back to the main DB + command.CommandText = "PRAGMA wal_checkpoint(PASSIVE);"; + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + if (!await reader.ReadAsync(cancellationToken)) + return new WalCheckpointStatus(false, 0, 0); + + var busy = reader.GetInt32(0) != 0; + var totalFrames = reader.GetInt32(1); + var checkpointed = reader.GetInt32(2); + + return new WalCheckpointStatus(busy, totalFrames, checkpointed); + } + + /// + /// Checks for a stale EF Core migration lock and optionally removes it. + /// + /// An open SQLite connection to the target database. + /// + /// When (the default), deletes the stale lock row so that + /// EF Core can proceed with migrations. When , only checks + /// for the lock without modifying the database. + /// + /// A cancellation token. + /// + /// if a stale migration lock was found (and released when + /// is ); + /// if the __EFMigrationsLock table does not exist or contains no rows. + /// + /// + /// + /// EF Core serializes concurrent migrations using a __EFMigrationsLock table. + /// If a migration process crashes or is killed after acquiring the lock but before + /// releasing it, subsequent migration attempts will block indefinitely waiting for + /// a lock that will never be freed. + /// + /// + /// This is especially relevant in multi-instance deployments where every app instance + /// calls Database.Migrate() at startup. The safest strategy is to run + /// migrations as a single, controlled step (e.g. an init container or a deployment + /// script) rather than from every instance concurrently. When a stale lock does + /// occur, call this method once with set to + /// before retrying the migration. + /// + /// + /// + /// // At startup, before calling Database.Migrate(): + /// var wasStale = await SqliteConnectionEnhancer.TryReleaseMigrationLockAsync(connection); + /// if (wasStale) + /// logger.LogWarning("Stale EF migration lock detected and released."); + /// await db.Database.MigrateAsync(); + /// + /// + /// + public static async Task TryReleaseMigrationLockAsync( + DbConnection connection, + bool release = true, + CancellationToken cancellationToken = default) + { + if (connection is not SqliteConnection) + return false; + + // Check whether the migrations lock table exists at all. + await using var tableCmd = connection.CreateCommand(); + tableCmd.CommandText = + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='__EFMigrationsLock';"; + var tableCount = (long)(await tableCmd.ExecuteScalarAsync(cancellationToken) ?? 0L); + if (tableCount == 0) + return false; + + // Check whether a lock row is present. + await using var lockCmd = connection.CreateCommand(); + lockCmd.CommandText = "SELECT COUNT(*) FROM __EFMigrationsLock;"; + var lockCount = (long)(await lockCmd.ExecuteScalarAsync(cancellationToken) ?? 0L); + if (lockCount == 0) + return false; + + // A stale lock exists β€” remove it if requested. + if (release) + { + await using var deleteCmd = connection.CreateCommand(); + deleteCmd.CommandText = "DELETE FROM __EFMigrationsLock;"; + await deleteCmd.ExecuteNonQueryAsync(cancellationToken); + } + + return true; + } +} diff --git a/EntityFrameworkCore.Sqlite.Concurrency/src/SqliteErrorCodes.cs b/EntityFrameworkCore.Sqlite.Concurrency/src/SqliteErrorCodes.cs new file mode 100644 index 0000000..54398dc --- /dev/null +++ b/EntityFrameworkCore.Sqlite.Concurrency/src/SqliteErrorCodes.cs @@ -0,0 +1,63 @@ +using Microsoft.Data.Sqlite; + +namespace EntityFrameworkCore.Sqlite.Concurrency; + +/// +/// SQLite primary and extended error code constants, with helpers for classifying exceptions. +/// +/// +/// SQLite error codes follow the pattern: extended = primary | (reason << 8). +/// holds the primary code (low 8 bits). +/// holds the full extended code. +/// +internal static class SqliteErrorCodes +{ + // Primary error codes + public const int Busy = 5; // SQLITE_BUSY + public const int Locked = 6; // SQLITE_LOCKED + + // Extended SQLITE_BUSY variants (5 | reason << 8) + public const int BusyRecovery = 261; // SQLITE_BUSY_RECOVERY β€” WAL recovery in progress; safe to wait and retry + public const int BusySnapshot = 517; // SQLITE_BUSY_SNAPSHOT β€” read snapshot too old; must rollback and restart + public const int BusyTimeout = 773; // SQLITE_BUSY_TIMEOUT β€” busy_timeout expired + + // Extended SQLITE_LOCKED variant (6 | reason << 8) + public const int LockedSharedCache = 262; // SQLITE_LOCKED_SHAREDCACHE + + /// + /// Returns for SQLITE_BUSY variants that can be resolved by + /// waiting and retrying the same operation β€” SQLITE_BUSY, SQLITE_BUSY_RECOVERY, + /// and SQLITE_BUSY_TIMEOUT. + /// + /// + /// SQLITE_BUSY_SNAPSHOT is excluded. It indicates that the connection's read snapshot + /// is older than the current database state, so the full transaction must be rolled + /// back and restarted rather than simply waited out. + /// + public static bool IsRetryableBusy(SqliteException ex) + => ex.SqliteErrorCode == Busy + && ex.SqliteExtendedErrorCode != BusySnapshot; + + /// + /// Returns when the exception is SQLITE_BUSY_SNAPSHOT, + /// meaning the connection's read snapshot is stale after another writer committed. + /// The transaction must be fully rolled back and restarted from scratch. + /// + public static bool IsBusySnapshot(SqliteException ex) + => ex.SqliteExtendedErrorCode == BusySnapshot; + + /// + /// Returns for any SQLITE_BUSY variant (retryable or not). + /// + public static bool IsAnyBusy(SqliteException ex) + => ex.SqliteErrorCode == Busy; + + /// + /// Returns for SQLITE_LOCKED, which indicates a conflict + /// within the same connection (e.g., a write attempted while another + /// statement on the same connection is still reading). This is an application-level + /// bug and should not be retried. + /// + public static bool IsLocked(SqliteException ex) + => ex.SqliteErrorCode == Locked; +} diff --git a/EntityFrameworkCore.Sqlite.Concurrency/src/ThreadSafeSqliteContext.cs b/EntityFrameworkCore.Sqlite.Concurrency/src/ThreadSafeSqliteContext.cs index e02821e..48385f9 100644 --- a/EntityFrameworkCore.Sqlite.Concurrency/src/ThreadSafeSqliteContext.cs +++ b/EntityFrameworkCore.Sqlite.Concurrency/src/ThreadSafeSqliteContext.cs @@ -31,7 +31,6 @@ public ThreadSafeSqliteContext(string connectionString) /// The options. public ThreadSafeSqliteContext(DbContextOptions options) : base(options) { - // Try to resolve connection string and lock from options var extension = options.FindExtension(); if (extension?.ConnectionString != null) { @@ -44,13 +43,13 @@ public ThreadSafeSqliteContext(DbContextOptions options) : base(options) _writeLock = SqliteConnectionEnhancer.GetWriteLock(_connectionString); } } - - private SemaphoreSlim WriteLock + + private SemaphoreSlim WriteLock { get { if (_writeLock != null) return _writeLock; - + // Fallback for cases where connection string wasn't available in constructor var connectionString = Database.GetDbConnection().ConnectionString; _writeLock = SqliteConnectionEnhancer.GetWriteLock(connectionString); @@ -74,12 +73,32 @@ protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) /// The operation to execute. /// The cancellation token. /// The result of the operation. + /// + /// + /// Two classes of SQLITE_BUSY are handled: + /// + /// + /// + /// SQLITE_BUSY / SQLITE_BUSY_RECOVERY / SQLITE_BUSY_TIMEOUT β€” the + /// transaction is rolled back and retried after exponential backoff with jitter. + /// + /// + /// SQLITE_BUSY_SNAPSHOT β€” the read snapshot became stale. The transaction + /// is rolled back and the entire operation lambda is restarted so it can re-query + /// any data that may now be stale. + /// + /// + /// + /// SQLITE_LOCKED (same-connection conflict) propagates immediately and is + /// not retried, as it indicates an application-level bug. + /// + /// public async Task ExecuteWriteAsync( Func> operation, CancellationToken ct = default) { - // Reentrancy check: if this execution flow already holds the lock, just execute. - // This avoids deadlocks in nested calls on the same thread/flow. + // Reentrancy check: if this execution flow already holds the lock, execute + // directly to avoid deadlocking on the same SemaphoreSlim. if (SqliteConnectionEnhancer.IsWriteLockHeld.Value) { return await operation((TContext)(object)this); @@ -95,7 +114,9 @@ public async Task ExecuteWriteAsync( try { - // Use explicit transaction. The interceptor will ensure BEGIN IMMEDIATE. + // The interceptor will upgrade this BEGIN to BEGIN IMMEDIATE, ensuring + // no later statement in the transaction fails with SQLITE_BUSY before + // commit (as long as UpgradeTransactionsToImmediate is true). await using var transaction = await Database.BeginTransactionAsync( System.Data.IsolationLevel.Serializable, ct); @@ -105,17 +126,34 @@ public async Task ExecuteWriteAsync( return result; } - catch (SqliteException ex) when (ex.SqliteErrorCode == 5) // SQLITE_BUSY + catch (SqliteException ex) when (SqliteErrorCodes.IsAnyBusy(ex)) { - // Release lock before retry wait + // Release the lock before sleeping so other writers can make progress. SqliteConnectionEnhancer.IsWriteLockHeld.Value = false; WriteLock.Release(); attempt++; if (attempt >= maxRetryAttempts) - throw new TimeoutException($"Database busy timeout after {attempt} retries", ex); + { + var kind = SqliteErrorCodes.IsBusySnapshot(ex) + ? "SQLITE_BUSY_SNAPSHOT (stale read snapshot β€” another writer committed after this transaction began)" + : $"SQLITE_BUSY (extended code {ex.SqliteExtendedErrorCode})"; + + throw new TimeoutException( + $"SQLite database busy after {attempt} retry attempt(s). " + + $"Error: {kind}. " + + $"Consider increasing MaxRetryAttempts or BusyTimeout.", + ex); + } - await Task.Delay(100 * (int)Math.Pow(2, attempt), ct); + // Exponential backoff with full jitter: sleep in [baseDelay, 2Γ—baseDelay]. + // Jitter prevents synchronized retry storms when multiple threads contend. + var baseDelay = 100 * Math.Pow(2, attempt); + var jitter = Random.Shared.NextDouble() * baseDelay; + await Task.Delay(TimeSpan.FromMilliseconds(baseDelay + jitter), ct); + + // Continue to next loop iteration β€” for BUSY_SNAPSHOT this correctly + // restarts the entire operation lambda so stale data is re-queried. } finally { @@ -145,12 +183,17 @@ await ExecuteWriteAsync(async ctx => } /// - /// Executes a read operation. No locking is performed. + /// Executes a read operation without locking. /// /// The result type. /// The operation to execute. /// The cancellation token. /// The result of the operation. + /// + /// WAL mode allows reads to proceed concurrently with writes. Keep read transactions + /// short to avoid blocking WAL checkpoint completion, which can cause the WAL file to + /// grow and degrade read performance over time. + /// public async Task ExecuteReadAsync( Func> operation, CancellationToken ct = default) @@ -170,7 +213,6 @@ public async Task BulkInsertSafeAsync( { await ExecuteWriteAsync(async ctx => { - // Batch inserts var batchSize = 1000; for (int i = 0; i < entities.Count; i += batchSize) { @@ -188,12 +230,7 @@ private SqliteConcurrencyOptions Options { get { - if (_options == null) - { - // Try to get options from context - _options = new SqliteConcurrencyOptions(); - } - + _options ??= new SqliteConcurrencyOptions(); return _options; } } @@ -203,4 +240,4 @@ public override async ValueTask DisposeAsync() { await base.DisposeAsync(); } -} \ No newline at end of file +}