diff --git a/CronSync.php.example b/CronSync.php.example index 8ff6ba1..d942e55 100644 --- a/CronSync.php.example +++ b/CronSync.php.example @@ -25,6 +25,7 @@ Settings::configure([ 'api_key' => 'put_your_key_here', 'db_path' => '/tmp/anticrawler.sqlite', 'visitor_forget_after' => 60 * 60 * 24 * 30, + // 'pending_requests_max_length' => 200000, // 'requests_backend' => 'keydb', // 'keydb_host' => '127.0.0.1', // 'keydb_port' => 6379, diff --git a/KeyDBManager.php b/KeyDBManager.php index 33e01c4..bee2155 100644 --- a/KeyDBManager.php +++ b/KeyDBManager.php @@ -50,7 +50,9 @@ public static function storeRequest(RequestDto $request, ResultDto $result): voi 'request_status' => $result->status, ], JSON_UNESCAPED_SLASHES); - self::connectFromSettings()->rPush(self::pendingKey(), $payload === false ? '{}' : $payload); + $redis = self::connectFromSettings(); + $redis->rPush(self::pendingKey(), $payload === false ? '{}' : $payload); + $redis->lTrim(self::pendingKey(), -self::pendingRequestsMaxLength(), -1); } public static function countPendingRequests(): int @@ -218,4 +220,9 @@ private static function visitorTtl(): int { return max(1, (int)Settings::$visitorForgetAfter); } + + private static function pendingRequestsMaxLength(): int + { + return max(1, (int)Settings::$pendingRequestsMaxLength); + } } diff --git a/README.md b/README.md index a806647..95c6043 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ List of settings: | min_sync_interval | int | Minimum time interval between synchronizations when using default sync behavior, in seconds | | max_sync_interval | int | Maximum time interval between synchronizations when using default sync behavior, in seconds | | visitor_forget_after | int | Time limit for storing visitor data in the library database, in seconds (decrease this if you have storage issues) | +| pending_requests_max_length | int | Maximum number of request entries retained in the KeyDB pending queue. Defaults to `100000`; older entries are trimmed away | | max_rows_before_sync | int | Maximum number of requests stored between synchronizations when using default sync behavior | | sync_by_cron | bool | Set this to true to use the cron synchronization mechanism. See `CronSync.php.example` | | requests_backend | string | Backend switch for both request logs and visitor-state data: `sqlite` (default) or `keydb` | @@ -93,5 +94,5 @@ SQLite can generate high I/O load when the traffic is high. If you see this, you ]); ``` -In this mode, request logs and visitor presence data are stored in KeyDB. `visitor_forget_after` is applied as visitor key TTL. +In this mode, request logs and visitor presence data are stored in KeyDB. `visitor_forget_after` is applied as visitor key TTL, and `pending_requests_max_length` caps the pending request list by count. SQLite is still used for `kv`, `lists`, and `user_agents` tables. diff --git a/Settings.php b/Settings.php index 0d5de63..4a76389 100644 --- a/Settings.php +++ b/Settings.php @@ -22,6 +22,9 @@ class Settings /** @var int */ public static $visitorForgetAfter = 60 * 60 * 24 * 30; // One month + /** @var int */ + public static $pendingRequestsMaxLength = 100000; + /** @var int */ public static $maxRowsBeforeSync = 20000; @@ -69,6 +72,8 @@ public static function configure(array $options): void self::$minSyncInterval = $options['min_sync_interval'] ?? self::$minSyncInterval; self::$maxSyncInterval = $options['max_sync_interval'] ?? self::$maxSyncInterval; self::$visitorForgetAfter = $options['visitor_forget_after'] ?? self::$visitorForgetAfter; + self::$pendingRequestsMaxLength = $options['pending_requests_max_length'] + ?? self::$pendingRequestsMaxLength; self::$maxRowsBeforeSync = $options['max_rows_before_sync'] ?? self::$maxRowsBeforeSync; self::$syncByCron = $options['sync_by_cron'] ?? self::$syncByCron; self::$requestsBackend = $options['requests_backend'] ?? self::$requestsBackend;