Merge tag 'pbs-0.2.3' into main

CMakeLists.txt

@@ -494,6 +494,7 @@ set(binsrv_source_files
 
   src/binsrv/storage_config_fwd.hpp
   src/binsrv/storage_config.hpp
+  src/binsrv/storage_config.cpp
 
   src/binsrv/storage_metadata_fwd.hpp
   src/binsrv/storage_metadata.hpp

README.md

@@ -175,7 +175,7 @@ For instance,
 ```
 may print
 ```
-0.1.0
+0.2.3
 ```
 
 #### 'search_by_timestamp' operation mode
@@ -197,7 +197,7 @@ may print
       "uri": "s3://binsrv-bucket/storage/binlog.000001",
       "min_timestamp": "2026-02-09T17:22:01",
       "max_timestamp": "2026-02-09T17:22:08",
-      "initial_gtids": "",
+      "previous_gtids": "",
       "added_gtids": "11111111-aaaa-1111-aaaa-111111111111:1-123456"
     },
     {
@@ -206,7 +206,7 @@ may print
       "uri": "s3://binsrv-bucket/storage/binlog.000002",
       "min_timestamp": "2026-02-09T17:22:08",
       "max_timestamp": "2026-02-09T17:22:09",
-      "initial_gtids": "11111111-aaaa-1111-aaaa-111111111111:1-123456",
+      "previous_gtids": "11111111-aaaa-1111-aaaa-111111111111:1-123456",
       "added_gtids": "11111111-aaaa-1111-aaaa-111111111111:123457-246912"
     }
   ]
@@ -243,7 +243,7 @@ may print
       "uri": "s3://binsrv-bucket/storage/binlog.000001",
       "min_timestamp": "2026-02-09T17:22:01",
       "max_timestamp": "2026-02-09T17:22:08",
-      "initial_gtids": "",
+      "previous_gtids": "",
       "added_gtids": "11111111-aaaa-1111-aaaa-111111111111:1-123456"
     }
   ]
@@ -264,7 +264,7 @@ may print
       "uri": "s3://binsrv-bucket/storage/binlog.000001",
       "min_timestamp": "2026-02-09T17:22:01",
       "max_timestamp": "2026-02-09T17:22:08",
-      "initial_gtids": "",
+      "previous_gtids": "",
       "added_gtids": "11111111-aaaa-1111-aaaa-111111111111:1-123456",
     },
     {
@@ -273,7 +273,7 @@ may print
       "uri": "s3://binsrv-bucket/storage/binlog.000002",
       "min_timestamp": "2026-02-09T17:22:08",
       "max_timestamp": "2026-02-09T17:22:09",
-      "initial_gtids": "11111111-aaaa-1111-aaaa-111111111111:1-123456",
+      "previous_gtids": "11111111-aaaa-1111-aaaa-111111111111:1-123456",
       "added_gtids": "11111111-aaaa-1111-aaaa-111111111111:123457-246912"
     }
   ]

mtr/binlog_streaming/r/gtid_purged.result

@@ -0,0 +1,50 @@
+*** Resetting replication at the very beginning of the test.
+
+*** Determining the first binary log name.
+
+*** Generating a configuration file in JSON format for the Binlog
+*** Server utility.
+
+*** Determining binlog file directory from the server.
+
+*** Creating a temporary directory <BINSRV_STORAGE_PATH> for storing
+*** binlog files downloaded via the Binlog Server utility.
+
+*** Creating a simple table.
+CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB;
+
+*** Filling the table with some data.
+INSERT INTO t1 VALUES();
+
+*** Flushing the first binary log and switching to the second one.
+FLUSH BINARY LOGS;
+
+*** Determining the second binary log name.
+
+*** Filling the table with more data.
+INSERT INTO t1 VALUES();
+
+*** Filling the table with more data.
+PURGE BINARY LOGS TO '<second_binlog>';
+
+*** Executing the Binlog Server utility and fetching all events.
+
+*** Executing the Binlog Server utility in the 'search_by_gtid_set'
+include/read_file_to_var.inc
+
+*** Executing the Binlog Server utility one more time (fetching nothing)
+
+*** Executing the Binlog Server utility in the 'search_by_gtid_set'
+*** one more time (expecting the same results)
+include/read_file_to_var.inc
+
+*** Removing the search result file.
+
+*** Dropping the table.
+DROP TABLE t1;
+
+*** Removing the Binlog Server utility storage directory.
+
+*** Removing the Binlog Server utility log file.
+
+*** Removing the Binlog Server utility configuration file.

mtr/binlog_streaming/r/thousand_binlogs.result

@@ -0,0 +1,36 @@
+*** Resetting replication at the very beginning of the test.
+
+*** Generating a configuration file in JSON format for the Binlog
+*** Server utility.
+
+*** Determining binlog file directory from the server.
+
+*** Creating a temporary directory <BINSRV_STORAGE_PATH> for storing
+*** binlog files downloaded via the Binlog Server utility.
+
+*** Creating a simple table.
+CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB;
+
+*** Generating 1000 binlogs
+
+*** Executing the Binlog Server utility and fetching all events.
+
+*** Executing the Binlog Server utility in the 'search_by_timestamp'
+include/read_file_to_var.inc
+
+*** Executing the Binlog Server utility one more time (fetching nothing)
+
+*** Executing the Binlog Server utility in the 'search_by_timestamp'
+*** one more time (expecting the same results)
+include/read_file_to_var.inc
+
+*** Removing the search result file.
+
+*** Dropping the table.
+DROP TABLE t1;
+
+*** Removing the Binlog Server utility storage directory.
+
+*** Removing the Binlog Server utility log file.
+
+*** Removing the Binlog Server utility configuration file.

mtr/binlog_streaming/t/data_directory_8_0_to_8_4_upgrade.test

@@ -42,7 +42,7 @@ if ($lts_series != v84)
 --echo *** Running 8.4 binaries on a 8.0 data directory
 --let $MYSQLD_LOG = $MYSQLTEST_VARDIR/log/upgrade.log
 --let $do_not_echo_parameters = 1
---let $restart_parameters = restart: --datadir=$CUSTOM_MYSQLD_DATADIR --log-error=$MYSQLD_LOG
+--let $restart_parameters = restart: --datadir=$CUSTOM_MYSQLD_DATADIR --log-error=$MYSQLD_LOG --binlog_expire_logs_auto_purge=OFF
 --source include/start_mysqld.inc
 
 --echo

mtr/binlog_streaming/t/gtid_purged-master.opt

@@ -0,0 +1,2 @@
+--gtid-mode=on
+--enforce-gtid-consistency

mtr/binlog_streaming/t/gtid_purged.test

@@ -0,0 +1,98 @@
+--source ../include/have_binsrv.inc
+
+--source ../include/v80_v84_compatibility_defines.inc
+
+# in case of --repeat=N, we need to start from a fresh binary log to make
+# this test deterministic
+--echo *** Resetting replication at the very beginning of the test.
+--disable_query_log
+eval $stmt_reset_binary_logs_and_gtids;
+--enable_query_log
+
+--echo
+--echo *** Determining the first binary log name.
+--let $first_binlog = query_get_value($stmt_show_binary_log_status, File, 1)
+
+# identifying backend storage type ('file' or 's3')
+--source ../include/identify_storage_backend.inc
+
+# creating data directory, configuration file, etc.
+--let $binsrv_connect_timeout = 20
+--let $binsrv_read_timeout = 60
+--let $binsrv_idle_time = 10
+--let $binsrv_verify_checksum = TRUE
+--let $binsrv_replication_mode = gtid
+--let $binsrv_checkpoint_size = 1
+--source ../include/set_up_binsrv_environment.inc
+
+--let $read_from_file = $MYSQL_TMP_DIR/search_result.json
+
+--echo
+--echo *** Creating a simple table.
+CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB;
+
+--echo
+--echo *** Filling the table with some data.
+INSERT INTO t1 VALUES();
+
+--echo
+--echo *** Flushing the first binary log and switching to the second one.
+FLUSH BINARY LOGS;
+
+--echo
+--echo *** Determining the second binary log name.
+--let $second_binlog = query_get_value($stmt_show_binary_log_status, File, 1)
+
+--echo
+--echo *** Filling the table with more data.
+INSERT INTO t1 VALUES();
+
+--echo
+--echo *** Filling the table with more data.
+--replace_result $second_binlog <second_binlog>
+eval PURGE BINARY LOGS TO '$second_binlog';
+
+--let $captured_gtid_purged = `SELECT @@global.gtid_purged`
+--let $captured_second_insert_gtid = `SELECT GTID_SUBTRACT(@@global.gtid_executed, @@global.gtid_purged)`
+
+--echo
+--echo *** Executing the Binlog Server utility and fetching all events.
+--exec $BINSRV fetch $binsrv_config_file_path > /dev/null
+
+--echo
+--echo *** Executing the Binlog Server utility in the 'search_by_gtid_set'
+--exec $BINSRV search_by_gtid_set $binsrv_config_file_path $captured_second_insert_gtid > $read_from_file
+
+--source include/read_file_to_var.inc
+--assert(`SELECT JSON_EXTRACT('$result', '$.status') = 'success'`)
+--assert(`SELECT JSON_LENGTH(JSON_EXTRACT('$result', '$.result')) = 1`)
+--assert(`SELECT JSON_EXTRACT('$result', '$.result[0].name') = '$second_binlog'`)
+--assert(`SELECT JSON_EXTRACT('$result', '$.result[0].previous_gtids') = '$captured_gtid_purged'`)
+--assert(`SELECT JSON_EXTRACT('$result', '$.result[0].added_gtids') = '$captured_second_insert_gtid'`)
+
+--echo
+--echo *** Executing the Binlog Server utility one more time (fetching nothing)
+--exec $BINSRV fetch $binsrv_config_file_path > /dev/null
+
+--echo
+--echo *** Executing the Binlog Server utility in the 'search_by_gtid_set'
+--echo *** one more time (expecting the same results)
+--exec $BINSRV search_by_gtid_set $binsrv_config_file_path $captured_second_insert_gtid > $read_from_file
+
+--source include/read_file_to_var.inc
+--assert(`SELECT JSON_EXTRACT('$result', '$.status') = 'success'`)
+--assert(`SELECT JSON_LENGTH(JSON_EXTRACT('$result', '$.result')) = 1`)
+--assert(`SELECT JSON_EXTRACT('$result', '$.result[0].name') = '$second_binlog'`)
+--assert(`SELECT JSON_EXTRACT('$result', '$.result[0].previous_gtids') = '$captured_gtid_purged'`)
+--assert(`SELECT JSON_EXTRACT('$result', '$.result[0].added_gtids') = '$captured_second_insert_gtid'`)
+
+--echo
+--echo *** Removing the search result file.
+--remove_file $read_from_file
+
+--echo
+--echo *** Dropping the table.
+DROP TABLE t1;
+
+# cleaning up
+--source ../include/tear_down_binsrv_environment.inc

mtr/binlog_streaming/t/thousand_binlogs.test

@@ -0,0 +1,90 @@
+--source ../include/have_binsrv.inc
+--source include/big_test.inc
+
+--source ../include/v80_v84_compatibility_defines.inc
+
+# in case of --repeat=N, we need to start from a fresh binary log to make
+# this test deterministic
+--echo *** Resetting replication at the very beginning of the test.
+--disable_query_log
+eval $stmt_reset_binary_logs_and_gtids;
+--enable_query_log
+
+# identifying backend storage type ('file' or 's3')
+--source ../include/identify_storage_backend.inc
+
+# creating data directory, configuration file, etc.
+--let $binsrv_connect_timeout = 20
+--let $binsrv_read_timeout = 60
+--let $binsrv_idle_time = 10
+--let $binsrv_verify_checksum = TRUE
+--let $binsrv_replication_mode = position
+--let $binsrv_checkpoint_size = 1
+--source ../include/set_up_binsrv_environment.inc
+
+--let $read_from_file = $MYSQL_TMP_DIR/search_result.json
+
+--echo
+--echo *** Creating a simple table.
+CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB;
+
+--let $number_of_binlogs = 1000
+--echo
+--echo *** Generating $number_of_binlogs binlogs
+--let $index = 0
+
+--disable_query_log
+while($index < $number_of_binlogs)
+{
+  INSERT INTO t1 VALUES();
+  FLUSH BINARY LOGS;
+  --inc $index
+}
+INSERT INTO t1 VALUES();
+--enable_query_log
+
+--echo
+--echo *** Executing the Binlog Server utility and fetching all events.
+--exec $BINSRV fetch $binsrv_config_file_path > /dev/null
+
+--let $current_timestamp = `SELECT DATE_FORMAT(CONVERT_TZ(NOW(), @@session.time_zone, '+00:00'),'%Y-%m-%dT%H:%i:%s')`
+
+--echo
+--echo *** Executing the Binlog Server utility in the 'search_by_timestamp'
+--exec $BINSRV search_by_timestamp $binsrv_config_file_path $current_timestamp > $read_from_file
+
+--source include/read_file_to_var.inc
+# the result will be too big, so storing it in a session variable
+--disable_query_log
+eval SET @search_result = '$result';
+--enable_query_log
+--assert(`SELECT JSON_EXTRACT(@search_result, '$.status') = 'success'`)
+--assert(`SELECT JSON_LENGTH(JSON_EXTRACT(@search_result, '$.result')) = $number_of_binlogs + 1`)
+
+--echo
+--echo *** Executing the Binlog Server utility one more time (fetching nothing)
+--exec $BINSRV fetch $binsrv_config_file_path > /dev/null
+
+--echo
+--echo *** Executing the Binlog Server utility in the 'search_by_timestamp'
+--echo *** one more time (expecting the same results)
+--exec $BINSRV search_by_timestamp $binsrv_config_file_path $current_timestamp > $read_from_file
+
+--source include/read_file_to_var.inc
+# the result will be too big, so storing it in a session variable
+--disable_query_log
+eval SET @search_result = '$result';
+--enable_query_log
+--assert(`SELECT JSON_EXTRACT('$result', '$.status') = 'success'`)
+--assert(`SELECT JSON_LENGTH(JSON_EXTRACT('$result', '$.result')) = $number_of_binlogs + 1`)
+
+--echo
+--echo *** Removing the search result file.
+--remove_file $read_from_file
+
+--echo
+--echo *** Dropping the table.
+DROP TABLE t1;
+
+# cleaning up
+--source ../include/tear_down_binsrv_environment.inc

packaging/debian/changelog

@@ -1,3 +1,29 @@
+percona-binlog-server (0.2.3-1) unstable; urgency=low
+
+  * PS-11033 Crash when S3 bucket accumulates large number of objects; recovery requires manual intervention.
+
+ -- Yura Sorokin <yura.sorokin@percona.com>  Fri, 24 Apr 2026 04:29:21 +0200
+
+percona-binlog-server (0.2.2-1) unstable; urgency=low
+
+  * PS-11054 Cannot replicate because the source purged required binary logs.
+
+ -- Yura Sorokin <yura.sorokin@percona.com>  Tue, 21 Apr 2026 19:09:19 +0200
+
+percona-binlog-server (0.2.1-1) unstable; urgency=low
+
+  * PS-10910 Bucket name is missing in search_* outputs.
+  * PS-11002 Changing storage prefix fails.
+  * PS-10911 Unexpected binlog position in artificial rotate event.
+
+ -- Yura Sorokin <yura.sorokin@percona.com>  Fri, 10 Apr 2026 16:13:14 +0200
+
+percona-binlog-server (0.2.0-1) unstable; urgency=low
+
+  * Packaging for 0.2.0-1
+
+ -- Yura Sorokin <yura.sorokin@percona.com>  Mon, 16 Mar 2026 15:33:00 +0200
+
 percona-binlog-server (0.1.0-1) unstable; urgency=low
 
   * Packaging for 0.1.0-1