PS-11001: Implement logic for purge_binlogs mode (local filesystem storage only)

CMakeLists.txt

@@ -125,6 +125,9 @@ set(util_source_files
 
   src/util/mixin_exception_adapter.hpp
 
+  src/util/native_file_operations_helpers.hpp
+  src/util/native_file_operations_helpers_posix.cpp
+
   src/util/nv_tuple_fwd.hpp
   src/util/nv_tuple.hpp
   src/util/nv_tuple_json_support.hpp

mtr/binlog_streaming/r/list.result

@@ -0,0 +1,51 @@
+*** Resetting replication at the very beginning of the test.
+
+*** Determining the first binary log name.
+
+*** Generating a configuration file in JSON format for the Binlog
+*** Server utility.
+
+*** Determining binlog file directory from the server.
+
+*** Creating a temporary directory <BINSRV_STORAGE_PATH> for storing
+*** binlog files downloaded via the Binlog Server utility.
+
+*** 1. Executing the Binlog Server utility in the 'list' mode on an
+***    empty storage and expecting an empty result array
+include/read_file_to_var.inc
+
+*** Creating a simple table.
+CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB;
+
+*** Filling the table with some data.
+INSERT INTO t1 VALUES();
+
+*** Flushing the first binary log and switching to the second one.
+FLUSH BINARY LOGS;
+
+*** Determining the second binary log name.
+
+*** Filling the table with more data.
+INSERT INTO t1 VALUES();
+
+*** Executing the Binlog Server utility and fetching all events.
+
+*** 2. Executing the Binlog Server utility in the 'list' mode on a
+***    non-empty storage and expecting both binlog files to be returned
+***    in chronological order
+include/read_file_to_var.inc
+
+*** 3. Executing the Binlog Server utility in the 'list' mode with a
+***    nonexistent configuration file path
+include/read_file_to_var.inc
+
+*** Removing the list result file.
+
+*** Dropping the table.
+DROP TABLE t1;
+
+*** Removing the Binlog Server utility storage directory.
+
+*** Removing the Binlog Server utility log file.
+
+*** Removing the Binlog Server utility configuration file.

mtr/binlog_streaming/r/purge_binlogs.result

@@ -0,0 +1,95 @@
+*** Resetting replication at the very beginning of the test.
+
+*** Generating a configuration file in JSON format for the Binlog
+*** Server utility.
+
+*** Determining binlog file directory from the server.
+
+*** Creating a temporary directory <BINSRV_STORAGE_PATH> for storing
+*** binlog files downloaded via the Binlog Server utility.
+
+*** 1. Executing 'purge_binlogs' on an empty storage and expecting
+***    an error.
+include/read_file_to_var.inc
+
+*** Determining the first binary log name.
+
+*** Creating a simple table.
+CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB;
+
+*** Filling the table with some data.
+INSERT INTO t1 VALUES();
+
+*** Flushing the first binary log and switching to the second one.
+FLUSH BINARY LOGS;
+
+*** Determining the second binary log name.
+
+*** Filling the table with more data.
+INSERT INTO t1 VALUES();
+
+*** Flushing the second binary log and switching to the third one.
+FLUSH BINARY LOGS;
+
+*** Determining the third binary log name.
+
+*** Filling the table with even more data.
+INSERT INTO t1 VALUES();
+
+*** Executing the Binlog Server utility and fetching all events.
+
+*** Sanity check: 'list' returns all three binlog files in chronological
+*** order.
+include/read_file_to_var.inc
+
+*** 2. 'purge_binlogs' with an unparseable target name fails with error.
+include/read_file_to_var.inc
+
+*** 3. 'purge_binlogs' with a target whose base name differs from the
+***    one in storage fails with error.
+include/read_file_to_var.inc
+
+*** 4. 'purge_binlogs' with a target not present in the storage fails
+***    with error.
+include/read_file_to_var.inc
+
+*** 5. 'purge_binlogs' with a nonexistent configuration file path fails
+***    with error.
+include/read_file_to_var.inc
+
+*** 6. 'purge_binlogs' the first binary log removes only that one file;
+***    the response lists the removed record and 'list' shows the two
+***    surviving files.
+include/read_file_to_var.inc
+include/read_file_to_var.inc
+
+*** 6a. Re-opening the purged storage with another 'fetch' must keep the
+***     surviving GTID metadata stable.
+include/read_file_to_var.inc
+
+*** 7. Re-running 'purge_binlogs' with the just-purged target now fails
+***    with the "not present" error - the operation is not idempotent
+***    by design.
+include/read_file_to_var.inc
+
+*** 8. 'purge_binlogs' the second binary log removes only that one (the
+***    first one is already gone); 'list' shows just the third file.
+include/read_file_to_var.inc
+include/read_file_to_var.inc
+
+*** 9. 'purge_binlogs' the last remaining (current tail) file is
+***    refused so that the resume position is preserved for the next
+***    'fetch' / 'pull'; 'list' still shows the third file unchanged.
+include/read_file_to_var.inc
+include/read_file_to_var.inc
+
+*** Removing the purge_binlogs result file.
+
+*** Dropping the table.
+DROP TABLE t1;
+
+*** Removing the Binlog Server utility storage directory.
+
+*** Removing the Binlog Server utility log file.
+
+*** Removing the Binlog Server utility configuration file.

mtr/binlog_streaming/t/list.combinations

@@ -0,0 +1,5 @@
+[position]
+
+[gtid]
+gtid-mode=on
+enforce-gtid-consistency

mtr/binlog_streaming/t/list.test

@@ -0,0 +1,94 @@
+--source ../include/have_binsrv.inc
+
+--source ../include/v80_v84_compatibility_defines.inc
+
+# in case of --repeat=N, we need to start from a fresh binary log to make
+# this test deterministic
+--echo *** Resetting replication at the very beginning of the test.
+--disable_query_log
+eval $stmt_reset_binary_logs_and_gtids;
+--enable_query_log
+
+--echo
+--echo *** Determining the first binary log name.
+--let $first_binlog = query_get_value($stmt_show_binary_log_status, File, 1)
+
+# identifying backend storage type ('file' or 's3')
+--source ../include/identify_storage_backend.inc
+
+# creating data directory, configuration file, etc.
+--let $binsrv_connect_timeout = 20
+--let $binsrv_read_timeout = 60
+--let $binsrv_idle_time = 10
+--let $binsrv_verify_checksum = TRUE
+--let $binsrv_replication_mode = `SELECT IF(@@global.gtid_mode = 'ON', 'gtid', 'position')`
+--let $binsrv_checkpoint_size = 1
+--source ../include/set_up_binsrv_environment.inc
+
+--let $read_from_file = $MYSQL_TMP_DIR/list_result.json
+
+--echo
+--echo *** 1. Executing the Binlog Server utility in the 'list' mode on an
+--echo ***    empty storage and expecting an empty result array
+--exec $BINSRV list $binsrv_config_file_path > $read_from_file
+--source include/read_file_to_var.inc
+--assert(`SELECT JSON_EXTRACT('$result', '$.status') = 'success'`)
+--assert(`SELECT JSON_LENGTH(JSON_EXTRACT('$result', '$.result')) = 0`)
+
+--echo
+--echo *** Creating a simple table.
+CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB;
+
+--echo
+--echo *** Filling the table with some data.
+INSERT INTO t1 VALUES();
+
+--echo
+--echo *** Flushing the first binary log and switching to the second one.
+FLUSH BINARY LOGS;
+
+--echo
+--echo *** Determining the second binary log name.
+--let $second_binlog = query_get_value($stmt_show_binary_log_status, File, 1)
+
+--echo
+--echo *** Filling the table with more data.
+INSERT INTO t1 VALUES();
+
+--echo
+--echo *** Executing the Binlog Server utility and fetching all events.
+--exec $BINSRV fetch $binsrv_config_file_path > /dev/null
+
+--echo
+--echo *** 2. Executing the Binlog Server utility in the 'list' mode on a
+--echo ***    non-empty storage and expecting both binlog files to be returned
+--echo ***    in chronological order
+--exec $BINSRV list $binsrv_config_file_path > $read_from_file
+--source include/read_file_to_var.inc
+--assert(`SELECT JSON_EXTRACT('$result', '$.status') = 'success'`)
+--assert(`SELECT JSON_LENGTH(JSON_EXTRACT('$result', '$.result')) = 2`)
+--assert(`SELECT JSON_EXTRACT('$result', '$.result[0].name') = '$first_binlog'`)
+--assert(`SELECT JSON_EXTRACT('$result', '$.result[1].name') = '$second_binlog'`)
+--assert(`SELECT JSON_EXTRACT('$result', '$.result[0].size') > 0`)
+--assert(`SELECT JSON_EXTRACT('$result', '$.result[1].size') > 0`)
+--assert(`SELECT JSON_EXTRACT('$result', '$.result[0].uri') IS NOT NULL`)
+--assert(`SELECT JSON_EXTRACT('$result', '$.result[1].uri') IS NOT NULL`)
+
+--echo
+--echo *** 3. Executing the Binlog Server utility in the 'list' mode with a
+--echo ***    nonexistent configuration file path
+--error 1
+--exec $BINSRV list $MYSQL_TMP_DIR/no_such_config.json > $read_from_file
+--source include/read_file_to_var.inc
+--assert(`SELECT JSON_EXTRACT('$result', '$.status') = 'error'`)
+
+--echo
+--echo *** Removing the list result file.
+--remove_file $read_from_file
+
+--echo
+--echo *** Dropping the table.
+DROP TABLE t1;
+
+# cleaning up
+--source ../include/tear_down_binsrv_environment.inc

mtr/binlog_streaming/t/purge_binlogs.combinations

@@ -0,0 +1,5 @@
+[position]
+
+[gtid]
+gtid-mode=on
+enforce-gtid-consistency