diff --git a/mtr/binlog_streaming/r/list.result b/mtr/binlog_streaming/r/list.result new file mode 100644 index 0000000..47fd111 --- /dev/null +++ b/mtr/binlog_streaming/r/list.result @@ -0,0 +1,51 @@ +*** Resetting replication at the very beginning of the test. + +*** Determining the first binary log name. + +*** Generating a configuration file in JSON format for the Binlog +*** Server utility. + +*** Determining binlog file directory from the server. + +*** Creating a temporary directory for storing +*** binlog files downloaded via the Binlog Server utility. + +*** 1. Executing the Binlog Server utility in the 'list' mode on an +*** empty storage and expecting an empty result array +include/read_file_to_var.inc + +*** Creating a simple table. +CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB; + +*** Filling the table with some data. +INSERT INTO t1 VALUES(); + +*** Flushing the first binary log and switching to the second one. +FLUSH BINARY LOGS; + +*** Determining the second binary log name. + +*** Filling the table with more data. +INSERT INTO t1 VALUES(); + +*** Executing the Binlog Server utility and fetching all events. + +*** 2. Executing the Binlog Server utility in the 'list' mode on a +*** non-empty storage and expecting both binlog files to be returned +*** in chronological order +include/read_file_to_var.inc + +*** 3. Executing the Binlog Server utility in the 'list' mode with a +*** nonexistent configuration file path +include/read_file_to_var.inc + +*** Removing the list result file. + +*** Dropping the table. +DROP TABLE t1; + +*** Removing the Binlog Server utility storage directory. + +*** Removing the Binlog Server utility log file. + +*** Removing the Binlog Server utility configuration file. diff --git a/mtr/binlog_streaming/t/list.combinations b/mtr/binlog_streaming/t/list.combinations new file mode 100644 index 0000000..411ce97 --- /dev/null +++ b/mtr/binlog_streaming/t/list.combinations @@ -0,0 +1,5 @@ +[position] + +[gtid] +gtid-mode=on +enforce-gtid-consistency diff --git a/mtr/binlog_streaming/t/list.test b/mtr/binlog_streaming/t/list.test new file mode 100644 index 0000000..d563a60 --- /dev/null +++ b/mtr/binlog_streaming/t/list.test @@ -0,0 +1,94 @@ +--source ../include/have_binsrv.inc + +--source ../include/v80_v84_compatibility_defines.inc + +# in case of --repeat=N, we need to start from a fresh binary log to make +# this test deterministic +--echo *** Resetting replication at the very beginning of the test. +--disable_query_log +eval $stmt_reset_binary_logs_and_gtids; +--enable_query_log + +--echo +--echo *** Determining the first binary log name. +--let $first_binlog = query_get_value($stmt_show_binary_log_status, File, 1) + +# identifying backend storage type ('file' or 's3') +--source ../include/identify_storage_backend.inc + +# creating data directory, configuration file, etc. +--let $binsrv_connect_timeout = 20 +--let $binsrv_read_timeout = 60 +--let $binsrv_idle_time = 10 +--let $binsrv_verify_checksum = TRUE +--let $binsrv_replication_mode = `SELECT IF(@@global.gtid_mode = 'ON', 'gtid', 'position')` +--let $binsrv_checkpoint_size = 1 +--source ../include/set_up_binsrv_environment.inc + +--let $read_from_file = $MYSQL_TMP_DIR/list_result.json + +--echo +--echo *** 1. Executing the Binlog Server utility in the 'list' mode on an +--echo *** empty storage and expecting an empty result array +--exec $BINSRV list $binsrv_config_file_path > $read_from_file +--source include/read_file_to_var.inc +--assert(`SELECT JSON_EXTRACT('$result', '$.status') = 'success'`) +--assert(`SELECT JSON_LENGTH(JSON_EXTRACT('$result', '$.result')) = 0`) + +--echo +--echo *** Creating a simple table. +CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB; + +--echo +--echo *** Filling the table with some data. +INSERT INTO t1 VALUES(); + +--echo +--echo *** Flushing the first binary log and switching to the second one. +FLUSH BINARY LOGS; + +--echo +--echo *** Determining the second binary log name. +--let $second_binlog = query_get_value($stmt_show_binary_log_status, File, 1) + +--echo +--echo *** Filling the table with more data. +INSERT INTO t1 VALUES(); + +--echo +--echo *** Executing the Binlog Server utility and fetching all events. +--exec $BINSRV fetch $binsrv_config_file_path > /dev/null + +--echo +--echo *** 2. Executing the Binlog Server utility in the 'list' mode on a +--echo *** non-empty storage and expecting both binlog files to be returned +--echo *** in chronological order +--exec $BINSRV list $binsrv_config_file_path > $read_from_file +--source include/read_file_to_var.inc +--assert(`SELECT JSON_EXTRACT('$result', '$.status') = 'success'`) +--assert(`SELECT JSON_LENGTH(JSON_EXTRACT('$result', '$.result')) = 2`) +--assert(`SELECT JSON_EXTRACT('$result', '$.result[0].name') = '$first_binlog'`) +--assert(`SELECT JSON_EXTRACT('$result', '$.result[1].name') = '$second_binlog'`) +--assert(`SELECT JSON_EXTRACT('$result', '$.result[0].size') > 0`) +--assert(`SELECT JSON_EXTRACT('$result', '$.result[1].size') > 0`) +--assert(`SELECT JSON_EXTRACT('$result', '$.result[0].uri') IS NOT NULL`) +--assert(`SELECT JSON_EXTRACT('$result', '$.result[1].uri') IS NOT NULL`) + +--echo +--echo *** 3. Executing the Binlog Server utility in the 'list' mode with a +--echo *** nonexistent configuration file path +--error 1 +--exec $BINSRV list $MYSQL_TMP_DIR/no_such_config.json > $read_from_file +--source include/read_file_to_var.inc +--assert(`SELECT JSON_EXTRACT('$result', '$.status') = 'error'`) + +--echo +--echo *** Removing the list result file. +--remove_file $read_from_file + +--echo +--echo *** Dropping the table. +DROP TABLE t1; + +# cleaning up +--source ../include/tear_down_binsrv_environment.inc diff --git a/src/app.cpp b/src/app.cpp index 60eccba..8e7ddda 100644 --- a/src/app.cpp +++ b/src/app.cpp @@ -115,6 +115,7 @@ check_cmd_args(const util::command_line_arg_view &cmd_args, switch (operation_mode) { case binsrv::operation_mode_type::fetch: case binsrv::operation_mode_type::pull: + case binsrv::operation_mode_type::list: if (number_of_cmd_args != expected_number_of_cmd_args_with_config) { return false; } @@ -952,6 +953,47 @@ bool handle_version() { return true; } +// shared by all 'handle_*' subcommands that build a 'search_response': +// translates a binlog record kept inside 'binsrv::storage' into +// a record of the response model +void append_record_to_response(binsrv::models::search_response &response, + const binsrv::storage &storage, + const auto &record) { + response.add_record(record.name.str(), record.size, + storage.get_binlog_uri(record.name), + record.previous_gtids, record.added_gtids, + record.timestamps.get_min_timestamp().get_value(), + record.timestamps.get_max_timestamp().get_value()); +} + +bool handle_list(std::string_view config_file_path) { + bool operation_successful{false}; + std::string result; + + try { + const binsrv::main_config config{config_file_path}; + const auto &storage_config = config.root().get<"storage">(); + const auto &replication_config = config.root().get<"replication">(); + const auto replication_mode{replication_config.get<"mode">()}; + + const binsrv::storage storage{ + storage_config, binsrv::storage_construction_mode_type::querying_only, + replication_mode}; + + binsrv::models::search_response response; + for (const auto &record : storage.get_binlog_records()) { + append_record_to_response(response, storage, record); + } + result = response.str(); + operation_successful = true; + } catch (const std::exception &e) { + const binsrv::models::error_response response{e.what()}; + result = response.str(); + } + std::cout << result << '\n'; + return operation_successful; +} + // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) bool handle_search_by_timestamp(std::string_view config_file_path, std::string_view subcommand_value) { @@ -984,11 +1026,7 @@ bool handle_search_by_timestamp(std::string_view config_file_path, if (record.timestamps.get_min_timestamp() > timestamp) { break; } - response.add_record(record.name.str(), record.size, - storage.get_binlog_uri(record.name), - record.previous_gtids, record.added_gtids, - record.timestamps.get_min_timestamp().get_value(), - record.timestamps.get_max_timestamp().get_value()); + append_record_to_response(response, storage, record); } if (response.root().get<"result">().empty()) { throw std::runtime_error("Timestamp is too old"); @@ -1044,11 +1082,7 @@ bool handle_search_by_gtid_set(std::string_view config_file_path, } remaining_gtids.subtract(*record.added_gtids); - response.add_record(record.name.str(), record.size, - storage.get_binlog_uri(record.name), - record.previous_gtids, record.added_gtids, - record.timestamps.get_min_timestamp().get_value(), - record.timestamps.get_max_timestamp().get_value()); + append_record_to_response(response, storage, record); } if (!remaining_gtids.is_empty()) { throw std::runtime_error("The specified GTID set cannot be covered"); @@ -1063,6 +1097,27 @@ bool handle_search_by_gtid_set(std::string_view config_file_path, return operation_successful; } +// dispatcher for the read-only subcommands that do not need logger / signal +// handler / replication setup; returns std::nullopt for streaming modes +// ('fetch' and 'pull') and the handler's success flag otherwise +std::optional +dispatch_stateless_command(binsrv::operation_mode_type operation_mode, + std::string_view config_file_path, + std::string_view subcommand_value) { + switch (operation_mode) { + case binsrv::operation_mode_type::version: + return handle_version(); + case binsrv::operation_mode_type::list: + return handle_list(config_file_path); + case binsrv::operation_mode_type::search_by_timestamp: + return handle_search_by_timestamp(config_file_path, subcommand_value); + case binsrv::operation_mode_type::search_by_gtid_set: + return handle_search_by_gtid_set(config_file_path, subcommand_value); + default: + return std::nullopt; + } +} + // since c++20 it is no longer needed to initialize std::atomic_flag with // ATOMIC_FLAG_INIT as this flag is modified from a signal handler it is marked // as volatile to make sure optimizer do optimizations which will be unsafe for @@ -1090,6 +1145,7 @@ int main(int argc, char *argv[]) { if (!cmd_args_checked) { std::cerr << "usage: " << executable_name << " (fetch|pull)) \n" + << " " << executable_name << " list \n" << " " << executable_name << " search_by_timestamp \n" << " " << executable_name @@ -1098,23 +1154,11 @@ int main(int argc, char *argv[]) { return EXIT_FAILURE; } - // handling the 'version' command - if (operation_mode == binsrv::operation_mode_type::version) { - return handle_version() ? EXIT_SUCCESS : EXIT_FAILURE; - } - - // handling the 'search_by_timestamp' command - if (operation_mode == binsrv::operation_mode_type::search_by_timestamp) { - return handle_search_by_timestamp(config_file_path, subcommand_value) - ? EXIT_SUCCESS - : EXIT_FAILURE; - } - - // handling the 'search_by_gtid_set' command - if (operation_mode == binsrv::operation_mode_type::search_by_gtid_set) { - return handle_search_by_gtid_set(config_file_path, subcommand_value) - ? EXIT_SUCCESS - : EXIT_FAILURE; + // handling the read-only subcommands ('version', 'list', 'search_by_*') + if (const auto stateless_result{dispatch_stateless_command( + operation_mode, config_file_path, subcommand_value)}; + stateless_result.has_value()) { + return *stateless_result ? EXIT_SUCCESS : EXIT_FAILURE; } int exit_code = EXIT_FAILURE; diff --git a/src/binsrv/operation_mode_type.hpp b/src/binsrv/operation_mode_type.hpp index 700f9f5..f729d10 100644 --- a/src/binsrv/operation_mode_type.hpp +++ b/src/binsrv/operation_mode_type.hpp @@ -34,6 +34,7 @@ namespace binsrv { #define BINSRV_OPERATION_MODE_TYPE_X_SEQUENCE() \ BINSRV_OPERATION_MODE_TYPE_X_MACRO(fetch ), \ BINSRV_OPERATION_MODE_TYPE_X_MACRO(pull ), \ + BINSRV_OPERATION_MODE_TYPE_X_MACRO(list ), \ BINSRV_OPERATION_MODE_TYPE_X_MACRO(search_by_timestamp), \ BINSRV_OPERATION_MODE_TYPE_X_MACRO(search_by_gtid_set ), \ BINSRV_OPERATION_MODE_TYPE_X_MACRO(version )