PS-11001: Implement logic for list mode

mtr/binlog_streaming/r/list.result

@@ -0,0 +1,51 @@
+*** Resetting replication at the very beginning of the test.
+
+*** Determining the first binary log name.
+
+*** Generating a configuration file in JSON format for the Binlog
+*** Server utility.
+
+*** Determining binlog file directory from the server.
+
+*** Creating a temporary directory <BINSRV_STORAGE_PATH> for storing
+*** binlog files downloaded via the Binlog Server utility.
+
+*** 1. Executing the Binlog Server utility in the 'list' mode on an
+***    empty storage and expecting an empty result array
+include/read_file_to_var.inc
+
+*** Creating a simple table.
+CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB;
+
+*** Filling the table with some data.
+INSERT INTO t1 VALUES();
+
+*** Flushing the first binary log and switching to the second one.
+FLUSH BINARY LOGS;
+
+*** Determining the second binary log name.
+
+*** Filling the table with more data.
+INSERT INTO t1 VALUES();
+
+*** Executing the Binlog Server utility and fetching all events.
+
+*** 2. Executing the Binlog Server utility in the 'list' mode on a
+***    non-empty storage and expecting both binlog files to be returned
+***    in chronological order
+include/read_file_to_var.inc
+
+*** 3. Executing the Binlog Server utility in the 'list' mode with a
+***    nonexistent configuration file path
+include/read_file_to_var.inc
+
+*** Removing the list result file.
+
+*** Dropping the table.
+DROP TABLE t1;
+
+*** Removing the Binlog Server utility storage directory.
+
+*** Removing the Binlog Server utility log file.
+
+*** Removing the Binlog Server utility configuration file.

mtr/binlog_streaming/t/list.combinations

@@ -0,0 +1,5 @@
+[position]
+
+[gtid]
+gtid-mode=on
+enforce-gtid-consistency

mtr/binlog_streaming/t/list.test

@@ -0,0 +1,94 @@
+--source ../include/have_binsrv.inc
+
+--source ../include/v80_v84_compatibility_defines.inc
+
+# in case of --repeat=N, we need to start from a fresh binary log to make
+# this test deterministic
+--echo *** Resetting replication at the very beginning of the test.
+--disable_query_log
+eval $stmt_reset_binary_logs_and_gtids;
+--enable_query_log
+
+--echo
+--echo *** Determining the first binary log name.
+--let $first_binlog = query_get_value($stmt_show_binary_log_status, File, 1)
+
+# identifying backend storage type ('file' or 's3')
+--source ../include/identify_storage_backend.inc
+
+# creating data directory, configuration file, etc.
+--let $binsrv_connect_timeout = 20
+--let $binsrv_read_timeout = 60
+--let $binsrv_idle_time = 10
+--let $binsrv_verify_checksum = TRUE
+--let $binsrv_replication_mode = `SELECT IF(@@global.gtid_mode = 'ON', 'gtid', 'position')`
+--let $binsrv_checkpoint_size = 1
+--source ../include/set_up_binsrv_environment.inc
+
+--let $read_from_file = $MYSQL_TMP_DIR/list_result.json
+
+--echo
+--echo *** 1. Executing the Binlog Server utility in the 'list' mode on an
+--echo ***    empty storage and expecting an empty result array
+--exec $BINSRV list $binsrv_config_file_path > $read_from_file
+--source include/read_file_to_var.inc
+--assert(`SELECT JSON_EXTRACT('$result', '$.status') = 'success'`)
+--assert(`SELECT JSON_LENGTH(JSON_EXTRACT('$result', '$.result')) = 0`)
+
+--echo
+--echo *** Creating a simple table.
+CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB;
+
+--echo
+--echo *** Filling the table with some data.
+INSERT INTO t1 VALUES();
+
+--echo
+--echo *** Flushing the first binary log and switching to the second one.
+FLUSH BINARY LOGS;
+
+--echo
+--echo *** Determining the second binary log name.
+--let $second_binlog = query_get_value($stmt_show_binary_log_status, File, 1)
+
+--echo
+--echo *** Filling the table with more data.
+INSERT INTO t1 VALUES();
+
+--echo
+--echo *** Executing the Binlog Server utility and fetching all events.
+--exec $BINSRV fetch $binsrv_config_file_path > /dev/null
+
+--echo
+--echo *** 2. Executing the Binlog Server utility in the 'list' mode on a
+--echo ***    non-empty storage and expecting both binlog files to be returned
+--echo ***    in chronological order
+--exec $BINSRV list $binsrv_config_file_path > $read_from_file
+--source include/read_file_to_var.inc
+--assert(`SELECT JSON_EXTRACT('$result', '$.status') = 'success'`)
+--assert(`SELECT JSON_LENGTH(JSON_EXTRACT('$result', '$.result')) = 2`)
+--assert(`SELECT JSON_EXTRACT('$result', '$.result[0].name') = '$first_binlog'`)
+--assert(`SELECT JSON_EXTRACT('$result', '$.result[1].name') = '$second_binlog'`)
+--assert(`SELECT JSON_EXTRACT('$result', '$.result[0].size') > 0`)
+--assert(`SELECT JSON_EXTRACT('$result', '$.result[1].size') > 0`)
+--assert(`SELECT JSON_EXTRACT('$result', '$.result[0].uri') IS NOT NULL`)
+--assert(`SELECT JSON_EXTRACT('$result', '$.result[1].uri') IS NOT NULL`)
+
+--echo
+--echo *** 3. Executing the Binlog Server utility in the 'list' mode with a
+--echo ***    nonexistent configuration file path
+--error 1
+--exec $BINSRV list $MYSQL_TMP_DIR/no_such_config.json > $read_from_file
+--source include/read_file_to_var.inc
+--assert(`SELECT JSON_EXTRACT('$result', '$.status') = 'error'`)
+
+--echo
+--echo *** Removing the list result file.
+--remove_file $read_from_file
+
+--echo
+--echo *** Dropping the table.
+DROP TABLE t1;
+
+# cleaning up
+--source ../include/tear_down_binsrv_environment.inc

src/app.cpp

@@ -115,6 +115,7 @@ check_cmd_args(const util::command_line_arg_view &cmd_args,
   switch (operation_mode) {
   case binsrv::operation_mode_type::fetch:
   case binsrv::operation_mode_type::pull:
+  case binsrv::operation_mode_type::list:
     if (number_of_cmd_args != expected_number_of_cmd_args_with_config) {
       return false;
     }
@@ -952,6 +953,47 @@ bool handle_version() {
   return true;
 }
 
+// shared by all 'handle_*' subcommands that build a 'search_response':
+// translates a binlog record kept inside 'binsrv::storage' into
+// a record of the response model
+void append_record_to_response(binsrv::models::search_response &response,
+                               const binsrv::storage &storage,
+                               const auto &record) {
+  response.add_record(record.name.str(), record.size,
+                      storage.get_binlog_uri(record.name),
+                      record.previous_gtids, record.added_gtids,
+                      record.timestamps.get_min_timestamp().get_value(),
+                      record.timestamps.get_max_timestamp().get_value());
+}
+
+bool handle_list(std::string_view config_file_path) {
+  bool operation_successful{false};
+  std::string result;
+
+  try {
+    const binsrv::main_config config{config_file_path};
+    const auto &storage_config = config.root().get<"storage">();
+    const auto &replication_config = config.root().get<"replication">();
+    const auto replication_mode{replication_config.get<"mode">()};
+
+    const binsrv::storage storage{
+        storage_config, binsrv::storage_construction_mode_type::querying_only,
+        replication_mode};
+
+    binsrv::models::search_response response;
+    for (const auto &record : storage.get_binlog_records()) {
+      append_record_to_response(response, storage, record);
+    }
+    result = response.str();
+    operation_successful = true;
+  } catch (const std::exception &e) {
+    const binsrv::models::error_response response{e.what()};
+    result = response.str();
+  }
+  std::cout << result << '\n';
+  return operation_successful;
+}
+
 // NOLINTNEXTLINE(bugprone-easily-swappable-parameters)
 bool handle_search_by_timestamp(std::string_view config_file_path,
                                 std::string_view subcommand_value) {
@@ -984,11 +1026,7 @@ bool handle_search_by_timestamp(std::string_view config_file_path,
       if (record.timestamps.get_min_timestamp() > timestamp) {
         break;
       }
-      response.add_record(record.name.str(), record.size,
-                          storage.get_binlog_uri(record.name),
-                          record.previous_gtids, record.added_gtids,
-                          record.timestamps.get_min_timestamp().get_value(),
-                          record.timestamps.get_max_timestamp().get_value());
+      append_record_to_response(response, storage, record);
     }
     if (response.root().get<"result">().empty()) {
       throw std::runtime_error("Timestamp is too old");
@@ -1044,11 +1082,7 @@ bool handle_search_by_gtid_set(std::string_view config_file_path,
       }
       remaining_gtids.subtract(*record.added_gtids);
 
-      response.add_record(record.name.str(), record.size,
-                          storage.get_binlog_uri(record.name),
-                          record.previous_gtids, record.added_gtids,
-                          record.timestamps.get_min_timestamp().get_value(),
-                          record.timestamps.get_max_timestamp().get_value());
+      append_record_to_response(response, storage, record);
     }
     if (!remaining_gtids.is_empty()) {
       throw std::runtime_error("The specified GTID set cannot be covered");
@@ -1063,6 +1097,27 @@ bool handle_search_by_gtid_set(std::string_view config_file_path,
   return operation_successful;
 }
 
+// dispatcher for the read-only subcommands that do not need logger / signal
+// handler / replication setup; returns std::nullopt for streaming modes
+// ('fetch' and 'pull') and the handler's success flag otherwise
+std::optional<bool>
+dispatch_stateless_command(binsrv::operation_mode_type operation_mode,
+                           std::string_view config_file_path,
+                           std::string_view subcommand_value) {
+  switch (operation_mode) {
+  case binsrv::operation_mode_type::version:
+    return handle_version();
+  case binsrv::operation_mode_type::list:
+    return handle_list(config_file_path);
+  case binsrv::operation_mode_type::search_by_timestamp:
+    return handle_search_by_timestamp(config_file_path, subcommand_value);
+  case binsrv::operation_mode_type::search_by_gtid_set:
+    return handle_search_by_gtid_set(config_file_path, subcommand_value);
+  default:
+    return std::nullopt;
+  }
+}
+
 // since c++20 it is no longer needed to initialize std::atomic_flag with
 // ATOMIC_FLAG_INIT as this flag is modified from a signal handler it is marked
 // as volatile to make sure optimizer do optimizations which will be unsafe for
@@ -1090,6 +1145,7 @@ int main(int argc, char *argv[]) {
   if (!cmd_args_checked) {
     std::cerr << "usage: " << executable_name
               << " (fetch|pull)) <json_config_file>\n"
+              << "       " << executable_name << " list <json_config_file>\n"
               << "       " << executable_name
               << " search_by_timestamp <json_config_file> <timestamp>\n"
               << "       " << executable_name
@@ -1098,23 +1154,11 @@ int main(int argc, char *argv[]) {
     return EXIT_FAILURE;
   }
 
-  // handling the 'version' command
-  if (operation_mode == binsrv::operation_mode_type::version) {
-    return handle_version() ? EXIT_SUCCESS : EXIT_FAILURE;
-  }
-
-  // handling the 'search_by_timestamp' command
-  if (operation_mode == binsrv::operation_mode_type::search_by_timestamp) {
-    return handle_search_by_timestamp(config_file_path, subcommand_value)
-               ? EXIT_SUCCESS
-               : EXIT_FAILURE;
-  }
-
-  // handling the 'search_by_gtid_set' command
-  if (operation_mode == binsrv::operation_mode_type::search_by_gtid_set) {
-    return handle_search_by_gtid_set(config_file_path, subcommand_value)
-               ? EXIT_SUCCESS
-               : EXIT_FAILURE;
+  // handling the read-only subcommands ('version', 'list', 'search_by_*')
+  if (const auto stateless_result{dispatch_stateless_command(
+          operation_mode, config_file_path, subcommand_value)};
+      stateless_result.has_value()) {
+    return *stateless_result ? EXIT_SUCCESS : EXIT_FAILURE;
   }
 
   int exit_code = EXIT_FAILURE;

src/binsrv/operation_mode_type.hpp

@@ -34,6 +34,7 @@ namespace binsrv {
 #define BINSRV_OPERATION_MODE_TYPE_X_SEQUENCE() \
   BINSRV_OPERATION_MODE_TYPE_X_MACRO(fetch              ),  \
   BINSRV_OPERATION_MODE_TYPE_X_MACRO(pull               ),  \
+  BINSRV_OPERATION_MODE_TYPE_X_MACRO(list               ),  \
   BINSRV_OPERATION_MODE_TYPE_X_MACRO(search_by_timestamp),  \
   BINSRV_OPERATION_MODE_TYPE_X_MACRO(search_by_gtid_set ),  \
   BINSRV_OPERATION_MODE_TYPE_X_MACRO(version            )