diff --git a/README.md b/README.md index 31d067ffb9..1d0798b691 100644 --- a/README.md +++ b/README.md @@ -457,8 +457,22 @@ The performance tests can similarly be enabled. To execute them and see their ou $ ctest --verbose -L performance ``` + ### Configuring -The 'conf' directory in the installation root contains a template config.yml document, minifi.properties, and minifi-log.properties. Please see our [Configuration document](CONFIGURE.md) for details on how to configure agents. +The 'conf' directory in the installation root contains all configuration files. + +The files conf/minifi.properties, conf/minifi-log.properties and conf/minifi-uid.properties contain key-value pair configuration settings; +these are the default settings supplied by the latest MiNiFi version. If you would like to modify these, you should create a corresponding +.d directory (e.g. conf/minifi.properties.d) and put your settings in a new file inside this directory. These files are read and applied +in lexicographic order, after the default settings file. +The Windows installer creates a conf/minifi.properties.d/10_installer.properties file, which contains C2 connection settings. +If C2 is enabled and settings are added/modified from the C2 server, these will be saved in conf/minifi.properties.d/90_c2.properties. + +The conf/config.yml file contains the flow definition (i.e. the layout of processors, controller services etc). When you start MiNiFi for +the first time, the flow will be fetched from the C2 server (if available), or a file containing an empty flow will be created by MiNiFi. + +Please see our [Configuration document](CONFIGURE.md) for details on how to configure agents. + ### Installing as a service diff --git a/conf/CMakeLists.txt b/conf/CMakeLists.txt index 309b5453b8..dcdcb80a16 100644 --- a/conf/CMakeLists.txt +++ b/conf/CMakeLists.txt @@ -41,12 +41,6 @@ else() message(FATAL_ERROR "Invalid MINIFI_PACKAGING_TYPE") endif() -configure_file( - config.yml.in - ${CMAKE_BINARY_DIR}/conf/config.yml - @ONLY -) - configure_file( minifi.properties.in ${CMAKE_BINARY_DIR}/conf/minifi.properties @@ -75,7 +69,6 @@ if (MINIFI_PACKAGING_TYPE STREQUAL "RPM") ${CMAKE_BINARY_DIR}/conf/minifi.properties ${CMAKE_BINARY_DIR}/conf/minifi-log.properties ${CMAKE_BINARY_DIR}/conf/minifi-uid.properties - ${CMAKE_BINARY_DIR}/conf/config.yml DESTINATION /${CMAKE_INSTALL_SYSCONFDIR}/${PROJECT_NAME} COMPONENT bin) elseif (MINIFI_PACKAGING_TYPE STREQUAL "TGZ") @@ -83,7 +76,6 @@ elseif (MINIFI_PACKAGING_TYPE STREQUAL "TGZ") ${CMAKE_BINARY_DIR}/conf/minifi.properties ${CMAKE_BINARY_DIR}/conf/minifi-log.properties ${CMAKE_BINARY_DIR}/conf/minifi-uid.properties - ${CMAKE_BINARY_DIR}/conf/config.yml DESTINATION conf COMPONENT bin) else() diff --git a/conf/config.yml.in b/conf/config.yml.in deleted file mode 100644 index 0bf85fc3d8..0000000000 --- a/conf/config.yml.in +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -Flow Controller: - name: MiNiFi Flow -Processors: [] -Connections: [] -Remote Processing Groups: [] -Provenance Reporting: diff --git a/core-framework/include/utils/file/FileUtils.h b/core-framework/include/utils/file/FileUtils.h index 06b1cc2bd1..07d5d610ff 100644 --- a/core-framework/include/utils/file/FileUtils.h +++ b/core-framework/include/utils/file/FileUtils.h @@ -242,6 +242,12 @@ inline int copy_file(const std::filesystem::path& path_from, const std::filesyst return 0; } +inline bool move_file(const std::filesystem::path& source_path, const std::filesystem::path& dest_path) { + std::error_code ec; + std::filesystem::rename(source_path, dest_path, ec); + return !ec; +} + inline void addFilesMatchingExtension(const std::shared_ptr &logger, const std::filesystem::path& originalPath, const std::filesystem::path& extension, diff --git a/docker/test/integration/cluster/containers/MinifiContainer.py b/docker/test/integration/cluster/containers/MinifiContainer.py index 387a3bd0d9..62c5564deb 100644 --- a/docker/test/integration/cluster/containers/MinifiContainer.py +++ b/docker/test/integration/cluster/containers/MinifiContainer.py @@ -238,9 +238,6 @@ def deploy(self): else: image = 'apacheminificpp:' + MinifiContainer.MINIFI_TAG_PREFIX + MinifiContainer.MINIFI_VERSION - if self.options.use_flow_config_from_url: - self.command = ["/bin/sh", "-c", "rm " + MinifiContainer.MINIFI_LOCATIONS.config_path + " && " + MinifiContainer.MINIFI_LOCATIONS.run_minifi_cmd] - ports = {} if self.options.enable_prometheus or self.options.enable_prometheus_with_ssl: ports = {'9936/tcp': 9936} diff --git a/extensions/standard-processors/tests/unit/ConfigurationTests.cpp b/extensions/standard-processors/tests/unit/ConfigurationTests.cpp index bc4db378a8..f8b078e42c 100644 --- a/extensions/standard-processors/tests/unit/ConfigurationTests.cpp +++ b/extensions/standard-processors/tests/unit/ConfigurationTests.cpp @@ -15,12 +15,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include + #include "unit/TestBase.h" #include "unit/Catch.h" - #include "properties/Configuration.h" #include "utils/Environment.h" +namespace { +bool settingsInFileAreAsExpected(const std::filesystem::path& file_name, const std::unordered_set& expected_contents) { + std::unordered_set actual_contents; + std::ifstream file{file_name}; + if (file.is_open()) { + for (std::string line; std::getline(file, line); ) { + actual_contents.insert(line); + } + } + return expected_contents == actual_contents; +} +} // namespace + namespace org::apache::nifi::minifi::test { TEST_CASE("Configuration can merge lists of property names", "[mergeProperties]") { @@ -59,41 +73,37 @@ TEST_CASE("Configuration can fix misconfigured timeperiod<->integer validated pr LogTestController::getInstance().setInfo(); TestController test_controller; - auto properties_path = test_controller.createTempDirectory() / "test.properties"; + const auto conf_directory = test_controller.createTempDirectory(); + const auto original_properties_path = conf_directory / "test.properties"; + const auto updated_properties_path = conf_directory / "test.properties.d" / PropertiesImpl::C2PropertiesFileName; - std::ofstream{properties_path} + std::ofstream{original_properties_path} << "nifi.c2.agent.heartbeat.period=1min\n" << "nifi.administrative.yield.duration=30000\n"; - auto properties_file_time_after_creation = std::filesystem::last_write_time(properties_path); + auto properties_file_time_after_creation = std::filesystem::last_write_time(original_properties_path); const std::shared_ptr configure = std::make_shared(); - configure->loadConfigureFile(properties_path); + configure->loadConfigureFile(original_properties_path); CHECK(configure->get("nifi.c2.agent.heartbeat.period") == "60000"); CHECK(configure->get("nifi.administrative.yield.duration") == "30000 ms"); { - CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(properties_path)); - std::ifstream properties_file(properties_path); - std::string first_line; - std::string second_line; - CHECK(std::getline(properties_file, first_line)); - CHECK(std::getline(properties_file, second_line)); - CHECK(first_line == "nifi.c2.agent.heartbeat.period=1min"); - CHECK(second_line == "nifi.administrative.yield.duration=30000"); + CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); + CHECK(settingsInFileAreAsExpected(original_properties_path, {"nifi.c2.agent.heartbeat.period=1min", "nifi.administrative.yield.duration=30000"})); } CHECK(configure->commitChanges()); { - CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(properties_path)); - std::ifstream properties_file(properties_path); - std::string first_line; - std::string second_line; - CHECK(std::getline(properties_file, first_line)); - CHECK(std::getline(properties_file, second_line)); - CHECK(first_line == "nifi.c2.agent.heartbeat.period=60000"); - CHECK(second_line == "nifi.administrative.yield.duration=30000 ms"); + CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); + CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(updated_properties_path)); + CHECK(settingsInFileAreAsExpected(updated_properties_path, {"nifi.c2.agent.heartbeat.period=60000", "nifi.administrative.yield.duration=30000 ms"})); } + + const std::shared_ptr configure_reread = std::make_shared(); + configure_reread->loadConfigureFile(original_properties_path); + CHECK(configure_reread->get("nifi.c2.agent.heartbeat.period") == "60000"); + CHECK(configure_reread->get("nifi.administrative.yield.duration") == "30000 ms"); } TEST_CASE("Configuration can fix misconfigured datasize<->integer validated properties") { @@ -101,80 +111,159 @@ TEST_CASE("Configuration can fix misconfigured datasize<->integer validated prop LogTestController::getInstance().setInfo(); TestController test_controller; - auto properties_path = test_controller.createTempDirectory() / "test.properties"; + const auto conf_directory = test_controller.createTempDirectory(); + const auto original_properties_path = conf_directory / "test.properties"; + const auto updated_properties_path = conf_directory / "test.properties.d" / PropertiesImpl::C2PropertiesFileName; { - std::ofstream properties_file(properties_path); + std::ofstream properties_file(original_properties_path); properties_file << "appender.rolling.max_file_size=6000" << std::endl; properties_file.close(); } - auto properties_file_time_after_creation = std::filesystem::last_write_time(properties_path); + auto properties_file_time_after_creation = std::filesystem::last_write_time(original_properties_path); const std::shared_ptr configure = std::make_shared(); - - configure->loadConfigureFile(properties_path, "nifi.log."); + configure->loadConfigureFile(original_properties_path, "nifi.log."); CHECK(configure->get("appender.rolling.max_file_size") == "6000 B"); { - CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(properties_path)); - std::ifstream properties_file(properties_path); - std::string first_line; - CHECK(std::getline(properties_file, first_line)); - CHECK(first_line == "appender.rolling.max_file_size=6000"); + CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); + CHECK(settingsInFileAreAsExpected(original_properties_path, {"appender.rolling.max_file_size=6000"})); } CHECK(configure->commitChanges()); { - CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(properties_path)); - std::ifstream properties_file(properties_path); - std::string first_line; - CHECK(std::getline(properties_file, first_line)); - CHECK(first_line == "appender.rolling.max_file_size=6000 B"); + CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); + CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(updated_properties_path)); + CHECK(settingsInFileAreAsExpected(updated_properties_path, {"appender.rolling.max_file_size=6000 B"})); } -} + const std::shared_ptr configure_reread = std::make_shared(); + configure_reread->loadConfigureFile(original_properties_path, "nifi.log."); + CHECK(configure_reread->get("appender.rolling.max_file_size") == "6000 B"); +} TEST_CASE("Configuration can fix misconfigured validated properties within environmental variables") { LogTestController::getInstance().setInfo(); LogTestController::getInstance().setInfo(); + TestController test_controller; - auto properties_path = test_controller.createTempDirectory() / "test.properties"; + const auto conf_directory = test_controller.createTempDirectory(); + const auto original_properties_path = conf_directory / "test.properties"; + const auto updated_properties_path = conf_directory / "test.properties.d" / PropertiesImpl::C2PropertiesFileName; CHECK(minifi::utils::Environment::setEnvironmentVariable("SOME_VARIABLE", "4000")); - std::ofstream{properties_path} + std::ofstream{original_properties_path} << "compression.cached.log.max.size=${SOME_VARIABLE}\n" << "compression.compressed.log.max.size=3000\n"; - auto properties_file_time_after_creation = std::filesystem::last_write_time(properties_path); + auto properties_file_time_after_creation = std::filesystem::last_write_time(original_properties_path); const std::shared_ptr configure = std::make_shared(); - configure->loadConfigureFile(properties_path, "nifi.log."); + configure->loadConfigureFile(original_properties_path, "nifi.log."); CHECK(configure->get("compression.cached.log.max.size") == "4000 B"); CHECK(configure->get("compression.compressed.log.max.size") == "3000 B"); { - CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(properties_path)); - std::ifstream properties_file(properties_path); - std::string first_line; - std::string second_line; - CHECK(std::getline(properties_file, first_line)); - CHECK(std::getline(properties_file, second_line)); - CHECK(first_line == "compression.cached.log.max.size=${SOME_VARIABLE}"); - CHECK(second_line == "compression.compressed.log.max.size=3000"); + CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); + CHECK(settingsInFileAreAsExpected(original_properties_path, {"compression.cached.log.max.size=${SOME_VARIABLE}", "compression.compressed.log.max.size=3000"})); } CHECK(configure->commitChanges()); { - CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(properties_path)); - std::ifstream properties_file(properties_path); - std::string first_line; - std::string second_line; - CHECK(std::getline(properties_file, first_line)); - CHECK(std::getline(properties_file, second_line)); - CHECK(first_line == "compression.cached.log.max.size=${SOME_VARIABLE}"); - CHECK(second_line == "compression.compressed.log.max.size=3000 B"); + CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); + CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(updated_properties_path)); + CHECK(settingsInFileAreAsExpected(updated_properties_path, {"compression.compressed.log.max.size=3000 B"})); } + + const std::shared_ptr configure_reread = std::make_shared(); + configure_reread->loadConfigureFile(original_properties_path, "nifi.log."); + CHECK(configure_reread->get("compression.cached.log.max.size") == "4000 B"); + CHECK(configure_reread->get("compression.compressed.log.max.size") == "3000 B"); +} + +TEST_CASE("Committing changes to a configuration creates a backup file") { + LogTestController::getInstance().setInfo(); + LogTestController::getInstance().setInfo(); + + TestController test_controller; + const auto conf_directory = test_controller.createTempDirectory(); + const auto original_properties_path = conf_directory / "test.properties"; + const auto updated_properties_path = conf_directory / "test.properties.d" / PropertiesImpl::C2PropertiesFileName; + const auto backup_properties_path = [&]() { + auto path = updated_properties_path; + path += ".bak"; + return path; + }(); + + std::ofstream{original_properties_path} + << "number.of.lions=7\n" + << "number.of.elephants=12\n" + << "number.of.giraffes=30\n"; + + const std::shared_ptr configure = std::make_shared(); + configure->loadConfigureFile(original_properties_path); + + CHECK(configure->get("number.of.lions") == "7"); + CHECK(configure->get("number.of.elephants") == "12"); + CHECK(configure->get("number.of.giraffes") == "30"); + + configure->set("number.of.lions", "8"); + CHECK(configure->commitChanges()); + CHECK(settingsInFileAreAsExpected(updated_properties_path, {"number.of.lions=8"})); + CHECK_FALSE(std::filesystem::exists(backup_properties_path)); + + const std::shared_ptr configure_2 = std::make_shared(); + configure_2->loadConfigureFile(original_properties_path); + CHECK(configure_2->get("number.of.lions") == "8"); + CHECK(configure_2->get("number.of.elephants") == "12"); + CHECK(configure_2->get("number.of.giraffes") == "30"); + + configure->set("number.of.giraffes", "29"); + CHECK(configure->commitChanges()); + CHECK(settingsInFileAreAsExpected(updated_properties_path, {"number.of.lions=8", "number.of.giraffes=29"})); + CHECK(settingsInFileAreAsExpected(backup_properties_path, {"number.of.lions=8"})); + + const std::shared_ptr configure_3 = std::make_shared(); + configure_3->loadConfigureFile(original_properties_path); + CHECK(configure_3->get("number.of.lions") == "8"); + CHECK(configure_3->get("number.of.elephants") == "12"); + CHECK(configure_3->get("number.of.giraffes") == "29"); +} + +TEST_CASE("Backup file are skipped when reading config files") { + LogTestController::getInstance().setInfo(); + LogTestController::getInstance().setInfo(); + + TestController test_controller; + const auto conf_directory = test_controller.createTempDirectory(); + const auto original_properties_path = conf_directory / "test.properties"; + const auto updated_properties_path = conf_directory / "test.properties.d" / PropertiesImpl::C2PropertiesFileName; + const auto backup_properties_path = [&]() { + auto path = updated_properties_path; + path += ".bak"; + return path; + }(); + + std::ofstream{original_properties_path} + << "number.of.lions=7\n" + << "number.of.elephants=12\n"; + + utils::file::create_dir(updated_properties_path.parent_path()); + std::ofstream{updated_properties_path} + << "number.of.lions=8\n"; + + std::ofstream{backup_properties_path} + << "number.of.elephants=20\n" + << "number.of.giraffes=30\n"; + + const std::shared_ptr configure = std::make_shared(); + configure->loadConfigureFile(original_properties_path); + + CHECK(configure->get("number.of.lions") == "8"); + CHECK(configure->get("number.of.elephants") == "12"); + CHECK_FALSE(configure->get("number.of.giraffes")); } } // namespace org::apache::nifi::minifi::test diff --git a/libminifi/include/core/logging/LoggerProperties.h b/libminifi/include/core/logging/LoggerProperties.h index 90098d4475..c776216cb4 100644 --- a/libminifi/include/core/logging/LoggerProperties.h +++ b/libminifi/include/core/logging/LoggerProperties.h @@ -32,7 +32,7 @@ namespace org::apache::nifi::minifi::core::logging { class LoggerProperties : public PropertiesImpl { public: explicit LoggerProperties(std::filesystem::path default_log_dir) - : PropertiesImpl("Logger properties"), + : PropertiesImpl(PersistTo::MultipleFiles, "Logger properties"), default_log_dir_(std::move(default_log_dir)) { } /** diff --git a/libminifi/include/properties/Configuration.h b/libminifi/include/properties/Configuration.h index 547aeb35d6..0dbdb434f1 100644 --- a/libminifi/include/properties/Configuration.h +++ b/libminifi/include/properties/Configuration.h @@ -34,7 +34,7 @@ class PropertyValidator; class ConfigurationImpl : public PropertiesImpl, public virtual Configuration { public: - ConfigurationImpl() : PropertiesImpl("MiNiFi configuration") {} + ConfigurationImpl() : PropertiesImpl(PersistTo::MultipleFiles, "MiNiFi configuration") {} }; } // namespace org::apache::nifi::minifi diff --git a/libminifi/include/properties/Properties.h b/libminifi/include/properties/Properties.h index f103d0565b..3c9c13380c 100644 --- a/libminifi/include/properties/Properties.h +++ b/libminifi/include/properties/Properties.h @@ -41,10 +41,14 @@ class PropertiesImpl : public virtual Properties { }; public: - explicit PropertiesImpl(std::string name = ""); + enum class PersistTo { SingleFile, MultipleFiles }; + + explicit PropertiesImpl(PersistTo persist_to, std::string name = ""); ~PropertiesImpl() override = default; + static constexpr std::string_view C2PropertiesFileName = "90_c2.properties"; + const std::string& getName() const override { return name_; } @@ -104,7 +108,6 @@ class PropertiesImpl : public virtual Properties { void loadConfigureFile(const std::filesystem::path& configuration_file, std::string_view prefix = "") override; - std::vector getConfiguredKeys() const override { std::vector keys; for (auto &property : properties_) { @@ -122,18 +125,17 @@ class PropertiesImpl : public virtual Properties { std::map getProperties() const override; private: - std::map properties_; + std::filesystem::path extra_properties_files_dir_name() const; + void setPropertiesFromFile(const std::filesystem::path& properties_file, std::string_view prefix); + std::map properties_; bool dirty_{false}; - - std::filesystem::path properties_file_; - + std::filesystem::path base_properties_file_; + std::vector properties_files_; utils::ChecksumCalculator checksum_calculator_; - - // Mutex for protection mutable std::mutex mutex_; std::shared_ptr logger_; - + PersistTo persist_to_; std::string name_; }; diff --git a/libminifi/include/utils/ChecksumCalculator.h b/libminifi/include/utils/ChecksumCalculator.h index 8cc1a4ed48..c11931d58e 100644 --- a/libminifi/include/utils/ChecksumCalculator.h +++ b/libminifi/include/utils/ChecksumCalculator.h @@ -20,7 +20,7 @@ #include #include #include -#include +#include namespace org::apache::nifi::minifi::utils { @@ -29,16 +29,15 @@ class ChecksumCalculator { static constexpr const char* CHECKSUM_TYPE = "SHA256"; static constexpr size_t LENGTH_OF_HASH_IN_BYTES = 32; - void setFileLocation(const std::filesystem::path& file_location); - [[nodiscard]] std::filesystem::path getFileName() const; + void setFileLocations(std::vector file_locations); + [[nodiscard]] std::filesystem::path getFileNameOfFirstFileLocation() const; std::string getChecksum(); void invalidateChecksum(); private: - static std::string computeChecksum(const std::filesystem::path& file_location); + static std::string computeChecksum(const std::vector& file_locations); - std::optional file_location_; - std::optional file_name_; + std::vector file_locations_; std::optional checksum_; }; diff --git a/libminifi/src/core/FlowConfiguration.cpp b/libminifi/src/core/FlowConfiguration.cpp index 2e7b56484e..11f58bd107 100644 --- a/libminifi/src/core/FlowConfiguration.cpp +++ b/libminifi/src/core/FlowConfiguration.cpp @@ -29,6 +29,19 @@ #include "minifi-cpp/SwapManager.h" #include "Connection.h" +namespace { +void createDefaultFlowConfigFile(const std::filesystem::path& path) { + std::ofstream ostream(path); + ostream.exceptions(std::ofstream::failbit | std::ofstream::badbit); + ostream << "Flow Controller:\n" + " name: MiNiFi Flow\n" + "Processors: []\n" + "Connections: []\n" + "Remote Processing Groups: []\n" + "Provenance Reporting:\n"; +} +} // namespace + namespace org::apache::nifi::minifi::core { FlowConfiguration::FlowConfiguration(ConfigurationContext ctx) @@ -52,12 +65,16 @@ FlowConfiguration::FlowConfiguration(ConfigurationContext ctx) if (!ctx.path) { logger_->log_error("Configuration path is not specified."); } else { + const bool c2_enabled = configuration_->get(Configure::nifi_c2_enable).and_then(&utils::string::toBool).value_or(false); + if (!c2_enabled && !ctx.path->empty() && !std::filesystem::exists(*ctx.path)) { + createDefaultFlowConfigFile(*ctx.path); + } config_path_ = utils::file::canonicalize(*ctx.path); if (!config_path_) { logger_->log_error("Couldn't find config file \"{}\".", ctx.path->string()); config_path_ = ctx.path; } - checksum_calculator_.setFileLocation(*config_path_); + checksum_calculator_.setFileLocations(std::vector{*config_path_}); } } diff --git a/libminifi/src/core/state/nodes/ConfigurationChecksums.cpp b/libminifi/src/core/state/nodes/ConfigurationChecksums.cpp index c2181e7fc5..f9b965c105 100644 --- a/libminifi/src/core/state/nodes/ConfigurationChecksums.cpp +++ b/libminifi/src/core/state/nodes/ConfigurationChecksums.cpp @@ -32,7 +32,7 @@ std::vector ConfigurationChecksums::serialize() { for (auto checksum_calculator : checksum_calculators_) { SerializedResponseNode file_checksum_node; - file_checksum_node.name = checksum_calculator->getFileName().string(); + file_checksum_node.name = checksum_calculator->getFileNameOfFirstFileLocation().string(); file_checksum_node.value = checksum_calculator->getChecksum(); checksums_node.children.push_back(file_checksum_node); } diff --git a/libminifi/src/properties/Properties.cpp b/libminifi/src/properties/Properties.cpp index b60e1499dd..a70da478f6 100644 --- a/libminifi/src/properties/Properties.cpp +++ b/libminifi/src/properties/Properties.cpp @@ -18,19 +18,20 @@ #include "properties/Properties.h" #include +#include #include #include "core/logging/LoggerConfiguration.h" #include "properties/Configuration.h" #include "properties/PropertiesFile.h" -#include "range/v3/algorithm/all_of.hpp" #include "utils/StringUtils.h" #include "utils/file/FileUtils.h" namespace org::apache::nifi::minifi { -PropertiesImpl::PropertiesImpl(std::string name) +PropertiesImpl::PropertiesImpl(PersistTo persist_to, std::string name) : logger_(core::logging::LoggerFactory::getLogger()), + persist_to_(persist_to), name_(std::move(name)) { } @@ -72,12 +73,12 @@ const core::PropertyValidator* getValidator(const std::string& lookup_value) { // isdigit requires unsigned chars as input bool allDigits(const std::string& value) { - return ranges::all_of(value, [](const unsigned char c){ return ::isdigit(c); }); + return std::ranges::all_of(value, [](const unsigned char c){ return ::isdigit(c); }); } // isdigit requires unsigned chars as input bool allDigitsOrSpaces(const std::string& value) { - return ranges::all_of(value, [](const unsigned char c) { return std::isdigit(c) || std::isspace(c);}); + return std::ranges::all_of(value, [](const unsigned char c) { return std::isdigit(c) || std::isspace(c);}); } std::optional ensureTimePeriodValidatedPropertyHasExplicitUnit(const core::PropertyValidator* const validator, const std::string& value) { @@ -157,8 +158,41 @@ void fixValidatedProperty(const std::string& property_name, needs_to_persist_new_value = false; } } + +auto getExtraPropertiesFileNames(const std::filesystem::path& extra_properties_files_dir, const std::shared_ptr& logger) { + std::vector extra_properties_file_names; + if (utils::file::exists(extra_properties_files_dir) && utils::file::is_directory(extra_properties_files_dir)) { + utils::file::list_dir(extra_properties_files_dir, [&](const std::filesystem::path&, const std::filesystem::path& file_name) { + if (!file_name.string().ends_with(".bak")) { + extra_properties_file_names.push_back(file_name); + } + return true; + }, logger, /* recursive = */ false); + } + std::ranges::sort(extra_properties_file_names); + return extra_properties_file_names; +} + +void updateChangedPropertiesInPropertiesFile(minifi::PropertiesFile& current_content, const auto& properties) { + for (const auto& prop : properties) { + if (!prop.second.need_to_persist_new_value) { + continue; + } + if (current_content.hasValue(prop.first)) { + current_content.update(prop.first, prop.second.persisted_value); + } else { + current_content.append(prop.first, prop.second.persisted_value); + } + } +} } // namespace +std::filesystem::path PropertiesImpl::extra_properties_files_dir_name() const { + auto extra_properties_files_dir = base_properties_file_; + extra_properties_files_dir += ".d"; + return extra_properties_files_dir; +} + void PropertiesImpl::loadConfigureFile(const std::filesystem::path& configuration_file, std::string_view prefix) { std::lock_guard lock(mutex_); if (configuration_file.empty()) { @@ -173,23 +207,42 @@ void PropertiesImpl::loadConfigureFile(const std::filesystem::path& configuratio } std::error_code ec; - properties_file_ = std::filesystem::canonical(configuration_file, ec); + base_properties_file_ = std::filesystem::canonical(configuration_file, ec); if (ec.value() != 0) { logger_->log_warn("Configuration file '{}' does not exist, and it could not be created", configuration_file); return; } + properties_files_ = { base_properties_file_ }; + const auto extra_properties_files_dir = extra_properties_files_dir_name(); + const auto extra_properties_file_names = getExtraPropertiesFileNames(extra_properties_files_dir, logger_); + for (const auto& file_name : extra_properties_file_names) { + properties_files_.push_back(extra_properties_files_dir / file_name); + } + logger_->log_info("Using configuration file to load configuration for {} from {} (located at {})", - getName().c_str(), configuration_file.string(), properties_file_.string()); + getName().c_str(), configuration_file.string(), base_properties_file_.string()); + if (!extra_properties_file_names.empty()) { + auto list_of_files = utils::string::join(", ", extra_properties_file_names, [](const auto& path) { return path.string(); }); + logger_->log_info("Also reading configuration from files {} in {}", list_of_files, extra_properties_files_dir.string()); + } + + properties_.clear(); + dirty_ = false; + for (const auto& properties_file : properties_files_) { + setPropertiesFromFile(properties_file, prefix); + } + + checksum_calculator_.setFileLocations(properties_files_); +} - std::ifstream file(properties_file_, std::ifstream::in); +void PropertiesImpl::setPropertiesFromFile(const std::filesystem::path& properties_file, std::string_view prefix) { + std::ifstream file(properties_file, std::ifstream::in); if (!file.good()) { - logger_->log_error("load configure file failed {}", properties_file_); + logger_->log_error("load configure file failed {}", properties_file); return; } - properties_.clear(); - dirty_ = false; for (const auto& line : PropertiesFile{file}) { auto key = line.getKey(); auto persisted_value = line.getValue(); @@ -199,59 +252,65 @@ void PropertiesImpl::loadConfigureFile(const std::filesystem::path& configuratio dirty_ = dirty_ || need_to_persist_new_value; properties_[key] = {persisted_value, value, need_to_persist_new_value}; } - checksum_calculator_.setFileLocation(properties_file_); } std::filesystem::path PropertiesImpl::getFilePath() const { std::lock_guard lock(mutex_); - return properties_file_; + return base_properties_file_; } bool PropertiesImpl::commitChanges() { std::lock_guard lock(mutex_); if (!dirty_) { - logger_->log_info("Attempt to persist, but properties are not updated"); + logger_->log_debug("commitChanges() called, but properties have not changed, nothing to do"); return true; } - std::ifstream file(properties_file_, std::ifstream::in); + const auto output_file = (persist_to_ == PersistTo::SingleFile ? base_properties_file_ : extra_properties_files_dir_name() / C2PropertiesFileName); + if (!std::filesystem::exists(output_file)) { + logger_->log_debug("Configuration file {} does not exist yet, creating it", output_file); + utils::file::create_dir(output_file.parent_path(), /* recursive = */ true); + std::ofstream file{output_file}; + } + + std::ifstream file(output_file, std::ifstream::in); if (!file) { - logger_->log_error("load configure file failed {}", properties_file_); + logger_->log_error("Failed to load configuration file {}", output_file); return false; } - - auto new_file = properties_file_; - new_file += ".new"; - PropertiesFile current_content{file}; - for (const auto& prop : properties_) { - if (!prop.second.need_to_persist_new_value) { - continue; - } - if (current_content.hasValue(prop.first)) { - current_content.update(prop.first, prop.second.persisted_value); - } else { - current_content.append(prop.first, prop.second.persisted_value); - } - } + file.close(); + + updateChangedPropertiesInPropertiesFile(current_content, properties_); + auto new_file = output_file; + new_file += ".new"; try { current_content.writeTo(new_file); } catch (const std::exception&) { - logger_->log_error("Could not update {}", properties_file_); + logger_->log_error("Could not write to {}", new_file); return false; } - auto backup = properties_file_; - backup += ".bak"; - if (utils::file::FileUtils::copy_file(properties_file_, backup) == 0 && utils::file::FileUtils::copy_file(new_file, properties_file_) == 0) { - logger_->log_info("Persisted {}", properties_file_); - checksum_calculator_.invalidateChecksum(); - dirty_ = false; - return true; + std::error_code ec; + const auto existing_file_size = std::filesystem::file_size(output_file, ec); + if (ec || existing_file_size == 0) { + if (!utils::file::move_file(new_file, output_file)) { + logger_->log_error("Could not create minifi properties file {}", output_file); + return false; + } + } else { + auto backup = output_file; + backup += ".bak"; + if (!utils::file::move_file(output_file, backup) || !utils::file::move_file(new_file, output_file)) { + logger_->log_error("Could not update minifi properties file {}", output_file); + return false; + } } - logger_->log_error("Could not update {}", properties_file_); - return false; + logger_->log_info("Persisted {}", output_file); + checksum_calculator_.invalidateChecksum(); + dirty_ = false; + return true; } std::map PropertiesImpl::getProperties() const { @@ -264,7 +323,7 @@ std::map PropertiesImpl::getProperties() const { } std::shared_ptr Properties::create() { - return std::make_shared(); + return std::make_shared(PropertiesImpl::PersistTo::SingleFile); } } // namespace org::apache::nifi::minifi diff --git a/libminifi/src/utils/ChecksumCalculator.cpp b/libminifi/src/utils/ChecksumCalculator.cpp index 14e1f831d2..ae32e8ee42 100644 --- a/libminifi/src/utils/ChecksumCalculator.cpp +++ b/libminifi/src/utils/ChecksumCalculator.cpp @@ -21,7 +21,6 @@ #include #include "sodium/crypto_hash_sha256.h" -#include "utils/file/FileUtils.h" #include "utils/StringUtils.h" #include "properties/Configuration.h" @@ -29,51 +28,59 @@ namespace { const std::string AGENT_IDENTIFIER_KEY = std::string(org::apache::nifi::minifi::Configuration::nifi_c2_agent_identifier) + "="; +namespace utils = org::apache::nifi::minifi::utils; + +void addFileToChecksum(const std::filesystem::path& file_path, crypto_hash_sha256_state& state) { + std::ifstream input_file{file_path, std::ios::in | std::ios::binary}; + if (!input_file.is_open()) { + throw std::runtime_error(utils::string::join_pack("Could not open config file '", file_path.string(), "' to compute the checksum: ", std::strerror(errno))); + } + + std::string line; + while (std::getline(input_file, line)) { + // skip lines containing the agent identifier, so agents in the same class will have the same checksum + if (line.starts_with(AGENT_IDENTIFIER_KEY)) { + continue; + } + if (!input_file.eof()) { // eof() means we have just read the last line, which was not terminated by a newline + line.append("\n"); + } + crypto_hash_sha256_update(&state, reinterpret_cast(line.data()), line.size()); + } + if (input_file.bad()) { + throw std::runtime_error(utils::string::join_pack("Error reading config file '", file_path.string(), "' while computing the checksum: ", std::strerror(errno))); + } +} + } // namespace namespace org::apache::nifi::minifi::utils { -void ChecksumCalculator::setFileLocation(const std::filesystem::path& file_location) { - file_location_ = file_location; - file_name_ = file_location.filename(); +void ChecksumCalculator::setFileLocations(std::vector file_locations) { + gsl_Expects(!file_locations.empty()); + file_locations_ = std::move(file_locations); invalidateChecksum(); } -std::filesystem::path ChecksumCalculator::getFileName() const { - gsl_Expects(file_name_); - return *file_name_; +std::filesystem::path ChecksumCalculator::getFileNameOfFirstFileLocation() const { + gsl_Expects(!file_locations_.empty()); + return file_locations_.front().filename(); } std::string ChecksumCalculator::getChecksum() { - gsl_Expects(file_location_); + gsl_Expects(!file_locations_.empty()); if (!checksum_) { - checksum_ = computeChecksum(*file_location_); + checksum_ = computeChecksum(file_locations_); } return *checksum_; } -std::string ChecksumCalculator::computeChecksum(const std::filesystem::path& file_location) { - std::ifstream input_file{file_location, std::ios::in | std::ios::binary}; - if (!input_file.is_open()) { - throw std::runtime_error(string::join_pack("Could not open config file '", file_location.string(), "' to compute the checksum: ", std::strerror(errno))); - } - +std::string ChecksumCalculator::computeChecksum(const std::vector& file_locations) { crypto_hash_sha256_state state; crypto_hash_sha256_init(&state); - std::string line; - while (std::getline(input_file, line)) { - // skip lines containing the agent identifier, so agents in the same class will have the same checksum - if (string::startsWith(line, AGENT_IDENTIFIER_KEY)) { - continue; - } - if (!input_file.eof()) { // eof() means we have just read the last line, which was not terminated by a newline - line.append("\n"); - } - crypto_hash_sha256_update(&state, reinterpret_cast(line.data()), line.size()); - } - if (input_file.bad()) { - throw std::runtime_error(string::join_pack("Error reading config file '", file_location.string(), "' while computing the checksum: ", std::strerror(errno))); + for (const auto& file_location : file_locations) { + addFileToChecksum(file_location, state); } std::array hash{}; diff --git a/libminifi/test/integration/C2PropertiesUpdateTests.cpp b/libminifi/test/integration/C2PropertiesUpdateTests.cpp index b762a2c867..e6cf2389d4 100644 --- a/libminifi/test/integration/C2PropertiesUpdateTests.cpp +++ b/libminifi/test/integration/C2PropertiesUpdateTests.cpp @@ -155,9 +155,9 @@ TEST_CASE("C2PropertiesUpdateTests", "[c2test]") { logger2->log_debug("DummyClass2::before"); logger3->log_debug("DummyClass3::before"); - REQUIRE(!log_test_controller->contains("DummyClass1::before", 0s)); - REQUIRE(!log_test_controller->contains("DummyClass2::before", 0s)); - REQUIRE(!log_test_controller->contains("DummyClass3::before", 0s)); + CHECK(!log_test_controller->contains("DummyClass1::before", 0s)); + CHECK(!log_test_controller->contains("DummyClass2::before", 0s)); + CHECK(!log_test_controller->contains("DummyClass3::before", 0s)); } // On msvc, the passed lambda can't capture a reference to the object under construction, so we need to late-init harness. @@ -194,23 +194,25 @@ TEST_CASE("C2PropertiesUpdateTests", "[c2test]") { logger2->log_debug("DummyClass2::after"); // this should still not log logger3->log_debug("DummyClass3::after"); } - REQUIRE(log_test_controller->contains("DummyClass1::after", 0s)); - REQUIRE(!log_test_controller->contains("DummyClass2::after", 0s)); - REQUIRE(log_test_controller->contains("DummyClass3::after", 0s)); + CHECK(log_test_controller->contains("DummyClass1::after", 0s)); + CHECK_FALSE(log_test_controller->contains("DummyClass2::after", 0s)); + CHECK(log_test_controller->contains("DummyClass3::after", 0s)); { - minifi::PropertiesFile minifi_properties(std::ifstream{home_dir / "conf/minifi.properties"}); - REQUIRE(!minifi_properties.hasValue("nifi.dummy.property")); - REQUIRE(minifi_properties.getValue("nifi.property.one") == "bush"); - REQUIRE(minifi_properties.getValue("nifi.property.two") == "ring"); - REQUIRE(!minifi_properties.hasValue(minifi::Configuration::nifi_c2_rest_heartbeat_minimize_updates)); - REQUIRE(minifi_properties.getValue(minifi::Configuration::minifi_disk_space_watchdog_enable) == "true"); + const std::shared_ptr minifi_properties = std::make_shared(); + minifi_properties->loadConfigureFile(home_dir / "conf" / "minifi.properties"); + CHECK_FALSE(minifi_properties->get("nifi.dummy.property")); + CHECK(minifi_properties->get("nifi.property.one") == "bush"); + CHECK(minifi_properties->get("nifi.property.two") == "ring"); + CHECK_FALSE(minifi_properties->get(minifi::Configuration::nifi_c2_rest_heartbeat_minimize_updates)); + CHECK(minifi_properties->get(minifi::Configuration::minifi_disk_space_watchdog_enable) == "true"); } { - minifi::PropertiesFile minifi_log_properties(std::ifstream{home_dir / "conf/minifi-log.properties"}); - REQUIRE(!minifi_log_properties.hasValue("logger.org::apache::nifi::minifi::test::dummy")); - REQUIRE(minifi_log_properties.getValue("logger.org::apache::nifi::minifi::test::DummyClass1") == "DEBUG,ostream"); + const std::shared_ptr minifi_log_properties = std::make_shared(); + minifi_log_properties->loadConfigureFile(home_dir / "conf" / "minifi-log.properties"); + CHECK_FALSE(minifi_log_properties->get("logger.org::apache::nifi::minifi::test::dummy")); + CHECK(minifi_log_properties->get("logger.org::apache::nifi::minifi::test::DummyClass1") == "DEBUG,ostream"); } }); diff --git a/libminifi/test/unit/ChecksumCalculatorTests.cpp b/libminifi/test/unit/ChecksumCalculatorTests.cpp index fabecddfe6..82623814c1 100644 --- a/libminifi/test/unit/ChecksumCalculatorTests.cpp +++ b/libminifi/test/unit/ChecksumCalculatorTests.cpp @@ -37,17 +37,33 @@ TEST_CASE("ChecksumCalculator can calculate the checksum, which is equal to sha2 REQUIRE(size_t{utils::ChecksumCalculator::LENGTH_OF_HASH_IN_BYTES} == size_t{32}); utils::ChecksumCalculator checksum_calculator; - checksum_calculator.setFileLocation(file_location); + checksum_calculator.setFileLocations(std::vector{file_location}); REQUIRE(checksum_calculator.getChecksum() == CHECKSUM_FOR_ONE_LINE_OF_TEXT); } +TEST_CASE("The input of ChecksumCalculator can be in multiple files", "[ChecksumCalculator]") { + TestController test_controller; + const auto test_dir = test_controller.createTempDirectory(); + const auto single_file_location = minifi::test::utils::putFileToDir(test_dir, "single.txt", "one line of text\nsecond line of text\n"); + const auto multiple_1_file_location = minifi::test::utils::putFileToDir(test_dir, "multiple_1.txt", "one line of text\n"); + const auto multiple_2_file_location = minifi::test::utils::putFileToDir(test_dir, "multiple_2.txt", "second line of text\n"); + + utils::ChecksumCalculator checksum_calculator_single; + checksum_calculator_single.setFileLocations(std::vector{single_file_location}); + + utils::ChecksumCalculator checksum_calculator_multiple; + checksum_calculator_multiple.setFileLocations(std::vector{multiple_1_file_location, multiple_2_file_location}); + + CHECK(checksum_calculator_single.getChecksum() == checksum_calculator_multiple.getChecksum()); +} + TEST_CASE("On Windows text files, the checksum calculated is also the same as sha256sum", "[ChecksumCalculator]") { TestController test_controller; auto test_dir = test_controller.createTempDirectory(); auto file_location = minifi::test::utils::putFileToDir(test_dir, "simple.txt", "one line of text\r\n"); utils::ChecksumCalculator checksum_calculator; - checksum_calculator.setFileLocation(file_location); + checksum_calculator.setFileLocations(std::vector{file_location}); REQUIRE(checksum_calculator.getChecksum() == "94fc46c62ef6cc5b45cbad9fd53116cfb15a80960a9b311c1c27e5b5265ad4b4"); } @@ -57,7 +73,7 @@ TEST_CASE("The checksum can be reset and recomputed", "[ChecksumCalculator]") { auto file_location = minifi::test::utils::putFileToDir(test_dir, "simple.txt", "one line of text\n"); utils::ChecksumCalculator checksum_calculator; - checksum_calculator.setFileLocation(file_location); + checksum_calculator.setFileLocations(std::vector{file_location}); REQUIRE(checksum_calculator.getChecksum() == CHECKSUM_FOR_ONE_LINE_OF_TEXT); std::ofstream append_to_file(file_location, std::ios::binary | std::ios::app); @@ -76,11 +92,11 @@ TEST_CASE("If the file location is updated, the checksum will be recomputed", "[ auto file_location = minifi::test::utils::putFileToDir(test_dir, "simple.txt", "one line of text\n"); utils::ChecksumCalculator checksum_calculator; - checksum_calculator.setFileLocation(file_location); + checksum_calculator.setFileLocations(std::vector{file_location}); REQUIRE(checksum_calculator.getChecksum() == CHECKSUM_FOR_ONE_LINE_OF_TEXT); auto other_file_location = minifi::test::utils::putFileToDir(test_dir, "long.txt", "one line of text\nanother line of text\n"); - checksum_calculator.setFileLocation(other_file_location); + checksum_calculator.setFileLocations(std::vector{other_file_location}); REQUIRE(checksum_calculator.getChecksum() == CHECKSUM_FOR_TWO_LINES_OF_TEXT); } @@ -92,7 +108,7 @@ TEST_CASE("Checksums can be computed for binary (eg. encrypted) files, too", "[C auto file_location = minifi::test::utils::putFileToDir(test_dir, "simple.txt", binary_data); utils::ChecksumCalculator checksum_calculator; - checksum_calculator.setFileLocation(file_location); + checksum_calculator.setFileLocations(std::vector{file_location}); REQUIRE(checksum_calculator.getChecksum() == "bdec77160c394c067419735de757e4daa1c4679ea45e82a33fa8f706eed87709"); } @@ -109,16 +125,16 @@ TEST_CASE("The agent identifier is excluded from the checksum", "[ChecksumCalcul "nifi.c2.agent.heartbeat.period=10 sec\n"); utils::ChecksumCalculator checksum_calculator_1; - checksum_calculator_1.setFileLocation(file_location_1); + checksum_calculator_1.setFileLocations(std::vector{file_location_1}); utils::ChecksumCalculator checksum_calculator_2; - checksum_calculator_2.setFileLocation(file_location_2); + checksum_calculator_2.setFileLocations(std::vector{file_location_2}); REQUIRE(checksum_calculator_1.getChecksum() == checksum_calculator_2.getChecksum()); } TEST_CASE("ChecksumCalculator::getChecksum will throw if the file does not exist", "[ChecksumCalculator]") { utils::ChecksumCalculator checksum_calculator; - checksum_calculator.setFileLocation("/this/file/does/not/exist/84a77fd9-16b3-49d2-aead-a1f9e58e530d"); + checksum_calculator.setFileLocations(std::vector{std::filesystem::path{"/this/file/does/not/exist/84a77fd9-16b3-49d2-aead-a1f9e58e530d"}}); REQUIRE_THROWS(checksum_calculator.getChecksum()); } diff --git a/libminifi/test/unit/ConfigurationChecksumsTests.cpp b/libminifi/test/unit/ConfigurationChecksumsTests.cpp index 6da67a66e6..956907ff2a 100644 --- a/libminifi/test/unit/ConfigurationChecksumsTests.cpp +++ b/libminifi/test/unit/ConfigurationChecksumsTests.cpp @@ -39,7 +39,7 @@ TEST_CASE("If one checksum calculator is added, we get a node with one child", " auto file_location = minifi::test::utils::putFileToDir(test_dir, "simple.txt", "one line of text\n"); utils::ChecksumCalculator checksum_calculator; - checksum_calculator.setFileLocation(file_location); + checksum_calculator.setFileLocations(std::vector{file_location}); ConfigurationChecksums configuration_checksums; configuration_checksums.addChecksumCalculator(checksum_calculator); @@ -62,9 +62,9 @@ TEST_CASE("If two checksum calculators are added, we get a node with two childre auto file_location_2 = minifi::test::utils::putFileToDir(test_dir, "second.txt", "this is the second file\n"); utils::ChecksumCalculator checksum_calculator_1; - checksum_calculator_1.setFileLocation(file_location_1); + checksum_calculator_1.setFileLocations(std::vector{file_location_1}); utils::ChecksumCalculator checksum_calculator_2; - checksum_calculator_2.setFileLocation(file_location_2); + checksum_calculator_2.setFileLocations(std::vector{file_location_2}); ConfigurationChecksums configuration_checksums; configuration_checksums.addChecksumCalculator(checksum_calculator_1); diff --git a/libminifi/test/unit/IdTests.cpp b/libminifi/test/unit/IdTests.cpp index 048072b5dc..aac8ef0fb5 100644 --- a/libminifi/test/unit/IdTests.cpp +++ b/libminifi/test/unit/IdTests.cpp @@ -39,7 +39,7 @@ TEST_CASE("Test default is time", "[id]") { LogTestController::getInstance().setDebug(); std::shared_ptr generator = utils::IdGenerator::getIdGenerator(); - generator->initialize(std::make_shared()); + generator->initialize(std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties")); REQUIRE(true == LogTestController::getInstance().contains("Using uuid_generate_time implementation for uids.")); LogTestController::getInstance().reset(); @@ -49,7 +49,7 @@ TEST_CASE("Test time", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "TiMe"); std::shared_ptr generator = utils::IdGenerator::getIdGenerator(); @@ -69,7 +69,7 @@ TEST_CASE("Test Generate Move", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "TiMe"); std::shared_ptr generator = utils::IdGenerator::getIdGenerator(); @@ -86,7 +86,7 @@ TEST_CASE("Test random", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "RaNDoM"); std::shared_ptr generator = utils::IdGenerator::getIdGenerator(); @@ -106,7 +106,7 @@ TEST_CASE("Test uuid_default", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "UUID_default"); std::shared_ptr generator = utils::IdGenerator::getIdGenerator(); @@ -120,7 +120,7 @@ TEST_CASE("Test invalid", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "InVaLiD"); std::shared_ptr generator = utils::IdGenerator::getIdGenerator(); @@ -134,7 +134,7 @@ TEST_CASE("Test parse", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "time"); std::shared_ptr generator = utils::IdGenerator::getIdGenerator(); @@ -160,7 +160,7 @@ TEST_CASE("Test parse invalid", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "time"); std::shared_ptr generator = utils::IdGenerator::getIdGenerator(); @@ -187,7 +187,7 @@ TEST_CASE("Test to_string", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "time"); std::shared_ptr generator = utils::IdGenerator::getIdGenerator(); @@ -229,7 +229,7 @@ TEST_CASE("Test Hex Device Segment 16 bits correct digits", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "minifi_uid"); id_props->set("uid.minifi.device.segment", "09aF"); @@ -255,7 +255,7 @@ TEST_CASE("Test Hex Device Segment 16 bits too many digits", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "minifi_uid"); id_props->set("uid.minifi.device.segment", "09aFee"); @@ -283,7 +283,7 @@ TEST_CASE("Test Hex Device Segment 18 bits", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "minifi_uid"); id_props->set("uid.minifi.device.segment.bits", "18"); id_props->set("uid.minifi.device.segment", "09aF8"); @@ -317,7 +317,7 @@ TEST_CASE("Collision", "[collision]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); SECTION("random") { id_props->set("uid.implementation", "random"); } @@ -357,7 +357,7 @@ TEST_CASE("Speed", "[speed]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); std::string implementation; SECTION("random") { implementation = "random"; diff --git a/minifi_main/MiNiFiMain.cpp b/minifi_main/MiNiFiMain.cpp index b344085765..278ecbefbd 100644 --- a/minifi_main/MiNiFiMain.cpp +++ b/minifi_main/MiNiFiMain.cpp @@ -316,7 +316,7 @@ int main(int argc, char **argv) { logger_configuration.initialize(log_properties); - std::shared_ptr uid_properties = std::make_shared("UID properties"); + std::shared_ptr uid_properties = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); uid_properties->loadConfigureFile(locations->uid_properties_path_); utils::IdGenerator::getIdGenerator()->initialize(uid_properties); diff --git a/packaging/msi/WixWin.wsi.in b/packaging/msi/WixWin.wsi.in index e294b547da..9c912f604e 100644 --- a/packaging/msi/WixWin.wsi.in +++ b/packaging/msi/WixWin.wsi.in @@ -58,10 +58,9 @@ ${WIX_EXTRA_FEATURES} - - - + + @@ -263,16 +262,16 @@ ${WIX_EXTRA_FEATURES} - - - - - - - - - - + + + + + + + + + + @@ -303,27 +302,22 @@ ${WIX_EXTRA_FEATURES} - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + - - "1"]]> + + diff --git a/packaging/rpm/expected-rpm-contents.in b/packaging/rpm/expected-rpm-contents.in index cfdd8fa0bc..b4eb8a37f1 100644 --- a/packaging/rpm/expected-rpm-contents.in +++ b/packaging/rpm/expected-rpm-contents.in @@ -1,5 +1,4 @@ /etc/nifi-minifi-cpp -/etc/nifi-minifi-cpp/config.yml /etc/nifi-minifi-cpp/fips /etc/nifi-minifi-cpp/fips/openssl.cnf /etc/nifi-minifi-cpp/minifi-log.properties