From f99212128db66ce0e4d573cb194232da9c8749ea Mon Sep 17 00:00:00 2001 From: Ferenc Gerlits Date: Thu, 16 Oct 2025 11:25:49 +0200 Subject: [PATCH 01/11] MINIFICPP-2342 Do not overwrite config files during an upgrade Currently, MiNiFi modifies the installed config files (minifi.properties and config.yml) while it runs. This confuses the Windows installer, so changes to these config files get lost when MiNiFi is upgraded. After this change, the minifi.properties, minifi-log.properties and minifi-uid.properties files are no longer modified by MiNiFi at runtime, so they can be safely replaced by new versions during upgrade. All changes to the settings should be put into new files in the minifi.properties.d (minifi-log.properties.d, minifi-uid.properties.d) directory; these new files will not be touched by the upgrade. The config.yml file will no longer be installed as part of MiNiFi. If C2 is enabled, config.yml will be fetched from the C2 server; otherwise, MiNiFi will create a new file with an empty flow, and the user can edit this. Either way, config.yml will not be touched by an upgrade. --- README.md | 16 +- conf/CMakeLists.txt | 8 - conf/config.yml.in | 21 -- core-framework/include/utils/file/FileUtils.h | 6 + .../cluster/containers/MinifiContainer.py | 3 - .../tests/unit/ConfigurationTests.cpp | 204 +++++++++++++----- .../include/core/logging/LoggerProperties.h | 2 +- libminifi/include/properties/Configuration.h | 2 +- libminifi/include/properties/Properties.h | 19 +- libminifi/include/utils/ChecksumCalculator.h | 9 +- libminifi/src/core/FlowConfiguration.cpp | 18 +- libminifi/src/properties/Properties.cpp | 124 +++++++---- libminifi/src/utils/ChecksumCalculator.cpp | 61 +++--- .../integration/C2PropertiesUpdateTests.cpp | 32 +-- .../test/unit/ChecksumCalculatorTests.cpp | 34 ++- .../test/unit/ConfigurationChecksumsTests.cpp | 6 +- libminifi/test/unit/IdTests.cpp | 28 +-- minifi_main/MiNiFiMain.cpp | 2 +- packaging/msi/WixWin.wsi.in | 60 +++--- packaging/rpm/expected-rpm-contents.in | 1 - 20 files changed, 407 insertions(+), 249 deletions(-) delete mode 100644 conf/config.yml.in diff --git a/README.md b/README.md index 31d067ffb9..96410e289a 100644 --- a/README.md +++ b/README.md @@ -457,8 +457,22 @@ The performance tests can similarly be enabled. To execute them and see their ou $ ctest --verbose -L performance ``` + ### Configuring -The 'conf' directory in the installation root contains a template config.yml document, minifi.properties, and minifi-log.properties. Please see our [Configuration document](CONFIGURE.md) for details on how to configure agents. +The 'conf' directory in the installation root contains all configuration files. + +The files conf/minifi.properties, conf/minifi-log.properties and conf/minifi-uid.properties contain key-value pair configuration settings; +these are the default settings supplied by the latest MiNiFi version. If you would like to modify these, you should create a corresponding +.d directory (e.g. conf/minifi.properties.d) and put your settings in a new file inside this directory. These files are read and applied +in lexicographic order, after the default settings file. +The Windows installer creates a conf/minifi.properties.d/10_installer_properties file, which contains C2 connection settings. +If C2 is enabled and settings are added/modified from the C2 server, these will be saved in conf/minifi.properties.d/90_c2_properties. + +The conf/config.yml file contains the flow definition (i.e. the layout of processors, controller services etc). When you start MiNiFi for +the first time, the flow will be fetched from the C2 server (if available), or a file containing an empty flow will be created by MiNiFi. + +Please see our [Configuration document](CONFIGURE.md) for details on how to configure agents. + ### Installing as a service diff --git a/conf/CMakeLists.txt b/conf/CMakeLists.txt index 309b5453b8..dcdcb80a16 100644 --- a/conf/CMakeLists.txt +++ b/conf/CMakeLists.txt @@ -41,12 +41,6 @@ else() message(FATAL_ERROR "Invalid MINIFI_PACKAGING_TYPE") endif() -configure_file( - config.yml.in - ${CMAKE_BINARY_DIR}/conf/config.yml - @ONLY -) - configure_file( minifi.properties.in ${CMAKE_BINARY_DIR}/conf/minifi.properties @@ -75,7 +69,6 @@ if (MINIFI_PACKAGING_TYPE STREQUAL "RPM") ${CMAKE_BINARY_DIR}/conf/minifi.properties ${CMAKE_BINARY_DIR}/conf/minifi-log.properties ${CMAKE_BINARY_DIR}/conf/minifi-uid.properties - ${CMAKE_BINARY_DIR}/conf/config.yml DESTINATION /${CMAKE_INSTALL_SYSCONFDIR}/${PROJECT_NAME} COMPONENT bin) elseif (MINIFI_PACKAGING_TYPE STREQUAL "TGZ") @@ -83,7 +76,6 @@ elseif (MINIFI_PACKAGING_TYPE STREQUAL "TGZ") ${CMAKE_BINARY_DIR}/conf/minifi.properties ${CMAKE_BINARY_DIR}/conf/minifi-log.properties ${CMAKE_BINARY_DIR}/conf/minifi-uid.properties - ${CMAKE_BINARY_DIR}/conf/config.yml DESTINATION conf COMPONENT bin) else() diff --git a/conf/config.yml.in b/conf/config.yml.in deleted file mode 100644 index 0bf85fc3d8..0000000000 --- a/conf/config.yml.in +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -Flow Controller: - name: MiNiFi Flow -Processors: [] -Connections: [] -Remote Processing Groups: [] -Provenance Reporting: diff --git a/core-framework/include/utils/file/FileUtils.h b/core-framework/include/utils/file/FileUtils.h index 06b1cc2bd1..07d5d610ff 100644 --- a/core-framework/include/utils/file/FileUtils.h +++ b/core-framework/include/utils/file/FileUtils.h @@ -242,6 +242,12 @@ inline int copy_file(const std::filesystem::path& path_from, const std::filesyst return 0; } +inline bool move_file(const std::filesystem::path& source_path, const std::filesystem::path& dest_path) { + std::error_code ec; + std::filesystem::rename(source_path, dest_path, ec); + return !ec; +} + inline void addFilesMatchingExtension(const std::shared_ptr &logger, const std::filesystem::path& originalPath, const std::filesystem::path& extension, diff --git a/docker/test/integration/cluster/containers/MinifiContainer.py b/docker/test/integration/cluster/containers/MinifiContainer.py index 387a3bd0d9..62c5564deb 100644 --- a/docker/test/integration/cluster/containers/MinifiContainer.py +++ b/docker/test/integration/cluster/containers/MinifiContainer.py @@ -238,9 +238,6 @@ def deploy(self): else: image = 'apacheminificpp:' + MinifiContainer.MINIFI_TAG_PREFIX + MinifiContainer.MINIFI_VERSION - if self.options.use_flow_config_from_url: - self.command = ["/bin/sh", "-c", "rm " + MinifiContainer.MINIFI_LOCATIONS.config_path + " && " + MinifiContainer.MINIFI_LOCATIONS.run_minifi_cmd] - ports = {} if self.options.enable_prometheus or self.options.enable_prometheus_with_ssl: ports = {'9936/tcp': 9936} diff --git a/extensions/standard-processors/tests/unit/ConfigurationTests.cpp b/extensions/standard-processors/tests/unit/ConfigurationTests.cpp index bc4db378a8..820b79e6cb 100644 --- a/extensions/standard-processors/tests/unit/ConfigurationTests.cpp +++ b/extensions/standard-processors/tests/unit/ConfigurationTests.cpp @@ -15,12 +15,27 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include + #include "unit/TestBase.h" #include "unit/Catch.h" - #include "properties/Configuration.h" #include "utils/Environment.h" +namespace { +bool fileContentsMatch(const std::filesystem::path& file_name, const std::unordered_set& expected_contents) { + std::unordered_set actual_contents; + std::ifstream file{file_name}; + if (file.is_open()) { + std::string line; + while (std::getline(file, line)) { + actual_contents.insert(line); + } + } + return expected_contents == actual_contents; +} +} // namespace + namespace org::apache::nifi::minifi::test { TEST_CASE("Configuration can merge lists of property names", "[mergeProperties]") { @@ -59,41 +74,37 @@ TEST_CASE("Configuration can fix misconfigured timeperiod<->integer validated pr LogTestController::getInstance().setInfo(); TestController test_controller; - auto properties_path = test_controller.createTempDirectory() / "test.properties"; + const auto conf_directory = test_controller.createTempDirectory(); + const auto original_properties_path = conf_directory / "test.properties"; + const auto updated_properties_path = conf_directory / "test.properties.d" / PropertiesImpl::C2PropertiesFileName; - std::ofstream{properties_path} + std::ofstream{original_properties_path} << "nifi.c2.agent.heartbeat.period=1min\n" << "nifi.administrative.yield.duration=30000\n"; - auto properties_file_time_after_creation = std::filesystem::last_write_time(properties_path); + auto properties_file_time_after_creation = std::filesystem::last_write_time(original_properties_path); const std::shared_ptr configure = std::make_shared(); - configure->loadConfigureFile(properties_path); + configure->loadConfigureFile(original_properties_path); CHECK(configure->get("nifi.c2.agent.heartbeat.period") == "60000"); CHECK(configure->get("nifi.administrative.yield.duration") == "30000 ms"); { - CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(properties_path)); - std::ifstream properties_file(properties_path); - std::string first_line; - std::string second_line; - CHECK(std::getline(properties_file, first_line)); - CHECK(std::getline(properties_file, second_line)); - CHECK(first_line == "nifi.c2.agent.heartbeat.period=1min"); - CHECK(second_line == "nifi.administrative.yield.duration=30000"); + CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); + CHECK(fileContentsMatch(original_properties_path, {"nifi.c2.agent.heartbeat.period=1min", "nifi.administrative.yield.duration=30000"})); } CHECK(configure->commitChanges()); { - CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(properties_path)); - std::ifstream properties_file(properties_path); - std::string first_line; - std::string second_line; - CHECK(std::getline(properties_file, first_line)); - CHECK(std::getline(properties_file, second_line)); - CHECK(first_line == "nifi.c2.agent.heartbeat.period=60000"); - CHECK(second_line == "nifi.administrative.yield.duration=30000 ms"); + CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); + CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(updated_properties_path)); + CHECK(fileContentsMatch(updated_properties_path, {"nifi.c2.agent.heartbeat.period=60000", "nifi.administrative.yield.duration=30000 ms"})); } + + const std::shared_ptr configure_reread = std::make_shared(); + configure_reread->loadConfigureFile(original_properties_path); + CHECK(configure_reread->get("nifi.c2.agent.heartbeat.period") == "60000"); + CHECK(configure_reread->get("nifi.administrative.yield.duration") == "30000 ms"); } TEST_CASE("Configuration can fix misconfigured datasize<->integer validated properties") { @@ -101,80 +112,159 @@ TEST_CASE("Configuration can fix misconfigured datasize<->integer validated prop LogTestController::getInstance().setInfo(); TestController test_controller; - auto properties_path = test_controller.createTempDirectory() / "test.properties"; + const auto conf_directory = test_controller.createTempDirectory(); + const auto original_properties_path = conf_directory / "test.properties"; + const auto updated_properties_path = conf_directory / "test.properties.d" / PropertiesImpl::C2PropertiesFileName; { - std::ofstream properties_file(properties_path); + std::ofstream properties_file(original_properties_path); properties_file << "appender.rolling.max_file_size=6000" << std::endl; properties_file.close(); } - auto properties_file_time_after_creation = std::filesystem::last_write_time(properties_path); + auto properties_file_time_after_creation = std::filesystem::last_write_time(original_properties_path); const std::shared_ptr configure = std::make_shared(); - - configure->loadConfigureFile(properties_path, "nifi.log."); + configure->loadConfigureFile(original_properties_path, "nifi.log."); CHECK(configure->get("appender.rolling.max_file_size") == "6000 B"); { - CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(properties_path)); - std::ifstream properties_file(properties_path); - std::string first_line; - CHECK(std::getline(properties_file, first_line)); - CHECK(first_line == "appender.rolling.max_file_size=6000"); + CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); + CHECK(fileContentsMatch(original_properties_path, {"appender.rolling.max_file_size=6000"})); } CHECK(configure->commitChanges()); { - CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(properties_path)); - std::ifstream properties_file(properties_path); - std::string first_line; - CHECK(std::getline(properties_file, first_line)); - CHECK(first_line == "appender.rolling.max_file_size=6000 B"); + CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); + CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(updated_properties_path)); + CHECK(fileContentsMatch(updated_properties_path, {"appender.rolling.max_file_size=6000 B"})); } -} + const std::shared_ptr configure_reread = std::make_shared(); + configure_reread->loadConfigureFile(original_properties_path, "nifi.log."); + CHECK(configure_reread->get("appender.rolling.max_file_size") == "6000 B"); +} TEST_CASE("Configuration can fix misconfigured validated properties within environmental variables") { LogTestController::getInstance().setInfo(); LogTestController::getInstance().setInfo(); + TestController test_controller; - auto properties_path = test_controller.createTempDirectory() / "test.properties"; + const auto conf_directory = test_controller.createTempDirectory(); + const auto original_properties_path = conf_directory / "test.properties"; + const auto updated_properties_path = conf_directory / "test.properties.d" / PropertiesImpl::C2PropertiesFileName; CHECK(minifi::utils::Environment::setEnvironmentVariable("SOME_VARIABLE", "4000")); - std::ofstream{properties_path} + std::ofstream{original_properties_path} << "compression.cached.log.max.size=${SOME_VARIABLE}\n" << "compression.compressed.log.max.size=3000\n"; - auto properties_file_time_after_creation = std::filesystem::last_write_time(properties_path); + auto properties_file_time_after_creation = std::filesystem::last_write_time(original_properties_path); const std::shared_ptr configure = std::make_shared(); - configure->loadConfigureFile(properties_path, "nifi.log."); + configure->loadConfigureFile(original_properties_path, "nifi.log."); CHECK(configure->get("compression.cached.log.max.size") == "4000 B"); CHECK(configure->get("compression.compressed.log.max.size") == "3000 B"); { - CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(properties_path)); - std::ifstream properties_file(properties_path); - std::string first_line; - std::string second_line; - CHECK(std::getline(properties_file, first_line)); - CHECK(std::getline(properties_file, second_line)); - CHECK(first_line == "compression.cached.log.max.size=${SOME_VARIABLE}"); - CHECK(second_line == "compression.compressed.log.max.size=3000"); + CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); + CHECK(fileContentsMatch(original_properties_path, {"compression.cached.log.max.size=${SOME_VARIABLE}", "compression.compressed.log.max.size=3000"})); } CHECK(configure->commitChanges()); { - CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(properties_path)); - std::ifstream properties_file(properties_path); - std::string first_line; - std::string second_line; - CHECK(std::getline(properties_file, first_line)); - CHECK(std::getline(properties_file, second_line)); - CHECK(first_line == "compression.cached.log.max.size=${SOME_VARIABLE}"); - CHECK(second_line == "compression.compressed.log.max.size=3000 B"); + CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); + CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(updated_properties_path)); + CHECK(fileContentsMatch(updated_properties_path, {"compression.compressed.log.max.size=3000 B"})); } + + const std::shared_ptr configure_reread = std::make_shared(); + configure_reread->loadConfigureFile(original_properties_path, "nifi.log."); + CHECK(configure_reread->get("compression.cached.log.max.size") == "4000 B"); + CHECK(configure_reread->get("compression.compressed.log.max.size") == "3000 B"); +} + +TEST_CASE("Committing changes to a configuration creates a backup file") { + LogTestController::getInstance().setInfo(); + LogTestController::getInstance().setInfo(); + + TestController test_controller; + const auto conf_directory = test_controller.createTempDirectory(); + const auto original_properties_path = conf_directory / "test.properties"; + const auto updated_properties_path = conf_directory / "test.properties.d" / PropertiesImpl::C2PropertiesFileName; + const auto backup_properties_path = [&]() { + auto path = updated_properties_path; + path += ".bak"; + return path; + }(); + + std::ofstream{original_properties_path} + << "number.of.lions=7\n" + << "number.of.elephants=12\n" + << "number.of.giraffes=30\n"; + + const std::shared_ptr configure = std::make_shared(); + configure->loadConfigureFile(original_properties_path); + + CHECK(configure->get("number.of.lions") == "7"); + CHECK(configure->get("number.of.elephants") == "12"); + CHECK(configure->get("number.of.giraffes") == "30"); + + configure->set("number.of.lions", "8"); + CHECK(configure->commitChanges()); + CHECK(fileContentsMatch(updated_properties_path, {"number.of.lions=8"})); + CHECK_FALSE(std::filesystem::exists(backup_properties_path)); + + const std::shared_ptr configure_2 = std::make_shared(); + configure_2->loadConfigureFile(original_properties_path); + CHECK(configure_2->get("number.of.lions") == "8"); + CHECK(configure_2->get("number.of.elephants") == "12"); + CHECK(configure_2->get("number.of.giraffes") == "30"); + + configure->set("number.of.giraffes", "29"); + CHECK(configure->commitChanges()); + CHECK(fileContentsMatch(updated_properties_path, {"number.of.lions=8", "number.of.giraffes=29"})); + CHECK(fileContentsMatch(backup_properties_path, {"number.of.lions=8"})); + + const std::shared_ptr configure_3 = std::make_shared(); + configure_3->loadConfigureFile(original_properties_path); + CHECK(configure_3->get("number.of.lions") == "8"); + CHECK(configure_3->get("number.of.elephants") == "12"); + CHECK(configure_3->get("number.of.giraffes") == "29"); +} + +TEST_CASE("Backup file are skipped when reading config files") { + LogTestController::getInstance().setInfo(); + LogTestController::getInstance().setInfo(); + + TestController test_controller; + const auto conf_directory = test_controller.createTempDirectory(); + const auto original_properties_path = conf_directory / "test.properties"; + const auto updated_properties_path = conf_directory / "test.properties.d" / PropertiesImpl::C2PropertiesFileName; + const auto backup_properties_path = [&]() { + auto path = updated_properties_path; + path += ".bak"; + return path; + }(); + + std::ofstream{original_properties_path} + << "number.of.lions=7\n" + << "number.of.elephants=12\n"; + + utils::file::create_dir(updated_properties_path.parent_path()); + std::ofstream{updated_properties_path} + << "number.of.lions=8\n"; + + std::ofstream{backup_properties_path} + << "number.of.elephants=20\n" + << "number.of.giraffes=30\n"; + + const std::shared_ptr configure = std::make_shared(); + configure->loadConfigureFile(original_properties_path); + + CHECK(configure->get("number.of.lions") == "8"); + CHECK(configure->get("number.of.elephants") == "12"); + CHECK_FALSE(configure->get("number.of.giraffes")); } } // namespace org::apache::nifi::minifi::test diff --git a/libminifi/include/core/logging/LoggerProperties.h b/libminifi/include/core/logging/LoggerProperties.h index 90098d4475..c776216cb4 100644 --- a/libminifi/include/core/logging/LoggerProperties.h +++ b/libminifi/include/core/logging/LoggerProperties.h @@ -32,7 +32,7 @@ namespace org::apache::nifi::minifi::core::logging { class LoggerProperties : public PropertiesImpl { public: explicit LoggerProperties(std::filesystem::path default_log_dir) - : PropertiesImpl("Logger properties"), + : PropertiesImpl(PersistTo::MultipleFiles, "Logger properties"), default_log_dir_(std::move(default_log_dir)) { } /** diff --git a/libminifi/include/properties/Configuration.h b/libminifi/include/properties/Configuration.h index 547aeb35d6..0dbdb434f1 100644 --- a/libminifi/include/properties/Configuration.h +++ b/libminifi/include/properties/Configuration.h @@ -34,7 +34,7 @@ class PropertyValidator; class ConfigurationImpl : public PropertiesImpl, public virtual Configuration { public: - ConfigurationImpl() : PropertiesImpl("MiNiFi configuration") {} + ConfigurationImpl() : PropertiesImpl(PersistTo::MultipleFiles, "MiNiFi configuration") {} }; } // namespace org::apache::nifi::minifi diff --git a/libminifi/include/properties/Properties.h b/libminifi/include/properties/Properties.h index f103d0565b..1f0e87d9b7 100644 --- a/libminifi/include/properties/Properties.h +++ b/libminifi/include/properties/Properties.h @@ -41,10 +41,14 @@ class PropertiesImpl : public virtual Properties { }; public: - explicit PropertiesImpl(std::string name = ""); + enum class PersistTo { SingleFile, MultipleFiles }; + + explicit PropertiesImpl(PersistTo persist_to, std::string name = ""); ~PropertiesImpl() override = default; + static constexpr std::string_view C2PropertiesFileName = "90_c2_properties"; + const std::string& getName() const override { return name_; } @@ -104,7 +108,6 @@ class PropertiesImpl : public virtual Properties { void loadConfigureFile(const std::filesystem::path& configuration_file, std::string_view prefix = "") override; - std::vector getConfiguredKeys() const override { std::vector keys; for (auto &property : properties_) { @@ -122,18 +125,16 @@ class PropertiesImpl : public virtual Properties { std::map getProperties() const override; private: - std::map properties_; + std::filesystem::path extra_properties_files_dir_name() const; + std::map properties_; bool dirty_{false}; - - std::filesystem::path properties_file_; - + std::filesystem::path base_properties_file_; + std::vector properties_files_; utils::ChecksumCalculator checksum_calculator_; - - // Mutex for protection mutable std::mutex mutex_; std::shared_ptr logger_; - + PersistTo persist_to_; std::string name_; }; diff --git a/libminifi/include/utils/ChecksumCalculator.h b/libminifi/include/utils/ChecksumCalculator.h index 8cc1a4ed48..bbea7faa80 100644 --- a/libminifi/include/utils/ChecksumCalculator.h +++ b/libminifi/include/utils/ChecksumCalculator.h @@ -20,7 +20,7 @@ #include #include #include -#include +#include namespace org::apache::nifi::minifi::utils { @@ -29,16 +29,15 @@ class ChecksumCalculator { static constexpr const char* CHECKSUM_TYPE = "SHA256"; static constexpr size_t LENGTH_OF_HASH_IN_BYTES = 32; - void setFileLocation(const std::filesystem::path& file_location); + void setFileLocations(std::vector file_locations); [[nodiscard]] std::filesystem::path getFileName() const; std::string getChecksum(); void invalidateChecksum(); private: - static std::string computeChecksum(const std::filesystem::path& file_location); + static std::string computeChecksum(const std::vector& file_locations); - std::optional file_location_; - std::optional file_name_; + std::vector file_locations_; std::optional checksum_; }; diff --git a/libminifi/src/core/FlowConfiguration.cpp b/libminifi/src/core/FlowConfiguration.cpp index 2e7b56484e..e480230ca4 100644 --- a/libminifi/src/core/FlowConfiguration.cpp +++ b/libminifi/src/core/FlowConfiguration.cpp @@ -29,6 +29,18 @@ #include "minifi-cpp/SwapManager.h" #include "Connection.h" +namespace { +void createDefaultFlowConfigFile(const std::filesystem::path& path) { + std::ofstream ostream(path); + ostream << "Flow Controller:\n" + " name: MiNiFi Flow\n" + "Processors: []\n" + "Connections: []\n" + "Remote Processing Groups: []\n" + "Provenance Reporting:\n"; +} +} // namespace + namespace org::apache::nifi::minifi::core { FlowConfiguration::FlowConfiguration(ConfigurationContext ctx) @@ -52,12 +64,16 @@ FlowConfiguration::FlowConfiguration(ConfigurationContext ctx) if (!ctx.path) { logger_->log_error("Configuration path is not specified."); } else { + const bool c2_enabled = configuration_->get(Configure::nifi_c2_enable).and_then(&utils::string::toBool).value_or(false); + if (!c2_enabled && !ctx.path->empty() && !std::filesystem::exists(*ctx.path)) { + createDefaultFlowConfigFile(*ctx.path); + } config_path_ = utils::file::canonicalize(*ctx.path); if (!config_path_) { logger_->log_error("Couldn't find config file \"{}\".", ctx.path->string()); config_path_ = ctx.path; } - checksum_calculator_.setFileLocation(*config_path_); + checksum_calculator_.setFileLocations(std::vector{*config_path_}); } } diff --git a/libminifi/src/properties/Properties.cpp b/libminifi/src/properties/Properties.cpp index b60e1499dd..98b43005cf 100644 --- a/libminifi/src/properties/Properties.cpp +++ b/libminifi/src/properties/Properties.cpp @@ -18,19 +18,20 @@ #include "properties/Properties.h" #include +#include #include #include "core/logging/LoggerConfiguration.h" #include "properties/Configuration.h" #include "properties/PropertiesFile.h" -#include "range/v3/algorithm/all_of.hpp" #include "utils/StringUtils.h" #include "utils/file/FileUtils.h" namespace org::apache::nifi::minifi { -PropertiesImpl::PropertiesImpl(std::string name) +PropertiesImpl::PropertiesImpl(PersistTo persist_to, std::string name) : logger_(core::logging::LoggerFactory::getLogger()), + persist_to_(persist_to), name_(std::move(name)) { } @@ -72,12 +73,12 @@ const core::PropertyValidator* getValidator(const std::string& lookup_value) { // isdigit requires unsigned chars as input bool allDigits(const std::string& value) { - return ranges::all_of(value, [](const unsigned char c){ return ::isdigit(c); }); + return std::ranges::all_of(value, [](const unsigned char c){ return ::isdigit(c); }); } // isdigit requires unsigned chars as input bool allDigitsOrSpaces(const std::string& value) { - return ranges::all_of(value, [](const unsigned char c) { return std::isdigit(c) || std::isspace(c);}); + return std::ranges::all_of(value, [](const unsigned char c) { return std::isdigit(c) || std::isspace(c);}); } std::optional ensureTimePeriodValidatedPropertyHasExplicitUnit(const core::PropertyValidator* const validator, const std::string& value) { @@ -159,6 +160,12 @@ void fixValidatedProperty(const std::string& property_name, } } // namespace +std::filesystem::path PropertiesImpl::extra_properties_files_dir_name() const { + auto extra_properties_files_dir = base_properties_file_; + extra_properties_files_dir += ".d"; + return extra_properties_files_dir; +} + void PropertiesImpl::loadConfigureFile(const std::filesystem::path& configuration_file, std::string_view prefix) { std::lock_guard lock(mutex_); if (configuration_file.empty()) { @@ -173,55 +180,82 @@ void PropertiesImpl::loadConfigureFile(const std::filesystem::path& configuratio } std::error_code ec; - properties_file_ = std::filesystem::canonical(configuration_file, ec); + base_properties_file_ = std::filesystem::canonical(configuration_file, ec); if (ec.value() != 0) { logger_->log_warn("Configuration file '{}' does not exist, and it could not be created", configuration_file); return; } - logger_->log_info("Using configuration file to load configuration for {} from {} (located at {})", - getName().c_str(), configuration_file.string(), properties_file_.string()); + properties_files_ = { base_properties_file_ }; + + const auto extra_properties_files_dir = extra_properties_files_dir_name(); + std::vector extra_properties_file_names; + if (utils::file::exists(extra_properties_files_dir) && utils::file::is_directory(extra_properties_files_dir)) { + utils::file::list_dir(extra_properties_files_dir, [&](const std::filesystem::path&, const std::filesystem::path& file_name) { + if (!file_name.string().ends_with(".bak")) { + extra_properties_file_names.push_back(file_name); + } + return true; + }, logger_, /* recursive = */ false); + } + std::ranges::sort(extra_properties_file_names); + for (const auto& file_name : extra_properties_file_names) { + properties_files_.push_back(extra_properties_files_dir / file_name); + } - std::ifstream file(properties_file_, std::ifstream::in); - if (!file.good()) { - logger_->log_error("load configure file failed {}", properties_file_); - return; + logger_->log_info("Using configuration file to load configuration for {} from {} (located at {})", + getName().c_str(), configuration_file.string(), base_properties_file_.string()); + if (!extra_properties_file_names.empty()) { + auto list_of_files = utils::string::join(", ", extra_properties_file_names, [](const auto& path) { return path.string(); }); + logger_->log_info("Also reading configuration from files {} in {}", list_of_files, extra_properties_files_dir.string()); } + properties_.clear(); dirty_ = false; - for (const auto& line : PropertiesFile{file}) { - auto key = line.getKey(); - auto persisted_value = line.getValue(); - auto value = utils::string::replaceEnvironmentVariables(persisted_value); - bool need_to_persist_new_value = false; - fixValidatedProperty(std::string(prefix) + key, persisted_value, value, need_to_persist_new_value, *logger_); - dirty_ = dirty_ || need_to_persist_new_value; - properties_[key] = {persisted_value, value, need_to_persist_new_value}; + for (const auto& properties_file : properties_files_) { + std::ifstream file(properties_file, std::ifstream::in); + if (!file.good()) { + logger_->log_error("load configure file failed {}", properties_file); + continue; + } + for (const auto& line : PropertiesFile{file}) { + auto key = line.getKey(); + auto persisted_value = line.getValue(); + auto value = utils::string::replaceEnvironmentVariables(persisted_value); + bool need_to_persist_new_value = false; + fixValidatedProperty(std::string(prefix) + key, persisted_value, value, need_to_persist_new_value, *logger_); + dirty_ = dirty_ || need_to_persist_new_value; + properties_[key] = {persisted_value, value, need_to_persist_new_value}; + } } - checksum_calculator_.setFileLocation(properties_file_); + + checksum_calculator_.setFileLocations(properties_files_); } std::filesystem::path PropertiesImpl::getFilePath() const { std::lock_guard lock(mutex_); - return properties_file_; + return base_properties_file_; } bool PropertiesImpl::commitChanges() { std::lock_guard lock(mutex_); if (!dirty_) { - logger_->log_info("Attempt to persist, but properties are not updated"); + logger_->log_debug("commitChanges() called, but properties have not changed, nothing to do"); return true; } - std::ifstream file(properties_file_, std::ifstream::in); + const auto output_file = (persist_to_ == PersistTo::SingleFile ? base_properties_file_ : extra_properties_files_dir_name() / C2PropertiesFileName); + if (!std::filesystem::exists(output_file)) { + logger_->log_debug("Configuration file {} does not exist yet, creating it", output_file); + utils::file::create_dir(output_file.parent_path(), /* recursive = */ true); + std::ofstream file{output_file}; + } + + std::ifstream file(output_file, std::ifstream::in); if (!file) { - logger_->log_error("load configure file failed {}", properties_file_); + logger_->log_error("Failed to load configuration file {}", output_file); return false; } - - auto new_file = properties_file_; - new_file += ".new"; - PropertiesFile current_content{file}; for (const auto& prop : properties_) { if (!prop.second.need_to_persist_new_value) { @@ -233,25 +267,37 @@ bool PropertiesImpl::commitChanges() { current_content.append(prop.first, prop.second.persisted_value); } } + file.close(); + auto new_file = output_file; + new_file += ".new"; try { current_content.writeTo(new_file); } catch (const std::exception&) { - logger_->log_error("Could not update {}", properties_file_); + logger_->log_error("Could not write to {}", new_file); return false; } - auto backup = properties_file_; - backup += ".bak"; - if (utils::file::FileUtils::copy_file(properties_file_, backup) == 0 && utils::file::FileUtils::copy_file(new_file, properties_file_) == 0) { - logger_->log_info("Persisted {}", properties_file_); - checksum_calculator_.invalidateChecksum(); - dirty_ = false; - return true; + std::error_code ec; + const auto existing_file_size = std::filesystem::file_size(output_file, ec); + if (ec || existing_file_size == 0) { + if (!utils::file::move_file(new_file, output_file)) { + logger_->log_error("Could not create minifi properties file {}", output_file); + return false; + } + } else { + auto backup = output_file; + backup += ".bak"; + if (!utils::file::move_file(output_file, backup) || !utils::file::move_file(new_file, output_file)) { + logger_->log_error("Could not update minifi properties file {}", output_file); + return false; + } } - logger_->log_error("Could not update {}", properties_file_); - return false; + logger_->log_info("Persisted {}", output_file); + checksum_calculator_.invalidateChecksum(); + dirty_ = false; + return true; } std::map PropertiesImpl::getProperties() const { @@ -264,7 +310,7 @@ std::map PropertiesImpl::getProperties() const { } std::shared_ptr Properties::create() { - return std::make_shared(); + return std::make_shared(PropertiesImpl::PersistTo::SingleFile); } } // namespace org::apache::nifi::minifi diff --git a/libminifi/src/utils/ChecksumCalculator.cpp b/libminifi/src/utils/ChecksumCalculator.cpp index 14e1f831d2..993c1b9633 100644 --- a/libminifi/src/utils/ChecksumCalculator.cpp +++ b/libminifi/src/utils/ChecksumCalculator.cpp @@ -21,7 +21,6 @@ #include #include "sodium/crypto_hash_sha256.h" -#include "utils/file/FileUtils.h" #include "utils/StringUtils.h" #include "properties/Configuration.h" @@ -29,51 +28,59 @@ namespace { const std::string AGENT_IDENTIFIER_KEY = std::string(org::apache::nifi::minifi::Configuration::nifi_c2_agent_identifier) + "="; +namespace utils = org::apache::nifi::minifi::utils; + +void addFileToChecksum(const std::filesystem::path& file_path, crypto_hash_sha256_state& state) { + std::ifstream input_file{file_path, std::ios::in | std::ios::binary}; + if (!input_file.is_open()) { + throw std::runtime_error(utils::string::join_pack("Could not open config file '", file_path.string(), "' to compute the checksum: ", std::strerror(errno))); + } + + std::string line; + while (std::getline(input_file, line)) { + // skip lines containing the agent identifier, so agents in the same class will have the same checksum + if (line.starts_with(AGENT_IDENTIFIER_KEY)) { + continue; + } + if (!input_file.eof()) { // eof() means we have just read the last line, which was not terminated by a newline + line.append("\n"); + } + crypto_hash_sha256_update(&state, reinterpret_cast(line.data()), line.size()); + } + if (input_file.bad()) { + throw std::runtime_error(utils::string::join_pack("Error reading config file '", file_path.string(), "' while computing the checksum: ", std::strerror(errno))); + } +} + } // namespace namespace org::apache::nifi::minifi::utils { -void ChecksumCalculator::setFileLocation(const std::filesystem::path& file_location) { - file_location_ = file_location; - file_name_ = file_location.filename(); +void ChecksumCalculator::setFileLocations(std::vector file_locations) { + gsl_Expects(!file_locations.empty()); + file_locations_ = std::move(file_locations); invalidateChecksum(); } std::filesystem::path ChecksumCalculator::getFileName() const { - gsl_Expects(file_name_); - return *file_name_; + gsl_Expects(!file_locations_.empty()); + return file_locations_.front().filename(); } std::string ChecksumCalculator::getChecksum() { - gsl_Expects(file_location_); + gsl_Expects(!file_locations_.empty()); if (!checksum_) { - checksum_ = computeChecksum(*file_location_); + checksum_ = computeChecksum(file_locations_); } return *checksum_; } -std::string ChecksumCalculator::computeChecksum(const std::filesystem::path& file_location) { - std::ifstream input_file{file_location, std::ios::in | std::ios::binary}; - if (!input_file.is_open()) { - throw std::runtime_error(string::join_pack("Could not open config file '", file_location.string(), "' to compute the checksum: ", std::strerror(errno))); - } - +std::string ChecksumCalculator::computeChecksum(const std::vector& file_locations) { crypto_hash_sha256_state state; crypto_hash_sha256_init(&state); - std::string line; - while (std::getline(input_file, line)) { - // skip lines containing the agent identifier, so agents in the same class will have the same checksum - if (string::startsWith(line, AGENT_IDENTIFIER_KEY)) { - continue; - } - if (!input_file.eof()) { // eof() means we have just read the last line, which was not terminated by a newline - line.append("\n"); - } - crypto_hash_sha256_update(&state, reinterpret_cast(line.data()), line.size()); - } - if (input_file.bad()) { - throw std::runtime_error(string::join_pack("Error reading config file '", file_location.string(), "' while computing the checksum: ", std::strerror(errno))); + for (const auto& file_location : file_locations) { + addFileToChecksum(file_location, state); } std::array hash{}; diff --git a/libminifi/test/integration/C2PropertiesUpdateTests.cpp b/libminifi/test/integration/C2PropertiesUpdateTests.cpp index b762a2c867..e6cf2389d4 100644 --- a/libminifi/test/integration/C2PropertiesUpdateTests.cpp +++ b/libminifi/test/integration/C2PropertiesUpdateTests.cpp @@ -155,9 +155,9 @@ TEST_CASE("C2PropertiesUpdateTests", "[c2test]") { logger2->log_debug("DummyClass2::before"); logger3->log_debug("DummyClass3::before"); - REQUIRE(!log_test_controller->contains("DummyClass1::before", 0s)); - REQUIRE(!log_test_controller->contains("DummyClass2::before", 0s)); - REQUIRE(!log_test_controller->contains("DummyClass3::before", 0s)); + CHECK(!log_test_controller->contains("DummyClass1::before", 0s)); + CHECK(!log_test_controller->contains("DummyClass2::before", 0s)); + CHECK(!log_test_controller->contains("DummyClass3::before", 0s)); } // On msvc, the passed lambda can't capture a reference to the object under construction, so we need to late-init harness. @@ -194,23 +194,25 @@ TEST_CASE("C2PropertiesUpdateTests", "[c2test]") { logger2->log_debug("DummyClass2::after"); // this should still not log logger3->log_debug("DummyClass3::after"); } - REQUIRE(log_test_controller->contains("DummyClass1::after", 0s)); - REQUIRE(!log_test_controller->contains("DummyClass2::after", 0s)); - REQUIRE(log_test_controller->contains("DummyClass3::after", 0s)); + CHECK(log_test_controller->contains("DummyClass1::after", 0s)); + CHECK_FALSE(log_test_controller->contains("DummyClass2::after", 0s)); + CHECK(log_test_controller->contains("DummyClass3::after", 0s)); { - minifi::PropertiesFile minifi_properties(std::ifstream{home_dir / "conf/minifi.properties"}); - REQUIRE(!minifi_properties.hasValue("nifi.dummy.property")); - REQUIRE(minifi_properties.getValue("nifi.property.one") == "bush"); - REQUIRE(minifi_properties.getValue("nifi.property.two") == "ring"); - REQUIRE(!minifi_properties.hasValue(minifi::Configuration::nifi_c2_rest_heartbeat_minimize_updates)); - REQUIRE(minifi_properties.getValue(minifi::Configuration::minifi_disk_space_watchdog_enable) == "true"); + const std::shared_ptr minifi_properties = std::make_shared(); + minifi_properties->loadConfigureFile(home_dir / "conf" / "minifi.properties"); + CHECK_FALSE(minifi_properties->get("nifi.dummy.property")); + CHECK(minifi_properties->get("nifi.property.one") == "bush"); + CHECK(minifi_properties->get("nifi.property.two") == "ring"); + CHECK_FALSE(minifi_properties->get(minifi::Configuration::nifi_c2_rest_heartbeat_minimize_updates)); + CHECK(minifi_properties->get(minifi::Configuration::minifi_disk_space_watchdog_enable) == "true"); } { - minifi::PropertiesFile minifi_log_properties(std::ifstream{home_dir / "conf/minifi-log.properties"}); - REQUIRE(!minifi_log_properties.hasValue("logger.org::apache::nifi::minifi::test::dummy")); - REQUIRE(minifi_log_properties.getValue("logger.org::apache::nifi::minifi::test::DummyClass1") == "DEBUG,ostream"); + const std::shared_ptr minifi_log_properties = std::make_shared(); + minifi_log_properties->loadConfigureFile(home_dir / "conf" / "minifi-log.properties"); + CHECK_FALSE(minifi_log_properties->get("logger.org::apache::nifi::minifi::test::dummy")); + CHECK(minifi_log_properties->get("logger.org::apache::nifi::minifi::test::DummyClass1") == "DEBUG,ostream"); } }); diff --git a/libminifi/test/unit/ChecksumCalculatorTests.cpp b/libminifi/test/unit/ChecksumCalculatorTests.cpp index fabecddfe6..82623814c1 100644 --- a/libminifi/test/unit/ChecksumCalculatorTests.cpp +++ b/libminifi/test/unit/ChecksumCalculatorTests.cpp @@ -37,17 +37,33 @@ TEST_CASE("ChecksumCalculator can calculate the checksum, which is equal to sha2 REQUIRE(size_t{utils::ChecksumCalculator::LENGTH_OF_HASH_IN_BYTES} == size_t{32}); utils::ChecksumCalculator checksum_calculator; - checksum_calculator.setFileLocation(file_location); + checksum_calculator.setFileLocations(std::vector{file_location}); REQUIRE(checksum_calculator.getChecksum() == CHECKSUM_FOR_ONE_LINE_OF_TEXT); } +TEST_CASE("The input of ChecksumCalculator can be in multiple files", "[ChecksumCalculator]") { + TestController test_controller; + const auto test_dir = test_controller.createTempDirectory(); + const auto single_file_location = minifi::test::utils::putFileToDir(test_dir, "single.txt", "one line of text\nsecond line of text\n"); + const auto multiple_1_file_location = minifi::test::utils::putFileToDir(test_dir, "multiple_1.txt", "one line of text\n"); + const auto multiple_2_file_location = minifi::test::utils::putFileToDir(test_dir, "multiple_2.txt", "second line of text\n"); + + utils::ChecksumCalculator checksum_calculator_single; + checksum_calculator_single.setFileLocations(std::vector{single_file_location}); + + utils::ChecksumCalculator checksum_calculator_multiple; + checksum_calculator_multiple.setFileLocations(std::vector{multiple_1_file_location, multiple_2_file_location}); + + CHECK(checksum_calculator_single.getChecksum() == checksum_calculator_multiple.getChecksum()); +} + TEST_CASE("On Windows text files, the checksum calculated is also the same as sha256sum", "[ChecksumCalculator]") { TestController test_controller; auto test_dir = test_controller.createTempDirectory(); auto file_location = minifi::test::utils::putFileToDir(test_dir, "simple.txt", "one line of text\r\n"); utils::ChecksumCalculator checksum_calculator; - checksum_calculator.setFileLocation(file_location); + checksum_calculator.setFileLocations(std::vector{file_location}); REQUIRE(checksum_calculator.getChecksum() == "94fc46c62ef6cc5b45cbad9fd53116cfb15a80960a9b311c1c27e5b5265ad4b4"); } @@ -57,7 +73,7 @@ TEST_CASE("The checksum can be reset and recomputed", "[ChecksumCalculator]") { auto file_location = minifi::test::utils::putFileToDir(test_dir, "simple.txt", "one line of text\n"); utils::ChecksumCalculator checksum_calculator; - checksum_calculator.setFileLocation(file_location); + checksum_calculator.setFileLocations(std::vector{file_location}); REQUIRE(checksum_calculator.getChecksum() == CHECKSUM_FOR_ONE_LINE_OF_TEXT); std::ofstream append_to_file(file_location, std::ios::binary | std::ios::app); @@ -76,11 +92,11 @@ TEST_CASE("If the file location is updated, the checksum will be recomputed", "[ auto file_location = minifi::test::utils::putFileToDir(test_dir, "simple.txt", "one line of text\n"); utils::ChecksumCalculator checksum_calculator; - checksum_calculator.setFileLocation(file_location); + checksum_calculator.setFileLocations(std::vector{file_location}); REQUIRE(checksum_calculator.getChecksum() == CHECKSUM_FOR_ONE_LINE_OF_TEXT); auto other_file_location = minifi::test::utils::putFileToDir(test_dir, "long.txt", "one line of text\nanother line of text\n"); - checksum_calculator.setFileLocation(other_file_location); + checksum_calculator.setFileLocations(std::vector{other_file_location}); REQUIRE(checksum_calculator.getChecksum() == CHECKSUM_FOR_TWO_LINES_OF_TEXT); } @@ -92,7 +108,7 @@ TEST_CASE("Checksums can be computed for binary (eg. encrypted) files, too", "[C auto file_location = minifi::test::utils::putFileToDir(test_dir, "simple.txt", binary_data); utils::ChecksumCalculator checksum_calculator; - checksum_calculator.setFileLocation(file_location); + checksum_calculator.setFileLocations(std::vector{file_location}); REQUIRE(checksum_calculator.getChecksum() == "bdec77160c394c067419735de757e4daa1c4679ea45e82a33fa8f706eed87709"); } @@ -109,16 +125,16 @@ TEST_CASE("The agent identifier is excluded from the checksum", "[ChecksumCalcul "nifi.c2.agent.heartbeat.period=10 sec\n"); utils::ChecksumCalculator checksum_calculator_1; - checksum_calculator_1.setFileLocation(file_location_1); + checksum_calculator_1.setFileLocations(std::vector{file_location_1}); utils::ChecksumCalculator checksum_calculator_2; - checksum_calculator_2.setFileLocation(file_location_2); + checksum_calculator_2.setFileLocations(std::vector{file_location_2}); REQUIRE(checksum_calculator_1.getChecksum() == checksum_calculator_2.getChecksum()); } TEST_CASE("ChecksumCalculator::getChecksum will throw if the file does not exist", "[ChecksumCalculator]") { utils::ChecksumCalculator checksum_calculator; - checksum_calculator.setFileLocation("/this/file/does/not/exist/84a77fd9-16b3-49d2-aead-a1f9e58e530d"); + checksum_calculator.setFileLocations(std::vector{std::filesystem::path{"/this/file/does/not/exist/84a77fd9-16b3-49d2-aead-a1f9e58e530d"}}); REQUIRE_THROWS(checksum_calculator.getChecksum()); } diff --git a/libminifi/test/unit/ConfigurationChecksumsTests.cpp b/libminifi/test/unit/ConfigurationChecksumsTests.cpp index 6da67a66e6..956907ff2a 100644 --- a/libminifi/test/unit/ConfigurationChecksumsTests.cpp +++ b/libminifi/test/unit/ConfigurationChecksumsTests.cpp @@ -39,7 +39,7 @@ TEST_CASE("If one checksum calculator is added, we get a node with one child", " auto file_location = minifi::test::utils::putFileToDir(test_dir, "simple.txt", "one line of text\n"); utils::ChecksumCalculator checksum_calculator; - checksum_calculator.setFileLocation(file_location); + checksum_calculator.setFileLocations(std::vector{file_location}); ConfigurationChecksums configuration_checksums; configuration_checksums.addChecksumCalculator(checksum_calculator); @@ -62,9 +62,9 @@ TEST_CASE("If two checksum calculators are added, we get a node with two childre auto file_location_2 = minifi::test::utils::putFileToDir(test_dir, "second.txt", "this is the second file\n"); utils::ChecksumCalculator checksum_calculator_1; - checksum_calculator_1.setFileLocation(file_location_1); + checksum_calculator_1.setFileLocations(std::vector{file_location_1}); utils::ChecksumCalculator checksum_calculator_2; - checksum_calculator_2.setFileLocation(file_location_2); + checksum_calculator_2.setFileLocations(std::vector{file_location_2}); ConfigurationChecksums configuration_checksums; configuration_checksums.addChecksumCalculator(checksum_calculator_1); diff --git a/libminifi/test/unit/IdTests.cpp b/libminifi/test/unit/IdTests.cpp index 048072b5dc..aac8ef0fb5 100644 --- a/libminifi/test/unit/IdTests.cpp +++ b/libminifi/test/unit/IdTests.cpp @@ -39,7 +39,7 @@ TEST_CASE("Test default is time", "[id]") { LogTestController::getInstance().setDebug(); std::shared_ptr generator = utils::IdGenerator::getIdGenerator(); - generator->initialize(std::make_shared()); + generator->initialize(std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties")); REQUIRE(true == LogTestController::getInstance().contains("Using uuid_generate_time implementation for uids.")); LogTestController::getInstance().reset(); @@ -49,7 +49,7 @@ TEST_CASE("Test time", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "TiMe"); std::shared_ptr generator = utils::IdGenerator::getIdGenerator(); @@ -69,7 +69,7 @@ TEST_CASE("Test Generate Move", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "TiMe"); std::shared_ptr generator = utils::IdGenerator::getIdGenerator(); @@ -86,7 +86,7 @@ TEST_CASE("Test random", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "RaNDoM"); std::shared_ptr generator = utils::IdGenerator::getIdGenerator(); @@ -106,7 +106,7 @@ TEST_CASE("Test uuid_default", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "UUID_default"); std::shared_ptr generator = utils::IdGenerator::getIdGenerator(); @@ -120,7 +120,7 @@ TEST_CASE("Test invalid", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "InVaLiD"); std::shared_ptr generator = utils::IdGenerator::getIdGenerator(); @@ -134,7 +134,7 @@ TEST_CASE("Test parse", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "time"); std::shared_ptr generator = utils::IdGenerator::getIdGenerator(); @@ -160,7 +160,7 @@ TEST_CASE("Test parse invalid", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "time"); std::shared_ptr generator = utils::IdGenerator::getIdGenerator(); @@ -187,7 +187,7 @@ TEST_CASE("Test to_string", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "time"); std::shared_ptr generator = utils::IdGenerator::getIdGenerator(); @@ -229,7 +229,7 @@ TEST_CASE("Test Hex Device Segment 16 bits correct digits", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "minifi_uid"); id_props->set("uid.minifi.device.segment", "09aF"); @@ -255,7 +255,7 @@ TEST_CASE("Test Hex Device Segment 16 bits too many digits", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "minifi_uid"); id_props->set("uid.minifi.device.segment", "09aFee"); @@ -283,7 +283,7 @@ TEST_CASE("Test Hex Device Segment 18 bits", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "minifi_uid"); id_props->set("uid.minifi.device.segment.bits", "18"); id_props->set("uid.minifi.device.segment", "09aF8"); @@ -317,7 +317,7 @@ TEST_CASE("Collision", "[collision]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); SECTION("random") { id_props->set("uid.implementation", "random"); } @@ -357,7 +357,7 @@ TEST_CASE("Speed", "[speed]") { TestController test_controller; LogTestController::getInstance().setDebug(); - std::shared_ptr id_props = std::make_shared(); + std::shared_ptr id_props = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); std::string implementation; SECTION("random") { implementation = "random"; diff --git a/minifi_main/MiNiFiMain.cpp b/minifi_main/MiNiFiMain.cpp index d6d32a50c8..f5c852cf14 100644 --- a/minifi_main/MiNiFiMain.cpp +++ b/minifi_main/MiNiFiMain.cpp @@ -306,7 +306,7 @@ int main(int argc, char **argv) { logger_configuration.initialize(log_properties); - std::shared_ptr uid_properties = std::make_shared("UID properties"); + std::shared_ptr uid_properties = std::make_shared(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); uid_properties->loadConfigureFile(locations->uid_properties_path_); utils::IdGenerator::getIdGenerator()->initialize(uid_properties); diff --git a/packaging/msi/WixWin.wsi.in b/packaging/msi/WixWin.wsi.in index e294b547da..cdbc9142a2 100644 --- a/packaging/msi/WixWin.wsi.in +++ b/packaging/msi/WixWin.wsi.in @@ -58,10 +58,9 @@ ${WIX_EXTRA_FEATURES} - - - + + @@ -263,16 +262,16 @@ ${WIX_EXTRA_FEATURES} - - - - - - - - - - + + + + + + + + + + @@ -303,27 +302,22 @@ ${WIX_EXTRA_FEATURES} - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + - - "1"]]> + + diff --git a/packaging/rpm/expected-rpm-contents.in b/packaging/rpm/expected-rpm-contents.in index cfdd8fa0bc..b4eb8a37f1 100644 --- a/packaging/rpm/expected-rpm-contents.in +++ b/packaging/rpm/expected-rpm-contents.in @@ -1,5 +1,4 @@ /etc/nifi-minifi-cpp -/etc/nifi-minifi-cpp/config.yml /etc/nifi-minifi-cpp/fips /etc/nifi-minifi-cpp/fips/openssl.cnf /etc/nifi-minifi-cpp/minifi-log.properties From e95af05b6a7622500f45fed4bc8747484c9aaef5 Mon Sep 17 00:00:00 2001 From: Ferenc Gerlits Date: Thu, 20 Nov 2025 18:56:03 +0100 Subject: [PATCH 02/11] rename fileContentsMatch to settingsInFileAreAsExpected ... and use a for loop instead of while --- .../tests/unit/ConfigurationTests.cpp | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/extensions/standard-processors/tests/unit/ConfigurationTests.cpp b/extensions/standard-processors/tests/unit/ConfigurationTests.cpp index 820b79e6cb..f8b078e42c 100644 --- a/extensions/standard-processors/tests/unit/ConfigurationTests.cpp +++ b/extensions/standard-processors/tests/unit/ConfigurationTests.cpp @@ -23,12 +23,11 @@ #include "utils/Environment.h" namespace { -bool fileContentsMatch(const std::filesystem::path& file_name, const std::unordered_set& expected_contents) { +bool settingsInFileAreAsExpected(const std::filesystem::path& file_name, const std::unordered_set& expected_contents) { std::unordered_set actual_contents; std::ifstream file{file_name}; if (file.is_open()) { - std::string line; - while (std::getline(file, line)) { + for (std::string line; std::getline(file, line); ) { actual_contents.insert(line); } } @@ -90,7 +89,7 @@ TEST_CASE("Configuration can fix misconfigured timeperiod<->integer validated pr { CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); - CHECK(fileContentsMatch(original_properties_path, {"nifi.c2.agent.heartbeat.period=1min", "nifi.administrative.yield.duration=30000"})); + CHECK(settingsInFileAreAsExpected(original_properties_path, {"nifi.c2.agent.heartbeat.period=1min", "nifi.administrative.yield.duration=30000"})); } CHECK(configure->commitChanges()); @@ -98,7 +97,7 @@ TEST_CASE("Configuration can fix misconfigured timeperiod<->integer validated pr { CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(updated_properties_path)); - CHECK(fileContentsMatch(updated_properties_path, {"nifi.c2.agent.heartbeat.period=60000", "nifi.administrative.yield.duration=30000 ms"})); + CHECK(settingsInFileAreAsExpected(updated_properties_path, {"nifi.c2.agent.heartbeat.period=60000", "nifi.administrative.yield.duration=30000 ms"})); } const std::shared_ptr configure_reread = std::make_shared(); @@ -128,7 +127,7 @@ TEST_CASE("Configuration can fix misconfigured datasize<->integer validated prop { CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); - CHECK(fileContentsMatch(original_properties_path, {"appender.rolling.max_file_size=6000"})); + CHECK(settingsInFileAreAsExpected(original_properties_path, {"appender.rolling.max_file_size=6000"})); } CHECK(configure->commitChanges()); @@ -136,7 +135,7 @@ TEST_CASE("Configuration can fix misconfigured datasize<->integer validated prop { CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(updated_properties_path)); - CHECK(fileContentsMatch(updated_properties_path, {"appender.rolling.max_file_size=6000 B"})); + CHECK(settingsInFileAreAsExpected(updated_properties_path, {"appender.rolling.max_file_size=6000 B"})); } const std::shared_ptr configure_reread = std::make_shared(); @@ -167,7 +166,7 @@ TEST_CASE("Configuration can fix misconfigured validated properties within envir { CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); - CHECK(fileContentsMatch(original_properties_path, {"compression.cached.log.max.size=${SOME_VARIABLE}", "compression.compressed.log.max.size=3000"})); + CHECK(settingsInFileAreAsExpected(original_properties_path, {"compression.cached.log.max.size=${SOME_VARIABLE}", "compression.compressed.log.max.size=3000"})); } CHECK(configure->commitChanges()); @@ -175,7 +174,7 @@ TEST_CASE("Configuration can fix misconfigured validated properties within envir { CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(updated_properties_path)); - CHECK(fileContentsMatch(updated_properties_path, {"compression.compressed.log.max.size=3000 B"})); + CHECK(settingsInFileAreAsExpected(updated_properties_path, {"compression.compressed.log.max.size=3000 B"})); } const std::shared_ptr configure_reread = std::make_shared(); @@ -212,7 +211,7 @@ TEST_CASE("Committing changes to a configuration creates a backup file") { configure->set("number.of.lions", "8"); CHECK(configure->commitChanges()); - CHECK(fileContentsMatch(updated_properties_path, {"number.of.lions=8"})); + CHECK(settingsInFileAreAsExpected(updated_properties_path, {"number.of.lions=8"})); CHECK_FALSE(std::filesystem::exists(backup_properties_path)); const std::shared_ptr configure_2 = std::make_shared(); @@ -223,8 +222,8 @@ TEST_CASE("Committing changes to a configuration creates a backup file") { configure->set("number.of.giraffes", "29"); CHECK(configure->commitChanges()); - CHECK(fileContentsMatch(updated_properties_path, {"number.of.lions=8", "number.of.giraffes=29"})); - CHECK(fileContentsMatch(backup_properties_path, {"number.of.lions=8"})); + CHECK(settingsInFileAreAsExpected(updated_properties_path, {"number.of.lions=8", "number.of.giraffes=29"})); + CHECK(settingsInFileAreAsExpected(backup_properties_path, {"number.of.lions=8"})); const std::shared_ptr configure_3 = std::make_shared(); configure_3->loadConfigureFile(original_properties_path); From f1861340eb95deea21344b6193cc7b4daf6bbfea Mon Sep 17 00:00:00 2001 From: Ferenc Gerlits Date: Thu, 20 Nov 2025 19:14:39 +0100 Subject: [PATCH 03/11] refactor loadConfigureFile() --- libminifi/include/properties/Properties.h | 1 + libminifi/src/properties/Properties.cpp | 58 +++++++++++++---------- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/libminifi/include/properties/Properties.h b/libminifi/include/properties/Properties.h index 1f0e87d9b7..edf530d929 100644 --- a/libminifi/include/properties/Properties.h +++ b/libminifi/include/properties/Properties.h @@ -126,6 +126,7 @@ class PropertiesImpl : public virtual Properties { private: std::filesystem::path extra_properties_files_dir_name() const; + void setPropertiesFromFile(const std::filesystem::path& configuration_file, std::string_view prefix); std::map properties_; bool dirty_{false}; diff --git a/libminifi/src/properties/Properties.cpp b/libminifi/src/properties/Properties.cpp index 98b43005cf..f2a470743a 100644 --- a/libminifi/src/properties/Properties.cpp +++ b/libminifi/src/properties/Properties.cpp @@ -158,6 +158,20 @@ void fixValidatedProperty(const std::string& property_name, needs_to_persist_new_value = false; } } + +auto getExtraPropertiesFileNames(const std::filesystem::path& extra_properties_files_dir, const std::shared_ptr& logger) { + std::vector extra_properties_file_names; + if (utils::file::exists(extra_properties_files_dir) && utils::file::is_directory(extra_properties_files_dir)) { + utils::file::list_dir(extra_properties_files_dir, [&](const std::filesystem::path&, const std::filesystem::path& file_name) { + if (!file_name.string().ends_with(".bak")) { + extra_properties_file_names.push_back(file_name); + } + return true; + }, logger, /* recursive = */ false); + } + std::ranges::sort(extra_properties_file_names); + return extra_properties_file_names; +} } // namespace std::filesystem::path PropertiesImpl::extra_properties_files_dir_name() const { @@ -188,18 +202,8 @@ void PropertiesImpl::loadConfigureFile(const std::filesystem::path& configuratio } properties_files_ = { base_properties_file_ }; - const auto extra_properties_files_dir = extra_properties_files_dir_name(); - std::vector extra_properties_file_names; - if (utils::file::exists(extra_properties_files_dir) && utils::file::is_directory(extra_properties_files_dir)) { - utils::file::list_dir(extra_properties_files_dir, [&](const std::filesystem::path&, const std::filesystem::path& file_name) { - if (!file_name.string().ends_with(".bak")) { - extra_properties_file_names.push_back(file_name); - } - return true; - }, logger_, /* recursive = */ false); - } - std::ranges::sort(extra_properties_file_names); + const auto extra_properties_file_names = getExtraPropertiesFileNames(extra_properties_files_dir, logger_); for (const auto& file_name : extra_properties_file_names) { properties_files_.push_back(extra_properties_files_dir / file_name); } @@ -214,25 +218,29 @@ void PropertiesImpl::loadConfigureFile(const std::filesystem::path& configuratio properties_.clear(); dirty_ = false; for (const auto& properties_file : properties_files_) { - std::ifstream file(properties_file, std::ifstream::in); - if (!file.good()) { - logger_->log_error("load configure file failed {}", properties_file); - continue; - } - for (const auto& line : PropertiesFile{file}) { - auto key = line.getKey(); - auto persisted_value = line.getValue(); - auto value = utils::string::replaceEnvironmentVariables(persisted_value); - bool need_to_persist_new_value = false; - fixValidatedProperty(std::string(prefix) + key, persisted_value, value, need_to_persist_new_value, *logger_); - dirty_ = dirty_ || need_to_persist_new_value; - properties_[key] = {persisted_value, value, need_to_persist_new_value}; - } + setPropertiesFromFile(properties_file, prefix); } checksum_calculator_.setFileLocations(properties_files_); } +void PropertiesImpl::setPropertiesFromFile(const std::filesystem::path& properties_file, std::string_view prefix) { + std::ifstream file(properties_file, std::ifstream::in); + if (!file.good()) { + logger_->log_error("load configure file failed {}", properties_file); + return; + } + for (const auto& line : PropertiesFile{file}) { + auto key = line.getKey(); + auto persisted_value = line.getValue(); + auto value = utils::string::replaceEnvironmentVariables(persisted_value); + bool need_to_persist_new_value = false; + fixValidatedProperty(std::string(prefix) + key, persisted_value, value, need_to_persist_new_value, *logger_); + dirty_ = dirty_ || need_to_persist_new_value; + properties_[key] = {persisted_value, value, need_to_persist_new_value}; + } +} + std::filesystem::path PropertiesImpl::getFilePath() const { std::lock_guard lock(mutex_); return base_properties_file_; From ad4597a61b9e13c77495e16775c36aef4281a833 Mon Sep 17 00:00:00 2001 From: Ferenc Gerlits Date: Thu, 20 Nov 2025 19:31:27 +0100 Subject: [PATCH 04/11] pull out part of commitChanges() --- libminifi/src/properties/Properties.cpp | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/libminifi/src/properties/Properties.cpp b/libminifi/src/properties/Properties.cpp index f2a470743a..a70da478f6 100644 --- a/libminifi/src/properties/Properties.cpp +++ b/libminifi/src/properties/Properties.cpp @@ -172,6 +172,19 @@ auto getExtraPropertiesFileNames(const std::filesystem::path& extra_properties_f std::ranges::sort(extra_properties_file_names); return extra_properties_file_names; } + +void updateChangedPropertiesInPropertiesFile(minifi::PropertiesFile& current_content, const auto& properties) { + for (const auto& prop : properties) { + if (!prop.second.need_to_persist_new_value) { + continue; + } + if (current_content.hasValue(prop.first)) { + current_content.update(prop.first, prop.second.persisted_value); + } else { + current_content.append(prop.first, prop.second.persisted_value); + } + } +} } // namespace std::filesystem::path PropertiesImpl::extra_properties_files_dir_name() const { @@ -265,18 +278,10 @@ bool PropertiesImpl::commitChanges() { return false; } PropertiesFile current_content{file}; - for (const auto& prop : properties_) { - if (!prop.second.need_to_persist_new_value) { - continue; - } - if (current_content.hasValue(prop.first)) { - current_content.update(prop.first, prop.second.persisted_value); - } else { - current_content.append(prop.first, prop.second.persisted_value); - } - } file.close(); + updateChangedPropertiesInPropertiesFile(current_content, properties_); + auto new_file = output_file; new_file += ".new"; try { From e5ab2ccbf65ed3ab89acd3bea0fc2f627da58b4f Mon Sep 17 00:00:00 2001 From: Ferenc Gerlits Date: Fri, 21 Nov 2025 09:15:50 +0100 Subject: [PATCH 05/11] clang-tidy fix --- libminifi/include/properties/Properties.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libminifi/include/properties/Properties.h b/libminifi/include/properties/Properties.h index edf530d929..c2f53e660f 100644 --- a/libminifi/include/properties/Properties.h +++ b/libminifi/include/properties/Properties.h @@ -126,7 +126,7 @@ class PropertiesImpl : public virtual Properties { private: std::filesystem::path extra_properties_files_dir_name() const; - void setPropertiesFromFile(const std::filesystem::path& configuration_file, std::string_view prefix); + void setPropertiesFromFile(const std::filesystem::path& properties_file, std::string_view prefix); std::map properties_; bool dirty_{false}; From 292d66ffddc5f6dcc72df79044aa6e114e0b630d Mon Sep 17 00:00:00 2001 From: Ferenc Gerlits Date: Fri, 5 Dec 2025 09:49:24 +0100 Subject: [PATCH 06/11] change _properties to .properties in minifi.properties.d --- README.md | 4 ++-- libminifi/include/properties/Properties.h | 2 +- packaging/msi/WixWin.wsi.in | 14 +++++++------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 96410e289a..1d0798b691 100644 --- a/README.md +++ b/README.md @@ -465,8 +465,8 @@ The files conf/minifi.properties, conf/minifi-log.properties and conf/minifi-uid these are the default settings supplied by the latest MiNiFi version. If you would like to modify these, you should create a corresponding .d directory (e.g. conf/minifi.properties.d) and put your settings in a new file inside this directory. These files are read and applied in lexicographic order, after the default settings file. -The Windows installer creates a conf/minifi.properties.d/10_installer_properties file, which contains C2 connection settings. -If C2 is enabled and settings are added/modified from the C2 server, these will be saved in conf/minifi.properties.d/90_c2_properties. +The Windows installer creates a conf/minifi.properties.d/10_installer.properties file, which contains C2 connection settings. +If C2 is enabled and settings are added/modified from the C2 server, these will be saved in conf/minifi.properties.d/90_c2.properties. The conf/config.yml file contains the flow definition (i.e. the layout of processors, controller services etc). When you start MiNiFi for the first time, the flow will be fetched from the C2 server (if available), or a file containing an empty flow will be created by MiNiFi. diff --git a/libminifi/include/properties/Properties.h b/libminifi/include/properties/Properties.h index c2f53e660f..3c9c13380c 100644 --- a/libminifi/include/properties/Properties.h +++ b/libminifi/include/properties/Properties.h @@ -47,7 +47,7 @@ class PropertiesImpl : public virtual Properties { ~PropertiesImpl() override = default; - static constexpr std::string_view C2PropertiesFileName = "90_c2_properties"; + static constexpr std::string_view C2PropertiesFileName = "90_c2.properties"; const std::string& getName() const override { return name_; diff --git a/packaging/msi/WixWin.wsi.in b/packaging/msi/WixWin.wsi.in index cdbc9142a2..9c912f604e 100644 --- a/packaging/msi/WixWin.wsi.in +++ b/packaging/msi/WixWin.wsi.in @@ -305,13 +305,13 @@ ${WIX_EXTRA_FEATURES} - - - - - - - + + + + + + + From f1032013fd0aeaa24c9b44b3cc7c87326aa8a050 Mon Sep 17 00:00:00 2001 From: Ferenc Gerlits Date: Wed, 10 Dec 2025 17:19:10 +0100 Subject: [PATCH 07/11] Update libminifi/src/core/FlowConfiguration.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Márton Szász --- libminifi/src/core/FlowConfiguration.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libminifi/src/core/FlowConfiguration.cpp b/libminifi/src/core/FlowConfiguration.cpp index e480230ca4..11f58bd107 100644 --- a/libminifi/src/core/FlowConfiguration.cpp +++ b/libminifi/src/core/FlowConfiguration.cpp @@ -32,6 +32,7 @@ namespace { void createDefaultFlowConfigFile(const std::filesystem::path& path) { std::ofstream ostream(path); + ostream.exceptions(std::ofstream::failbit | std::ofstream::badbit); ostream << "Flow Controller:\n" " name: MiNiFi Flow\n" "Processors: []\n" From 12f3c4d7b5a469ef0ba29b280cdecde02a616f58 Mon Sep 17 00:00:00 2001 From: Ferenc Gerlits Date: Wed, 10 Dec 2025 17:37:07 +0100 Subject: [PATCH 08/11] improve the name of this function --- libminifi/include/utils/ChecksumCalculator.h | 2 +- libminifi/src/core/state/nodes/ConfigurationChecksums.cpp | 2 +- libminifi/src/utils/ChecksumCalculator.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libminifi/include/utils/ChecksumCalculator.h b/libminifi/include/utils/ChecksumCalculator.h index bbea7faa80..c11931d58e 100644 --- a/libminifi/include/utils/ChecksumCalculator.h +++ b/libminifi/include/utils/ChecksumCalculator.h @@ -30,7 +30,7 @@ class ChecksumCalculator { static constexpr size_t LENGTH_OF_HASH_IN_BYTES = 32; void setFileLocations(std::vector file_locations); - [[nodiscard]] std::filesystem::path getFileName() const; + [[nodiscard]] std::filesystem::path getFileNameOfFirstFileLocation() const; std::string getChecksum(); void invalidateChecksum(); diff --git a/libminifi/src/core/state/nodes/ConfigurationChecksums.cpp b/libminifi/src/core/state/nodes/ConfigurationChecksums.cpp index c2181e7fc5..f9b965c105 100644 --- a/libminifi/src/core/state/nodes/ConfigurationChecksums.cpp +++ b/libminifi/src/core/state/nodes/ConfigurationChecksums.cpp @@ -32,7 +32,7 @@ std::vector ConfigurationChecksums::serialize() { for (auto checksum_calculator : checksum_calculators_) { SerializedResponseNode file_checksum_node; - file_checksum_node.name = checksum_calculator->getFileName().string(); + file_checksum_node.name = checksum_calculator->getFileNameOfFirstFileLocation().string(); file_checksum_node.value = checksum_calculator->getChecksum(); checksums_node.children.push_back(file_checksum_node); } diff --git a/libminifi/src/utils/ChecksumCalculator.cpp b/libminifi/src/utils/ChecksumCalculator.cpp index 993c1b9633..ae32e8ee42 100644 --- a/libminifi/src/utils/ChecksumCalculator.cpp +++ b/libminifi/src/utils/ChecksumCalculator.cpp @@ -62,7 +62,7 @@ void ChecksumCalculator::setFileLocations(std::vector fil invalidateChecksum(); } -std::filesystem::path ChecksumCalculator::getFileName() const { +std::filesystem::path ChecksumCalculator::getFileNameOfFirstFileLocation() const { gsl_Expects(!file_locations_.empty()); return file_locations_.front().filename(); } From 683915e4351f8d3e63a5f9f9c9e8547515d86ce1 Mon Sep 17 00:00:00 2001 From: Ferenc Gerlits Date: Tue, 13 Jan 2026 18:12:37 +0100 Subject: [PATCH 09/11] use an allow-list instead of a deny-list --- libminifi/src/properties/Properties.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libminifi/src/properties/Properties.cpp b/libminifi/src/properties/Properties.cpp index a70da478f6..190cfb689f 100644 --- a/libminifi/src/properties/Properties.cpp +++ b/libminifi/src/properties/Properties.cpp @@ -163,7 +163,7 @@ auto getExtraPropertiesFileNames(const std::filesystem::path& extra_properties_f std::vector extra_properties_file_names; if (utils::file::exists(extra_properties_files_dir) && utils::file::is_directory(extra_properties_files_dir)) { utils::file::list_dir(extra_properties_files_dir, [&](const std::filesystem::path&, const std::filesystem::path& file_name) { - if (!file_name.string().ends_with(".bak")) { + if (file_name.string().ends_with(".properties")) { extra_properties_file_names.push_back(file_name); } return true; From be4f9d117061b7530e47cf8fd2e17a32087a02a2 Mon Sep 17 00:00:00 2001 From: Ferenc Gerlits Date: Wed, 14 Jan 2026 11:33:39 +0100 Subject: [PATCH 10/11] Add "do not modify" comments to the default properties files. --- conf/minifi-log.properties.in | 4 ++++ conf/minifi-uid.properties.in | 4 ++++ conf/minifi.properties.in | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/conf/minifi-log.properties.in b/conf/minifi-log.properties.in index 9d99467152..5fcbce77c8 100644 --- a/conf/minifi-log.properties.in +++ b/conf/minifi-log.properties.in @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +### NOTE: Changes to this file are overwritten on upgrade. Add your custom settings to new files ### +### in the conf/minifi-log.properties.d directory instead. These custom settings files should have ### +### a name ending with ".properties", and they are applied in lexicographic order, after this file. ### + #More verbose pattern by default #Format details at https://github.com/gabime/spdlog/wiki/3.-Custom-formatting spdlog.pattern=[%Y-%m-%d %H:%M:%S.%e] [%n] [%l] %v diff --git a/conf/minifi-uid.properties.in b/conf/minifi-uid.properties.in index fb49014d2e..006eebddbd 100644 --- a/conf/minifi-uid.properties.in +++ b/conf/minifi-uid.properties.in @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +### NOTE: Changes to this file are overwritten on upgrade. Add your custom settings to new files ### +### in the conf/minifi-uid.properties.d directory instead. These custom settings files should have ### +### a name ending with ".properties", and they are applied in lexicographic order, after this file. ### + # Implementation for uid generation. # Valid values: # time - use uuid_generate_time diff --git a/conf/minifi.properties.in b/conf/minifi.properties.in index 7eceafc607..f12d8704bf 100644 --- a/conf/minifi.properties.in +++ b/conf/minifi.properties.in @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +### NOTE: Changes to this file are overwritten on upgrade. Add your custom settings to new files ### +### in the conf/minifi.properties.d directory instead. These custom settings files should have ### +### a name ending with ".properties", and they are applied in lexicographic order, after this file. ### + # Core Properties # nifi.flow.configuration.file=@MINIFI_PATH_FLOW_CONFIG@ nifi.administrative.yield.duration=30 sec From 2f48d44464d70bb9e9834447ed1f3af07470da82 Mon Sep 17 00:00:00 2001 From: Ferenc Gerlits Date: Wed, 14 Jan 2026 17:40:26 +0100 Subject: [PATCH 11/11] Add example file names --- conf/minifi-log.properties.in | 1 + conf/minifi-uid.properties.in | 1 + conf/minifi.properties.in | 1 + 3 files changed, 3 insertions(+) diff --git a/conf/minifi-log.properties.in b/conf/minifi-log.properties.in index 5fcbce77c8..3e9930fb4f 100644 --- a/conf/minifi-log.properties.in +++ b/conf/minifi-log.properties.in @@ -16,6 +16,7 @@ ### NOTE: Changes to this file are overwritten on upgrade. Add your custom settings to new files ### ### in the conf/minifi-log.properties.d directory instead. These custom settings files should have ### ### a name ending with ".properties", and they are applied in lexicographic order, after this file. ### +### Example filename: conf/minifi-log.properties.d/90-log-to-stderr.properties ### #More verbose pattern by default #Format details at https://github.com/gabime/spdlog/wiki/3.-Custom-formatting diff --git a/conf/minifi-uid.properties.in b/conf/minifi-uid.properties.in index 006eebddbd..e07ea6837b 100644 --- a/conf/minifi-uid.properties.in +++ b/conf/minifi-uid.properties.in @@ -16,6 +16,7 @@ ### NOTE: Changes to this file are overwritten on upgrade. Add your custom settings to new files ### ### in the conf/minifi-uid.properties.d directory instead. These custom settings files should have ### ### a name ending with ".properties", and they are applied in lexicographic order, after this file. ### +### Example filename: conf/minifi-uid.properties.d/10-use-random.properties ### # Implementation for uid generation. # Valid values: diff --git a/conf/minifi.properties.in b/conf/minifi.properties.in index f12d8704bf..7ae002fbce 100644 --- a/conf/minifi.properties.in +++ b/conf/minifi.properties.in @@ -16,6 +16,7 @@ ### NOTE: Changes to this file are overwritten on upgrade. Add your custom settings to new files ### ### in the conf/minifi.properties.d directory instead. These custom settings files should have ### ### a name ending with ".properties", and they are applied in lexicographic order, after this file. ### +### Example filenames: conf/minifi.properties.d/10-c2.properties, conf/minifi.properties.d/90-repository-tweaks.properties ### # Core Properties # nifi.flow.configuration.file=@MINIFI_PATH_FLOW_CONFIG@