MINIFICPP-2715 - Use symbols to check api compatibility

Extensions.md

@@ -17,8 +17,39 @@
 To enable all extensions for your platform, you may use -DENABLE_ALL=TRUE OR select the option to "Enable all Extensions" in the bootstrap script. [ReadMe](https://github.com/apache/nifi-minifi-cpp/#bootstrapping)
 
 # Extension internals
-Extensions are dynamic libraries loaded at runtime by the agent. An extension makes its
-capabilities (classes) available to the system through registrars. Registration must happen in source files, not headers.
+Extensions are dynamic libraries loaded at runtime by the agent.
+
+## C extensions
+You can build a shared library depending on the C capabilities of the agent as given in the `minifi-c.h` file.
+For the shared library to be considered a valid extension, it has to have a global symbol with the name `MinifiCApiVersion`
+with its value as a null terminated string (`const char*`) of the macro `MINIFI_API_VERSION` from `minifi-c.h`.
+
+Moreover the actual resource registration (processors/controller services) has to happen during the `MinifiInitExtension` call.
+One possible example of this is:
+
+```C++
+extern "C" const char* const MinifiApiVersion = MINIFI_API_VERSION;
+
+extern "C" void MinifiInitExtension(MinifiExtension* extension, MinifiConfig* /*config*/) {
+  minifi::api::core::useProcessorClassDescription<minifi::extensions::llamacpp::processors::RunLlamaCppInference>([&] (const MinifiProcessorClassDefinition& description) {
+    MinifiExtensionCreateInfo ext_create_info{
+      .name = minifi::api::utils::toStringView(MAKESTRING(EXTENSION_NAME)),
+      .version = minifi::api::utils::toStringView(MAKESTRING(EXTENSION_VERSION)),
+      .deinit = nullptr,
+      .user_data = nullptr,
+      .processors_count = 1,
+      .processors_ptr = &description,
+    };
+    MinifiCreateExtension(extension, &ext_create_info);
+  });
+}
+```
+
+## C++ extensions
+You can utilize the C++ api, linking to `minifi-api` and possibly using the helpers in `extension-framework`.
+No compatibilities are guaranteed beyond what extensions are built together with the agent at the same time.
+
+An extension makes its capabilities (classes) available to the system through registrars. Registration must happen in source files, not headers.
 
 ```C++
 // register user-facing classes as
@@ -33,10 +64,10 @@ REGISTER_RESOURCE(RESTSender, DescriptionOnly);
 ```
 
 Some extensions (e.g. `OpenCVExtension`) require initialization before use.
-You need to define an `InitExtension` function of type `MinifiExtension*(MinifiConfig*)` to be called.
+You need to define an `MinifiInitCppExtension` function of type `MinifiExtension*(MinifiConfig*)` to be called.
 
 ```C++
-extern "C" MinifiExtension* InitExtension(MinifiConfig* /*config*/) {
+extern "C" void MinifiInitCppExtension(MinifiExtension* extension, MinifiConfig* /*config*/) {
   const auto success = org::apache::nifi::minifi::utils::Environment::setEnvironmentVariable("OPENCV_FFMPEG_CAPTURE_OPTIONS", "rtsp_transport;udp", false /*overwrite*/);
   if (!success) {
     return nullptr;
@@ -49,7 +80,7 @@ extern "C" MinifiExtension* InitExtension(MinifiConfig* /*config*/) {
     .processors_count = 0,
     .processors_ptr = nullptr
   };
-  return MinifiCreateExtension(minifi::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
+  minifi::utils::MinifiCreateCppExtension(extension, &ext_create_info);
 }
 ```
 

cmake/Extensions.cmake

@@ -22,32 +22,16 @@ define_property(GLOBAL PROPERTY EXTENSION-OPTIONS
 
 set_property(GLOBAL PROPERTY EXTENSION-OPTIONS "")
 
-set(extension-build-info-file "${CMAKE_CURRENT_BINARY_DIR}/ExtensionBuildInfo.cpp")
-file(GENERATE OUTPUT ${extension-build-info-file}
-    CONTENT "\
-    #include \"minifi-cpp/utils/Export.h\"\n\
-    #ifdef BUILD_ID_VARIABLE_NAME\n\
-    EXTENSIONAPI extern const char* const BUILD_ID_VARIABLE_NAME = \"__EXTENSION_BUILD_IDENTIFIER_BEGIN__${BUILD_IDENTIFIER}__EXTENSION_BUILD_IDENTIFIER_END__\";\n\
-    #else\n\
-    static_assert(false, \"BUILD_ID_VARIABLE_NAME is not defined\");\n\
-    #endif\n")
-
-function(get_build_id_variable_name extension-name output)
-    string(REPLACE "-" "_" result ${extension-name})
-    string(APPEND result "_build_identifier")
-    set("${output}" "${result}" PARENT_SCOPE)
-endfunction()
-
 macro(register_extension extension-name extension-display-name extension-guard description)
     set(${extension-guard} ${extension-name} PARENT_SCOPE)
     get_property(extensions GLOBAL PROPERTY EXTENSION-OPTIONS)
     set_property(GLOBAL APPEND PROPERTY EXTENSION-OPTIONS ${extension-name})
-    get_build_id_variable_name(${extension-name} build-id-variable-name)
-    set_source_files_properties(${extension-build-info-file} PROPERTIES GENERATED TRUE)
-    target_sources(${extension-name} PRIVATE ${extension-build-info-file})
+    get_target_property(has_custom_initializer ${extension-name} HAS_CUSTOM_INITIALIZER)
+    if (NOT has_custom_initializer)
+        target_sources(${extension-name} PRIVATE ${CMAKE_SOURCE_DIR}/extensions/ExtensionInitializer.cpp)
+    endif()
     target_compile_definitions(${extension-name}
-        PRIVATE "MODULE_NAME=${extension-name}"
-        PRIVATE "BUILD_ID_VARIABLE_NAME=${build-id-variable-name}")
+        PRIVATE "MODULE_NAME=${extension-name}")
     set_target_properties(${extension-name} PROPERTIES
         ENABLE_EXPORTS True
         POSITION_INDEPENDENT_CODE ON)

extension-framework/cpp-extension-lib/include/api/utils/minifi-c-utils.h

@@ -24,6 +24,12 @@
 #include "utils/StringUtils.h"
 #include "minifi-cpp/core/PropertyDefinition.h"
 
+#ifdef WIN32
+  #define CEXTENSIONAPI extern "C" __declspec(dllexport)
+#else
+  #define CEXTENSIONAPI extern "C"
+#endif
+
 namespace org::apache::nifi::minifi::api::utils {
 
 inline MinifiStringView toStringView(std::string_view str) {

extension-framework/include/utils/ExtensionInitUtils.h

@@ -30,4 +30,8 @@ inline MinifiStringView toStringView(std::string_view str) {
 
 using ConfigReader = std::function<std::optional<std::string>(std::string_view key)>;
 
+static inline void MinifiCreateCppExtension(MinifiExtension* extension, const MinifiExtensionCreateInfo* create_info) {
+  MINIFI_CREATE_EXTENSION_FN(extension, create_info);
+}
+
 }  // namespace org::apache::nifi::minifi::utils

extensions/ExtensionInitializer.cpp

@@ -0,0 +1,35 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "minifi-c/minifi-c.h"
+#include "utils/ExtensionInitUtils.h"
+#include "minifi-cpp/agent/agent_version.h"
+#include "core/Resource.h"
+
+namespace minifi = org::apache::nifi::minifi;
+
+extern "C" void MinifiInitCppExtension(MinifiExtension* extension, MinifiConfig* /*config*/) {
+  MinifiExtensionCreateInfo ext_create_info{
+    .name = minifi::utils::toStringView(MAKESTRING(MODULE_NAME)),
+    .version = minifi::utils::toStringView(minifi::AgentBuild::VERSION),
+    .deinit = nullptr,
+    .user_data = nullptr,
+    .processors_count = 0,
+    .processors_ptr = nullptr
+  };
+  minifi::utils::MinifiCreateCppExtension(extension, &ext_create_info);
+}

extensions/llamacpp/processors/ExtensionInitializer.cpp

@@ -24,8 +24,9 @@
 
 namespace minifi = org::apache::nifi::minifi;
 
-extern "C" MinifiExtension* InitExtension(MinifiConfig* /*config*/) {
-  MinifiExtension* extension = nullptr;
+CEXTENSIONAPI const uint32_t MinifiApiVersion = MINIFI_API_VERSION;
+
+CEXTENSIONAPI void MinifiInitExtension(MinifiExtension* extension, MinifiConfig* /*config*/) {
   minifi::api::core::useProcessorClassDescription<minifi::extensions::llamacpp::processors::RunLlamaCppInference>([&] (const MinifiProcessorClassDefinition& description) {
     MinifiExtensionCreateInfo ext_create_info{
       .name = minifi::api::utils::toStringView(MAKESTRING(EXTENSION_NAME)),
@@ -35,9 +36,6 @@ extern "C" MinifiExtension* InitExtension(MinifiConfig* /*config*/) {
       .processors_count = 1,
       .processors_ptr = &description,
     };
-    extension = MinifiCreateExtension(minifi::api::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
+    MinifiCreateExtension(extension, &ext_create_info);
   });
-  return extension;
 }
-
-extern const char* const MINIFI_API_VERSION_TAG_var = MINIFI_API_VERSION_TAG;

extensions/opencv/CMakeLists.txt

@@ -29,6 +29,7 @@ include(${CMAKE_SOURCE_DIR}/extensions/ExtensionHeader.txt)
 file(GLOB SOURCES  "*.cpp")
 
 add_minifi_library(minifi-opencv SHARED ${SOURCES})
+set_target_properties(minifi-opencv PROPERTIES HAS_CUSTOM_INITIALIZER TRUE)
 
 target_link_libraries(minifi-opencv ${LIBMINIFI})
 target_link_libraries(minifi-opencv OPENCV::libopencv)

extensions/opencv/OpenCVLoader.cpp

@@ -24,15 +24,15 @@
 
 namespace minifi = org::apache::nifi::minifi;
 
-extern "C" MinifiExtension* InitExtension(MinifiConfig* /*config*/) {
+extern "C" void MinifiInitCppExtension(MinifiExtension* extension, MinifiConfig* /*config*/) {
   // By default in OpenCV, ffmpeg capture is hardcoded to use TCP and this is a workaround
   // also if UDP timeout, ffmpeg will retry with TCP
   // Note:
   // 1. OpenCV community are trying to find a better approach than setenv.
   // 2. The command will not overwrite value if "OPENCV_FFMPEG_CAPTURE_OPTIONS" already exists.
   const auto success = org::apache::nifi::minifi::utils::Environment::setEnvironmentVariable("OPENCV_FFMPEG_CAPTURE_OPTIONS", "rtsp_transport;udp", false /*overwrite*/);
   if (!success) {
-    return nullptr;
+    return;
   }
   MinifiExtensionCreateInfo ext_create_info{
     .name = minifi::utils::toStringView(MAKESTRING(MODULE_NAME)),
@@ -42,5 +42,5 @@ extern "C" MinifiExtension* InitExtension(MinifiConfig* /*config*/) {
     .processors_count = 0,
     .processors_ptr = nullptr
   };
-  return MinifiCreateExtension(minifi::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
+  minifi::utils::MinifiCreateCppExtension(extension, &ext_create_info);
 }

extensions/python/CMakeLists.txt

@@ -25,12 +25,14 @@ include(${CMAKE_SOURCE_DIR}/extensions/ExtensionHeader.txt)
 
 if (NOT WIN32)
     add_minifi_library(minifi-python-lib-loader-extension SHARED pythonlibloader/PythonLibLoader.cpp)
+    set_target_properties(minifi-python-lib-loader-extension PROPERTIES HAS_CUSTOM_INITIALIZER TRUE)
     target_link_libraries(minifi-python-lib-loader-extension PRIVATE ${LIBMINIFI})
 endif()
 
 file(GLOB SOURCES "*.cpp" "types/*.cpp" "pythonloader/PyProcLoader.cpp")
 
 add_minifi_library(minifi-python-script-extension SHARED ${SOURCES})
+set_target_properties(minifi-python-script-extension PROPERTIES HAS_CUSTOM_INITIALIZER TRUE)
 
 target_link_libraries(minifi-python-script-extension PRIVATE ${LIBMINIFI} Threads::Threads)
 

extensions/python/pythonlibloader/PythonLibLoader.cpp

@@ -98,7 +98,7 @@ class PythonLibLoader {
   std::shared_ptr<minifi::core::logging::Logger> logger_ = minifi::core::logging::LoggerFactory<PythonLibLoader>::getLogger();
 };
 
-extern "C" MinifiExtension* InitExtension(MinifiConfig* config) {
+extern "C" void MinifiInitCppExtension(MinifiExtension* extension, MinifiConfig* config) {
   static PythonLibLoader python_lib_loader([&] (std::string_view key) -> std::optional<std::string> {
     std::optional<std::string> result;
     MinifiConfigGet(config, minifi::utils::toStringView(key), [] (void* user_data, MinifiStringView value) {
@@ -114,5 +114,5 @@ extern "C" MinifiExtension* InitExtension(MinifiConfig* config) {
     .processors_count = 0,
     .processors_ptr = nullptr
   };
-  return MinifiCreateExtension(minifi::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
+  minifi::utils::MinifiCreateCppExtension(extension, &ext_create_info);
 }

extensions/python/pythonloader/PyProcLoader.cpp

@@ -33,7 +33,7 @@ static minifi::extensions::python::PythonCreator& getPythonCreator() {
 // the symbols of the python library
 extern "C" const int LOAD_MODULE_AS_GLOBAL = 1;
 
-extern "C" MinifiExtension* InitExtension(MinifiConfig* config) {
+extern "C" void MinifiInitCppExtension(MinifiExtension* extension, MinifiConfig* config) {
   getPythonCreator().configure([&] (std::string_view key) -> std::optional<std::string> {
     std::optional<std::string> result;
     MinifiConfigGet(config, minifi::utils::toStringView(key), [] (void* user_data, MinifiStringView value) {
@@ -49,5 +49,5 @@ extern "C" MinifiExtension* InitExtension(MinifiConfig* config) {
     .processors_count = 0,
     .processors_ptr = nullptr
   };
-  return MinifiCreateExtension(minifi::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
+  minifi::utils::MinifiCreateCppExtension(extension, &ext_create_info);
 }

extensions/sftp/CMakeLists.txt

@@ -31,6 +31,7 @@ include_directories(client processors)
 file(GLOB SOURCES  "*.cpp" "client/*.cpp" "processors/*.cpp")
 
 add_minifi_library(minifi-sftp SHARED ${SOURCES})
+set_target_properties(minifi-sftp PROPERTIES HAS_CUSTOM_INITIALIZER TRUE)
 
 target_link_libraries(minifi-sftp ${LIBMINIFI} Threads::Threads)
 target_link_libraries(minifi-sftp libssh2 RapidJSON)

extensions/sftp/SFTPLoader.cpp

@@ -25,13 +25,13 @@
 
 namespace minifi = org::apache::nifi::minifi;
 
-extern "C" MinifiExtension* InitExtension(MinifiConfig* /*config*/) {
+extern "C" void MinifiInitCppExtension(MinifiExtension* extension, MinifiConfig* /*config*/) {
   if (libssh2_init(0) != 0) {
-    return nullptr;
+    return;
   }
   if (curl_global_init(CURL_GLOBAL_DEFAULT) != CURLE_OK) {
     libssh2_exit();
-    return nullptr;
+    return;
   }
   MinifiExtensionCreateInfo ext_create_info{
     .name = minifi::utils::toStringView(MAKESTRING(MODULE_NAME)),
@@ -44,5 +44,5 @@ extern "C" MinifiExtension* InitExtension(MinifiConfig* /*config*/) {
     .processors_count = 0,
     .processors_ptr = nullptr
   };
-  return MinifiCreateExtension(minifi::utils::toStringView(MINIFI_API_VERSION), &ext_create_info);
+  minifi::utils::MinifiCreateCppExtension(extension, &ext_create_info);
 }

libminifi/include/core/extension/ApiVersion.h

@@ -0,0 +1,30 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <cinttypes>
+
+namespace org::apache::nifi::minifi::core::extension {
+
+uint32_t getAgentApiVersion();
+uint32_t getMinSupportedApiVersion();
+void test_setAgentApiVersion(uint32_t api_version);
+void test_setMinSupportedApiVersion(uint32_t min_api_version);
+
+}  // namespace org::apache::nifi::minifi::core::extension

libminifi/include/core/extension/Extension.h

@@ -17,13 +17,15 @@
 
 #pragma once
 
-#include <memory>
+#include <filesystem>
 #include <map>
+#include <memory>
 #include <string>
-#include <filesystem>
 
+#include "minifi-c/minifi-c.h"
 #include "minifi-cpp/core/logging/Logger.h"
 #include "minifi-cpp/properties/Configure.h"
+#include "ApiVersion.h"
 
 namespace org::apache::nifi::minifi::core::extension {
 
@@ -49,6 +51,8 @@ class Extension {
 
   bool initialize(const std::shared_ptr<minifi::Configure>& configure);
 
+  bool setInfo(Info info);
+
  private:
 #ifdef WIN32
   std::map<void*, std::string> resource_mapping_;
@@ -67,11 +71,12 @@ class Extension {
   bool unload();
   void* findSymbol(const char* name);
 
-  std::string name_;
+  std::string library_name_;
   std::filesystem::path library_path_;
   gsl::owner<void*> handle_ = nullptr;
 
-  std::unique_ptr<Info> info_;
+  std::optional<Info> info_;
+  uint32_t api_version_{0};
 
   const std::shared_ptr<logging::Logger> logger_;
 };

libminifi/include/core/extension/ExtensionManager.h

@@ -37,6 +37,8 @@ class ExtensionManager {
   ExtensionManager& operator=(const ExtensionManager&) = delete;
   ExtensionManager& operator=(ExtensionManager&&) = delete;
 
+  static Extension* getExtensionBeingInitialized();
+
  private:
   std::vector<std::unique_ptr<Extension>> extensions_;