Add OpenSSL native QUIC backend

CMakeLists.txt

@@ -212,6 +212,7 @@ set(ENABLE_TPROXY
         'X' where X is a number to use as the IP_TRANSPARENT sockopt,
         anything else to enable."
 )
+option(ENABLE_OPENSSL_QUIC "Use OpenSSL native QUIC (default OFF)")
 option(ENABLE_QUICHE "Use quiche (default OFF)")
 
 option(ENABLE_EXAMPLE "Build example directory (default OFF)")
@@ -280,6 +281,8 @@ pkg_check_modules(PCRE2 REQUIRED IMPORTED_TARGET libpcre2-8)
 include(CheckOpenSSLIsBoringSSL)
 include(CheckOpenSSLIsQuictls)
 include(CheckOpenSSLIsAwsLc)
+include(CheckOpenSSLHasQuicTlsCbs)
+include(CheckOpenSSLHasNativeQuic)
 find_package(OpenSSL REQUIRED)
 check_openssl_is_boringssl(SSLLIB_IS_BORINGSSL BORINGSSL_VERSION "${OPENSSL_INCLUDE_DIR}")
 check_openssl_is_awslc(SSLLIB_IS_AWSLC AWSLC_VERSION "${OPENSSL_INCLUDE_DIR}")
@@ -309,6 +312,18 @@ if(OPENSSL_VERSION VERSION_GREATER_EQUAL "3.0.0")
   add_compile_definitions(OPENSSL_API_COMPAT=10002 OPENSSL_IS_OPENSSL3)
 endif()
 
+check_openssl_has_quic_tls_cbs(SSLLIB_HAS_QUIC_TLS_CBS "${OPENSSL_INCLUDE_DIR}")
+if(SSLLIB_HAS_QUIC_TLS_CBS
+   AND NOT SSLLIB_IS_BORINGSSL
+   AND NOT SSLLIB_IS_AWSLC
+   AND NOT SSLLIB_IS_QUICTLS
+)
+  set(TS_OPENSSL_QUIC_TLS_CBS_COMPAT TRUE)
+  add_compile_definitions(TS_OPENSSL_QUIC_TLS_CBS_COMPAT)
+endif()
+
+check_openssl_has_native_quic(SSLLIB_HAS_NATIVE_QUIC "${OPENSSL_INCLUDE_DIR}")
+
 if(ENABLE_PROFILER)
   find_package(profiler REQUIRED)
   set(TS_HAS_PROFILER ${profiler_FOUND})
@@ -324,13 +339,40 @@ elseif(TS_HAS_MIMALLOC)
   link_libraries(mimalloc)
 endif()
 
+if(ENABLE_OPENSSL_QUIC AND ENABLE_QUICHE)
+  message(FATAL_ERROR "ENABLE_OPENSSL_QUIC and ENABLE_QUICHE are mutually exclusive QUIC backends")
+endif()
+
+if(ENABLE_OPENSSL_QUIC)
+  if(NOT SSLLIB_HAS_NATIVE_QUIC)
+    message(FATAL_ERROR "OpenSSL native QUIC support requires OpenSSL 3.5 or newer with OSSL_QUIC_server_method")
+  endif()
+  if(SSLLIB_IS_BORINGSSL
+     OR SSLLIB_IS_AWSLC
+     OR SSLLIB_IS_QUICTLS
+  )
+    message(FATAL_ERROR "OpenSSL native QUIC support requires upstream OpenSSL 3.5 or newer")
+  endif()
+  set(TS_HAS_OPENSSL_QUIC TRUE)
+  set(TS_USE_QUIC TRUE)
+  message(STATUS "Using OpenSSL native QUIC")
+endif()
+
 if(ENABLE_QUICHE)
+  if(TS_OPENSSL_QUIC_TLS_CBS_COMPAT)
+    set(quiche_USE_STATIC TRUE)
+  endif()
   find_package(quiche REQUIRED)
 
   set(TS_HAS_QUICHE ${quiche_FOUND})
   set(TS_USE_QUIC ${TS_HAS_QUICHE})
-  if(NOT SSLLIB_IS_BORINGSSL AND NOT SSLLIB_IS_QUICTLS)
-    message(FATAL_ERROR "Use of BoringSSL or OPENSSL/QUICTLS is required if quiche is used.")
+  if(NOT SSLLIB_IS_BORINGSSL
+     AND NOT SSLLIB_IS_QUICTLS
+     AND NOT TS_OPENSSL_QUIC_TLS_CBS_COMPAT
+  )
+    message(
+      FATAL_ERROR "Use of BoringSSL, OPENSSL/QUICTLS, or OpenSSL QUIC TLS callbacks is required if quiche is used."
+    )
   endif()
 
   if(SSLLIB_IS_QUICTLS)
@@ -339,6 +381,8 @@ if(ENABLE_QUICHE)
     message(
       "WARNING - Using quictls requires using a special version of quiche. Make sure quictls is supported in quiche."
     )
+  elseif(TS_OPENSSL_QUIC_TLS_CBS_COMPAT)
+    message(STATUS "Using OpenSSL QUIC TLS callbacks compatibility for quiche")
   endif()
 endif()
 

cmake/CheckOpenSSLHasNativeQuic.cmake

@@ -0,0 +1,59 @@
+#######################
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+#  agreements.  See the NOTICE file distributed with this work for additional information regarding
+#  copyright ownership.  The ASF licenses this file to you under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with the License.  You may obtain
+#  a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing permissions and limitations under
+#  the License.
+#
+#######################
+
+function(CHECK_OPENSSL_HAS_NATIVE_QUIC OUT_VAR OPENSSL_INCLUDE_DIR)
+  set(CHECK_PROGRAM
+      "
+        #include <openssl/ssl.h>
+        #include <openssl/quic.h>
+        #include <cstdint>
+
+        int main() {
+            const SSL_METHOD *method = OSSL_QUIC_server_method();
+            SSL *ssl = nullptr;
+            uint64_t value = 0;
+            return method == nullptr ||
+                   SSL_new_listener == nullptr ||
+                   SSL_listen == nullptr ||
+                   SSL_handle_events == nullptr ||
+                   SSL_accept_connection == nullptr ||
+                   SSL_accept_stream == nullptr ||
+                   SSL_new_stream == nullptr ||
+                   SSL_stream_conclude == nullptr ||
+                   SSL_get_stream_id == nullptr ||
+                   SSL_get_stream_type == nullptr ||
+                   SSL_get_stream_read_state == nullptr ||
+                   SSL_get_stream_write_buf_avail(ssl, &value) ||
+                   SSL_get_conn_close_info == nullptr ||
+                   SSL_shutdown_ex == nullptr ||
+                   SSL_set_default_stream_mode == nullptr ||
+                   SSL_set_blocking_mode == nullptr ||
+                   SSL_set_event_handling_mode(ssl, SSL_VALUE_EVENT_HANDLING_MODE_EXPLICIT) ||
+                   SSL_set_feature_request_uint(ssl, SSL_VALUE_QUIC_IDLE_TIMEOUT, value) ||
+                   SSL_set_incoming_stream_policy == nullptr;
+        }
+        "
+  )
+  set(CMAKE_REQUIRED_INCLUDES "${OPENSSL_INCLUDE_DIR}")
+  set(CMAKE_REQUIRED_LIBRARIES OpenSSL::SSL OpenSSL::Crypto)
+  include(CheckCXXSourceCompiles)
+  check_cxx_source_compiles("${CHECK_PROGRAM}" ${OUT_VAR})
+  set(${OUT_VAR}
+      ${${OUT_VAR}}
+      PARENT_SCOPE
+  )
+endfunction()

cmake/CheckOpenSSLHasQuicTlsCbs.cmake

@@ -0,0 +1,55 @@
+#######################
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+#  agreements.  See the NOTICE file distributed with this work for additional information regarding
+#  copyright ownership.  The ASF licenses this file to you under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with the License.  You may obtain
+#  a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software distributed under the License
+#  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  or implied. See the License for the specific language governing permissions and limitations under
+#  the License.
+#
+#######################
+
+function(CHECK_OPENSSL_HAS_QUIC_TLS_CBS OUT_VAR OPENSSL_INCLUDE_DIR)
+  set(CHECK_PROGRAM
+      "
+        #include <openssl/ssl.h>
+        #include <openssl/core_dispatch.h>
+
+        int main() {
+            OSSL_FUNC_SSL_QUIC_TLS_crypto_send_fn *crypto_send = nullptr;
+            OSSL_FUNC_SSL_QUIC_TLS_crypto_recv_rcd_fn *crypto_recv = nullptr;
+            OSSL_FUNC_SSL_QUIC_TLS_crypto_release_rcd_fn *crypto_release = nullptr;
+            OSSL_FUNC_SSL_QUIC_TLS_yield_secret_fn *yield_secret = nullptr;
+            OSSL_FUNC_SSL_QUIC_TLS_got_transport_params_fn *got_transport_params = nullptr;
+            OSSL_FUNC_SSL_QUIC_TLS_alert_fn *alert = nullptr;
+            const OSSL_DISPATCH callbacks[] = {
+                {OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_SEND, reinterpret_cast<void (*)(void)>(crypto_send)},
+                {OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_RECV_RCD, reinterpret_cast<void (*)(void)>(crypto_recv)},
+                {OSSL_FUNC_SSL_QUIC_TLS_CRYPTO_RELEASE_RCD, reinterpret_cast<void (*)(void)>(crypto_release)},
+                {OSSL_FUNC_SSL_QUIC_TLS_YIELD_SECRET, reinterpret_cast<void (*)(void)>(yield_secret)},
+                {OSSL_FUNC_SSL_QUIC_TLS_GOT_TRANSPORT_PARAMS, reinterpret_cast<void (*)(void)>(got_transport_params)},
+                {OSSL_FUNC_SSL_QUIC_TLS_ALERT, reinterpret_cast<void (*)(void)>(alert)},
+                {0, nullptr},
+            };
+
+            return callbacks[0].function_id == 0 ||
+                   SSL_set_quic_tls_cbs == nullptr ||
+                   SSL_set_quic_tls_transport_params == nullptr;
+        }
+        "
+  )
+  set(CMAKE_REQUIRED_INCLUDES "${OPENSSL_INCLUDE_DIR}")
+  set(CMAKE_REQUIRED_LIBRARIES OpenSSL::SSL OpenSSL::Crypto)
+  include(CheckCXXSourceCompiles)
+  check_cxx_source_compiles("${CHECK_PROGRAM}" ${OUT_VAR})
+  set(${OUT_VAR}
+      ${${OUT_VAR}}
+      PARENT_SCOPE
+  )
+endfunction()

cmake/Findquiche.cmake

@@ -28,7 +28,24 @@
 #     quiche::quiche
 #
 
+if(quiche_USE_STATIC
+   AND quiche_LIBRARY
+   AND NOT quiche_LIBRARY MATCHES "\\${CMAKE_STATIC_LIBRARY_SUFFIX}$"
+)
+  unset(quiche_LIBRARY CACHE)
+endif()
+
+if(quiche_USE_STATIC)
+  set(_quiche_ORIGINAL_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
+  set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_STATIC_LIBRARY_SUFFIX})
+endif()
+
 find_library(quiche_LIBRARY NAMES quiche)
+
+if(quiche_USE_STATIC)
+  set(CMAKE_FIND_LIBRARY_SUFFIXES ${_quiche_ORIGINAL_FIND_LIBRARY_SUFFIXES})
+endif()
+
 find_path(
   quiche_INCLUDE_DIR
   NAMES quiche.h
@@ -40,8 +57,18 @@ mark_as_advanced(quiche_FOUND quiche_LIBRARY quiche_INCLUDE_DIR)
 include(FindPackageHandleStandardArgs)
 find_package_handle_standard_args(quiche REQUIRED_VARS quiche_LIBRARY quiche_INCLUDE_DIR)
 
+if(quiche_USE_STATIC
+   AND quiche_FOUND
+   AND NOT quiche_LIBRARY MATCHES "\\${CMAKE_STATIC_LIBRARY_SUFFIX}$"
+)
+  message(FATAL_ERROR "Static quiche was requested, but ${quiche_LIBRARY} is not a static library")
+endif()
+
 if(quiche_FOUND)
   set(quiche_INCLUDE_DIRS "${quiche_INCLUDE_DIR}")
+  if(quiche_USE_STATIC)
+    message(STATUS "Using static quiche library: ${quiche_LIBRARY}")
+  endif()
 endif()
 
 if(quiche_FOUND AND NOT TARGET quiche::quiche)

include/iocore/net/QUICMultiCertConfigLoader.h

@@ -25,6 +25,7 @@
 
 #include "iocore/net/SSLMultiCertConfigLoader.h"
 #include "iocore/eventsystem/ConfigProcessor.h"
+#include "tscore/ink_config.h"
 
 class QUICCertConfig
 {
@@ -48,7 +49,11 @@ class QUICMultiCertConfigLoader : public SSLMultiCertConfigLoader
   virtual SSL_CTX *default_server_ssl_ctx() override;
 
 private:
-  const char  *_debug_tag() const override;
+  const char *_debug_tag() const override;
+#if TS_HAS_OPENSSL_QUIC
+  virtual void _set_handshake_callbacks(SSL_CTX *ctx) override;
+  virtual bool _set_alpn_callback(SSL_CTX *ctx) override;
+#endif
   virtual bool _setup_session_cache(SSL_CTX *ctx) override;
   virtual bool _set_cipher_suites_for_legacy_versions(SSL_CTX *ctx) override;
   virtual bool _set_info_callback(SSL_CTX *ctx) override;

include/iocore/net/quic/QUICConfig.h

@@ -24,7 +24,11 @@
 #pragma once
 
 #include <openssl/ssl.h>
+#include "tscore/ink_config.h"
+
+#if TS_HAS_QUICHE
 #include <quiche.h>
+#endif
 
 #include "iocore/eventsystem/ConfigProcessor.h"
 #include "iocore/net/SSLTypes.h"
@@ -88,7 +92,9 @@ class QUICConfigParams : public ConfigInfo
 
   bool disable_http_0_9() const;
 
+#if TS_HAS_QUICHE
   quiche_cc_algorithm get_cc_algorithm() const;
+#endif
 
 private:
   static int _connection_table_size;
@@ -162,3 +168,4 @@ class QUICConfig
 };
 
 SSL_CTX *quic_new_ssl_ctx();
+SSL_CTX *quic_new_server_ssl_ctx();

include/iocore/net/quic/QUICStream.h

@@ -30,11 +30,25 @@
 #include "iocore/net/quic/QUICConnection.h"
 #include "iocore/net/quic/QUICDebugNames.h"
 
-#include <quiche.h>
+#include <cstddef>
+#include <cstdint>
 
 class QUICStreamAdapter;
 class QUICStreamStateListener;
 
+class QUICStreamIO
+{
+public:
+  using ErrorCode = uint64_t;
+
+  virtual ~QUICStreamIO() = default;
+
+  virtual int64_t read_stream(QUICStreamId stream_id, uint8_t *buf, size_t len, bool &fin, ErrorCode &error_code)       = 0;
+  virtual bool    stream_read_finished(QUICStreamId stream_id)                                                          = 0;
+  virtual int64_t stream_write_capacity(QUICStreamId stream_id)                                                         = 0;
+  virtual int64_t write_stream(QUICStreamId stream_id, uint8_t const *buf, size_t len, bool fin, ErrorCode &error_code) = 0;
+};
+
 /**
  * @brief QUIC Stream
  * TODO: This is similar to Http2Stream. Need to think some integration.
@@ -53,14 +67,15 @@ class QUICStream
   QUICStreamDirection               direction() const;
   bool                              is_bidirectional() const;
   bool                              has_no_more_data() const;
+  bool                              has_data_to_send();
 
   QUICOffset final_offset() const;
 
   void stop_sending(QUICStreamErrorUPtr error);
   void reset(QUICStreamErrorUPtr error);
 
-  void receive_data(quiche_conn *quiche_con);
-  void send_data(quiche_conn *quiche_con);
+  void    receive_data(QUICStreamIO &stream_io);
+  int64_t send_data(QUICStreamIO &stream_io);
 
   /*
    * QUICApplication need to call one of these functions when it process VC_EVENT_*

include/tscore/ink_config.h.cmake.in

@@ -145,6 +145,7 @@ const int DEFAULT_STACKSIZE = @DEFAULT_STACK_SIZE@;
 #cmakedefine01 TS_HAS_JEMALLOC
 #cmakedefine01 TS_HAS_MIMALLOC
 #cmakedefine01 TS_HAS_PROFILER
+#cmakedefine01 TS_HAS_OPENSSL_QUIC
 #cmakedefine01 TS_HAS_QUICHE
 #cmakedefine01 TS_HAS_SO_MARK
 #cmakedefine01 TS_HAS_SO_PEERCRED

src/iocore/net/CMakeLists.txt

@@ -80,18 +80,17 @@ if(TS_USE_QUIC)
   add_subdirectory(quic)
 
   target_sources(
-    inknet
-    PRIVATE QUICClosedConCollector.cc
-            QUICMultiCertConfigLoader.cc
-            QUICNet.cc
-            QUICNetProcessor.cc
-            QUICNetVConnection.cc
-            QUICNextProtocolAccept.cc
-            QUICPacketHandler.cc
-            QUICSupport.cc
+    inknet PRIVATE QUICClosedConCollector.cc QUICMultiCertConfigLoader.cc QUICNextProtocolAccept.cc QUICSupport.cc
   )
 
-  target_link_libraries(inknet PUBLIC quiche::quiche ts::quic)
+  if(TS_HAS_OPENSSL_QUIC)
+    target_sources(inknet PRIVATE OpenSSLQUICNetProcessor.cc OpenSSLQUICNetVConnection.cc OpenSSLQUICPacketHandler.cc)
+  elseif(TS_HAS_QUICHE)
+    target_sources(inknet PRIVATE QUICNet.cc QUICNetProcessor.cc QUICNetVConnection.cc QUICPacketHandler.cc)
+    target_link_libraries(inknet PUBLIC quiche::quiche)
+  endif()
+
+  target_link_libraries(inknet PUBLIC ts::quic)
 endif()
 
 if(BUILD_REGRESSION_TESTING OR BUILD_TESTING)
@@ -155,6 +154,9 @@ if(BUILD_TESTING)
   )
   if(TS_USE_QUIC)
     list(APPEND LINK_GROUP_LIBS quic http3)
+    if(TS_HAS_QUICHE)
+      list(APPEND LINK_GROUP_LIBS quiche::quiche)
+    endif()
   endif()
   if(CMAKE_LINK_GROUP_USING_RESCAN_SUPPORTED OR CMAKE_CXX_LINK_GROUP_USING_RESCAN_SUPPORTED)
     string(JOIN "," LINK_GROUP_LIBS_CSV ${LINK_GROUP_LIBS})