feat: Add experimental H.265 support via NVIDIA NVENC

BuildScripts~/build_libwebrtc_win.cmd

@@ -1,86 +1,88 @@
-@echo off
-
-if not exist depot_tools (
-  git clone --depth 1 https://chromium.googlesource.com/chromium/tools/depot_tools.git
-)
-
-set COMMAND_DIR=%~dp0
-set PATH=%cd%\depot_tools;%PATH%
-set WEBRTC_VERSION=5845
-set DEPOT_TOOLS_WIN_TOOLCHAIN=0
-set GYP_GENERATORS=ninja,msvs-ninja
-set GYP_MSVS_VERSION=2022
-set OUTPUT_DIR=out
-set ARTIFACTS_DIR=%cd%\artifacts
-set PYPI_URL=https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
-set vs2022_install=C:\Program Files\Microsoft Visual Studio\2022\Professional
-
-if not exist src (
-  call fetch.bat --nohooks webrtc
-  cd src
-  call git.bat config --system core.longpaths true
-  call git.bat checkout  refs/remotes/branch-heads/%WEBRTC_VERSION%
-  cd ..
-  call gclient.bat sync -D --force --reset
-)
-
-rem add jsoncpp
-patch -N "src\BUILD.gn" < "%COMMAND_DIR%\patches\add_jsoncpp.patch"
-
-rem fix towupper
-patch -N "src\modules\desktop_capture\win\full_screen_win_application_handler.cc" < "%COMMAND_DIR%\patches\fix_towupper.patch"
-
-rem fix abseil
-patch -N "src\third_party\abseil-cpp/absl/base/config.h" < "%COMMAND_DIR%\patches\fix_abseil.patch"
-
-rem fix task_queue_base
-patch -N "src\api\task_queue\task_queue_base.h" < "%COMMAND_DIR%\patches\fix_task_queue_base.patch"
-
-rem fix SetRawImagePlanes() in LibvpxVp8Encoder
-patch -N "src\modules\video_coding\codecs\vp8\libvpx_vp8_encoder.cc" < "%COMMAND_DIR%\patches\libvpx_vp8_encoder.patch"
-
-mkdir "%ARTIFACTS_DIR%\lib"
-
-setlocal enabledelayedexpansion
-
-for %%i in (x64) do (
-  mkdir "%ARTIFACTS_DIR%/lib/%%i"
-  for %%j in (true false) do (
-
-    rem generate ninja for release
-    call gn.bat gen %OUTPUT_DIR% --root="src" ^
-      --args="is_debug=%%j is_clang=true target_cpu=\"%%i\" use_custom_libcxx=false rtc_include_tests=false rtc_build_examples=false rtc_use_h264=false symbol_level=0 enable_iterator_debugging=false use_cxx17=true"
-
-    rem build
-    call ninja.bat -C %OUTPUT_DIR% webrtc
-
-    set filename=
-    if true==%%j (
-      set filename=webrtcd.lib
-    ) else (
-      set filename=webrtc.lib
-    )
-
-    rem copy static library for release build
-    copy "%OUTPUT_DIR%\obj\webrtc.lib" "%ARTIFACTS_DIR%\lib\%%i\!filename!"
-  )
-)
-
-endlocal
-
-rem generate license
-call python.bat "%cd%\src\tools_webrtc\libs\generate_licenses.py" ^
-  --target :webrtc %OUTPUT_DIR% %OUTPUT_DIR%
-
-rem unescape license
-powershell -File "%COMMAND_DIR%\Unescape.ps1" "%OUTPUT_DIR%\LICENSE.md"
-
-rem copy header
-xcopy src\*.h "%ARTIFACTS_DIR%\include" /C /S /I /F /H
-
-rem copy license
-copy "%OUTPUT_DIR%\LICENSE.md" "%ARTIFACTS_DIR%"
-
-rem create zip
-cd %ARTIFACTS_DIR%
+@echo off
+
+if not exist depot_tools (
+  git clone --depth 1 https://chromium.googlesource.com/chromium/tools/depot_tools.git
+)
+
+set COMMAND_DIR=%~dp0
+set PATH=%cd%\depot_tools;%PATH%
+set WEBRTC_VERSION=7103
+set DEPOT_TOOLS_WIN_TOOLCHAIN=0
+set GYP_GENERATORS=ninja,msvs-ninja
+set GYP_MSVS_VERSION=2022
+set OUTPUT_DIR=out
+set ARTIFACTS_DIR=%cd%\artifacts
+set PYPI_URL=https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
+set vs2022_install=C:\Program Files\Microsoft Visual Studio\2022\Community
+
+if not exist src (
+  call fetch.bat --nohooks webrtc
+  cd src
+  call git.bat config --system core.longpaths true
+  call git.bat checkout  refs/remotes/branch-heads/%WEBRTC_VERSION%
+  cd ..
+  call gclient.bat sync -D --force --reset
+)
+
+rem add jsoncpp
+patch -N "src\BUILD.gn" < "%COMMAND_DIR%\patches\add_jsoncpp.patch"
+
+rem fix towupper
+patch -N "src\modules\desktop_capture\win\full_screen_win_application_handler.cc" < "%COMMAND_DIR%\patches\fix_towupper.patch"
+
+rem fix abseil
+patch -N "src\third_party\abseil-cpp/absl/base/config.h" < "%COMMAND_DIR%\patches\fix_abseil.patch"
+
+rem fix task_queue_base
+patch -N "src\api\task_queue\task_queue_base.h" < "%COMMAND_DIR%\patches\fix_task_queue_base.patch"
+
+rem fix SetRawImagePlanes() in LibvpxVp8Encoder
+patch -N "src\modules\video_coding\codecs\vp8\libvpx_vp8_encoder.cc" < "%COMMAND_DIR%\patches\libvpx_vp8_encoder.patch"
+
+mkdir "%ARTIFACTS_DIR%\lib"
+
+setlocal enabledelayedexpansion
+
+for %%i in (x64) do (
+  mkdir "%ARTIFACTS_DIR%/lib/%%i"
+  for %%j in (true false) do (
+
+    rem generate ninja for release
+    call gn.bat gen %OUTPUT_DIR% --root="src" ^
+      --args="is_debug=%%j is_clang=true target_cpu=\"%%i\" use_custom_libcxx=false rtc_include_tests=false rtc_build_examples=false rtc_use_h264=false rtc_use_h265=true proprietary_codecs=true symbol_level=0 enable_iterator_debugging=false use_cxx17=true"
+
+    rem build
+    call ninja.bat -C %OUTPUT_DIR% webrtc
+
+    set filename=
+    if true==%%j (
+      set filename=webrtcd.lib
+    ) else (
+      set filename=webrtc.lib
+    )
+
+    rem copy static library for release build
+    copy "%OUTPUT_DIR%\obj\webrtc.lib" "%ARTIFACTS_DIR%\lib\%%i\!filename!"
+  )
+)
+
+endlocal
+
+rem generate license
+call python.bat "%cd%\src\tools_webrtc\libs\generate_licenses.py" ^
+  --target :webrtc %OUTPUT_DIR% %OUTPUT_DIR%
+
+rem unescape license
+powershell -File "%COMMAND_DIR%\Unescape.ps1" "%OUTPUT_DIR%\LICENSE.md"
+
+rem copy header
+xcopy src\*.h "%ARTIFACTS_DIR%\include" /C /S /I /F /H
+
+xcopy src\*.inc "%ARTIFACTS_DIR%\include" /C /S /I /F /H
+
+rem copy license
+copy "%OUTPUT_DIR%\LICENSE.md" "%ARTIFACTS_DIR%"
+
+rem create zip
+cd %ARTIFACTS_DIR%
 7z a -tzip webrtc-win.zip *

Plugin~/CMakeLists.txt

@@ -74,7 +74,7 @@ endif()
 set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
 find_package(Threads REQUIRED)
 
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 
 find_package(WebRTC REQUIRED)
 

Plugin~/NvCodec/CMakeLists.txt

@@ -16,6 +16,15 @@ if(Linux)
   add_subdirectory(implib)
 endif()
 
+# set iterator debug level and compiler flags.
+if(Windows AND CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
+    set(CMAKE_CXX_FLAGS_DEBUG
+            "${CMAKE_CXX_FLAGS_DEBUG} -D_ITERATOR_DEBUG_LEVEL=0 /Zi /Od")
+else()
+    set(CMAKE_CXX_FLAGS_DEBUG
+            "${CMAKE_CXX_FLAGS_DEBUG} -D_ITERATOR_DEBUG_LEVEL=0")
+endif()
+
 target_include_directories(
   NvCodec PRIVATE ${CUDA_INCLUDE_DIRS} ${Vulkan_INCLUDE_DIR}
                   ${GLEW_INCLUDE_DIRS} ${NVCODEC_INCLUDE_DIR})

Plugin~/WebRTCPlugin/AudioTrackSinkAdapter.cpp

@@ -34,10 +34,12 @@ namespace webrtc
         // the `number_of_channels` argument is passed as `1`. However, Unity expects
         // to receive audio which is stereo channel usually, in this case we need to
         // resample audio monoural to stereo.
+        const InterleavedView<const int16_t> view(static_cast<const int16_t*>(audio_data), number_of_frames, number_of_channels);
         webrtc::voe::RemixAndResample(
-            static_cast<const int16_t*>(audio_data),
-            number_of_frames,
-            number_of_channels,
+            // static_cast<const int16_t*>(audio_data),
+            // number_of_frames,
+            // number_of_channels,
+            view,
             sample_rate,
             &_resampler,
             &_frame);

Plugin~/WebRTCPlugin/Codec/CMakeLists.txt

@@ -2,6 +2,7 @@ target_sources(
   WebRTCLib
   PRIVATE CreateVideoCodecFactory.cpp CreateVideoCodecFactory.h
           H264ProfileLevelId.cpp H264ProfileLevelId.h
+          H265ProfileLevelId.cpp H265ProfileLevelId.h
           SimulcastEncoderFactory.cpp SimulcastEncoderFactory.h)
 
 if(Windows OR Linux)

Plugin~/WebRTCPlugin/Codec/H264ProfileLevelId.cpp

@@ -4,6 +4,8 @@
 
 #include "H264ProfileLevelId.h"
 
+#include "api/video_codecs/h265_profile_tier_level.h"
+
 using namespace ::webrtc;
 
 namespace unity

Plugin~/WebRTCPlugin/Codec/H265ProfileLevelId.cpp

@@ -0,0 +1,101 @@
+#include "pch.h"
+#include "H265ProfileLevelId.h"
+#include "rtc_base/checks.h"
+#include <rtc_base/arraysize.h>
+
+namespace unity
+{
+namespace webrtc
+{
+
+    // H.265 Level constraints based on ITU-T H.265 Table A-6
+    // max_luma_ps: Max number of luma samples per second (maxLumaSr)
+    // max_luma_ps: Max number of luma samples per picture (maxLumaPs)
+    // max_br: Max bitrate (in kbps)
+    struct H265LevelConstraint {
+        uint32_t max_luma_sr;        // Max luma samples per second
+        uint32_t max_luma_ps;        // Max luma samples per picture
+        uint32_t max_br_main;        // Max bitrate for Main tier (in kbps)
+        uint32_t max_br_high;        // Max bitrate for High tier (in kbps)
+        H265Level level;
+    };
+
+    // H.265 level constraints from ITU-T H.265 Table A-6 and A-7
+    // Values are based on the H.265 standard specifications
+    static constexpr H265LevelConstraint kH265LevelConstraints[] = {
+        // Level 1.0
+        { 552960, 36864, 128, 64, H265Level::kLevel1 },
+        // Level 2.0
+        { 3686400, 122880, 1500, 750, H265Level::kLevel2 },
+        // Level 2.1
+        { 7372800, 245760, 3000, 1500, H265Level::kLevel2_1 },
+        // Level 3.0
+        { 16588800, 552960, 6000, 3000, H265Level::kLevel3 },
+        // Level 3.1
+        { 33177600, 1228800, 10000, 5000, H265Level::kLevel3_1 },
+        // Level 4.0
+        { 62914560, 2097152, 12000, 6000, H265Level::kLevel4 },
+        // Level 4.1
+        { 62914560, 2097152, 20000, 10000, H265Level::kLevel4_1 },
+        // Level 5.0
+        { 116640000, 4194304, 25000, 12500, H265Level::kLevel5 },
+        // Level 5.1
+        { 233280000, 8388608, 40000, 20000, H265Level::kLevel5_1 },
+        // Level 5.2
+        { 466560000, 16777216, 60000, 30000, H265Level::kLevel5_2 },
+        // Level 6.0
+        { 833972800, 33554432, 60000, 30000, H265Level::kLevel6 },
+        // Level 6.1
+        { 1667942400, 67108864, 120000, 60000, H265Level::kLevel6_1 },
+        // Level 6.2
+        { 3349708800, 134217728, 240000, 120000, H265Level::kLevel6_2 }
+    };
+
+    absl::optional<H265Level> H265SupportedLevel(int maxFramePixelCount, int maxFramerate, int maxBitrate)
+    {
+        if (maxFramePixelCount <= 0 || maxFramerate <= 0 || maxBitrate <= 0)
+            return absl::nullopt;
+
+        // Calculate samples per second required
+        uint32_t requiredSamplesPerSec = static_cast<uint32_t>(maxFramePixelCount) * static_cast<uint32_t>(maxFramerate);
+
+        for (size_t i = 0; i < arraysize(kH265LevelConstraints); i++)
+        {
+            const H265LevelConstraint& constraint = kH265LevelConstraints[i];
+
+            // Check if this level can support the required parameters
+            // Use Main tier constraints by default (more restrictive)
+            if (constraint.max_luma_ps >= static_cast<uint32_t>(maxFramePixelCount) &&
+                constraint.max_luma_sr >= requiredSamplesPerSec &&
+                constraint.max_br_main >= static_cast<uint32_t>(maxBitrate))
+            {
+                return constraint.level;
+            }
+        }
+
+        // No level supported
+        return absl::nullopt;
+    }
+
+    int SupportedMaxFramerate(H265Level level, int maxFramePixelCount)
+    {
+        if (maxFramePixelCount <= 0)
+            return 0;
+
+        for (size_t i = 0; i < arraysize(kH265LevelConstraints); i++)
+        {
+            const H265LevelConstraint& constraint = kH265LevelConstraints[i];
+            if (constraint.level == level)
+            {
+                // Calculate max framerate based on max luma samples per second
+                // max_framerate = max_luma_sr / maxFramePixelCount
+                return static_cast<int>(constraint.max_luma_sr / maxFramePixelCount);
+            }
+        }
+
+        // Level not found
+        return 0;
+    }
+
+} // end namespace webrtc
+} // end namespace unity

Plugin~/WebRTCPlugin/Codec/H265ProfileLevelId.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <absl/types/optional.h>
+#include <api/video_codecs/h265_profile_tier_level.h>
+#include <api/video/resolution.h>
+
+namespace unity
+{
+namespace webrtc
+{
+    using namespace ::webrtc;
+
+    // Returns the minimum level which can support given parameters.
+    absl::optional<H265Level> H265SupportedLevel(int maxFramePixelCount, int maxFramerate, int maxBitrate);
+
+    // Returns the max framerate that calculated by maxFramePixelCount for H.265.
+    int SupportedMaxFramerate(H265Level level, int maxFramePixelCount);
+
+} // end namespace webrtc
+} // end namespace unity

Plugin~/WebRTCPlugin/Codec/NvCodec/CMakeLists.txt

@@ -2,9 +2,11 @@ target_sources(
   WebRTCLib
   PRIVATE NvCodec.cpp
           NvCodec.h
-          NvDecoderImpl.cpp
+          NvDecoderImplH264.cpp
+          NvDecoderImplH265.cpp
           NvDecoderImpl.h
           NvEncoderCudaWithCUarray.cpp
           NvEncoderCudaWithCUarray.h
-          NvEncoderImpl.cpp
+          NvEncoderImplH264.cpp
+          NvEncoderImplH265.cpp
           NvEncoderImpl.h)