Skip to content

Consolidate Video Encoder Processing #1634

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mhoyer-streamlabs wants to merge 13 commits into
base: staging
Choose a base branch
from
Open

Consolidate Video Encoder Processing #1634

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
0223b4c
Consolidate encoder processing
mhoyer-streamlabs Feb 17, 2026
11543a6
Update CMakeLists.txt to add osn-encoders
mhoyer-streamlabs Feb 18, 2026
69a0ea9
Updates
mhoyer-streamlabs Feb 19, 2026
284a9c8
Update nodeobs_service.cpp
mhoyer-streamlabs Feb 25, 2026
accca08
Remove logging
mhoyer-streamlabs Feb 25, 2026
970d8f1
Updates
mhoyer-streamlabs Feb 17, 2026
ee9a056
formatting
mhoyer-streamlabs Feb 25, 2026
8b2bc3c
Merge branch 'mh-consolidate-encoders' of https://github.com/streamla…
mhoyer-streamlabs Feb 25, 2026
2f74fe3
Update osn-encoders.cpp
mhoyer-streamlabs Feb 25, 2026
c45512e
Code review updates
mhoyer-streamlabs Feb 26, 2026
e6bf282
formatting
mhoyer-streamlabs Feb 26, 2026
ab8b588
Code review updates
mhoyer-streamlabs Feb 27, 2026
13e18b1
formatting
mhoyer-streamlabs Feb 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions obs-studio-server/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,8 @@ SET(osn-server_SOURCES
"${PROJECT_SOURCE_DIR}/source/osn-advanced-replay-buffer.hpp"
"${PROJECT_SOURCE_DIR}/source/osn-output-signals.cpp"
"${PROJECT_SOURCE_DIR}/source/osn-output-signals.hpp"
"${PROJECT_SOURCE_DIR}/source/osn-encoders.cpp"
"${PROJECT_SOURCE_DIR}/source/osn-encoders.hpp"

###### utlity graphics ######
"${PROJECT_SOURCE_DIR}/source/gs-limits.h"
Expand Down
2 changes: 1 addition & 1 deletion obs-studio-server/source/nodeobs_api.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1008,7 +1008,7 @@ void OBS_API::OBS_API_initAPI(void *data, const int64_t id, const std::vector<ip

OBS_service::createVideoStreamingEncoder(StreamServiceId::Main);
OBS_service::createVideoStreamingEncoder(StreamServiceId::Second);
OBS_service::createVideoRecordingEncoder();
OBS_service::createDefaultSimpleVideoRecordingEncoder();

OBS_service::resetAudioContext();

Expand Down
1 change: 1 addition & 0 deletions obs-studio-server/source/nodeobs_autoconfig.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
#include <future>
#include "osn-error.hpp"
#include "shared.hpp"
#include "osn-encoders.hpp"

enum class Type { Invalid, Streaming, Recording };

Expand Down
1 change: 1 addition & 0 deletions obs-studio-server/source/nodeobs_configManager.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
#include <util/platform.h>
#include "shared.hpp"
#include "nodeobs_service.h"
#include "osn-encoders.hpp"

void ConfigManager::setAppdataPath(const std::string &path)
{
Expand Down
136 changes: 40 additions & 96 deletions obs-studio-server/source/nodeobs_service.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
#include "utility.hpp"
#include <osn-video.hpp>
#include "osn-vcam.hpp"
#include "osn-encoders.hpp"

#include "osn-multitrack-video-configuration.hpp"
#include "osn-audio-bitrate.hpp"
Expand Down Expand Up @@ -827,7 +828,7 @@ bool OBS_service::createVideoStreamingEncoder(StreamServiceId serviceId)
encoderId = config_get_string(ConfigManager::getInstance().getBasic(), "AdvOut", "Encoder");
}

if (encoderId == NULL || !EncoderAvailable(encoderId) || isInvalidEncoder(encoderId)) {
if (encoderId == NULL || !osn::EncoderUtils::isEncoderRegistered(encoderId) || osn::EncoderUtils::isInvalidAppleEncoder(encoderId)) {
encoderId = ADVANCED_ENCODER_X264;
}

Expand Down Expand Up @@ -1029,7 +1030,7 @@ static void remove_reserved_file_characters(std::string &s)
replace(s.begin(), s.end(), '<', '_');
}

bool OBS_service::createVideoRecordingEncoder()
bool OBS_service::createDefaultSimpleVideoRecordingEncoder()
{
std::string encoderName = GetVideoEncoderName(StreamServiceId::Main, true, true, ADVANCED_ENCODER_X264);
obs_encoder_t *newRecordingEncoder = obs_video_encoder_create(ADVANCED_ENCODER_X264, encoderName.c_str(), nullptr, nullptr);
Expand Down Expand Up @@ -1549,46 +1550,15 @@ void OBS_service::LoadRecordingPreset_Lossy(const char *encoderId)
throw "Failed to create video recording encoder (simple output)";
}

const char *get_simple_output_encoder(const char *encoder)
{
if (strcmp(encoder, SIMPLE_ENCODER_X264) == 0) {
return ADVANCED_ENCODER_X264;
} else if (strcmp(encoder, SIMPLE_ENCODER_X264_LOWCPU) == 0) {
return ADVANCED_ENCODER_X264;
} else if (strcmp(encoder, SIMPLE_ENCODER_QSV) == 0) {
return "obs_qsv11_v2";
} else if (strcmp(encoder, SIMPLE_ENCODER_QSV_AV1) == 0) {
return "obs_qsv11_av1";
} else if (strcmp(encoder, SIMPLE_ENCODER_AMD) == 0) {
return ADVANCED_ENCODER_AMD;
} else if (strcmp(encoder, SIMPLE_ENCODER_AMD_HEVC) == 0) {
return ADVANCED_ENCODER_AMD_HEVC;
} else if (strcmp(encoder, SIMPLE_ENCODER_AMD_AV1) == 0) {
return "av1_texture_amf";
} else if ((strcmp(encoder, SIMPLE_ENCODER_NVENC) == 0) || (strcmp(encoder, ENCODER_NVENC_H264_TEX) == 0)) {
return EncoderAvailable(ENCODER_NVENC_H264_TEX) ? ENCODER_NVENC_H264_TEX : ADVANCED_ENCODER_NVENC;
} else if (strcmp(encoder, SIMPLE_ENCODER_NVENC_HEVC) == 0) {
return EncoderAvailable(ENCODER_NVENC_HEVC_TEX) ? ENCODER_NVENC_HEVC_TEX : "ffmpeg_hevc_nvenc";
} else if (strcmp(encoder, SIMPLE_ENCODER_NVENC_AV1) == 0) {
return ENCODER_NVENC_AV1_TEX;
} else if (strcmp(encoder, SIMPLE_ENCODER_APPLE_H264) == 0) {
return APPLE_HARDWARE_VIDEO_ENCODER_M1;
} else if (strcmp(encoder, SIMPLE_ENCODER_APPLE_HEVC) == 0) {
return "com.apple.videotoolbox.videoencoder.ave.hevc";
}

blog(LOG_WARNING, "get_simple_output_encoder - encoder %s is not found, creating a default one", encoder);

return ADVANCED_ENCODER_X264;
}

void OBS_service::updateVideoRecordingEncoder(bool isSimpleMode)
{
if (isRecording && rpUsesRec)
return;

const char *section = isSimpleMode ? "SimpleOutput" : "AdvOut";

const char *quality = config_get_string(ConfigManager::getInstance().getBasic(), "SimpleOutput", "RecQuality");
const char *encoder = config_get_string(ConfigManager::getInstance().getBasic(), "SimpleOutput", "RecEncoder");
const char *encoder = config_get_string(ConfigManager::getInstance().getBasic(), section, "RecEncoder");

videoEncoder = encoder;
videoQuality = quality;
Expand All @@ -1598,12 +1568,11 @@ void OBS_service::updateVideoRecordingEncoder(bool isSimpleMode)
lowCPUx264 = false;
if (strcmp(encoder, SIMPLE_ENCODER_X264_LOWCPU) == 0)
lowCPUx264 = true;
LoadRecordingPreset_Lossy(get_simple_output_encoder(encoder));
LoadRecordingPreset_Lossy((osn::EncoderUtils::getInternalEncoderFromSimple(encoder)).c_str());
usingRecordingPreset = true;
updateVideoRecordingEncoderSettings();
} else {
const char *recordingEncoder = config_get_string(ConfigManager::getInstance().getBasic(), "AdvOut", "RecEncoder");
if (recordingEncoder && strcmp(recordingEncoder, ENCODER_NVENC_H264_TEX) != 0) {
if (encoder && strcmp(encoder, ENCODER_NVENC_H264_TEX) != 0) {
unsigned int cx = 0;
unsigned int cy = 0;

Expand Down Expand Up @@ -2036,39 +2005,33 @@ void OBS_service::updateVideoStreamingEncoder(bool isSimpleMode, StreamServiceId
bool enforceBitrate = config_get_bool(ConfigManager::getInstance().getBasic(), "SimpleOutput", "EnforceBitrate");
const char *custom = config_get_string(ConfigManager::getInstance().getBasic(), "SimpleOutput", "x264Settings");
const char *encoder = config_get_string(ConfigManager::getInstance().getBasic(), "SimpleOutput", "StreamEncoder");
const char *encoderID = nullptr;
const char *presetType = nullptr;
std::string encoderID = "";
std::string presetType = "";
const char *preset = nullptr;

if (encoder != NULL) {
if (strcmp(encoder, SIMPLE_ENCODER_QSV) == 0 || strcmp(encoder, ADVANCED_ENCODER_QSV) == 0) {
presetType = "QSVPreset";
encoderID = ADVANCED_ENCODER_QSV;
} else if (strcmp(encoder, SIMPLE_ENCODER_AMD) == 0 || strcmp(encoder, ADVANCED_ENCODER_AMD) == 0) {
presetType = "AMDPreset";
UpdateStreamingSettings_amd(h264Settings, videoBitrate);
encoderID = ADVANCED_ENCODER_AMD;
} else if (strcmp(encoder, SIMPLE_ENCODER_NVENC) == 0 || strcmp(encoder, ADVANCED_ENCODER_NVENC) == 0) {
presetType = "NVENCPreset";
encoderID = ADVANCED_ENCODER_NVENC;
} else if (strcmp(encoder, ENCODER_NVENC_H264_TEX) == 0) {
presetType = "NVENCPreset";
encoderID = ENCODER_NVENC_H264_TEX;
} else if (strcmp(encoder, APPLE_HARDWARE_VIDEO_ENCODER) == 0) {
encoderID = APPLE_HARDWARE_VIDEO_ENCODER;
} else if (strcmp(encoder, APPLE_HARDWARE_VIDEO_ENCODER_M1) == 0) {
encoderID = APPLE_HARDWARE_VIDEO_ENCODER_M1;
} else {
presetType = "Preset";
encoderID = ADVANCED_ENCODER_X264;
std::string presetType = osn::EncoderUtils::getEncoderPreset(encoder);
encoderID = osn::EncoderUtils::getInternalEncoderFromSimple(encoder);

if (!presetType.empty()) {
preset = config_get_string(ConfigManager::getInstance().getBasic(), "SimpleOutput", presetType.c_str());
//if this calls fails and preset type is NVENC, use legacy NVENC preset for backward compatibility
if (preset == NULL && presetType == PRESET_NVENC) {
presetType = PRESET_NVENC_DEP;
preset = config_get_string(ConfigManager::getInstance().getBasic(), "SimpleOutput", presetType.c_str());
if (preset != NULL) {
//convert the old preset to new
const char *oldValue = preset;
preset = osn::EncoderUtils::convertNvencSimplePreset(oldValue);
blog(LOG_INFO, "NVENC preset converted from %s to %s", oldValue, preset);
}
}
}
if (presetType)
preset = config_get_string(ConfigManager::getInstance().getBasic(), "SimpleOutput", presetType);

// Here and in other places we repeat the same pattern.
// Avoiding case when to an output there might not be any attached video encoder which can lead to crash.
std::string encoder_name = GetVideoEncoderName(serviceId, true, false, encoderID);
obs_encoder_t *streamingEncoder = obs_video_encoder_create(encoderID, encoder_name.c_str(), nullptr, nullptr);
std::string encoder_name = GetVideoEncoderName(serviceId, true, false, encoderID.c_str());
obs_encoder_t *streamingEncoder = obs_video_encoder_create(encoderID.c_str(), encoder_name.c_str(), nullptr, nullptr);
setStreamingEncoder(streamingEncoder, serviceId);
}

Expand Down Expand Up @@ -2109,8 +2072,7 @@ void OBS_service::updateVideoStreamingEncoder(bool isSimpleMode, StreamServiceId
obs_encoder_set_preferred_video_format(videoStreamingEncoder[serviceId], VIDEO_FORMAT_NV12);
}

if (strcmp(encoder, APPLE_SOFTWARE_VIDEO_ENCODER) == 0 || strcmp(encoder, APPLE_HARDWARE_VIDEO_ENCODER) == 0 ||
strcmp(encoder, APPLE_HARDWARE_VIDEO_ENCODER_M1) == 0) {
if (osn::EncoderUtils::getEncoderFamily(encoderID.c_str()) == FAMILY_APPLE) {
const char *profile = config_get_string(ConfigManager::getInstance().getBasic(), "SimpleOutput", "Profile");
if (profile)
obs_data_set_string(h264Settings, "profile", profile);
Expand Down Expand Up @@ -2575,29 +2537,22 @@ void OBS_service::updateVideoRecordingEncoderSettings()
{
bool ultra_hq = (videoQuality == "HQ");
int crf = CalcCRF(ultra_hq ? 16 : 23);
std::string encFamily = osn::EncoderUtils::getEncoderFamily(videoEncoder.c_str());

if (videoEncoder.compare(SIMPLE_ENCODER_X264) == 0 || videoEncoder.compare(ADVANCED_ENCODER_X264) == 0 ||
videoEncoder.compare(SIMPLE_ENCODER_X264_LOWCPU) == 0) {
if (encFamily == FAMILY_OBS)
UpdateRecordingSettings_x264_crf(crf);

} else if (videoEncoder.compare(SIMPLE_ENCODER_QSV) == 0 || videoEncoder.compare(ADVANCED_ENCODER_QSV) == 0) {
UpdateRecordingSettings_qsv11(crf);

} else if (videoEncoder.compare(SIMPLE_ENCODER_AMD) == 0 || videoEncoder.compare(SIMPLE_ENCODER_AMD_HEVC) == 0 ||
videoEncoder.compare(ADVANCED_ENCODER_AMD) == 0) {
UpdateRecordingSettings_amd_cqp(crf);

} else if (videoEncoder.compare(SIMPLE_ENCODER_NVENC) == 0 || videoEncoder.compare(ADVANCED_ENCODER_NVENC) == 0) {
else if (encFamily == FAMILY_NVENC)
UpdateRecordingSettings_nvenc(crf);
} else if (videoEncoder.compare(ENCODER_NVENC_H264_TEX) == 0) {
UpdateRecordingSettings_nvenc(crf);
} else if (videoEncoder.compare(SIMPLE_ENCODER_NVENC_HEVC) == 0) {
else if (encFamily == FAMILY_NVENC_HEVC)
UpdateRecordingSettings_nvenc_hevc(crf);
} else if (videoEncoder.compare(APPLE_SOFTWARE_VIDEO_ENCODER) == 0 || videoEncoder.compare(APPLE_HARDWARE_VIDEO_ENCODER) == 0 ||
videoEncoder.compare(APPLE_HARDWARE_VIDEO_ENCODER_M1) == 0) {
/* These are magic numbers. 0 - 100, more is better. */
else if (encFamily == FAMILY_QSV)
UpdateRecordingSettings_qsv11(crf);
else if (encFamily == FAMILY_AMD)
UpdateRecordingSettings_amd_cqp(crf);
else if (encFamily == FAMILY_APPLE)
UpdateRecordingSettings_apple(ultra_hq ? 70 : 50);
}
else
blog(LOG_WARNING, "Unable to update settings with unknown encoder family.");
}

obs_encoder_t *OBS_service::getStreamingEncoder(StreamServiceId serviceId)
Expand Down Expand Up @@ -3351,17 +3306,6 @@ void OBS_service::setupVodTrack(bool isSimpleMode)
}
}
}

bool OBS_service::isInvalidEncoder(const char *encoderID)
{
#if defined(__APPLE__)
// disable this encoder; not functioning properly
return strcmp(encoderID, APPLE_SOFTWARE_VIDEO_ENCODER) == 0;
#else
return false;
#endif
}

std::string GetFormatExt(const std::string container)
{
std::string ext = container;
Expand Down
44 changes: 1 addition & 43 deletions obs-studio-server/source/nodeobs_service.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,48 +44,7 @@
#include <unistd.h>
#endif

#ifdef WIN32
#define SIMPLE_ENCODER_X264 "x264"
#elif __APPLE__
#define SIMPLE_ENCODER_X264 "obs_x264"
#endif
#define SIMPLE_ENCODER_X264 "x264"
#define SIMPLE_ENCODER_X264_LOWCPU "x264_lowcpu"
#define SIMPLE_ENCODER_QSV "qsv"
#define SIMPLE_ENCODER_QSV_AV1 "qsv_av1"
#define SIMPLE_ENCODER_NVENC "nvenc"
#define SIMPLE_ENCODER_NVENC_AV1 "nvenc_av1"
#define SIMPLE_ENCODER_NVENC_HEVC "nvenc_hevc"
#define SIMPLE_ENCODER_AMD "amd"
#define SIMPLE_ENCODER_AMD_HEVC "amd_hevc"
#define SIMPLE_ENCODER_AMD_AV1 "amd_av1"
#define SIMPLE_ENCODER_APPLE_H264 "apple_h264"
#define SIMPLE_ENCODER_APPLE_HEVC "apple_hevc"

#define ADVANCED_ENCODER_X264 "obs_x264"
#define ADVANCED_ENCODER_QSV "obs_qsv11"
#define ADVANCED_ENCODER_NVENC "ffmpeg_nvenc"
#define ADVANCED_ENCODER_AMD "h264_texture_amf"
#define ADVANCED_ENCODER_AMD_HEVC "h265_texture_amf"

#define ENCODER_NVENC_H264_TEX "obs_nvenc_h264_tex"
#define ENCODER_NVENC_HEVC_TEX "obs_nvenc_hevc_tex"
#define ENCODER_NVENC_AV1_TEX "obs_nvenc_av1_tex"

// These 3 are deprecated
#define ENCODER_JIM_NVENC "jim_nvenc"
#define ENCODER_JIM_HEVC_NVENC "jim_hevc_nvenc"
#define ENCODER_JIM_AV1_NVENC "jim_av1_nvenc"

#define ENCODER_AV1_SVT_FFMPEG "ffmpeg_svt_av1"
#define ENCODER_AV1_AOM_FFMPEG "ffmpeg_aom_av1"

#define APPLE_SOFTWARE_VIDEO_ENCODER "com.apple.videotoolbox.videoencoder.h264"
#define APPLE_HARDWARE_VIDEO_ENCODER "com.apple.videotoolbox.videoencoder.h264.gva"
#define APPLE_HARDWARE_VIDEO_ENCODER_M1 "com.apple.videotoolbox.videoencoder.ave.avc"

#define ARCHIVE_NAME "archive_aac"

#define SIMPLE_AUDIO_ENCODER_AAC "ffmpeg_aac"
#define SIMPLE_AUDIO_ENCODER_OPUS "ffmpeg_opus"

Expand Down Expand Up @@ -211,7 +170,7 @@ class OBS_service {
static bool createVideoStreamingEncoder(StreamServiceId serviceId);
static std::string GetVideoEncoderName(StreamServiceId serviceId, bool isSimpleMode, bool recording, const char *encoder);
static void createAudioStreamingEncoder(StreamServiceId serviceId, bool isSimpleMode, const std::string &encoder_id);
static bool createVideoRecordingEncoder();
static bool createDefaultSimpleVideoRecordingEncoder();
static obs_encoder_t *getStreamingEncoder(StreamServiceId serviceId);
static void setStreamingEncoder(obs_encoder_t *encoder, StreamServiceId serviceId);
static obs_encoder_t *getRecordingEncoder(void);
Expand Down Expand Up @@ -293,5 +252,4 @@ class OBS_service {
static void stopAllOutputs(void);
static void setupVodTrack(bool isSimpleMode);
static void clearArchiveVodEncoder();
static bool isInvalidEncoder(const char *encoderID);
};
Loading
Loading