UI: Show warning on startup if selected encoder is missing

UI/data/locale/en-US.ini

@@ -1600,3 +1600,27 @@ MultitrackVideo.IncompatibleSettings.UpdateAndStartStreaming="Update Settings an
 MultitrackVideo.IncompatibleSettings.AudioChannels="%1 is not currently compatible with [Audio → General → Channels] set to '%2', %3"
 MultitrackVideo.IncompatibleSettings.AudioChannelsSingle="[Audio → General → Channels] needs to be set to '%1'"
 MultitrackVideo.IncompatibleSettings.AudioChannelsMultiple="%1 requires multiple different settings for [Audio → General → Channels]"
+
+# Missing Encoder Warning
+EncoderMissing.Title="Missing Encoders"
+EncoderMissing.Text="The following configured encoders are missing:\n<ul>\n%1\n</ul>The following errors were encountered:\n<ul>\n%2\n</ul>\nSee the <a href=\"%3\">Knowledge Base Article</a> for further information."
+EncoderMissing.Unknown="Unknown"
+
+## Nvenc specific errors
+EncoderMissing.NVENC.TestProgramFailedStartup="NVENC check program start failure"
+EncoderMissing.NVENC.TestProgramExitWithError="NVENC check process failure, exit code: %1"
+EncoderMissing.NVENC.TestProgramReadFailure="NVENC check process output read failure, exit code: %1"
+EncoderMissing.NVENC.TestProgramError="NVENC check failed with reason: %1 (code: %2)"
+EncoderMissing.NVENC.NoDevices="No NVIDIA GPUs found"
+EncoderMissing.NVENC.Unsupported.Kepler="The architecture (Kepler) of your GPU (%1) is no longer supported. Please use OBS 30.2 or earlier instead."
+EncoderMissing.NVENC.Reason.NvmlLoad="Failed loading NVML library"
+EncoderMissing.NVENC.Reason.NvmlInit="Failed initializing NVML"
+EncoderMissing.NVENC.Reason.NvencLoad="Failed loading NVENC library"
+EncoderMissing.NVENC.Reason.NvencInit="Failed initializing NVENC"
+EncoderMissing.NVENC.Reason.NvencVersion="NVENC version empty"
+EncoderMissing.NVENC.Reason.CudaLoad="Failed loading CUDA library"
+EncoderMissing.NVENC.Reason.CudaInit="Failed initializing CUDA"
+EncoderMissing.NVENC.Reason.CudaVersion="Invalid/Missing CUDA version"
+EncoderMissing.NVENC.Reason.DriverOutdated="Outdated driver"
+EncoderMissing.NVENC.Reason.NoSupportedDevices="No NVIDIA devices with NVENC support found"
+EncoderMissing.NVENC.Reason.SessionLimitExceeded="Encoder session limit exceeded"

UI/window-basic-main-profiles.cpp

@@ -20,13 +20,16 @@
 #include <string>
 #include <map>
 #include <tuple>
+#include <unordered_set>
 #include <obs.hpp>
+#include <util/pipe.h>
 #include <util/platform.h>
 #include <util/util.hpp>
 #include <QMessageBox>
 #include <QVariant>
 #include <QFileDialog>
 #include <qt-wrappers.hpp>
+
 #include "window-basic-main.hpp"
 #include "window-basic-auto-config.hpp"
 #include "window-namedialog.hpp"
@@ -670,6 +673,7 @@ void OBSBasic::ActivateProfile(const OBSProfile &profile, bool reset)
 		ResetProfileData();
 	}
 
+	CheckForMissingEncoders();
 	CheckForSimpleModeX264Fallback();
 
 	RefreshProfiles();
@@ -863,3 +867,156 @@ void OBSBasic::CheckForSimpleModeX264Fallback()
 		activeConfiguration.SaveSafe("tmp");
 	}
 }
+
+#ifndef __APPLE__
+/* These may also be useful for QSV/AMF tests so keep them here. */
+static auto args_deleter = [](os_process_args_t *args) {
+	os_process_args_destroy(args);
+};
+using OsProcessArgs = std::unique_ptr<os_process_args_t, decltype(args_deleter)>;
+
+static std::optional<std::pair<QStringView, QString>> CheckNVENCInternal()
+{
+	const std::vector<std::pair<std::string_view, std::string_view>> nvencFailureReasons = {
+		{"nvml_lib", "EncoderMissing.NVENC.Reason.NvmlLoad"},
+		// Some error messages are a prefix + numeric error code
+		{"nvml_init", "EncoderMissing.NVENC.Reason.NvmlInit"},
+		{"nvenc_lib", "EncoderMissing.NVENC.Reason.NvencLoad"},
+		{"nvenc_init", "EncoderMissing.NVENC.Reason.NvencInit"},
+		{"cuda_lib", "EncoderMissing.NVENC.Reason.CudaLoad"},
+		{"cuda_init", "EncoderMissing.NVENC.Reason.CudaInit"},
+		{"no_cuda_version", "EncoderMissing.NVENC.Reason.CudaVersion"},
+		{"no_devices", "EncoderMissing.NVENC.NoDevices"},
+		{"no_nvenc_version", "EncoderMissing.NVENC.Reason.NvencVersion"},
+		{"outdated_driver", "EncoderMissing.NVENC.Reason.DriverOutdated"},
+		{"no_supported_devices", "EncoderMissing.NVENC.Reason.NoSupportedDevices"},
+		{"session_limit", "EncoderMissing.NVENC.Reason.SessionLimitExceeded"},
+	};
+
+#ifdef _WIN32
+	BPtr test_exe = os_get_executable_path_ptr("obs-nvenc-test.exe");
+#else
+	BPtr test_exe = os_get_executable_path_ptr("obs-nvenc-test");
+#endif
+
+	OsProcessArgs args{os_process_args_create(test_exe), args_deleter};
+
+	os_process_pipe_t *pp = os_process_pipe_create2(args.get(), "r");
+	if (!pp)
+		return std::make_pair(u"", QTStr("EncoderMissing.NVENC.TestProgramFailedStartup"));
+
+	std::string caps_result;
+	for (;;) {
+		char data[2048];
+		size_t len = os_process_pipe_read(pp, (uint8_t *)data, sizeof(data));
+		if (!len)
+			break;
+
+		caps_result.append(data, len);
+	}
+
+	int exit_code = os_process_pipe_destroy(pp);
+
+	if (caps_result.empty())
+		return std::make_pair(u"", QTStr("EncoderMissing.NVENC.TestProgramExitWithError").arg(exit_code));
+
+	auto config = ConfigFile();
+	if (config.OpenString(caps_result.c_str()) != CONFIG_SUCCESS)
+		return std::make_pair(u"", QTStr("EncoderMissing.NVENC.TestProgramReadFailure").arg(exit_code));
+
+	/* Read devices and attempt to find reason for failure */
+	auto numDevices = config_get_uint(config, "general", "cuda_devices");
+	if (!numDevices)
+		return std::make_pair(u"", QTStr("EncoderMissing.NVENC.NoDevices"));
+
+	/* Check device generation (architecture) */
+	for (uint64_t i = 0; i < numDevices; i++) {
+		std::string section = "device." + std::to_string(i);
+
+		if (!config_has_user_value(config, section.c_str(), "name"))
+			continue;
+
+		const QString name(config_get_string(config, section.c_str(), "name"));
+		const std::string_view arch = config_get_string(config, section.c_str(), "architecture_name");
+
+		if (arch == "Kepler")
+			return std::make_pair(u"kepler", QTStr("EncoderMissing.NVENC.Unsupported.Kepler").arg(name));
+	}
+
+	const std::string_view reason = config_get_string(config, "general", "reason");
+
+	/* All generic errors */
+	if (!reason.empty()) {
+		for (auto &[code, desc] : nvencFailureReasons) {
+			if (reason.find(code) == std::string_view::npos)
+				continue;
+
+			QString ret = QTStr("EncoderMissing.NVENC.TestProgramError").arg(QTStr(desc.data())).arg(code.data());
+			QString crumb = QString::fromUtf8(code.data());
+			return std::make_pair(crumb, ret);
+		}
+	}
+
+	return std::nullopt;
+}
+
+static std::optional<std::pair<QStringView, QString>> CheckNVENC()
+{
+	static auto result = CheckNVENCInternal();
+	return result;
+}
+#endif
+
+extern const char *get_simple_output_encoder(const char *name);
+
+void OBSBasic::CheckForMissingEncoders()
+{
+	constexpr QStringView kbURL(u"https://obsproject.com/kb/encoder-missing#%1");
+	constexpr QStringView encoderListItem(u"<li><code>%1<code></li>\n");
+	constexpr QStringView errorListItem(u"<li>%1</li>\n");
+
+	std::unordered_set<std::string_view> missing_encoders = {
+		config_get_string(activeConfiguration, "AdvOut", "Encoder"),
+		config_get_string(activeConfiguration, "AdvOut", "RecEncoder"),
+		get_simple_output_encoder(config_get_string(activeConfiguration, "SimpleOutput", "StreamEncoder")),
+		get_simple_output_encoder(config_get_string(activeConfiguration, "SimpleOutput", "RecEncoder")),
+	};
+
+	size_t idx = 0;
+	const char *id;
+	while (obs_enum_encoder_types(idx++, &id))
+		missing_encoders.erase(id);
+
+	if (missing_encoders.empty())
+		return;
+
+	QStringView crumb;
+	QString encoderList;
+	QString errorList;
+
+	for (auto &encoder : missing_encoders) {
+		encoderList += encoderListItem.arg(encoder.data());
+
+		std::optional<std::pair<QStringView, QString>> res;
+#ifndef __APPLE__
+		if (encoder.find("nvenc") != std::string_view::npos)
+			res = CheckNVENC();
+#endif
+
+		if (res) {
+			if (!errorList.contains(res->second))
+				errorList += errorListItem.arg(res->second);
+			if (crumb.empty())
+				crumb = res->first;
+		} else {
+			auto unknown = QTStr("EncoderMissing.Unknown");
+			if (!errorList.contains(unknown))
+				errorList += errorListItem.arg(unknown);
+		}
+	}
+
+	// Format and show error message
+	const QString encoderMissingMessage =
+		QTStr("EncoderMissing.Text").arg(encoderList).arg(errorList).arg(kbURL.arg(crumb));
+	OBSMessageBox::warning(this, QTStr("EncoderMissing.Title"), encoderMissingMessage, true);
+}

UI/window-basic-main.cpp

@@ -2121,6 +2121,7 @@ void OBSBasic::OBSInit()
 
 	InitBasicConfigDefaults2();
 
+	CheckForMissingEncoders();
 	CheckForSimpleModeX264Fallback();
 
 	LogEncoders();

UI/window-basic-main.hpp

@@ -1304,6 +1304,7 @@ public slots:
 	std::vector<std::string> GetRestartRequirements(const ConfigFile &config) const;
 	void ResetProfileData();
 	void CheckForSimpleModeX264Fallback();
+	void CheckForMissingEncoders();
 
 public:
 	inline const OBSProfileCache &GetProfileCache() const noexcept { return profiles; };