From 02e32b220ad8a5057feb0ae45be3a2ac56bb09f3 Mon Sep 17 00:00:00 2001 From: shinyquagsire23 Date: Wed, 2 Oct 2024 17:44:17 -0600 Subject: [PATCH 1/8] Initial attempt at AV1, ffplay is happy with the AV1 stream, but the Quest 3 is not. --- .../cpp/platform/win32/VideoEncoderNVENC.cpp | 43 +++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/alvr/server_openvr/cpp/platform/win32/VideoEncoderNVENC.cpp b/alvr/server_openvr/cpp/platform/win32/VideoEncoderNVENC.cpp index a0c19644fd..772335f64b 100644 --- a/alvr/server_openvr/cpp/platform/win32/VideoEncoderNVENC.cpp +++ b/alvr/server_openvr/cpp/platform/win32/VideoEncoderNVENC.cpp @@ -155,8 +155,7 @@ void VideoEncoderNVENC::FillEncodeConfig( encoderGUID = NV_ENC_CODEC_HEVC_GUID; break; case ALVR_CODEC_AV1: - Warn("AV1 is not supported yet. Using HEVC instead."); - encoderGUID = NV_ENC_CODEC_HEVC_GUID; + encoderGUID = NV_ENC_CODEC_AV1_GUID; break; } @@ -265,6 +264,7 @@ void VideoEncoderNVENC::FillEncodeConfig( config.h264VUIParameters.colourMatrix = NV_ENC_VUI_MATRIX_COEFFS_BT709; } } + break; case ALVR_CODEC_HEVC: { auto& config = encodeConfig.encodeCodecConfig.hevcConfig; config.repeatSPSPPS = 1; @@ -305,9 +305,46 @@ void VideoEncoderNVENC::FillEncodeConfig( config.hevcVUIParameters.colourMatrix = NV_ENC_VUI_MATRIX_COEFFS_BT709; } } + break; case ALVR_CODEC_AV1: { - // todo + auto& config = encodeConfig.encodeCodecConfig.av1Config; + config.repeatSeqHdr = 1; + config.enableIntraRefresh = Settings::Instance().m_nvencEnableIntraRefresh; + + if (Settings::Instance().m_nvencIntraRefreshPeriod != -1) { + config.intraRefreshPeriod = Settings::Instance().m_nvencIntraRefreshPeriod; + } + if (Settings::Instance().m_nvencIntraRefreshCount != -1) { + config.intraRefreshCnt = Settings::Instance().m_nvencIntraRefreshCount; + } + + config.maxNumRefFramesInDPB = maxNumRefFrames; + config.idrPeriod = gopLength; + + if (Settings::Instance().m_use10bitEncoder) { + config.pixelBitDepthMinus8 = 2; + } + + if (Settings::Instance().m_fillerData) { + config.enableBitstreamPadding = Settings::Instance().m_rateControlMode == ALVR_CBR; + } + + config.chromaFormatIDC = 1; // 4:2:0, 4:4:4 currently not supported + config.colorRange + = Settings::Instance().m_useFullRangeEncoding ? 1 : 0; + if (Settings::Instance().m_enableHdr) { + config.colorPrimaries = NV_ENC_VUI_COLOR_PRIMARIES_BT2020; + config.transferCharacteristics + = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SRGB; + config.matrixCoefficients = NV_ENC_VUI_MATRIX_COEFFS_BT2020_NCL; + } else { + config.colorPrimaries = NV_ENC_VUI_COLOR_PRIMARIES_BT709; + config.transferCharacteristics + = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SRGB; + config.matrixCoefficients = NV_ENC_VUI_MATRIX_COEFFS_BT709; + } } + break; } // Disable automatic IDR insertion by NVENC. We need to manually insert IDR when packet is From 6a809d1e89cbd88894d2135d7352af7ac2162cb1 Mon Sep 17 00:00:00 2001 From: shinyquagsire23 Date: Fri, 4 Oct 2024 01:02:33 -0600 Subject: [PATCH 2/8] Trim out IVF bits for NVENC AV1 --- .../cpp/platform/win32/VideoEncoderNVENC.cpp | 42 +++++++++++++------ 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/alvr/server_openvr/cpp/platform/win32/VideoEncoderNVENC.cpp b/alvr/server_openvr/cpp/platform/win32/VideoEncoderNVENC.cpp index 772335f64b..35480dbb2a 100644 --- a/alvr/server_openvr/cpp/platform/win32/VideoEncoderNVENC.cpp +++ b/alvr/server_openvr/cpp/platform/win32/VideoEncoderNVENC.cpp @@ -133,7 +133,29 @@ void VideoEncoderNVENC::Transmit( fpOut.write(reinterpret_cast(packet.data()), packet.size()); } - ParseFrameNals(m_codec, packet.data(), (int)packet.size(), targetTimestampNs, insertIDR); + uint8_t* buf = packet.data(); + int len = (int)packet.size(); + + // NVENC's AV1 encoding includes a bunch of IVF wrapping, + // so we need to strip it down to just the OBUs + if (m_codec == ALVR_CODEC_AV1) { + const uint8_t ivf_magic[4] = { 0x44, 0x4B, 0x49, 0x46 }; + if (!memcmp(buf, ivf_magic, 4)) { + buf += 0x20; + len -= 0x20; + } + if (len <= 0xC) { + continue; + } + buf += 0xC; // skip past the IVF packet size header thing + len -= 0xC; + } + + if (len <= 0) { + continue; + } + + ParseFrameNals(m_codec, buf, len, targetTimestampNs, insertIDR); } } @@ -263,8 +285,7 @@ void VideoEncoderNVENC::FillEncodeConfig( = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SRGB; config.h264VUIParameters.colourMatrix = NV_ENC_VUI_MATRIX_COEFFS_BT709; } - } - break; + } break; case ALVR_CODEC_HEVC: { auto& config = encodeConfig.encodeCodecConfig.hevcConfig; config.repeatSPSPPS = 1; @@ -304,8 +325,7 @@ void VideoEncoderNVENC::FillEncodeConfig( = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SRGB; config.hevcVUIParameters.colourMatrix = NV_ENC_VUI_MATRIX_COEFFS_BT709; } - } - break; + } break; case ALVR_CODEC_AV1: { auto& config = encodeConfig.encodeCodecConfig.av1Config; config.repeatSeqHdr = 1; @@ -330,21 +350,17 @@ void VideoEncoderNVENC::FillEncodeConfig( } config.chromaFormatIDC = 1; // 4:2:0, 4:4:4 currently not supported - config.colorRange - = Settings::Instance().m_useFullRangeEncoding ? 1 : 0; + config.colorRange = Settings::Instance().m_useFullRangeEncoding ? 1 : 0; if (Settings::Instance().m_enableHdr) { config.colorPrimaries = NV_ENC_VUI_COLOR_PRIMARIES_BT2020; - config.transferCharacteristics - = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SRGB; + config.transferCharacteristics = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SRGB; config.matrixCoefficients = NV_ENC_VUI_MATRIX_COEFFS_BT2020_NCL; } else { config.colorPrimaries = NV_ENC_VUI_COLOR_PRIMARIES_BT709; - config.transferCharacteristics - = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SRGB; + config.transferCharacteristics = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SRGB; config.matrixCoefficients = NV_ENC_VUI_MATRIX_COEFFS_BT709; } - } - break; + } break; } // Disable automatic IDR insertion by NVENC. We need to manually insert IDR when packet is From ff717b528de1922e4cb30e703f821b6bab8779b0 Mon Sep 17 00:00:00 2001 From: shinyquagsire23 Date: Fri, 4 Oct 2024 01:03:40 -0600 Subject: [PATCH 3/8] Working OSLog for iOS/macOS --- alvr/client_core/src/lib.rs | 5 +++++ alvr/client_core/src/logging_backend.rs | 19 ++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/alvr/client_core/src/lib.rs b/alvr/client_core/src/lib.rs index a1ef98f008..b31d7d09e3 100644 --- a/alvr/client_core/src/lib.rs +++ b/alvr/client_core/src/lib.rs @@ -89,6 +89,11 @@ pub struct ClientCoreContext { impl ClientCoreContext { pub fn new(capabilities: ClientCapabilities) -> Self { + #[cfg(any(target_os = "ios", target_os = "macos"))] + { + init_logging(); + } + dbg_client_core!("Create"); // Make sure to reset config in case of version compat mismatch. diff --git a/alvr/client_core/src/logging_backend.rs b/alvr/client_core/src/logging_backend.rs index 39dc601cfa..fccb2f1f40 100644 --- a/alvr/client_core/src/logging_backend.rs +++ b/alvr/client_core/src/logging_backend.rs @@ -106,7 +106,24 @@ pub fn init_logging() { .with_max_level(alvr_common::log::LevelFilter::Info), ); } - #[cfg(not(target_os = "android"))] + #[cfg(any(target_os = "ios", target_os = "macos"))] + { + use std::io::Write; + env_logger::builder() + .format(|f, record| { + if send_log(record) { + println!("[ALVR NATIVE-RUST] {}", record.args()); + Ok(()) + } else { + println!("[ALVR NATIVE-RUST] {}", record.args()); + Ok(()) + } + }) + .filter_level(alvr_common::log::LevelFilter::Info) + .try_init() + .ok(); + } + #[cfg(not(any(target_os = "android", target_os = "ios", target_os = "macos")))] { use std::io::Write; env_logger::builder() From 3763db6040353857de2bc68c6966ef21c0587b80 Mon Sep 17 00:00:00 2001 From: shinyquagsire23 Date: Fri, 4 Oct 2024 01:24:34 -0600 Subject: [PATCH 4/8] Not AMD only any more --- alvr/session/src/settings.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index 569290a157..2d2c8a0c98 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -488,7 +488,7 @@ pub enum CodecType { H264 = 0, #[schema(strings(display_name = "HEVC"))] Hevc = 1, - #[schema(strings(display_name = "AV1 (AMD only)"))] + #[schema(strings(display_name = "AV1"))] AV1 = 2, } From c02e3d2470c8d614caceb6d3cc1ddac125b0111c Mon Sep 17 00:00:00 2001 From: shinyquagsire23 Date: Fri, 4 Oct 2024 01:36:43 -0600 Subject: [PATCH 5/8] One more bounds check --- alvr/server_openvr/cpp/platform/win32/VideoEncoderNVENC.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alvr/server_openvr/cpp/platform/win32/VideoEncoderNVENC.cpp b/alvr/server_openvr/cpp/platform/win32/VideoEncoderNVENC.cpp index 35480dbb2a..51651cb570 100644 --- a/alvr/server_openvr/cpp/platform/win32/VideoEncoderNVENC.cpp +++ b/alvr/server_openvr/cpp/platform/win32/VideoEncoderNVENC.cpp @@ -140,7 +140,7 @@ void VideoEncoderNVENC::Transmit( // so we need to strip it down to just the OBUs if (m_codec == ALVR_CODEC_AV1) { const uint8_t ivf_magic[4] = { 0x44, 0x4B, 0x49, 0x46 }; - if (!memcmp(buf, ivf_magic, 4)) { + if (len >= 4 && !memcmp(buf, ivf_magic, 4)) { buf += 0x20; len -= 0x20; } From fb163337da97cda6d104b9eca6970da00094b7a7 Mon Sep 17 00:00:00 2001 From: shinyquagsire23 Date: Fri, 4 Oct 2024 01:48:50 -0600 Subject: [PATCH 6/8] Fix writing AV1 to files --- .../cpp/platform/win32/VideoEncoderNVENC.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/alvr/server_openvr/cpp/platform/win32/VideoEncoderNVENC.cpp b/alvr/server_openvr/cpp/platform/win32/VideoEncoderNVENC.cpp index 51651cb570..6fcbd3c2e4 100644 --- a/alvr/server_openvr/cpp/platform/win32/VideoEncoderNVENC.cpp +++ b/alvr/server_openvr/cpp/platform/win32/VideoEncoderNVENC.cpp @@ -129,10 +129,6 @@ void VideoEncoderNVENC::Transmit( m_NvNecoder->EncodeFrame(vPacket, &picParams); for (std::vector& packet : vPacket) { - if (fpOut) { - fpOut.write(reinterpret_cast(packet.data()), packet.size()); - } - uint8_t* buf = packet.data(); int len = (int)packet.size(); @@ -155,6 +151,10 @@ void VideoEncoderNVENC::Transmit( continue; } + if (fpOut) { + fpOut.write(reinterpret_cast(buf), len); + } + ParseFrameNals(m_codec, buf, len, targetTimestampNs, insertIDR); } } From cfa72d44f6011370fb9ba521c83041572b6bdff0 Mon Sep 17 00:00:00 2001 From: shinyquagsire23 Date: Fri, 4 Oct 2024 10:09:07 -0600 Subject: [PATCH 7/8] Remove logging changes --- alvr/client_core/src/lib.rs | 5 ----- alvr/client_core/src/logging_backend.rs | 19 +------------------ 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/alvr/client_core/src/lib.rs b/alvr/client_core/src/lib.rs index b31d7d09e3..a1ef98f008 100644 --- a/alvr/client_core/src/lib.rs +++ b/alvr/client_core/src/lib.rs @@ -89,11 +89,6 @@ pub struct ClientCoreContext { impl ClientCoreContext { pub fn new(capabilities: ClientCapabilities) -> Self { - #[cfg(any(target_os = "ios", target_os = "macos"))] - { - init_logging(); - } - dbg_client_core!("Create"); // Make sure to reset config in case of version compat mismatch. diff --git a/alvr/client_core/src/logging_backend.rs b/alvr/client_core/src/logging_backend.rs index fccb2f1f40..39dc601cfa 100644 --- a/alvr/client_core/src/logging_backend.rs +++ b/alvr/client_core/src/logging_backend.rs @@ -106,24 +106,7 @@ pub fn init_logging() { .with_max_level(alvr_common::log::LevelFilter::Info), ); } - #[cfg(any(target_os = "ios", target_os = "macos"))] - { - use std::io::Write; - env_logger::builder() - .format(|f, record| { - if send_log(record) { - println!("[ALVR NATIVE-RUST] {}", record.args()); - Ok(()) - } else { - println!("[ALVR NATIVE-RUST] {}", record.args()); - Ok(()) - } - }) - .filter_level(alvr_common::log::LevelFilter::Info) - .try_init() - .ok(); - } - #[cfg(not(any(target_os = "android", target_os = "ios", target_os = "macos")))] + #[cfg(not(target_os = "android"))] { use std::io::Write; env_logger::builder() From ae78f9e0a2891c5c76d674c202c49b6b10e34d78 Mon Sep 17 00:00:00 2001 From: shinyquagsire23 Date: Fri, 4 Oct 2024 10:11:16 -0600 Subject: [PATCH 8/8] No hex --- .../cpp/platform/win32/VideoEncoderNVENC.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/alvr/server_openvr/cpp/platform/win32/VideoEncoderNVENC.cpp b/alvr/server_openvr/cpp/platform/win32/VideoEncoderNVENC.cpp index 6fcbd3c2e4..82d93aba0e 100644 --- a/alvr/server_openvr/cpp/platform/win32/VideoEncoderNVENC.cpp +++ b/alvr/server_openvr/cpp/platform/win32/VideoEncoderNVENC.cpp @@ -137,14 +137,14 @@ void VideoEncoderNVENC::Transmit( if (m_codec == ALVR_CODEC_AV1) { const uint8_t ivf_magic[4] = { 0x44, 0x4B, 0x49, 0x46 }; if (len >= 4 && !memcmp(buf, ivf_magic, 4)) { - buf += 0x20; - len -= 0x20; + buf += 32; + len -= 32; } - if (len <= 0xC) { + if (len <= 12) { continue; } - buf += 0xC; // skip past the IVF packet size header thing - len -= 0xC; + buf += 12; // skip past the IVF packet size header thing + len -= 12; } if (len <= 0) {