From ab2eb3b2851e07e24c1cecd2b877381c9c9041a8 Mon Sep 17 00:00:00 2001 From: Yury Date: Tue, 17 Jan 2023 01:05:59 +0200 Subject: [PATCH 1/5] Request configuration headers from encoder --- alvr/client_core/src/c_api.rs | 6 +++ alvr/client_core/src/decoder.rs | 2 +- alvr/client_core/src/lib.rs | 7 +++ .../cpp/alvr_server/ClientConnection.cpp | 53 ------------------- alvr/server/cpp/alvr_server/alvr_server.cpp | 6 +++ alvr/server/cpp/alvr_server/bindings.h | 1 + alvr/server/cpp/platform/linux/CEncoder.cpp | 12 +++-- alvr/server/cpp/platform/linux/CEncoder.h | 2 + .../cpp/platform/linux/EncodePipeline.h | 1 + .../cpp/platform/linux/EncodePipelineAMF.cpp | 17 +++--- .../cpp/platform/linux/EncodePipelineAMF.h | 2 +- .../platform/linux/EncodePipelineNvEnc.cpp | 6 +++ .../cpp/platform/linux/EncodePipelineNvEnc.h | 1 + .../cpp/platform/linux/EncodePipelineSW.cpp | 6 +++ .../cpp/platform/linux/EncodePipelineSW.h | 1 + .../platform/linux/EncodePipelineVAAPI.cpp | 6 +++ .../cpp/platform/linux/EncodePipelineVAAPI.h | 1 + alvr/server/cpp/platform/win32/CEncoder.cpp | 4 ++ alvr/server/cpp/platform/win32/CEncoder.h | 2 + alvr/server/cpp/platform/win32/VideoEncoder.h | 1 + .../cpp/platform/win32/VideoEncoderNVENC.cpp | 8 ++- .../cpp/platform/win32/VideoEncoderNVENC.h | 2 + .../cpp/platform/win32/VideoEncoderSW.cpp | 7 ++- .../cpp/platform/win32/VideoEncoderSW.h | 2 + .../cpp/platform/win32/VideoEncoderVCE.cpp | 23 ++++---- .../cpp/platform/win32/VideoEncoderVCE.h | 3 ++ alvr/server/src/connection.rs | 1 + alvr/sockets/src/packets.rs | 1 + 28 files changed, 103 insertions(+), 81 deletions(-) diff --git a/alvr/client_core/src/c_api.rs b/alvr/client_core/src/c_api.rs index 7407966612..3cd3fd4d98 100644 --- a/alvr/client_core/src/c_api.rs +++ b/alvr/client_core/src/c_api.rs @@ -443,6 +443,12 @@ pub extern "C" fn alvr_request_idr() { crate::request_idr(); } +/// Call only with external decoder +#[no_mangle] +pub extern "C" fn alvr_request_config_nal() { + crate::request_config_nal(); +} + /// Call only with external decoder #[no_mangle] pub extern "C" fn alvr_report_frame_decoded(target_timestamp_ns: u64) { diff --git a/alvr/client_core/src/decoder.rs b/alvr/client_core/src/decoder.rs index 6ba1456896..87105b56c3 100644 --- a/alvr/client_core/src/decoder.rs +++ b/alvr/client_core/src/decoder.rs @@ -85,7 +85,7 @@ pub extern "C" fn push_nal(buffer: *const c_char, length: i32, timestamp_ns: u64 show_err(decoder.push_frame_nal(timestamp, &nal, Duration::from_millis(500))); } else if let Some(sender) = &*crate::CONTROL_CHANNEL_SENDER.lock() { sender - .send(alvr_sockets::ClientControlPacket::RequestIdr) + .send(alvr_sockets::ClientControlPacket::RequestConfigNAL) .ok(); } } diff --git a/alvr/client_core/src/lib.rs b/alvr/client_core/src/lib.rs index 81dac1b1fc..b6147ebbaa 100644 --- a/alvr/client_core/src/lib.rs +++ b/alvr/client_core/src/lib.rs @@ -221,6 +221,13 @@ pub fn request_idr() { } } +/// Call only with external decoder +pub fn request_config_nal() { + if let Some(sender) = &*CONTROL_CHANNEL_SENDER.lock() { + sender.send(ClientControlPacket::RequestConfigNAL).ok(); + } +} + /// Call only with external decoder pub fn report_frame_decoded(target_timestamp: Duration) { if let Some(stats) = &mut *STATISTICS_MANAGER.lock() { diff --git a/alvr/server/cpp/alvr_server/ClientConnection.cpp b/alvr/server/cpp/alvr_server/ClientConnection.cpp index d21dc6f3f1..7f69bc9788 100644 --- a/alvr/server/cpp/alvr_server/ClientConnection.cpp +++ b/alvr/server/cpp/alvr_server/ClientConnection.cpp @@ -8,9 +8,6 @@ #include "Utils.h" #include "Settings.h" -static const uint8_t NAL_TYPE_SPS = 7; -static const uint8_t H265_NAL_TYPE_VPS = 32; - ClientConnection::ClientConnection() { m_Statistics = std::make_shared(); @@ -21,31 +18,6 @@ ClientConnection::ClientConnection() { m_fecPercentage = INITIAL_FEC_PERCENTAGE; } -int findVPSSPS(const uint8_t *frameBuffer, int frameByteSize) { - int zeroes = 0; - int foundNals = 0; - for (int i = 0; i < frameByteSize; i++) { - if (frameBuffer[i] == 0) { - zeroes++; - } else if (frameBuffer[i] == 1) { - if (zeroes >= 2) { - foundNals++; - if (Settings::Instance().m_codec == ALVR_CODEC_H264 && foundNals >= 3) { - // Find end of SPS+PPS on H.264. - return i - 3; - } else if (Settings::Instance().m_codec == ALVR_CODEC_H265 && foundNals >= 4) { - // Find end of VPS+SPS+PPS on H.264. - return i - 3; - } - } - zeroes = 0; - } else { - zeroes = 0; - } - } - return -1; -} - void ClientConnection::FECSend(uint8_t *buf, int len, uint64_t targetTimestampNs, uint64_t videoFrameIndex) { int shardPackets = CalculateFECShardPackets(len, m_fecPercentage); @@ -133,31 +105,6 @@ void ClientConnection::SendVideo(uint8_t *buf, int len, uint64_t targetTimestamp // Report before the frame is packetized ReportEncoded(targetTimestampNs); - uint8_t NALType; - if (Settings::Instance().m_codec == ALVR_CODEC_H264) - NALType = buf[4] & 0x1F; - else - NALType = (buf[4] >> 1) & 0x3F; - - if ((Settings::Instance().m_codec == ALVR_CODEC_H264 && NALType == NAL_TYPE_SPS) || - (Settings::Instance().m_codec == ALVR_CODEC_H265 && NALType == H265_NAL_TYPE_VPS)) { - // This frame contains (VPS + )SPS + PPS + IDR on NVENC H.264 (H.265) stream. - // (VPS + )SPS + PPS has short size (8bytes + 28bytes in some environment), so we can - // assume SPS + PPS is contained in first fragment. - - int end = findVPSSPS(buf, len); - if (end == -1) { - // Invalid frame. - return; - } - - InitializeDecoder((const unsigned char *)buf, end); - - // move the cursor forward excluding config NALs - buf = &buf[end]; - len = len - end; - } - if (Settings::Instance().m_enableFec) { FECSend(buf, len, targetTimestampNs, mVideoFrameIndex); } else { diff --git a/alvr/server/cpp/alvr_server/alvr_server.cpp b/alvr/server/cpp/alvr_server/alvr_server.cpp index e936a628d6..6b8b757df6 100644 --- a/alvr/server/cpp/alvr_server/alvr_server.cpp +++ b/alvr/server/cpp/alvr_server/alvr_server.cpp @@ -242,6 +242,12 @@ void RequestIDR() { } } +void RequestConfigNAL() { + if (g_driver_provider.hmd && g_driver_provider.hmd->m_encoder) { + g_driver_provider.hmd->m_encoder->GetConfigNAL(); + } +} + void SetTracking(unsigned long long targetTimestampNs, float controllerPoseTimeOffsetS, const AlvrDeviceMotion *deviceMotions, diff --git a/alvr/server/cpp/alvr_server/bindings.h b/alvr/server/cpp/alvr_server/bindings.h index 9027ab17ee..064db1680c 100644 --- a/alvr/server/cpp/alvr_server/bindings.h +++ b/alvr/server/cpp/alvr_server/bindings.h @@ -140,6 +140,7 @@ extern "C" void InitializeStreaming(); extern "C" void DeinitializeStreaming(); extern "C" void SendVSync(float frameIntervalS); extern "C" void RequestIDR(); +extern "C" void RequestConfigNAL(); extern "C" void SetTracking(unsigned long long targetTimestampNs, float controllerPoseTimeOffsetS, const AlvrDeviceMotion *deviceMotions, diff --git a/alvr/server/cpp/platform/linux/CEncoder.cpp b/alvr/server/cpp/platform/linux/CEncoder.cpp index 09df8dea0e..108a2c4199 100644 --- a/alvr/server/cpp/platform/linux/CEncoder.cpp +++ b/alvr/server/cpp/platform/linux/CEncoder.cpp @@ -220,7 +220,7 @@ void CEncoder::Run() { alvr::VkFrameCtx vk_frame_ctx(vk_ctx, output.imageInfo); alvr::VkFrame frame(vk_ctx, output.image, output.imageInfo, output.size, output.memory, output.drm); - auto encode_pipeline = alvr::EncodePipeline::Create(&render, vk_ctx, frame, vk_frame_ctx, render.GetEncodingWidth(), render.GetEncodingHeight()); + m_encodePipeline = alvr::EncodePipeline::Create(&render, vk_ctx, frame, vk_frame_ctx, render.GetEncodingWidth(), render.GetEncodingHeight()); fprintf(stderr, "CEncoder starting to read present packets"); present_packet frame_info; @@ -229,7 +229,7 @@ void CEncoder::Run() { read_latest(client, (char *)&frame_info, sizeof(frame_info), m_exiting); if (m_listener->GetStatistics()->CheckBitrateUpdated()) { - encode_pipeline->SetBitrate(m_listener->GetStatistics()->GetBitrate() * 1000000L); // in bits; + m_encodePipeline->SetBitrate(m_listener->GetStatistics()->GetBitrate() * 1000000L); // in bits; } auto pose = m_poseHistory->GetBestPoseMatch((const vr::HmdMatrix34_t&)frame_info.pose); @@ -246,19 +246,19 @@ void CEncoder::Run() { render.Render(frame_info.image, frame_info.semaphore_value); - encode_pipeline->PushFrame(pose->targetTimestampNs, m_scheduler.CheckIDRInsertion()); + m_encodePipeline->PushFrame(pose->targetTimestampNs, m_scheduler.CheckIDRInsertion()); static_assert(sizeof(frame_info.pose) == sizeof(vr::HmdMatrix34_t&)); encoded_data.clear(); uint64_t pts; - if (!encode_pipeline->GetEncoded(encoded_data, &pts)) { + if (!m_encodePipeline->GetEncoded(encoded_data, &pts)) { Error("Failed to get encoded data!"); continue; } auto render_timestamps = render.GetTimestamps(); - auto encode_timestamp = encode_pipeline->GetTimestamp(); + auto encode_timestamp = m_encodePipeline->GetTimestamp(); uint64_t present_offset = render_timestamps.now - render_timestamps.renderBegin; uint64_t composed_offset = 0; @@ -306,4 +306,6 @@ void CEncoder::OnPacketLoss() { m_scheduler.OnPacketLoss(); } void CEncoder::InsertIDR() { m_scheduler.InsertIDR(); } +void CEncoder::GetConfigNAL() { m_encodePipeline->GetConfigNAL(); } + void CEncoder::CaptureFrame() { m_captureFrame = true; } diff --git a/alvr/server/cpp/platform/linux/CEncoder.h b/alvr/server/cpp/platform/linux/CEncoder.h index 75f2513b79..d6c67b2ca8 100644 --- a/alvr/server/cpp/platform/linux/CEncoder.h +++ b/alvr/server/cpp/platform/linux/CEncoder.h @@ -20,6 +20,7 @@ class CEncoder : public CThread { void Stop(); void OnPacketLoss(); void InsertIDR(); + void GetConfigNAL(); bool IsConnected() { return m_connected; } void CaptureFrame(); @@ -27,6 +28,7 @@ class CEncoder : public CThread { void GetFds(int client, int (*fds)[6]); std::shared_ptr m_listener; std::shared_ptr m_poseHistory; + std::unique_ptr m_encodePipeline; std::atomic_bool m_exiting{false}; IDRScheduler m_scheduler; pollfd m_socket; diff --git a/alvr/server/cpp/platform/linux/EncodePipeline.h b/alvr/server/cpp/platform/linux/EncodePipeline.h index bf651e875d..e75a3ecee0 100644 --- a/alvr/server/cpp/platform/linux/EncodePipeline.h +++ b/alvr/server/cpp/platform/linux/EncodePipeline.h @@ -26,6 +26,7 @@ class EncodePipeline virtual void PushFrame(uint64_t targetTimestampNs, bool idr) = 0; virtual bool GetEncoded(std::vector & out, uint64_t *pts); + virtual void GetConfigNAL() = 0; virtual Timestamp GetTimestamp() { return timestamp; } virtual void SetBitrate(int64_t bitrate); diff --git a/alvr/server/cpp/platform/linux/EncodePipelineAMF.cpp b/alvr/server/cpp/platform/linux/EncodePipelineAMF.cpp index 59e9ee473c..2b35127102 100644 --- a/alvr/server/cpp/platform/linux/EncodePipelineAMF.cpp +++ b/alvr/server/cpp/platform/linux/EncodePipelineAMF.cpp @@ -500,18 +500,12 @@ void EncodePipelineAMF::ApplyFrameProperties(const amf::AMFSurfacePtr &surface, case ALVR_CODEC_H264: surface->SetProperty(AMF_VIDEO_ENCODER_INSERT_AUD, false); if (insertIDR) { - Debug("Inserting IDR frame for H.264.\n"); - surface->SetProperty(AMF_VIDEO_ENCODER_INSERT_SPS, true); - surface->SetProperty(AMF_VIDEO_ENCODER_INSERT_PPS, true); surface->SetProperty(AMF_VIDEO_ENCODER_FORCE_PICTURE_TYPE, AMF_VIDEO_ENCODER_PICTURE_TYPE_IDR); } break; case ALVR_CODEC_H265: surface->SetProperty(AMF_VIDEO_ENCODER_HEVC_INSERT_AUD, false); if (insertIDR) { - Debug("Inserting IDR frame for H.265.\n"); - // Insert VPS,SPS,PPS - surface->SetProperty(AMF_VIDEO_ENCODER_HEVC_INSERT_HEADER, true); surface->SetProperty(AMF_VIDEO_ENCODER_HEVC_FORCE_PICTURE_TYPE, AMF_VIDEO_ENCODER_HEVC_PICTURE_TYPE_IDR); } break; @@ -520,4 +514,15 @@ void EncodePipelineAMF::ApplyFrameProperties(const amf::AMFSurfacePtr &surface, } } +void EncodePipelineAMF::GetConfigNAL() { + amf::AMFVariant var; + if (m_codec == ALVR_CODEC_H264) { + m_amfComponents.back()->GetProperty(AMF_VIDEO_ENCODER_EXTRADATA, &var); + } else { + m_amfComponents.back()->GetProperty(AMF_VIDEO_ENCODER_HEVC_EXTRADATA, &var); + } + amf::AMFBufferPtr buffer(var.pInterface); + InitializeDecoder(reinterpret_cast(buffer->GetNative()), buffer->GetSize()); +} + }; diff --git a/alvr/server/cpp/platform/linux/EncodePipelineAMF.h b/alvr/server/cpp/platform/linux/EncodePipelineAMF.h index 6bc94a0df7..1d34d5a013 100644 --- a/alvr/server/cpp/platform/linux/EncodePipelineAMF.h +++ b/alvr/server/cpp/platform/linux/EncodePipelineAMF.h @@ -70,7 +70,7 @@ class EncodePipelineAMF : public EncodePipeline void PushFrame(uint64_t targetTimestampNs, bool idr) override; bool GetEncoded(std::vector &out, uint64_t *pts) override; void SetBitrate(int64_t bitrate) override; - + void GetConfigNAL() override; private: amf::AMFComponentPtr MakeConverter(amf::AMF_SURFACE_FORMAT inputFormat, int width, int height, amf::AMF_SURFACE_FORMAT outputFormat); amf::AMFComponentPtr MakePreprocessor(amf::AMF_SURFACE_FORMAT inputFormat, int width, int height); diff --git a/alvr/server/cpp/platform/linux/EncodePipelineNvEnc.cpp b/alvr/server/cpp/platform/linux/EncodePipelineNvEnc.cpp index a75a6afade..646839cfb2 100644 --- a/alvr/server/cpp/platform/linux/EncodePipelineNvEnc.cpp +++ b/alvr/server/cpp/platform/linux/EncodePipelineNvEnc.cpp @@ -120,6 +120,8 @@ alvr::EncodePipelineNvEnc::EncodePipelineNvEnc(Renderer *render, encoder_ctx->gop_size = INT16_MAX; encoder_ctx->bit_rate = settings.mEncodeBitrateMBs * 1000 * 1000; + encoder_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; + err = avcodec_open2(encoder_ctx, codec, NULL); if (err < 0) { throw alvr::AvException("Cannot open video encoder codec:", err); @@ -148,3 +150,7 @@ void alvr::EncodePipelineNvEnc::PushFrame(uint64_t targetTimestampNs, bool idr) throw alvr::AvException("avcodec_send_frame failed:", err); } } + +void alvr::EncodePipelineNvEnc::GetConfigNAL() { + InitializeDecoder(encoder_ctx->extradata, encoder_ctx->extradata_size); +} diff --git a/alvr/server/cpp/platform/linux/EncodePipelineNvEnc.h b/alvr/server/cpp/platform/linux/EncodePipelineNvEnc.h index c5e2d41f4d..975dd6b097 100644 --- a/alvr/server/cpp/platform/linux/EncodePipelineNvEnc.h +++ b/alvr/server/cpp/platform/linux/EncodePipelineNvEnc.h @@ -19,6 +19,7 @@ class EncodePipelineNvEnc: public EncodePipeline EncodePipelineNvEnc(Renderer *render, VkFrame &input_frame, VkFrameCtx& vk_frame_ctx, uint32_t width, uint32_t height); void PushFrame(uint64_t targetTimestampNs, bool idr) override; + void GetConfigNAL() override; private: Renderer *r = nullptr; diff --git a/alvr/server/cpp/platform/linux/EncodePipelineSW.cpp b/alvr/server/cpp/platform/linux/EncodePipelineSW.cpp index e23cbca918..57dbe68568 100644 --- a/alvr/server/cpp/platform/linux/EncodePipelineSW.cpp +++ b/alvr/server/cpp/platform/linux/EncodePipelineSW.cpp @@ -92,6 +92,8 @@ alvr::EncodePipelineSW::EncodePipelineSW(Renderer *render, uint32_t width, uint3 encoder_ctx->thread_type = FF_THREAD_SLICE; encoder_ctx->thread_count = settings.m_swThreadCount; + encoder_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; + int err = avcodec_open2(encoder_ctx, codec, &opt); if (err < 0) { throw alvr::AvException("Cannot open video encoder codec:", err); @@ -127,3 +129,7 @@ void alvr::EncodePipelineSW::PushFrame(uint64_t targetTimestampNs, bool idr) throw alvr::AvException("avcodec_send_frame failed:", err); } } + +void alvr::EncodePipelineSW::GetConfigNAL() { + InitializeDecoder(encoder_ctx->extradata, encoder_ctx->extradata_size); +} diff --git a/alvr/server/cpp/platform/linux/EncodePipelineSW.h b/alvr/server/cpp/platform/linux/EncodePipelineSW.h index e243e44475..c039290503 100644 --- a/alvr/server/cpp/platform/linux/EncodePipelineSW.h +++ b/alvr/server/cpp/platform/linux/EncodePipelineSW.h @@ -16,6 +16,7 @@ class EncodePipelineSW: public EncodePipeline EncodePipelineSW(Renderer *render, uint32_t width, uint32_t height); void PushFrame(uint64_t targetTimestampNs, bool idr) override; + void GetConfigNAL() override; private: AVFrame *encoder_frame = nullptr; diff --git a/alvr/server/cpp/platform/linux/EncodePipelineVAAPI.cpp b/alvr/server/cpp/platform/linux/EncodePipelineVAAPI.cpp index 753f18eebc..db5ab9316f 100644 --- a/alvr/server/cpp/platform/linux/EncodePipelineVAAPI.cpp +++ b/alvr/server/cpp/platform/linux/EncodePipelineVAAPI.cpp @@ -196,6 +196,8 @@ alvr::EncodePipelineVAAPI::EncodePipelineVAAPI(Renderer *render, VkContext &vk_c encoder_ctx->rc_max_rate = encoder_ctx->bit_rate; encoder_ctx->rc_buffer_size = encoder_ctx->bit_rate / settings.m_refreshRate; + encoder_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; + vlVaQualityBits quality = {}; quality.valid_setting = 1; quality.vbaq_mode = Settings::Instance().m_enableVbaq; //No noticable performance difference and should improve subjective quality by allocating more bits to smooth areas @@ -318,3 +320,7 @@ void alvr::EncodePipelineVAAPI::PushFrame(uint64_t targetTimestampNs, bool idr) } av_frame_unref(encoder_frame); } + +void alvr::EncodePipelineVAAPI::GetConfigNAL() { + InitializeDecoder(encoder_ctx->extradata, encoder_ctx->extradata_size); +} diff --git a/alvr/server/cpp/platform/linux/EncodePipelineVAAPI.h b/alvr/server/cpp/platform/linux/EncodePipelineVAAPI.h index bb946542c0..9401dd10e4 100644 --- a/alvr/server/cpp/platform/linux/EncodePipelineVAAPI.h +++ b/alvr/server/cpp/platform/linux/EncodePipelineVAAPI.h @@ -30,6 +30,7 @@ class EncodePipelineVAAPI: public EncodePipeline EncodePipelineVAAPI(Renderer *render, VkContext &vk_ctx, VkFrame &input_frame, uint32_t width, uint32_t height); void PushFrame(uint64_t targetTimestampNs, bool idr) override; + void GetConfigNAL() override; private: Renderer *r = nullptr; diff --git a/alvr/server/cpp/platform/win32/CEncoder.cpp b/alvr/server/cpp/platform/win32/CEncoder.cpp index 30cde9a706..10030af057 100644 --- a/alvr/server/cpp/platform/win32/CEncoder.cpp +++ b/alvr/server/cpp/platform/win32/CEncoder.cpp @@ -137,5 +137,9 @@ m_scheduler.InsertIDR(); } + void CEncoder::GetConfigNAL() { + m_videoEncoder->GetConfigNAL(); + } + void CEncoder::CaptureFrame() { } diff --git a/alvr/server/cpp/platform/win32/CEncoder.h b/alvr/server/cpp/platform/win32/CEncoder.h index 30511d4742..f91f7f9a95 100644 --- a/alvr/server/cpp/platform/win32/CEncoder.h +++ b/alvr/server/cpp/platform/win32/CEncoder.h @@ -55,6 +55,8 @@ void CaptureFrame(); + void GetConfigNAL(); + private: CThreadEvent m_newFrameReady, m_encodeFinished; std::shared_ptr m_videoEncoder; diff --git a/alvr/server/cpp/platform/win32/VideoEncoder.h b/alvr/server/cpp/platform/win32/VideoEncoder.h index 96faf71fc1..a71030e0a0 100644 --- a/alvr/server/cpp/platform/win32/VideoEncoder.h +++ b/alvr/server/cpp/platform/win32/VideoEncoder.h @@ -12,4 +12,5 @@ class VideoEncoder virtual void Shutdown() = 0; virtual void Transmit(ID3D11Texture2D *pTexture, uint64_t presentationTime, uint64_t targetTimestampNs, bool insertIDR) = 0; + virtual void GetConfigNAL() = 0; }; diff --git a/alvr/server/cpp/platform/win32/VideoEncoderNVENC.cpp b/alvr/server/cpp/platform/win32/VideoEncoderNVENC.cpp index 1d2949ba52..8ccd39e634 100644 --- a/alvr/server/cpp/platform/win32/VideoEncoderNVENC.cpp +++ b/alvr/server/cpp/platform/win32/VideoEncoderNVENC.cpp @@ -268,4 +268,10 @@ void VideoEncoderNVENC::FillEncodeConfig(NV_ENC_INITIALIZE_PARAMS &initializePar if (Settings::Instance().m_nvencRcAverageBitrate != -1) { encodeConfig.rcParams.averageBitRate = Settings::Instance().m_nvencRcAverageBitrate; } -} \ No newline at end of file +} + +void VideoEncoderNVENC::GetConfigNAL() { + std::vector header; + m_NvNecoder->GetSequenceParams(header); + InitializeDecoder(header.data(), header.size()); +} diff --git a/alvr/server/cpp/platform/win32/VideoEncoderNVENC.h b/alvr/server/cpp/platform/win32/VideoEncoderNVENC.h index fe7bb54991..67cb33400f 100644 --- a/alvr/server/cpp/platform/win32/VideoEncoderNVENC.h +++ b/alvr/server/cpp/platform/win32/VideoEncoderNVENC.h @@ -24,6 +24,8 @@ class VideoEncoderNVENC : public VideoEncoder void Shutdown(); void Transmit(ID3D11Texture2D *pTexture, uint64_t presentationTime, uint64_t targetTimestampNs, bool insertIDR); + + void GetConfigNAL(); private: void FillEncodeConfig(NV_ENC_INITIALIZE_PARAMS &initializeParams, int refreshRate, int renderWidth, int renderHeight, uint64_t bitrateBits); diff --git a/alvr/server/cpp/platform/win32/VideoEncoderSW.cpp b/alvr/server/cpp/platform/win32/VideoEncoderSW.cpp index d2a2b945ab..c06c54c6bd 100644 --- a/alvr/server/cpp/platform/win32/VideoEncoderSW.cpp +++ b/alvr/server/cpp/platform/win32/VideoEncoderSW.cpp @@ -93,8 +93,7 @@ void VideoEncoderSW::Initialize() { av_dict_set(&opt, "nal-hrd", "vbr", 0); break; } - m_codecContext->rc_max_rate = Settings::Instance().mEncodeBitrateMBs * 1'000'000L; - m_codecContext->thread_count = Settings::Instance().m_swThreadCount; + m_codecContext->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; if((err = avcodec_open2(m_codecContext, codec, &opt))) throw MakeException("Cannot open video encoder codec: %d", err); @@ -297,4 +296,8 @@ AVCodecID VideoEncoderSW::ToFFMPEGCodec(ALVR_CODEC codec) { } } +void VideoEncoderSW::GetConfigNAL() { + InitializeDecoder(m_codecContext->extradata, m_codecContext->extradata_size); +} + #endif // ALVR_GPL \ No newline at end of file diff --git a/alvr/server/cpp/platform/win32/VideoEncoderSW.h b/alvr/server/cpp/platform/win32/VideoEncoderSW.h index 6494d7bcb6..174879bd20 100644 --- a/alvr/server/cpp/platform/win32/VideoEncoderSW.h +++ b/alvr/server/cpp/platform/win32/VideoEncoderSW.h @@ -40,6 +40,8 @@ class VideoEncoderSW : public VideoEncoder void Transmit(ID3D11Texture2D *pTexture, uint64_t presentationTime, uint64_t targetTimestampNs, bool insertIDR); HRESULT SetupStagingTexture(ID3D11Texture2D *pTexture); HRESULT CopyTexture(ID3D11Texture2D *pTexture); + + void GetConfigNAL(); private: std::shared_ptr m_d3dRender; std::shared_ptr m_Listener; diff --git a/alvr/server/cpp/platform/win32/VideoEncoderVCE.cpp b/alvr/server/cpp/platform/win32/VideoEncoderVCE.cpp index 5cd30ff941..2ff675ac78 100644 --- a/alvr/server/cpp/platform/win32/VideoEncoderVCE.cpp +++ b/alvr/server/cpp/platform/win32/VideoEncoderVCE.cpp @@ -158,8 +158,6 @@ amf::AMFComponentPtr VideoEncoderVCE::MakeEncoder( switch (Settings::Instance().m_rateControlMode) { case ALVR_CBR: amfEncoder->SetProperty(AMF_VIDEO_ENCODER_RATE_CONTROL_METHOD, AMF_VIDEO_ENCODER_RATE_CONTROL_METHOD_CBR); - // Required for CBR to work correctly - amfEncoder->SetProperty(AMF_VIDEO_ENCODER_FILLER_DATA_ENABLE, true); break; case ALVR_VBR: amfEncoder->SetProperty(AMF_VIDEO_ENCODER_RATE_CONTROL_METHOD, AMF_VIDEO_ENCODER_RATE_CONTROL_METHOD_LATENCY_CONSTRAINED_VBR); @@ -222,8 +220,6 @@ amf::AMFComponentPtr VideoEncoderVCE::MakeEncoder( switch (Settings::Instance().m_rateControlMode) { case ALVR_CBR: amfEncoder->SetProperty(AMF_VIDEO_ENCODER_HEVC_RATE_CONTROL_METHOD, AMF_VIDEO_ENCODER_HEVC_RATE_CONTROL_METHOD_CBR); - // Required for CBR to work correctly - amfEncoder->SetProperty(AMF_VIDEO_ENCODER_HEVC_FILLER_DATA_ENABLE, true); break; case ALVR_VBR: amfEncoder->SetProperty(AMF_VIDEO_ENCODER_HEVC_RATE_CONTROL_METHOD, AMF_VIDEO_ENCODER_HEVC_RATE_CONTROL_METHOD_LATENCY_CONSTRAINED_VBR); @@ -463,9 +459,6 @@ void VideoEncoderVCE::ApplyFrameProperties(const amf::AMFSurfacePtr &surface, bo // FIXME: This option doesn't work in drivers 22.3.1 - 22.5.1, but works in 22.10.3 surface->SetProperty(AMF_VIDEO_ENCODER_INSERT_AUD, false); if (insertIDR) { - Debug("Inserting IDR frame for H.264.\n"); - surface->SetProperty(AMF_VIDEO_ENCODER_INSERT_SPS, true); - surface->SetProperty(AMF_VIDEO_ENCODER_INSERT_PPS, true); surface->SetProperty(AMF_VIDEO_ENCODER_FORCE_PICTURE_TYPE, AMF_VIDEO_ENCODER_PICTURE_TYPE_IDR); } break; @@ -473,11 +466,6 @@ void VideoEncoderVCE::ApplyFrameProperties(const amf::AMFSurfacePtr &surface, bo // FIXME: This option works with 22.10.3, but may not work with older drivers surface->SetProperty(AMF_VIDEO_ENCODER_HEVC_INSERT_AUD, false); if (insertIDR) { - Debug("Inserting IDR frame for H.265.\n"); - // Insert VPS,SPS,PPS - // These options don't work properly on older AMD driver (Radeon Software 17.7, AMF Runtime 1.4.4) - // Fixed in 18.9.2 & 1.4.9 - surface->SetProperty(AMF_VIDEO_ENCODER_HEVC_INSERT_HEADER, true); surface->SetProperty(AMF_VIDEO_ENCODER_HEVC_FORCE_PICTURE_TYPE, AMF_VIDEO_ENCODER_HEVC_PICTURE_TYPE_IDR); } break; @@ -528,3 +516,14 @@ void VideoEncoderVCE::SkipAUD(char **buffer, int *length) { *buffer += m_audNalSize; *length -= m_audNalSize; } + +void VideoEncoderVCE::GetConfigNAL() { + amf::AMFVariant var; + if (m_codec == ALVR_CODEC_H264) { + m_amfComponents.back()->GetProperty(AMF_VIDEO_ENCODER_EXTRADATA, &var); + } else { + m_amfComponents.back()->GetProperty(AMF_VIDEO_ENCODER_HEVC_EXTRADATA, &var); + } + amf::AMFBufferPtr buffer(var.pInterface); + InitializeDecoder(reinterpret_cast(buffer->GetNative()), buffer->GetSize()); +} \ No newline at end of file diff --git a/alvr/server/cpp/platform/win32/VideoEncoderVCE.h b/alvr/server/cpp/platform/win32/VideoEncoderVCE.h index de8c7cf448..b8db2dc732 100644 --- a/alvr/server/cpp/platform/win32/VideoEncoderVCE.h +++ b/alvr/server/cpp/platform/win32/VideoEncoderVCE.h @@ -6,6 +6,7 @@ #include "amf/public/include/components/VideoEncoderHEVC.h" #include "amf/public/include/components/VideoConverter.h" #include "amf/public/include/components/PreProcessing.h" +#include "amf/public/include/core/Buffer.h" #include "amf/public/common/AMFSTL.h" #include "amf/public/common/Thread.h" @@ -63,6 +64,8 @@ class VideoEncoderVCE : public VideoEncoder void Transmit(ID3D11Texture2D *pTexture, uint64_t presentationTime, uint64_t targetTimestampNs, bool insertIDR); void Receive(AMFDataPtr data); + + void GetConfigNAL(); private: static const wchar_t *START_TIME_PROPERTY; static const wchar_t *FRAME_INDEX_PROPERTY; diff --git a/alvr/server/src/connection.rs b/alvr/server/src/connection.rs index 0f3fb16273..95ea43e185 100644 --- a/alvr/server/src/connection.rs +++ b/alvr/server/src/connection.rs @@ -1023,6 +1023,7 @@ async fn connection_pipeline( } } Ok(ClientControlPacket::RequestIdr) => unsafe { crate::RequestIDR() }, + Ok(ClientControlPacket::RequestConfigNAL) => unsafe { crate::RequestConfigNAL() }, Ok(ClientControlPacket::VideoErrorReport) => unsafe { crate::VideoErrorReportReceive() }, diff --git a/alvr/sockets/src/packets.rs b/alvr/sockets/src/packets.rs index ecdc9aeb6a..baf18c6481 100644 --- a/alvr/sockets/src/packets.rs +++ b/alvr/sockets/src/packets.rs @@ -67,6 +67,7 @@ pub struct BatteryPacket { pub enum ClientControlPacket { PlayspaceSync(Vec2), RequestIdr, + RequestConfigNAL, KeepAlive, StreamReady, ViewsConfig(ViewsConfig), From 6a65144701ba496b0de68d69629bd8d4a2953614 Mon Sep 17 00:00:00 2001 From: Vixea <112600048+Vixea@users.noreply.github.com> Date: Wed, 18 Jan 2023 20:29:22 -0600 Subject: [PATCH 2/5] fix linux builds --- alvr/server/cpp/platform/linux/CEncoder.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/alvr/server/cpp/platform/linux/CEncoder.h b/alvr/server/cpp/platform/linux/CEncoder.h index d6c67b2ca8..8acdc98529 100644 --- a/alvr/server/cpp/platform/linux/CEncoder.h +++ b/alvr/server/cpp/platform/linux/CEncoder.h @@ -7,6 +7,8 @@ #include #include +#include "EncodePipeline.h" + class ClientConnection; class PoseHistory; From 2fe0bfa884a0de5a4d4d30e16f2c9a9536ad4e9d Mon Sep 17 00:00:00 2001 From: Yury Date: Sun, 2 Jul 2023 14:02:51 +0300 Subject: [PATCH 3/5] Revert accidental filler data removal --- alvr/server/cpp/platform/win32/VideoEncoderAMF.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/alvr/server/cpp/platform/win32/VideoEncoderAMF.cpp b/alvr/server/cpp/platform/win32/VideoEncoderAMF.cpp index 6be17be6b6..b604c8fb16 100644 --- a/alvr/server/cpp/platform/win32/VideoEncoderAMF.cpp +++ b/alvr/server/cpp/platform/win32/VideoEncoderAMF.cpp @@ -154,6 +154,8 @@ amf::AMFComponentPtr VideoEncoderAMF::MakeEncoder( switch (Settings::Instance().m_rateControlMode) { case ALVR_CBR: amfEncoder->SetProperty(AMF_VIDEO_ENCODER_RATE_CONTROL_METHOD, AMF_VIDEO_ENCODER_RATE_CONTROL_METHOD_CBR); + // Required for CBR to work correctly + amfEncoder->SetProperty(AMF_VIDEO_ENCODER_FILLER_DATA_ENABLE, Settings::Instance().m_fillerData); break; case ALVR_VBR: amfEncoder->SetProperty(AMF_VIDEO_ENCODER_RATE_CONTROL_METHOD, AMF_VIDEO_ENCODER_RATE_CONTROL_METHOD_LATENCY_CONSTRAINED_VBR); @@ -216,6 +218,8 @@ amf::AMFComponentPtr VideoEncoderAMF::MakeEncoder( switch (Settings::Instance().m_rateControlMode) { case ALVR_CBR: amfEncoder->SetProperty(AMF_VIDEO_ENCODER_HEVC_RATE_CONTROL_METHOD, AMF_VIDEO_ENCODER_HEVC_RATE_CONTROL_METHOD_CBR); + // Required for CBR to work correctly + amfEncoder->SetProperty(AMF_VIDEO_ENCODER_HEVC_FILLER_DATA_ENABLE, Settings::Instance().m_fillerData); break; case ALVR_VBR: amfEncoder->SetProperty(AMF_VIDEO_ENCODER_HEVC_RATE_CONTROL_METHOD, AMF_VIDEO_ENCODER_HEVC_RATE_CONTROL_METHOD_LATENCY_CONSTRAINED_VBR); From 81b3b17f9a9cdbeae31c3b2c31b62fb6ff78202e Mon Sep 17 00:00:00 2001 From: Yury Date: Sun, 2 Jul 2023 14:11:49 +0300 Subject: [PATCH 4/5] Add missing codec param --- alvr/server/cpp/platform/linux/EncodePipelineAMF.cpp | 2 +- alvr/server/cpp/platform/linux/EncodePipelineNvEnc.cpp | 2 +- alvr/server/cpp/platform/linux/EncodePipelineSW.cpp | 2 +- alvr/server/cpp/platform/linux/EncodePipelineVAAPI.cpp | 2 +- alvr/server/cpp/platform/win32/VideoEncoderNVENC.cpp | 2 +- alvr/server/cpp/platform/win32/VideoEncoderSW.cpp | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/alvr/server/cpp/platform/linux/EncodePipelineAMF.cpp b/alvr/server/cpp/platform/linux/EncodePipelineAMF.cpp index f2edc5a247..735613ac60 100644 --- a/alvr/server/cpp/platform/linux/EncodePipelineAMF.cpp +++ b/alvr/server/cpp/platform/linux/EncodePipelineAMF.cpp @@ -458,7 +458,7 @@ void EncodePipelineAMF::GetConfigNAL() { m_amfComponents.back()->GetProperty(AMF_VIDEO_ENCODER_HEVC_EXTRADATA, &var); } amf::AMFBufferPtr buffer(var.pInterface); - InitializeDecoder(reinterpret_cast(buffer->GetNative()), buffer->GetSize()); + InitializeDecoder(reinterpret_cast(buffer->GetNative()), buffer->GetSize(), GetCodec()); } }; diff --git a/alvr/server/cpp/platform/linux/EncodePipelineNvEnc.cpp b/alvr/server/cpp/platform/linux/EncodePipelineNvEnc.cpp index bf5f200bc7..a2ac88efce 100644 --- a/alvr/server/cpp/platform/linux/EncodePipelineNvEnc.cpp +++ b/alvr/server/cpp/platform/linux/EncodePipelineNvEnc.cpp @@ -148,5 +148,5 @@ void alvr::EncodePipelineNvEnc::PushFrame(uint64_t targetTimestampNs, bool idr) } void alvr::EncodePipelineNvEnc::GetConfigNAL() { - InitializeDecoder(encoder_ctx->extradata, encoder_ctx->extradata_size); + InitializeDecoder(encoder_ctx->extradata, encoder_ctx->extradata_size, GetCodec()); } diff --git a/alvr/server/cpp/platform/linux/EncodePipelineSW.cpp b/alvr/server/cpp/platform/linux/EncodePipelineSW.cpp index 88bc313027..edb8256efd 100644 --- a/alvr/server/cpp/platform/linux/EncodePipelineSW.cpp +++ b/alvr/server/cpp/platform/linux/EncodePipelineSW.cpp @@ -98,7 +98,7 @@ void alvr::EncodePipelineSW::PushFrame(uint64_t targetTimestampNs, bool idr) } void alvr::EncodePipelineSW::GetConfigNAL() { - InitializeDecoder(encoder_ctx->extradata, encoder_ctx->extradata_size); + InitializeDecoder(encoder_ctx->extradata, encoder_ctx->extradata_size, GetCodec()); } bool alvr::EncodePipelineSW::GetEncoded(FramePacket &packet) diff --git a/alvr/server/cpp/platform/linux/EncodePipelineVAAPI.cpp b/alvr/server/cpp/platform/linux/EncodePipelineVAAPI.cpp index 68d6eb68a4..eaed37fb24 100644 --- a/alvr/server/cpp/platform/linux/EncodePipelineVAAPI.cpp +++ b/alvr/server/cpp/platform/linux/EncodePipelineVAAPI.cpp @@ -334,7 +334,7 @@ void alvr::EncodePipelineVAAPI::PushFrame(uint64_t targetTimestampNs, bool idr) } void alvr::EncodePipelineVAAPI::GetConfigNAL() { - InitializeDecoder(encoder_ctx->extradata, encoder_ctx->extradata_size); + InitializeDecoder(encoder_ctx->extradata, encoder_ctx->extradata_size, GetCodec()); } void alvr::EncodePipelineVAAPI::SetParams(FfiDynamicEncoderParams params) diff --git a/alvr/server/cpp/platform/win32/VideoEncoderNVENC.cpp b/alvr/server/cpp/platform/win32/VideoEncoderNVENC.cpp index c3f2017b10..e1acce7775 100644 --- a/alvr/server/cpp/platform/win32/VideoEncoderNVENC.cpp +++ b/alvr/server/cpp/platform/win32/VideoEncoderNVENC.cpp @@ -290,5 +290,5 @@ void VideoEncoderNVENC::FillEncodeConfig(NV_ENC_INITIALIZE_PARAMS &initializePar void VideoEncoderNVENC::GetConfigNAL() { std::vector header; m_NvNecoder->GetSequenceParams(header); - InitializeDecoder(header.data(), header.size()); + InitializeDecoder(header.data(), header.size(), m_codec); } diff --git a/alvr/server/cpp/platform/win32/VideoEncoderSW.cpp b/alvr/server/cpp/platform/win32/VideoEncoderSW.cpp index 455dbbc145..e3556f8446 100644 --- a/alvr/server/cpp/platform/win32/VideoEncoderSW.cpp +++ b/alvr/server/cpp/platform/win32/VideoEncoderSW.cpp @@ -247,7 +247,7 @@ AVCodecID VideoEncoderSW::ToFFMPEGCodec(ALVR_CODEC codec) { } void VideoEncoderSW::GetConfigNAL() { - InitializeDecoder(m_codecContext->extradata, m_codecContext->extradata_size); + InitializeDecoder(m_codecContext->extradata, m_codecContext->extradata_size, m_codec); } #endif // ALVR_GPL \ No newline at end of file From 37beb7f9915fb65dbc5583178b256247547dfef8 Mon Sep 17 00:00:00 2001 From: Yury Date: Sun, 2 Jul 2023 14:14:52 +0300 Subject: [PATCH 5/5] Add missing RequestConfigNAL --- alvr/packets/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/alvr/packets/src/lib.rs b/alvr/packets/src/lib.rs index 3076cb40ad..46bd7c5f23 100644 --- a/alvr/packets/src/lib.rs +++ b/alvr/packets/src/lib.rs @@ -89,6 +89,7 @@ pub struct ButtonEntry { pub enum ClientControlPacket { PlayspaceSync(Option), RequestIdr, + RequestConfigNAL, KeepAlive, StreamReady, ViewsConfig(ViewsConfig),