From 183228bbc0b8ae09666627f855cd6c6fd3c9f7a8 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Thu, 15 Oct 2026 18:57:00 +0200 Subject: [PATCH] feat: encode AV1 temporal layers with VA-API The VA-API AV1 encoder now takes L1T2 and L1T3, with WebRTC's reference structure and a rate control for each layer, rather than leaving them to libaom. A frame it cannot encode now falls back to libaom. --- docs/guide/advanced/video-codecs.md | 2 +- .../video/codec/EncoderOutputProcessor.h | 31 +- .../media/video/codec/linux/VaapiAv1Encoder.h | 53 +++- .../video/codec/linux/VaapiEncodeSession.h | 25 +- .../video/codec/EncoderOutputProcessor.cpp | 62 +++- .../video/codec/linux/VaapiAv1Encoder.cpp | 284 ++++++++++++++---- .../video/codec/linux/VaapiEncodeSession.cpp | 75 ++++- .../codec/HardwareVideoEncoderFactory.java | 3 +- .../HardwareVideoEncoderIntegrationTest.java | 74 +++++ 9 files changed, 507 insertions(+), 102 deletions(-) diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index c1063f01..e87bc22e 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -30,7 +30,7 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder() | Linux | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) where it encodes H.265 | NVENC on NVIDIA RTX 40 and newer, otherwise the VA-API encoder of the GPU driver where it encodes AV1 (e.g. AMD RDNA3 and newer) | | macOS | VideoToolbox, as with `DefaultVideoEncoderFactory` | VideoToolbox, on Macs with a hardware HEVC encoder, such as all with Apple silicon | Software | -Hardware encoders produce a single layer. A stream that asks for SVC, such as an AV1 stream with a scalability mode like `L1T3`, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom. +Hardware encoders produce a single layer, but for the VA-API AV1 encoder, which encodes temporal layers too (`L1T2` and `L1T3`), with a rate control of each layer, so that an SFU can forward a lower frame rate. A stream that asks for spatial layers, or for temporal layers from another hardware encoder, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom. VP9 can be encoded in hardware too, with the VP9 encoder of the GPU driver, which Intel GPUs have: through VA-API on Linux, and through Media Foundation on Windows. On Windows it is off by default, since it has not been tested on such a GPU yet; turn it on with a system property before the factory is created: diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h b/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h index 6b0f8ddf..0e158255 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h @@ -26,9 +26,13 @@ #include "modules/video_coding/codecs/h264/include/h264_globals.h" #include "modules/video_coding/codecs/vp9/include/vp9_globals.h" #include "modules/video_coding/include/video_codec_interface.h" -#include "modules/video_coding/svc/scalable_video_controller_no_layering.h" +#include "api/video/video_bitrate_allocation.h" +#include "api/video_codecs/scalability_mode.h" +#include "modules/video_coding/svc/scalable_video_controller.h" #include +#include +#include #include #include @@ -43,9 +47,10 @@ namespace jni // or the sequence header of AV1. Encoders put them in front of the first // key frame, not necessarily in front of every one, so they are kept and // put back where missing. The frame dependencies WebRTC needs for AV1 and - // H.265 are those of a stream without layers, which is all hardware - // encoders produce here. VP9 frames go out as they come, with the VP9 - // information of a single layer, as the Java encoders get it. + // H.265 are those of a stream without layers, unless an AV1 encoder that + // makes temporal layers sets its scalability mode and asks for the layer + // of each frame before it encodes it. VP9 frames go out as they come, with + // the VP9 information of a single layer, as the Java encoders get it. class EncoderOutputProcessor { public: @@ -54,6 +59,15 @@ namespace jni // Forgets the parameter sets, for a stream that starts over. void Reset(); + // Sets the layers of the stream, temporal ones of AV1 only, as in + // L1T3. Returns false for a mode WebRTC has no structure of. + bool SetScalabilityMode(webrtc::ScalabilityMode mode); + // Lets the layers follow the bitrate WebRTC gives each. + void OnRatesUpdated(const webrtc::VideoBitrateAllocation & bitrates); + // The layer of the next frame, which buffers it refers to and which + // it replaces, or nothing if no layer is to be encoded now. + std::optional NextFrameConfig(bool keyFrame); + // Fills the image with the bitstream of one frame, and the codec // specific information WebRTC needs to send it. The caller sets // the size of the image before, which the AV1 frame dependencies @@ -64,6 +78,10 @@ namespace jni // frame lacks parameter sets and there are none to put back. bool Process(std::span bitstream, bool keyFrame, webrtc::EncodedImage & image, webrtc::CodecSpecificInfo & info); + // Does what the above does, for a frame encoded as the given layer. + bool Process(std::span bitstream, bool keyFrame, + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::EncodedImage & image, + webrtc::CodecSpecificInfo & info); // The quality scaling settings that suit the codec: the thresholds // of WebRTC's H.264 encoder, for H.265 as well, those for VP9 of @@ -77,7 +95,7 @@ namespace jni bool ProcessAv1(std::span bitstream, bool keyFrame, std::vector & output); void FillCodecSpecificInfo(const webrtc::EncodedImage & image, bool keyFrame, - webrtc::CodecSpecificInfo & info); + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::CodecSpecificInfo & info); private: const webrtc::VideoCodecType codec; @@ -89,7 +107,8 @@ namespace jni webrtc::H264BitstreamParser h264Parser; webrtc::H265BitstreamParser h265Parser; - webrtc::ScalableVideoControllerNoLayering svcController; + webrtc::ScalabilityMode scalabilityMode; + std::unique_ptr svcController; // The group of frames VP9 describes: one layer, every frame // referring to the one before. diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h index 9e5dbbad..e4885707 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h @@ -25,10 +25,12 @@ #include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/svc/scalable_video_controller.h" #include #include +#include #include #include #include @@ -39,12 +41,16 @@ namespace jni // drivers provide it on Linux. // // It works as FFmpeg's VA-API encoder does without B-frames: synchronous, - // one tile, inter frames that refer to the frame before them only, which - // alternate between two reference slots. The sequence and frame headers + // one tile, inter frames that refer to one frame before them. Without + // layers that is the frame just before; with temporal layers (L1T2, L1T3) + // it is the one of the same layer or a lower one that WebRTC's structure + // of the mode names, so that a receiver can do without the upper layers. + // Each reference buffer of the structure is the reference slot of the + // same number. The sequence and frame headers // are written here and handed to the driver as packed headers, with where // the quantizer, loop filter and CDEF parameters are in the frame header, - // which the driver's rate control fills in. Anything that fails is an - // error, upon which libaom takes over. + // which the driver's rate control fills in. Spatial layers, and anything + // that fails, are left to libaom. class VaapiAv1Encoder : public webrtc::VideoEncoder { public: @@ -74,10 +80,32 @@ namespace jni // Reads which headers the driver takes and how it codes them. void QueryCapabilities(); - bool Submit(bool keyFrame); + // What a frame refers to: the slot each of the seven reference + // frames names, the second reference, if any, being golden; the + // slots the frame replaces; and its temporal layer. + struct FrameReferences + { + std::array refFrameIdx = {}; + bool golden = false; + uint8_t refreshFrameFlags = 0; + int temporalId = 0; + }; + + // Turns the buffers of the layer frame into reference slots. + // Returns false if it refers to a slot that holds no frame, or to + // one beyond those kept. + bool PlanReferences(bool keyFrame, const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, + FrameReferences & references) const; + // A reconstructed surface that no reference slot holds. + int FreeSurface() const; + // Adds the rate control of the stream, or of each temporal layer. + bool AddRateControl(bool keyFrame); + + bool Submit(bool keyFrame, const FrameReferences & references, int surface); void WriteSequenceHeader(std::vector & obu) const; - void WriteFrameHeader(std::vector & obu, FrameHeaderLayout & layout, bool keyFrame) const; + void WriteFrameHeader(std::vector & obu, FrameHeaderLayout & layout, bool keyFrame, + const FrameReferences & references) const; // The seq_level_idx the stream's size, frame rate and bitrate fit. uint8_t Level() const; @@ -100,9 +128,16 @@ namespace jni // The sequence header, kept for the key frames. std::vector sequenceHeader; - // Which of the reconstructed surfaces, and so which reference slot, - // holds the current frame. - int current; + // The temporal layers, one without layers, and the bitrate of each, + // counting the layers below it. + int temporalLayers; + std::vector layerBitratesBps; + + // The reconstructed surface each reference slot holds, -1 for + // none: those of the two buffers WebRTC's structures use, up to + // L1T3. A key frame fills the other six slots too, but no frame + // refers to them. + std::array slots; bool referenceValid; // The frames since the last key frame, whose low bits are the // order hint. diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h index 42dd6621..23a4ec43 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h @@ -30,8 +30,8 @@ namespace jni { // What every VA-API encoder of a single codec needs: a configuration and - // a context with an input surface and two reconstructed ones (the frame - // being encoded and the one it refers to), the output buffer, the + // a context with an input surface and reconstructed ones (by default two: + // the frame being encoded and the one it refers to), the output buffer, the // parameter buffers of a frame, uploading a frame as NV12, encoding it // and reading what came out. The codec's encoder fills in the parameters. // @@ -49,12 +49,14 @@ namespace jni // Creates the session for 4:2:0 at constant bitrate, for pictures // of the given size, in surfaces of the given size, which is that - // of the picture aligned as the codec needs, and with the - // VA_ENC_PACKED_HEADER_* the encoder hands the driver. Mesa takes - // the size of the coded picture from the context, so the context - // has the picture's. + // of the picture aligned as the codec needs, with the + // VA_ENC_PACKED_HEADER_* the encoder hands the driver, and with + // the given number of reconstructed surfaces. Mesa takes the size + // of the coded picture from the context, so the context has the + // picture's. bool Create(VAProfile profile, VAEntrypoint entrypoint, uint32_t pictureWidth, uint32_t pictureHeight, - uint32_t surfaceWidth, uint32_t surfaceHeight, uint32_t packedHeaders = VA_ENC_PACKED_HEADER_NONE); + uint32_t surfaceWidth, uint32_t surfaceHeight, uint32_t packedHeaders = VA_ENC_PACKED_HEADER_NONE, + int reconstructedCount = 2); void Destroy(); VASurfaceID InputSurface() const; @@ -75,6 +77,12 @@ namespace jni // The rate control, frame rate and HRD parameters, for constant // bitrate with the given initial quantizer. bool AddRateControl(uint32_t bitrateBps, uint32_t framerate, uint32_t initialQp); + // The same, with a rate control of each temporal layer: their + // bitrates, each counting the layers below it too, and the layer + // of each frame after a key frame, a pattern that repeats, which + // the driver is told along with the rate control of a key frame. + bool AddLayerRateControl(const std::vector & bitratesBps, uint32_t framerate, + const std::vector & pattern, bool keyFrame); // Encodes the input surface with the parameter buffers added, and // waits for it. @@ -92,7 +100,8 @@ namespace jni VAConfigID config; VAContextID context; - VASurfaceID surfaces[3]; + // The input surface, then the reconstructed ones. + std::vector surfaces; VABufferID codedBuffer; std::vector frameBuffers; }; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp index 26c7198f..1609dd16 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp @@ -21,6 +21,8 @@ #include "common_video/h264/h264_common.h" #include "common_video/h265/h265_common.h" #include "modules/video_coding/codecs/interface/common_constants.h" +#include "modules/video_coding/svc/create_scalability_structure.h" +#include "modules/video_coding/svc/scalable_video_controller_no_layering.h" #include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" #include "rtc_base/logging.h" @@ -119,7 +121,9 @@ namespace jni EncoderOutputProcessor::EncoderOutputProcessor(webrtc::VideoCodecType codec, const webrtc::SdpVideoFormat & format) : codec(codec), - packetizationMode(webrtc::H264PacketizationMode::NonInterleaved) + packetizationMode(webrtc::H264PacketizationMode::NonInterleaved), + scalabilityMode(webrtc::ScalabilityMode::kL1T1), + svcController(std::make_unique()) { auto mode = format.parameters.find("packetization-mode"); @@ -136,8 +140,49 @@ namespace jni gofIndex = 0; } + bool EncoderOutputProcessor::SetScalabilityMode(webrtc::ScalabilityMode mode) + { + std::unique_ptr controller = webrtc::CreateScalabilityStructure(mode); + + if (!controller) { + return false; + } + + scalabilityMode = mode; + svcController = std::move(controller); + + return true; + } + + void EncoderOutputProcessor::OnRatesUpdated(const webrtc::VideoBitrateAllocation & bitrates) + { + svcController->OnRatesUpdated(bitrates); + } + + std::optional EncoderOutputProcessor::NextFrameConfig( + bool keyFrame) + { + auto layerFrames = svcController->NextFrameConfig(/*restart=*/keyFrame); + + if (layerFrames.empty()) { + return std::nullopt; + } + + return layerFrames[0]; + } + bool EncoderOutputProcessor::Process(std::span bitstream, bool keyFrame, webrtc::EncodedImage & image, webrtc::CodecSpecificInfo & info) + { + // A stream without layers has a frame for every one encoded. + auto layerFrame = NextFrameConfig(keyFrame); + + return layerFrame && Process(bitstream, keyFrame, *layerFrame, image, info); + } + + bool EncoderOutputProcessor::Process(std::span bitstream, bool keyFrame, + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::EncodedImage & image, + webrtc::CodecSpecificInfo & info) { std::vector output; bool processed; @@ -189,7 +234,11 @@ namespace jni image.qp_ = -1; } - FillCodecSpecificInfo(image, keyFrame, info); + if (scalabilityMode != webrtc::ScalabilityMode::kL1T1) { + image.SetTemporalIndex(layerFrame.TemporalId()); + } + + FillCodecSpecificInfo(image, keyFrame, layerFrame, info); return true; } @@ -335,7 +384,7 @@ namespace jni } void EncoderOutputProcessor::FillCodecSpecificInfo(const webrtc::EncodedImage & image, bool keyFrame, - webrtc::CodecSpecificInfo & info) + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::CodecSpecificInfo & info) { info.codecType = codec; @@ -350,12 +399,11 @@ namespace jni // AV1 and H.265 have no codec specific information; WebRTC sends // their frame dependencies, and those of VP9, from these, in the // dependency descriptor. - auto layerFrames = svcController.NextFrameConfig(/*restart=*/keyFrame); - info.generic_frame_info = svcController.OnEncodeDone(layerFrames[0]); - info.scalability_mode = webrtc::ScalabilityMode::kL1T1; + info.generic_frame_info = svcController->OnEncodeDone(layerFrame); + info.scalability_mode = scalabilityMode; if (keyFrame) { - info.template_structure = svcController.DependencyStructure(); + info.template_structure = svcController->DependencyStructure(); info.template_structure->resolutions = { webrtc::RenderResolution(image._encodedWidth, image._encodedHeight) }; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp index 277bc3b2..ede55634 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp @@ -23,6 +23,7 @@ #include "rtc_base/logging.h" #include +#include namespace jni { @@ -55,7 +56,33 @@ namespace jni constexpr int kRefsPerFrame = 7; constexpr uint8_t kPrimaryRefNone = 7; + // The reference frames LAST and GOLDEN, and their index among the + // seven. constexpr uint32_t kLastFrame = 1; + constexpr uint32_t kGoldenFrame = 4; + constexpr int kGoldenIndex = 3; + // LAST is the first reference of a frame, and the one its probabilities + // and other state carry over from. + constexpr uint8_t kPrimaryRefLast = 0; + + // The slots of the two reference buffers L1T3 has, and the frame being + // encoded. + constexpr int kReconstructedSurfaces = 3; + + // The layers of the frames after a key frame, as + // VAEncMiscParameterTemporalLayerStructure takes them, of L1T2 and + // L1T3 as WebRTC's structures make them: L1T3 is 0, 2, 1, 2. + const std::vector kL1T2Pattern = { 1, 0 }; + const std::vector kL1T3Pattern = { 2, 1, 2, 0 }; + + // The share of the bitrate up to each layer until WebRTC gives its + // own, as WebRTC splits it for temporal layers. + constexpr double kL1T2Shares[] = { 0.6, 1.0 }; + constexpr double kL1T3Shares[] = { 0.4, 0.6, 1.0 }; + + // What a frame that cannot be encoded returns: libaom takes over the + // stream, which stays in AV1, rather than WebRTC changing the codec. + constexpr int32_t kEncodeFailed = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; constexpr uint32_t kOrderHintBits = 8; // TxMode: largest, or chosen per block. @@ -180,13 +207,20 @@ namespace jni uint32_t count = 0; }; - // An OBU of the payload: the header, without an extension, and - // obu_size as a leb128 of the given fixed length, which is what the - // driver rewrites the size of the frame header OBU as. - std::vector MakeObu(uint8_t type, const std::vector & payload, uint32_t sizeLength) + // An OBU of the payload: the header, with the extension that names + // the temporal layer if there is one, and obu_size as a leb128 of the + // given fixed length, which is what the driver rewrites the size of + // the frame header OBU as. + std::vector MakeObu(uint8_t type, const std::vector & payload, uint32_t sizeLength, + std::optional temporalId = std::nullopt) { std::vector obu; - obu.push_back(static_cast((type << 3) | 0x02)); + obu.push_back(static_cast((type << 3) | (temporalId ? 0x04 : 0) | 0x02)); + + if (temporalId) { + // temporal_id, and spatial_id 0. + obu.push_back(static_cast(*temporalId << 5)); + } const uint32_t size = static_cast(payload.size()); @@ -219,7 +253,7 @@ namespace jni ratesChanged(false), obuSizeLength(0), txMode(0), - current(0), + temporalLayers(1), referenceValid(false), frameNumber(0), callback(nullptr), @@ -279,8 +313,12 @@ namespace jni if (settings->numberOfSimulcastStreams > 1) { return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED; } - // A single layer; spatial or temporal layers are left to libaom. - if (settings->GetScalabilityMode().value_or(webrtc::ScalabilityMode::kL1T1) != webrtc::ScalabilityMode::kL1T1) { + // Temporal layers; spatial ones are left to libaom. + const webrtc::ScalabilityMode mode = settings->GetScalabilityMode().value_or(webrtc::ScalabilityMode::kL1T1); + + if (mode != webrtc::ScalabilityMode::kL1T1 && mode != webrtc::ScalabilityMode::kL1T2 + && mode != webrtc::ScalabilityMode::kL1T3) + { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } if (obuSizeLength == 0) { @@ -309,16 +347,28 @@ namespace jni bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; framerate = std::max(1u, codecSettings.maxFramerate); ratesChanged = true; - current = 0; + temporalLayers = mode == webrtc::ScalabilityMode::kL1T3 ? 3 : mode == webrtc::ScalabilityMode::kL1T2 ? 2 : 1; + slots.fill(-1); referenceValid = false; frameNumber = 0; outputProcessor.Reset(); + layerBitratesBps.clear(); + + for (int layer = 0; layer < temporalLayers && temporalLayers > 1; layer++) { + const double share = temporalLayers == 3 ? kL1T3Shares[layer] : kL1T2Shares[layer]; + layerBitratesBps.push_back(static_cast(bitrateBps * share)); + } + + if (!outputProcessor.SetScalabilityMode(mode)) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + const uint32_t surfaceWidth = Align(codecSettings.width, kSuperblockSize); const uint32_t surfaceHeight = Align(codecSettings.height, kSuperblockSize); if (!session.Create(VAProfileAV1Profile0, entrypoint, codecSettings.width, codecSettings.height, surfaceWidth, - surfaceHeight, kPackedHeaders)) + surfaceHeight, kPackedHeaders, kReconstructedSurfaces)) { Release(); return WEBRTC_VIDEO_CODEC_ERROR; @@ -329,7 +379,7 @@ namespace jni initialized = true; RTC_LOG(LS_INFO) << implementationName << " AV1 initialized: " << codecSettings.width << "x" - << codecSettings.height << " at " << bitrateBps << " bps"; + << codecSettings.height << " at " << bitrateBps << " bps, " << webrtc::ScalabilityModeToString(mode); return WEBRTC_VIDEO_CODEC_OK; } @@ -370,18 +420,33 @@ namespace jni frameNumber = 0; } + const auto layerFrame = outputProcessor.NextFrameConfig(keyFrame); + + // No layer is encoded now, as when WebRTC gives none a bitrate. + if (!layerFrame) { + return WEBRTC_VIDEO_CODEC_OK; + } + + FrameReferences references; + const int surface = FreeSurface(); + + if (surface < 0 || !PlanReferences(keyFrame, *layerFrame, references)) { + RTC_LOG(LS_WARNING) << implementationName << " AV1: a frame refers to a slot that holds no frame"; + return kEncodeFailed; + } + std::vector output; // A frame up to the alignment larger is cropped; one of another size, // which raced with a change of size, is scaled. const bool encoded = session.Upload(frame, static_cast(codecSettings.width), - static_cast(codecSettings.height), kSizeAlignment - 1) && Submit(keyFrame) + static_cast(codecSettings.height), kSizeAlignment - 1) && Submit(keyFrame, references, surface) && session.ReadOutput(output); session.DropBuffers(); if (!encoded) { - return WEBRTC_VIDEO_CODEC_ERROR; + return kEncodeFailed; } webrtc::EncodedImage image; @@ -394,13 +459,18 @@ namespace jni webrtc::CodecSpecificInfo info; - if (!outputProcessor.Process(output, keyFrame, image, info)) { - return WEBRTC_VIDEO_CODEC_ERROR; + if (!outputProcessor.Process(output, keyFrame, *layerFrame, image, info)) { + return kEncodeFailed; + } + + // The slots the frame replaces hold it now. + for (int slot = 0; slot < static_cast(slots.size()); slot++) { + if (references.refreshFrameFlags & (1 << slot)) { + slots[slot] = surface; + } } - // The frame just encoded is the reference of the next one. referenceValid = true; - current = 1 - current; frameNumber++; ratesChanged = false; @@ -409,13 +479,81 @@ namespace jni return WEBRTC_VIDEO_CODEC_OK; } - bool VaapiAv1Encoder::Submit(bool keyFrame) + bool VaapiAv1Encoder::PlanReferences(bool keyFrame, + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, FrameReferences & references) const + { + references.temporalId = layerFrame.TemporalId(); + + // A key frame replaces every slot. + if (keyFrame) { + references.refreshFrameFlags = 0xFF; + return true; + } + + std::vector referenced; + + for (const webrtc::CodecBufferUsage & buffer : layerFrame.Buffers()) { + if (buffer.id < 0 || buffer.id >= static_cast(slots.size())) { + return false; + } + if (buffer.referenced) { + if (slots[buffer.id] < 0) { + return false; + } + + referenced.push_back(buffer.id); + } + if (buffer.updated) { + references.refreshFrameFlags |= static_cast(1 << buffer.id); + } + } + + if (referenced.empty()) { + return false; + } + + // Every reference frame names the first reference, but golden, which + // names the second where there is one. + references.refFrameIdx.fill(static_cast(referenced[0])); + + if (referenced.size() > 1) { + references.refFrameIdx[kGoldenIndex] = static_cast(referenced[1]); + references.golden = true; + } + + return true; + } + + bool VaapiAv1Encoder::AddRateControl(bool keyFrame) + { + // No initial quantizer: the rate control picks its own, as FFmpeg + // leaves it. + if (temporalLayers == 1) { + return session.AddRateControl(bitrateBps, framerate, 0); + } + + // Without a rate control of each layer, AMD's driver gives the upper + // layers next to no bits. + return session.AddLayerRateControl(layerBitratesBps, framerate, + temporalLayers == 3 ? kL1T3Pattern : kL1T2Pattern, keyFrame); + } + + int VaapiAv1Encoder::FreeSurface() const + { + for (int surface = 0; surface < kReconstructedSurfaces; surface++) { + if (std::find(slots.begin(), slots.end(), surface) == slots.end()) { + return surface; + } + } + + return -1; + } + + bool VaapiAv1Encoder::Submit(bool keyFrame, const FrameReferences & references, int surface) { const uint32_t sbCols = Align(codecSettings.width, kSuperblockSize) / kSuperblockSize; const uint32_t sbRows = Align(codecSettings.height, kSuperblockSize) / kSuperblockSize; - // The slot of the frame this one refers to, which the reconstructed - // surfaces stand for. - const int referenceSlot = 1 - current; + const bool layered = temporalLayers > 1; if (keyFrame) { VAEncSequenceParameterBufferAV1 sequence = {}; @@ -425,6 +563,7 @@ namespace jni // Key frames only where asked for. sequence.intra_period = kKeyFrameInterval; sequence.ip_period = 1; + sequence.hierarchical_flag = layered ? 1 : 0; sequence.bits_per_second = bitrateBps; sequence.seq_fields.bits.enable_order_hint = 1; sequence.seq_fields.bits.enable_cdef = 1; @@ -435,21 +574,20 @@ namespace jni } } - // No initial quantizer: the rate control picks its own, as FFmpeg - // leaves it. - if ((keyFrame || ratesChanged) && !session.AddRateControl(bitrateBps, framerate, 0)) { + if ((keyFrame || ratesChanged) && !AddRateControl(keyFrame)) { return false; } std::vector frameHeader; FrameHeaderLayout layout; - WriteFrameHeader(frameHeader, layout, keyFrame); + WriteFrameHeader(frameHeader, layout, keyFrame, references); VAEncPictureParameterBufferAV1 picture = {}; picture.frame_width_minus_1 = static_cast(codecSettings.width - 1); picture.frame_height_minus_1 = static_cast(codecSettings.height - 1); - picture.reconstructed_frame = session.Reconstructed(current); + picture.reconstructed_frame = session.Reconstructed(surface); picture.coded_buf = session.CodedBuffer(); + picture.refresh_frame_flags = references.refreshFrameFlags; for (VASurfaceID & reference : picture.reference_frames) { reference = VA_INVALID_SURFACE; @@ -457,25 +595,35 @@ namespace jni if (keyFrame) { picture.primary_ref_frame = kPrimaryRefNone; - picture.refresh_frame_flags = 0xFF; picture.picture_flags.bits.frame_type = kKeyFrameType; picture.picture_flags.bits.error_resilient_mode = 1; } else { - picture.reference_frames[referenceSlot] = session.Reconstructed(referenceSlot); - - for (int i = 0; i < kRefsPerFrame; i++) { - picture.ref_frame_idx[i] = static_cast(referenceSlot); + for (int slot = 0; slot < static_cast(slots.size()); slot++) { + if (slots[slot] >= 0) { + picture.reference_frames[slot] = session.Reconstructed(slots[slot]); + } } - // Every reference frame names the same slot, so any of them is - // the primary one; FFmpeg takes the index of the slot. - picture.primary_ref_frame = static_cast(referenceSlot); - picture.refresh_frame_flags = static_cast(1 << current); + std::copy(references.refFrameIdx.begin(), references.refFrameIdx.end(), picture.ref_frame_idx); + + picture.primary_ref_frame = kPrimaryRefLast; picture.ref_frame_ctrl_l0.fields.search_idx0 = kLastFrame; + + if (references.golden) { + picture.ref_frame_ctrl_l0.fields.search_idx1 = kGoldenFrame; + } + picture.picture_flags.bits.frame_type = kInterFrameType; } + if (layered) { + picture.temporal_id = static_cast(references.temporalId); + picture.hierarchical_level_plus1 = static_cast(references.temporalId + 1); + picture.tile_group_obu_hdr_info.bits.obu_extension_flag = 1; + picture.tile_group_obu_hdr_info.bits.temporal_id = static_cast(references.temporalId); + } + picture.order_hint = static_cast(frameNumber); picture.base_qindex = kQIndex; picture.min_base_qindex = kMinQIndex; @@ -496,7 +644,8 @@ namespace jni picture.bit_offset_cdef_params = layout.cdefOffset; picture.size_in_bits_cdef_params = layout.cdefSize; picture.size_in_bits_frame_hdr_obu = layout.size; - picture.byte_offset_frame_hdr_obu_size = (keyFrame ? static_cast(sequenceHeader.size()) : 0) + 1; + picture.byte_offset_frame_hdr_obu_size = (keyFrame ? static_cast(sequenceHeader.size()) : 0) + + (layered ? 2 : 1); VAEncTileGroupBufferAV1 tileGroup = {}; tileGroup.tg_start = 0; @@ -522,17 +671,24 @@ namespace jni writer.Put(0, 3); writer.Put(0, 1); writer.Put(0, 1); - // timing_info_present_flag, initial_display_delay_present_flag, - // operating_points_cnt_minus_1, operating_point_idc[0]. + // timing_info_present_flag, initial_display_delay_present_flag. writer.Put(0, 1); writer.Put(0, 1); - writer.Put(0, 5); - writer.Put(0, 12); - // seq_level_idx[0], and seq_tier[0] above level 3.3. - writer.Put(level, 5); + // operating_points_cnt_minus_1: with temporal layers, an operating + // point for each, from all layers down to the lowest. Then + // operating_point_idc, of spatial layer 0 and those temporal layers, + // seq_level_idx, and seq_tier above level 3.3. + writer.Put(temporalLayers - 1, 5); - if (level > 7) { - writer.Put(0, 1); + for (int point = 0; point < temporalLayers; point++) { + const uint32_t idc = temporalLayers > 1 ? 0x100 | ((1u << (temporalLayers - point)) - 1) : 0; + + writer.Put(idc, 12); + writer.Put(level, 5); + + if (level > 7) { + writer.Put(0, 1); + } } writer.Put(widthBits - 1, 4); @@ -574,17 +730,18 @@ namespace jni obu = MakeObu(kObuSequenceHeader, writer.Bytes(), obuSizeLength); } - void VaapiAv1Encoder::WriteFrameHeader(std::vector & obu, FrameHeaderLayout & layout, bool keyFrame) const + void VaapiAv1Encoder::WriteFrameHeader(std::vector & obu, FrameHeaderLayout & layout, bool keyFrame, + const FrameReferences & references) const { // uncompressed_header, section 5.9, as FFmpeg's VA-API encoder writes // it, for the sequence header above. - const int referenceSlot = 1 - current; + const bool layered = temporalLayers > 1; const uint32_t miCols = 2 * ((codecSettings.width + 7) >> 3); const uint32_t miRows = 2 * ((codecSettings.height + 7) >> 3); const uint32_t sbCols = (miCols + 15) >> 4; const uint32_t sbRows = (miRows + 15) >> 4; - // The OBU header and obu_size come before the payload. - const uint32_t payloadOffset = (1 + obuSizeLength) * 8; + // The OBU header, its extension, and obu_size come before the payload. + const uint32_t payloadOffset = ((layered ? 2 : 1) + obuSizeLength) * 8; BitWriter writer; // show_existing_frame, frame_type, show_frame. @@ -604,14 +761,13 @@ namespace jni if (!keyFrame) { // primary_ref_frame, refresh_frame_flags. - writer.Put(referenceSlot, 3); - writer.Put(1u << current, 8); - // frame_refs_short_signaling, then ref_frame_idx: every reference - // is the frame before. + writer.Put(kPrimaryRefLast, 3); + writer.Put(references.refreshFrameFlags, 8); + // frame_refs_short_signaling, then ref_frame_idx. writer.Put(0, 1); for (int i = 0; i < kRefsPerFrame; i++) { - writer.Put(referenceSlot, 3); + writer.Put(references.refFrameIdx[i], 3); } } @@ -692,7 +848,8 @@ namespace jni writer.TrailingBits(); - obu = MakeObu(kObuFrameHeader, writer.Bytes(), obuSizeLength); + obu = MakeObu(kObuFrameHeader, writer.Bytes(), obuSizeLength, + layered ? std::optional(references.temporalId) : std::nullopt); layout.size = static_cast(obu.size() * 8); } @@ -724,6 +881,17 @@ namespace jni } bitrateBps = bitrate; + outputProcessor.OnRatesUpdated(parameters.bitrate); + + // The bitrate of each temporal layer, counting those below it. + uint32_t sum = 0; + + for (int layer = 0; layer < static_cast(layerBitratesBps.size()); layer++) { + sum += parameters.bitrate.GetBitrate(0, layer); + // A layer WebRTC gives no bitrate is not encoded; it keeps the + // rate of the one below, which the driver takes. + layerBitratesBps[layer] = std::max(sum, 1u); + } if (parameters.framerate_fps >= 1.0) { framerate = static_cast(parameters.framerate_fps + 0.5); @@ -743,6 +911,14 @@ namespace jni info.scaling_settings = outputProcessor.GetScalingSettings(); info.requested_resolution_alignment = kSizeAlignment; + // With temporal layers, the share of the frame rate up to each. + if (temporalLayers > 1) { + for (int layer = 0; layer < temporalLayers; layer++) { + info.fps_allocation[0].push_back(static_cast( + EncoderInfo::kMaxFramerateFraction >> (temporalLayers - 1 - layer))); + } + } + return info; } } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiEncodeSession.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiEncodeSession.cpp index 4f0bd54e..dab2bdb4 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiEncodeSession.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiEncodeSession.cpp @@ -38,7 +38,6 @@ namespace jni name(std::move(name)), config(VA_INVALID_ID), context(VA_INVALID_ID), - surfaces{ VA_INVALID_SURFACE, VA_INVALID_SURFACE, VA_INVALID_SURFACE }, codedBuffer(VA_INVALID_ID) { } @@ -60,7 +59,8 @@ namespace jni } bool VaapiEncodeSession::Create(VAProfile profile, VAEntrypoint entrypoint, uint32_t pictureWidth, - uint32_t pictureHeight, uint32_t surfaceWidth, uint32_t surfaceHeight, uint32_t packedHeaders) + uint32_t pictureHeight, uint32_t surfaceWidth, uint32_t surfaceHeight, uint32_t packedHeaders, + int reconstructedCount) { Destroy(); @@ -83,18 +83,18 @@ namespace jni return false; } - if (!Check(va.CreateSurfaces(display, VA_RT_FORMAT_YUV420, surfaceWidth, surfaceHeight, surfaces, 3, nullptr, 0), - "creating surfaces")) - { - for (VASurfaceID & surface : surfaces) { - surface = VA_INVALID_SURFACE; - } + const int surfaceCount = 1 + reconstructedCount; + surfaces.assign(surfaceCount, VA_INVALID_SURFACE); + if (!Check(va.CreateSurfaces(display, VA_RT_FORMAT_YUV420, surfaceWidth, surfaceHeight, surfaces.data(), + surfaceCount, nullptr, 0), "creating surfaces")) + { + surfaces.clear(); return false; } - if (!Check(va.CreateContext(display, config, pictureWidth, pictureHeight, VA_PROGRESSIVE, surfaces, 3, &context), - "creating the context")) + if (!Check(va.CreateContext(display, config, pictureWidth, pictureHeight, VA_PROGRESSIVE, surfaces.data(), + surfaceCount, &context), "creating the context")) { context = VA_INVALID_ID; return false; @@ -127,12 +127,9 @@ namespace jni va.DestroyContext(display, context); context = VA_INVALID_ID; } - if (surfaces[0] != VA_INVALID_SURFACE) { - va.DestroySurfaces(display, surfaces, 3); - } - - for (VASurfaceID & surface : surfaces) { - surface = VA_INVALID_SURFACE; + if (!surfaces.empty()) { + va.DestroySurfaces(display, surfaces.data(), static_cast(surfaces.size())); + surfaces.clear(); } if (config != VA_INVALID_ID) { @@ -283,6 +280,52 @@ namespace jni && AddMiscParameter(VAEncMiscParameterTypeHRD, &hrd, sizeof(hrd)); } + bool VaapiEncodeSession::AddLayerRateControl(const std::vector & bitratesBps, uint32_t framerate, + const std::vector & pattern, bool keyFrame) + { + const uint32_t layers = static_cast(bitratesBps.size()); + + if (keyFrame) { + VAEncMiscParameterTemporalLayerStructure structure = {}; + structure.number_of_layers = layers; + structure.periodicity = static_cast(pattern.size()); + + for (size_t i = 0; i < pattern.size() && i < 32; i++) { + structure.layer_id[i] = pattern[i]; + } + + if (!AddMiscParameter(VAEncMiscParameterTypeTemporalLayerStructure, &structure, sizeof(structure))) { + return false; + } + } + + for (uint32_t layer = 0; layer < layers; layer++) { + VAEncMiscParameterRateControl rateControl = {}; + rateControl.bits_per_second = bitratesBps[layer]; + rateControl.target_percentage = 100; + rateControl.window_size = 1000; + rateControl.rc_flags.bits.temporal_id = layer; + + // Each layer down has half the frame rate of the one above it. + VAEncMiscParameterFrameRate frameRate = {}; + frameRate.framerate = framerate | ((1u << (layers - 1 - layer)) << 16); + frameRate.framerate_flags.bits.temporal_id = layer; + + if (!AddMiscParameter(VAEncMiscParameterTypeRateControl, &rateControl, sizeof(rateControl)) + || !AddMiscParameter(VAEncMiscParameterTypeFrameRate, &frameRate, sizeof(frameRate))) + { + return false; + } + } + + VAEncMiscParameterHRD hrd = {}; + hrd.buffer_size = static_cast( + static_cast(bitratesBps.back()) * kHrdBufferNumerator / kHrdBufferDenominator); + hrd.initial_buffer_fullness = hrd.buffer_size / 2; + + return AddMiscParameter(VAEncMiscParameterTypeHRD, &hrd, sizeof(hrd)); + } + bool VaapiEncodeSession::Encode() { VADisplay display = library.Display(); diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java index 88d922aa..84c517f0 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java @@ -38,7 +38,8 @@ * other GPUs it is encoded with the Media Foundation encoder of the driver on * Windows, and with its VA-API encoder on Linux. AV1 is encoded on GPUs that * have an AV1 encoder, with NVENC, Media Foundation or VA-API, as a single - * layer; a stream that asks for SVC is encoded in software. H.265 (Main profile) is + * layer, or with VA-API in temporal layers too (L1T2, L1T3); a stream that + * asks for other SVC modes is encoded in software. H.265 (Main profile) is * encoded with NVENC or Media Foundation on GPUs that have an H.265 encoder. * VP9 (profile 0) is encoded on GPUs that have a VP9 encoder, which are * Intel's: with VA-API on Linux, and with Media Foundation on Windows only diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java index c2a7f2f4..5486c161 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java @@ -18,6 +18,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeFalse; @@ -41,6 +42,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -520,6 +522,78 @@ void av1RestartsCleanly() throws Exception { } } + @Test + void av1EncodesTemporalLayers() throws Exception { + // Temporal layers are references between frames of one size, which a + // hardware encoder can make; an SFU can then forward a lower frame + // rate without asking for key frames. + for (String mode : new String[] { "L1T2", "L1T3" }) { + PeerConnectionFactory hardware = hardwareFactory(); + CountDownLatch received = new CountDownLatch(30); + + try (TestMediaCall call = new TestMediaCall(hardware, true, false, AV1)) { + call.setVideoSize(640, 360); + call.negotiate(); + + RTCRtpReceiver receiver = call.getReceiver("video"); + VideoTrack track = (VideoTrack) receiver.getTrack(); + VideoTrackSink sink = frame -> { + frame.release(); + received.countDown(); + }; + track.addSink(sink); + + call.awaitConnected(); + + RTCRtpSendParameters parameters = call.getVideoSender().getParameters(); + parameters.encodings.get(0).scalabilityMode = mode; + call.getVideoSender().setParameters(parameters); + + call.startMedia(); + + assertTrue(received.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), "too few frames received in " + mode); + + Map outbound = awaitOutboundStats(call, "encoderImplementation", "scalabilityMode"); + Map inbound = call.getInboundVideoStats(); + Object pliCount = inbound == null ? null : inbound.get("pliCount"); + + track.removeSink(sink); + receiver.dispose(); + + assertHardware(String.valueOf(outbound.get("encoderImplementation")), HARDWARE_AV1_REQUIRED); + assertEquals(mode, outbound.get("scalabilityMode")); + assertTrue(!(pliCount instanceof Number) || ((Number) pliCount).longValue() <= 1, + "key frames requested: " + pliCount); + } + finally { + hardware.dispose(); + } + } + } + + /** + * Returns the outbound video statistics once they have all the given + * members, which come shortly after the encoder. + */ + private static Map awaitOutboundStats(TestMediaCall call, String... members) + throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + Map outbound = null; + + while (System.nanoTime() < deadline) { + outbound = call.getOutboundVideoStats(); + + if (outbound != null && Arrays.stream(members).allMatch(outbound::containsKey)) { + return outbound; + } + + Thread.sleep(100); + } + + assertNotNull(outbound, "no outbound statistics"); + return outbound; + } + @Test void av1BelowHardwareMinimumStaysAv1() throws Exception { PeerConnectionFactory hardware = hardwareFactory();