From 183228bbc0b8ae09666627f855cd6c6fd3c9f7a8 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Thu, 15 Oct 2026 18:57:00 +0200 Subject: [PATCH 1/2] feat: encode AV1 temporal layers with VA-API The VA-API AV1 encoder now takes L1T2 and L1T3, with WebRTC's reference structure and a rate control for each layer, rather than leaving them to libaom. A frame it cannot encode now falls back to libaom. --- docs/guide/advanced/video-codecs.md | 2 +- .../video/codec/EncoderOutputProcessor.h | 31 +- .../media/video/codec/linux/VaapiAv1Encoder.h | 53 +++- .../video/codec/linux/VaapiEncodeSession.h | 25 +- .../video/codec/EncoderOutputProcessor.cpp | 62 +++- .../video/codec/linux/VaapiAv1Encoder.cpp | 284 ++++++++++++++---- .../video/codec/linux/VaapiEncodeSession.cpp | 75 ++++- .../codec/HardwareVideoEncoderFactory.java | 3 +- .../HardwareVideoEncoderIntegrationTest.java | 74 +++++ 9 files changed, 507 insertions(+), 102 deletions(-) diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index c1063f01..e87bc22e 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -30,7 +30,7 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder() | Linux | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) where it encodes H.265 | NVENC on NVIDIA RTX 40 and newer, otherwise the VA-API encoder of the GPU driver where it encodes AV1 (e.g. AMD RDNA3 and newer) | | macOS | VideoToolbox, as with `DefaultVideoEncoderFactory` | VideoToolbox, on Macs with a hardware HEVC encoder, such as all with Apple silicon | Software | -Hardware encoders produce a single layer. A stream that asks for SVC, such as an AV1 stream with a scalability mode like `L1T3`, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom. +Hardware encoders produce a single layer, but for the VA-API AV1 encoder, which encodes temporal layers too (`L1T2` and `L1T3`), with a rate control of each layer, so that an SFU can forward a lower frame rate. A stream that asks for spatial layers, or for temporal layers from another hardware encoder, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom. VP9 can be encoded in hardware too, with the VP9 encoder of the GPU driver, which Intel GPUs have: through VA-API on Linux, and through Media Foundation on Windows. On Windows it is off by default, since it has not been tested on such a GPU yet; turn it on with a system property before the factory is created: diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h b/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h index 6b0f8ddf..0e158255 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h @@ -26,9 +26,13 @@ #include "modules/video_coding/codecs/h264/include/h264_globals.h" #include "modules/video_coding/codecs/vp9/include/vp9_globals.h" #include "modules/video_coding/include/video_codec_interface.h" -#include "modules/video_coding/svc/scalable_video_controller_no_layering.h" +#include "api/video/video_bitrate_allocation.h" +#include "api/video_codecs/scalability_mode.h" +#include "modules/video_coding/svc/scalable_video_controller.h" #include +#include +#include #include #include @@ -43,9 +47,10 @@ namespace jni // or the sequence header of AV1. Encoders put them in front of the first // key frame, not necessarily in front of every one, so they are kept and // put back where missing. The frame dependencies WebRTC needs for AV1 and - // H.265 are those of a stream without layers, which is all hardware - // encoders produce here. VP9 frames go out as they come, with the VP9 - // information of a single layer, as the Java encoders get it. + // H.265 are those of a stream without layers, unless an AV1 encoder that + // makes temporal layers sets its scalability mode and asks for the layer + // of each frame before it encodes it. VP9 frames go out as they come, with + // the VP9 information of a single layer, as the Java encoders get it. class EncoderOutputProcessor { public: @@ -54,6 +59,15 @@ namespace jni // Forgets the parameter sets, for a stream that starts over. void Reset(); + // Sets the layers of the stream, temporal ones of AV1 only, as in + // L1T3. Returns false for a mode WebRTC has no structure of. + bool SetScalabilityMode(webrtc::ScalabilityMode mode); + // Lets the layers follow the bitrate WebRTC gives each. + void OnRatesUpdated(const webrtc::VideoBitrateAllocation & bitrates); + // The layer of the next frame, which buffers it refers to and which + // it replaces, or nothing if no layer is to be encoded now. + std::optional NextFrameConfig(bool keyFrame); + // Fills the image with the bitstream of one frame, and the codec // specific information WebRTC needs to send it. The caller sets // the size of the image before, which the AV1 frame dependencies @@ -64,6 +78,10 @@ namespace jni // frame lacks parameter sets and there are none to put back. bool Process(std::span bitstream, bool keyFrame, webrtc::EncodedImage & image, webrtc::CodecSpecificInfo & info); + // Does what the above does, for a frame encoded as the given layer. + bool Process(std::span bitstream, bool keyFrame, + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::EncodedImage & image, + webrtc::CodecSpecificInfo & info); // The quality scaling settings that suit the codec: the thresholds // of WebRTC's H.264 encoder, for H.265 as well, those for VP9 of @@ -77,7 +95,7 @@ namespace jni bool ProcessAv1(std::span bitstream, bool keyFrame, std::vector & output); void FillCodecSpecificInfo(const webrtc::EncodedImage & image, bool keyFrame, - webrtc::CodecSpecificInfo & info); + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::CodecSpecificInfo & info); private: const webrtc::VideoCodecType codec; @@ -89,7 +107,8 @@ namespace jni webrtc::H264BitstreamParser h264Parser; webrtc::H265BitstreamParser h265Parser; - webrtc::ScalableVideoControllerNoLayering svcController; + webrtc::ScalabilityMode scalabilityMode; + std::unique_ptr svcController; // The group of frames VP9 describes: one layer, every frame // referring to the one before. diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h index 9e5dbbad..e4885707 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h @@ -25,10 +25,12 @@ #include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/svc/scalable_video_controller.h" #include #include +#include #include #include #include @@ -39,12 +41,16 @@ namespace jni // drivers provide it on Linux. // // It works as FFmpeg's VA-API encoder does without B-frames: synchronous, - // one tile, inter frames that refer to the frame before them only, which - // alternate between two reference slots. The sequence and frame headers + // one tile, inter frames that refer to one frame before them. Without + // layers that is the frame just before; with temporal layers (L1T2, L1T3) + // it is the one of the same layer or a lower one that WebRTC's structure + // of the mode names, so that a receiver can do without the upper layers. + // Each reference buffer of the structure is the reference slot of the + // same number. The sequence and frame headers // are written here and handed to the driver as packed headers, with where // the quantizer, loop filter and CDEF parameters are in the frame header, - // which the driver's rate control fills in. Anything that fails is an - // error, upon which libaom takes over. + // which the driver's rate control fills in. Spatial layers, and anything + // that fails, are left to libaom. class VaapiAv1Encoder : public webrtc::VideoEncoder { public: @@ -74,10 +80,32 @@ namespace jni // Reads which headers the driver takes and how it codes them. void QueryCapabilities(); - bool Submit(bool keyFrame); + // What a frame refers to: the slot each of the seven reference + // frames names, the second reference, if any, being golden; the + // slots the frame replaces; and its temporal layer. + struct FrameReferences + { + std::array refFrameIdx = {}; + bool golden = false; + uint8_t refreshFrameFlags = 0; + int temporalId = 0; + }; + + // Turns the buffers of the layer frame into reference slots. + // Returns false if it refers to a slot that holds no frame, or to + // one beyond those kept. + bool PlanReferences(bool keyFrame, const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, + FrameReferences & references) const; + // A reconstructed surface that no reference slot holds. + int FreeSurface() const; + // Adds the rate control of the stream, or of each temporal layer. + bool AddRateControl(bool keyFrame); + + bool Submit(bool keyFrame, const FrameReferences & references, int surface); void WriteSequenceHeader(std::vector & obu) const; - void WriteFrameHeader(std::vector & obu, FrameHeaderLayout & layout, bool keyFrame) const; + void WriteFrameHeader(std::vector & obu, FrameHeaderLayout & layout, bool keyFrame, + const FrameReferences & references) const; // The seq_level_idx the stream's size, frame rate and bitrate fit. uint8_t Level() const; @@ -100,9 +128,16 @@ namespace jni // The sequence header, kept for the key frames. std::vector sequenceHeader; - // Which of the reconstructed surfaces, and so which reference slot, - // holds the current frame. - int current; + // The temporal layers, one without layers, and the bitrate of each, + // counting the layers below it. + int temporalLayers; + std::vector layerBitratesBps; + + // The reconstructed surface each reference slot holds, -1 for + // none: those of the two buffers WebRTC's structures use, up to + // L1T3. A key frame fills the other six slots too, but no frame + // refers to them. + std::array slots; bool referenceValid; // The frames since the last key frame, whose low bits are the // order hint. diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h index 42dd6621..23a4ec43 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h @@ -30,8 +30,8 @@ namespace jni { // What every VA-API encoder of a single codec needs: a configuration and - // a context with an input surface and two reconstructed ones (the frame - // being encoded and the one it refers to), the output buffer, the + // a context with an input surface and reconstructed ones (by default two: + // the frame being encoded and the one it refers to), the output buffer, the // parameter buffers of a frame, uploading a frame as NV12, encoding it // and reading what came out. The codec's encoder fills in the parameters. // @@ -49,12 +49,14 @@ namespace jni // Creates the session for 4:2:0 at constant bitrate, for pictures // of the given size, in surfaces of the given size, which is that - // of the picture aligned as the codec needs, and with the - // VA_ENC_PACKED_HEADER_* the encoder hands the driver. Mesa takes - // the size of the coded picture from the context, so the context - // has the picture's. + // of the picture aligned as the codec needs, with the + // VA_ENC_PACKED_HEADER_* the encoder hands the driver, and with + // the given number of reconstructed surfaces. Mesa takes the size + // of the coded picture from the context, so the context has the + // picture's. bool Create(VAProfile profile, VAEntrypoint entrypoint, uint32_t pictureWidth, uint32_t pictureHeight, - uint32_t surfaceWidth, uint32_t surfaceHeight, uint32_t packedHeaders = VA_ENC_PACKED_HEADER_NONE); + uint32_t surfaceWidth, uint32_t surfaceHeight, uint32_t packedHeaders = VA_ENC_PACKED_HEADER_NONE, + int reconstructedCount = 2); void Destroy(); VASurfaceID InputSurface() const; @@ -75,6 +77,12 @@ namespace jni // The rate control, frame rate and HRD parameters, for constant // bitrate with the given initial quantizer. bool AddRateControl(uint32_t bitrateBps, uint32_t framerate, uint32_t initialQp); + // The same, with a rate control of each temporal layer: their + // bitrates, each counting the layers below it too, and the layer + // of each frame after a key frame, a pattern that repeats, which + // the driver is told along with the rate control of a key frame. + bool AddLayerRateControl(const std::vector & bitratesBps, uint32_t framerate, + const std::vector & pattern, bool keyFrame); // Encodes the input surface with the parameter buffers added, and // waits for it. @@ -92,7 +100,8 @@ namespace jni VAConfigID config; VAContextID context; - VASurfaceID surfaces[3]; + // The input surface, then the reconstructed ones. + std::vector surfaces; VABufferID codedBuffer; std::vector frameBuffers; }; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp index 26c7198f..1609dd16 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp @@ -21,6 +21,8 @@ #include "common_video/h264/h264_common.h" #include "common_video/h265/h265_common.h" #include "modules/video_coding/codecs/interface/common_constants.h" +#include "modules/video_coding/svc/create_scalability_structure.h" +#include "modules/video_coding/svc/scalable_video_controller_no_layering.h" #include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" #include "rtc_base/logging.h" @@ -119,7 +121,9 @@ namespace jni EncoderOutputProcessor::EncoderOutputProcessor(webrtc::VideoCodecType codec, const webrtc::SdpVideoFormat & format) : codec(codec), - packetizationMode(webrtc::H264PacketizationMode::NonInterleaved) + packetizationMode(webrtc::H264PacketizationMode::NonInterleaved), + scalabilityMode(webrtc::ScalabilityMode::kL1T1), + svcController(std::make_unique()) { auto mode = format.parameters.find("packetization-mode"); @@ -136,8 +140,49 @@ namespace jni gofIndex = 0; } + bool EncoderOutputProcessor::SetScalabilityMode(webrtc::ScalabilityMode mode) + { + std::unique_ptr controller = webrtc::CreateScalabilityStructure(mode); + + if (!controller) { + return false; + } + + scalabilityMode = mode; + svcController = std::move(controller); + + return true; + } + + void EncoderOutputProcessor::OnRatesUpdated(const webrtc::VideoBitrateAllocation & bitrates) + { + svcController->OnRatesUpdated(bitrates); + } + + std::optional EncoderOutputProcessor::NextFrameConfig( + bool keyFrame) + { + auto layerFrames = svcController->NextFrameConfig(/*restart=*/keyFrame); + + if (layerFrames.empty()) { + return std::nullopt; + } + + return layerFrames[0]; + } + bool EncoderOutputProcessor::Process(std::span bitstream, bool keyFrame, webrtc::EncodedImage & image, webrtc::CodecSpecificInfo & info) + { + // A stream without layers has a frame for every one encoded. + auto layerFrame = NextFrameConfig(keyFrame); + + return layerFrame && Process(bitstream, keyFrame, *layerFrame, image, info); + } + + bool EncoderOutputProcessor::Process(std::span bitstream, bool keyFrame, + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::EncodedImage & image, + webrtc::CodecSpecificInfo & info) { std::vector output; bool processed; @@ -189,7 +234,11 @@ namespace jni image.qp_ = -1; } - FillCodecSpecificInfo(image, keyFrame, info); + if (scalabilityMode != webrtc::ScalabilityMode::kL1T1) { + image.SetTemporalIndex(layerFrame.TemporalId()); + } + + FillCodecSpecificInfo(image, keyFrame, layerFrame, info); return true; } @@ -335,7 +384,7 @@ namespace jni } void EncoderOutputProcessor::FillCodecSpecificInfo(const webrtc::EncodedImage & image, bool keyFrame, - webrtc::CodecSpecificInfo & info) + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::CodecSpecificInfo & info) { info.codecType = codec; @@ -350,12 +399,11 @@ namespace jni // AV1 and H.265 have no codec specific information; WebRTC sends // their frame dependencies, and those of VP9, from these, in the // dependency descriptor. - auto layerFrames = svcController.NextFrameConfig(/*restart=*/keyFrame); - info.generic_frame_info = svcController.OnEncodeDone(layerFrames[0]); - info.scalability_mode = webrtc::ScalabilityMode::kL1T1; + info.generic_frame_info = svcController->OnEncodeDone(layerFrame); + info.scalability_mode = scalabilityMode; if (keyFrame) { - info.template_structure = svcController.DependencyStructure(); + info.template_structure = svcController->DependencyStructure(); info.template_structure->resolutions = { webrtc::RenderResolution(image._encodedWidth, image._encodedHeight) }; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp index 277bc3b2..ede55634 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp @@ -23,6 +23,7 @@ #include "rtc_base/logging.h" #include +#include namespace jni { @@ -55,7 +56,33 @@ namespace jni constexpr int kRefsPerFrame = 7; constexpr uint8_t kPrimaryRefNone = 7; + // The reference frames LAST and GOLDEN, and their index among the + // seven. constexpr uint32_t kLastFrame = 1; + constexpr uint32_t kGoldenFrame = 4; + constexpr int kGoldenIndex = 3; + // LAST is the first reference of a frame, and the one its probabilities + // and other state carry over from. + constexpr uint8_t kPrimaryRefLast = 0; + + // The slots of the two reference buffers L1T3 has, and the frame being + // encoded. + constexpr int kReconstructedSurfaces = 3; + + // The layers of the frames after a key frame, as + // VAEncMiscParameterTemporalLayerStructure takes them, of L1T2 and + // L1T3 as WebRTC's structures make them: L1T3 is 0, 2, 1, 2. + const std::vector kL1T2Pattern = { 1, 0 }; + const std::vector kL1T3Pattern = { 2, 1, 2, 0 }; + + // The share of the bitrate up to each layer until WebRTC gives its + // own, as WebRTC splits it for temporal layers. + constexpr double kL1T2Shares[] = { 0.6, 1.0 }; + constexpr double kL1T3Shares[] = { 0.4, 0.6, 1.0 }; + + // What a frame that cannot be encoded returns: libaom takes over the + // stream, which stays in AV1, rather than WebRTC changing the codec. + constexpr int32_t kEncodeFailed = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; constexpr uint32_t kOrderHintBits = 8; // TxMode: largest, or chosen per block. @@ -180,13 +207,20 @@ namespace jni uint32_t count = 0; }; - // An OBU of the payload: the header, without an extension, and - // obu_size as a leb128 of the given fixed length, which is what the - // driver rewrites the size of the frame header OBU as. - std::vector MakeObu(uint8_t type, const std::vector & payload, uint32_t sizeLength) + // An OBU of the payload: the header, with the extension that names + // the temporal layer if there is one, and obu_size as a leb128 of the + // given fixed length, which is what the driver rewrites the size of + // the frame header OBU as. + std::vector MakeObu(uint8_t type, const std::vector & payload, uint32_t sizeLength, + std::optional temporalId = std::nullopt) { std::vector obu; - obu.push_back(static_cast((type << 3) | 0x02)); + obu.push_back(static_cast((type << 3) | (temporalId ? 0x04 : 0) | 0x02)); + + if (temporalId) { + // temporal_id, and spatial_id 0. + obu.push_back(static_cast(*temporalId << 5)); + } const uint32_t size = static_cast(payload.size()); @@ -219,7 +253,7 @@ namespace jni ratesChanged(false), obuSizeLength(0), txMode(0), - current(0), + temporalLayers(1), referenceValid(false), frameNumber(0), callback(nullptr), @@ -279,8 +313,12 @@ namespace jni if (settings->numberOfSimulcastStreams > 1) { return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED; } - // A single layer; spatial or temporal layers are left to libaom. - if (settings->GetScalabilityMode().value_or(webrtc::ScalabilityMode::kL1T1) != webrtc::ScalabilityMode::kL1T1) { + // Temporal layers; spatial ones are left to libaom. + const webrtc::ScalabilityMode mode = settings->GetScalabilityMode().value_or(webrtc::ScalabilityMode::kL1T1); + + if (mode != webrtc::ScalabilityMode::kL1T1 && mode != webrtc::ScalabilityMode::kL1T2 + && mode != webrtc::ScalabilityMode::kL1T3) + { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } if (obuSizeLength == 0) { @@ -309,16 +347,28 @@ namespace jni bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; framerate = std::max(1u, codecSettings.maxFramerate); ratesChanged = true; - current = 0; + temporalLayers = mode == webrtc::ScalabilityMode::kL1T3 ? 3 : mode == webrtc::ScalabilityMode::kL1T2 ? 2 : 1; + slots.fill(-1); referenceValid = false; frameNumber = 0; outputProcessor.Reset(); + layerBitratesBps.clear(); + + for (int layer = 0; layer < temporalLayers && temporalLayers > 1; layer++) { + const double share = temporalLayers == 3 ? kL1T3Shares[layer] : kL1T2Shares[layer]; + layerBitratesBps.push_back(static_cast(bitrateBps * share)); + } + + if (!outputProcessor.SetScalabilityMode(mode)) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + const uint32_t surfaceWidth = Align(codecSettings.width, kSuperblockSize); const uint32_t surfaceHeight = Align(codecSettings.height, kSuperblockSize); if (!session.Create(VAProfileAV1Profile0, entrypoint, codecSettings.width, codecSettings.height, surfaceWidth, - surfaceHeight, kPackedHeaders)) + surfaceHeight, kPackedHeaders, kReconstructedSurfaces)) { Release(); return WEBRTC_VIDEO_CODEC_ERROR; @@ -329,7 +379,7 @@ namespace jni initialized = true; RTC_LOG(LS_INFO) << implementationName << " AV1 initialized: " << codecSettings.width << "x" - << codecSettings.height << " at " << bitrateBps << " bps"; + << codecSettings.height << " at " << bitrateBps << " bps, " << webrtc::ScalabilityModeToString(mode); return WEBRTC_VIDEO_CODEC_OK; } @@ -370,18 +420,33 @@ namespace jni frameNumber = 0; } + const auto layerFrame = outputProcessor.NextFrameConfig(keyFrame); + + // No layer is encoded now, as when WebRTC gives none a bitrate. + if (!layerFrame) { + return WEBRTC_VIDEO_CODEC_OK; + } + + FrameReferences references; + const int surface = FreeSurface(); + + if (surface < 0 || !PlanReferences(keyFrame, *layerFrame, references)) { + RTC_LOG(LS_WARNING) << implementationName << " AV1: a frame refers to a slot that holds no frame"; + return kEncodeFailed; + } + std::vector output; // A frame up to the alignment larger is cropped; one of another size, // which raced with a change of size, is scaled. const bool encoded = session.Upload(frame, static_cast(codecSettings.width), - static_cast(codecSettings.height), kSizeAlignment - 1) && Submit(keyFrame) + static_cast(codecSettings.height), kSizeAlignment - 1) && Submit(keyFrame, references, surface) && session.ReadOutput(output); session.DropBuffers(); if (!encoded) { - return WEBRTC_VIDEO_CODEC_ERROR; + return kEncodeFailed; } webrtc::EncodedImage image; @@ -394,13 +459,18 @@ namespace jni webrtc::CodecSpecificInfo info; - if (!outputProcessor.Process(output, keyFrame, image, info)) { - return WEBRTC_VIDEO_CODEC_ERROR; + if (!outputProcessor.Process(output, keyFrame, *layerFrame, image, info)) { + return kEncodeFailed; + } + + // The slots the frame replaces hold it now. + for (int slot = 0; slot < static_cast(slots.size()); slot++) { + if (references.refreshFrameFlags & (1 << slot)) { + slots[slot] = surface; + } } - // The frame just encoded is the reference of the next one. referenceValid = true; - current = 1 - current; frameNumber++; ratesChanged = false; @@ -409,13 +479,81 @@ namespace jni return WEBRTC_VIDEO_CODEC_OK; } - bool VaapiAv1Encoder::Submit(bool keyFrame) + bool VaapiAv1Encoder::PlanReferences(bool keyFrame, + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, FrameReferences & references) const + { + references.temporalId = layerFrame.TemporalId(); + + // A key frame replaces every slot. + if (keyFrame) { + references.refreshFrameFlags = 0xFF; + return true; + } + + std::vector referenced; + + for (const webrtc::CodecBufferUsage & buffer : layerFrame.Buffers()) { + if (buffer.id < 0 || buffer.id >= static_cast(slots.size())) { + return false; + } + if (buffer.referenced) { + if (slots[buffer.id] < 0) { + return false; + } + + referenced.push_back(buffer.id); + } + if (buffer.updated) { + references.refreshFrameFlags |= static_cast(1 << buffer.id); + } + } + + if (referenced.empty()) { + return false; + } + + // Every reference frame names the first reference, but golden, which + // names the second where there is one. + references.refFrameIdx.fill(static_cast(referenced[0])); + + if (referenced.size() > 1) { + references.refFrameIdx[kGoldenIndex] = static_cast(referenced[1]); + references.golden = true; + } + + return true; + } + + bool VaapiAv1Encoder::AddRateControl(bool keyFrame) + { + // No initial quantizer: the rate control picks its own, as FFmpeg + // leaves it. + if (temporalLayers == 1) { + return session.AddRateControl(bitrateBps, framerate, 0); + } + + // Without a rate control of each layer, AMD's driver gives the upper + // layers next to no bits. + return session.AddLayerRateControl(layerBitratesBps, framerate, + temporalLayers == 3 ? kL1T3Pattern : kL1T2Pattern, keyFrame); + } + + int VaapiAv1Encoder::FreeSurface() const + { + for (int surface = 0; surface < kReconstructedSurfaces; surface++) { + if (std::find(slots.begin(), slots.end(), surface) == slots.end()) { + return surface; + } + } + + return -1; + } + + bool VaapiAv1Encoder::Submit(bool keyFrame, const FrameReferences & references, int surface) { const uint32_t sbCols = Align(codecSettings.width, kSuperblockSize) / kSuperblockSize; const uint32_t sbRows = Align(codecSettings.height, kSuperblockSize) / kSuperblockSize; - // The slot of the frame this one refers to, which the reconstructed - // surfaces stand for. - const int referenceSlot = 1 - current; + const bool layered = temporalLayers > 1; if (keyFrame) { VAEncSequenceParameterBufferAV1 sequence = {}; @@ -425,6 +563,7 @@ namespace jni // Key frames only where asked for. sequence.intra_period = kKeyFrameInterval; sequence.ip_period = 1; + sequence.hierarchical_flag = layered ? 1 : 0; sequence.bits_per_second = bitrateBps; sequence.seq_fields.bits.enable_order_hint = 1; sequence.seq_fields.bits.enable_cdef = 1; @@ -435,21 +574,20 @@ namespace jni } } - // No initial quantizer: the rate control picks its own, as FFmpeg - // leaves it. - if ((keyFrame || ratesChanged) && !session.AddRateControl(bitrateBps, framerate, 0)) { + if ((keyFrame || ratesChanged) && !AddRateControl(keyFrame)) { return false; } std::vector frameHeader; FrameHeaderLayout layout; - WriteFrameHeader(frameHeader, layout, keyFrame); + WriteFrameHeader(frameHeader, layout, keyFrame, references); VAEncPictureParameterBufferAV1 picture = {}; picture.frame_width_minus_1 = static_cast(codecSettings.width - 1); picture.frame_height_minus_1 = static_cast(codecSettings.height - 1); - picture.reconstructed_frame = session.Reconstructed(current); + picture.reconstructed_frame = session.Reconstructed(surface); picture.coded_buf = session.CodedBuffer(); + picture.refresh_frame_flags = references.refreshFrameFlags; for (VASurfaceID & reference : picture.reference_frames) { reference = VA_INVALID_SURFACE; @@ -457,25 +595,35 @@ namespace jni if (keyFrame) { picture.primary_ref_frame = kPrimaryRefNone; - picture.refresh_frame_flags = 0xFF; picture.picture_flags.bits.frame_type = kKeyFrameType; picture.picture_flags.bits.error_resilient_mode = 1; } else { - picture.reference_frames[referenceSlot] = session.Reconstructed(referenceSlot); - - for (int i = 0; i < kRefsPerFrame; i++) { - picture.ref_frame_idx[i] = static_cast(referenceSlot); + for (int slot = 0; slot < static_cast(slots.size()); slot++) { + if (slots[slot] >= 0) { + picture.reference_frames[slot] = session.Reconstructed(slots[slot]); + } } - // Every reference frame names the same slot, so any of them is - // the primary one; FFmpeg takes the index of the slot. - picture.primary_ref_frame = static_cast(referenceSlot); - picture.refresh_frame_flags = static_cast(1 << current); + std::copy(references.refFrameIdx.begin(), references.refFrameIdx.end(), picture.ref_frame_idx); + + picture.primary_ref_frame = kPrimaryRefLast; picture.ref_frame_ctrl_l0.fields.search_idx0 = kLastFrame; + + if (references.golden) { + picture.ref_frame_ctrl_l0.fields.search_idx1 = kGoldenFrame; + } + picture.picture_flags.bits.frame_type = kInterFrameType; } + if (layered) { + picture.temporal_id = static_cast(references.temporalId); + picture.hierarchical_level_plus1 = static_cast(references.temporalId + 1); + picture.tile_group_obu_hdr_info.bits.obu_extension_flag = 1; + picture.tile_group_obu_hdr_info.bits.temporal_id = static_cast(references.temporalId); + } + picture.order_hint = static_cast(frameNumber); picture.base_qindex = kQIndex; picture.min_base_qindex = kMinQIndex; @@ -496,7 +644,8 @@ namespace jni picture.bit_offset_cdef_params = layout.cdefOffset; picture.size_in_bits_cdef_params = layout.cdefSize; picture.size_in_bits_frame_hdr_obu = layout.size; - picture.byte_offset_frame_hdr_obu_size = (keyFrame ? static_cast(sequenceHeader.size()) : 0) + 1; + picture.byte_offset_frame_hdr_obu_size = (keyFrame ? static_cast(sequenceHeader.size()) : 0) + + (layered ? 2 : 1); VAEncTileGroupBufferAV1 tileGroup = {}; tileGroup.tg_start = 0; @@ -522,17 +671,24 @@ namespace jni writer.Put(0, 3); writer.Put(0, 1); writer.Put(0, 1); - // timing_info_present_flag, initial_display_delay_present_flag, - // operating_points_cnt_minus_1, operating_point_idc[0]. + // timing_info_present_flag, initial_display_delay_present_flag. writer.Put(0, 1); writer.Put(0, 1); - writer.Put(0, 5); - writer.Put(0, 12); - // seq_level_idx[0], and seq_tier[0] above level 3.3. - writer.Put(level, 5); + // operating_points_cnt_minus_1: with temporal layers, an operating + // point for each, from all layers down to the lowest. Then + // operating_point_idc, of spatial layer 0 and those temporal layers, + // seq_level_idx, and seq_tier above level 3.3. + writer.Put(temporalLayers - 1, 5); - if (level > 7) { - writer.Put(0, 1); + for (int point = 0; point < temporalLayers; point++) { + const uint32_t idc = temporalLayers > 1 ? 0x100 | ((1u << (temporalLayers - point)) - 1) : 0; + + writer.Put(idc, 12); + writer.Put(level, 5); + + if (level > 7) { + writer.Put(0, 1); + } } writer.Put(widthBits - 1, 4); @@ -574,17 +730,18 @@ namespace jni obu = MakeObu(kObuSequenceHeader, writer.Bytes(), obuSizeLength); } - void VaapiAv1Encoder::WriteFrameHeader(std::vector & obu, FrameHeaderLayout & layout, bool keyFrame) const + void VaapiAv1Encoder::WriteFrameHeader(std::vector & obu, FrameHeaderLayout & layout, bool keyFrame, + const FrameReferences & references) const { // uncompressed_header, section 5.9, as FFmpeg's VA-API encoder writes // it, for the sequence header above. - const int referenceSlot = 1 - current; + const bool layered = temporalLayers > 1; const uint32_t miCols = 2 * ((codecSettings.width + 7) >> 3); const uint32_t miRows = 2 * ((codecSettings.height + 7) >> 3); const uint32_t sbCols = (miCols + 15) >> 4; const uint32_t sbRows = (miRows + 15) >> 4; - // The OBU header and obu_size come before the payload. - const uint32_t payloadOffset = (1 + obuSizeLength) * 8; + // The OBU header, its extension, and obu_size come before the payload. + const uint32_t payloadOffset = ((layered ? 2 : 1) + obuSizeLength) * 8; BitWriter writer; // show_existing_frame, frame_type, show_frame. @@ -604,14 +761,13 @@ namespace jni if (!keyFrame) { // primary_ref_frame, refresh_frame_flags. - writer.Put(referenceSlot, 3); - writer.Put(1u << current, 8); - // frame_refs_short_signaling, then ref_frame_idx: every reference - // is the frame before. + writer.Put(kPrimaryRefLast, 3); + writer.Put(references.refreshFrameFlags, 8); + // frame_refs_short_signaling, then ref_frame_idx. writer.Put(0, 1); for (int i = 0; i < kRefsPerFrame; i++) { - writer.Put(referenceSlot, 3); + writer.Put(references.refFrameIdx[i], 3); } } @@ -692,7 +848,8 @@ namespace jni writer.TrailingBits(); - obu = MakeObu(kObuFrameHeader, writer.Bytes(), obuSizeLength); + obu = MakeObu(kObuFrameHeader, writer.Bytes(), obuSizeLength, + layered ? std::optional(references.temporalId) : std::nullopt); layout.size = static_cast(obu.size() * 8); } @@ -724,6 +881,17 @@ namespace jni } bitrateBps = bitrate; + outputProcessor.OnRatesUpdated(parameters.bitrate); + + // The bitrate of each temporal layer, counting those below it. + uint32_t sum = 0; + + for (int layer = 0; layer < static_cast(layerBitratesBps.size()); layer++) { + sum += parameters.bitrate.GetBitrate(0, layer); + // A layer WebRTC gives no bitrate is not encoded; it keeps the + // rate of the one below, which the driver takes. + layerBitratesBps[layer] = std::max(sum, 1u); + } if (parameters.framerate_fps >= 1.0) { framerate = static_cast(parameters.framerate_fps + 0.5); @@ -743,6 +911,14 @@ namespace jni info.scaling_settings = outputProcessor.GetScalingSettings(); info.requested_resolution_alignment = kSizeAlignment; + // With temporal layers, the share of the frame rate up to each. + if (temporalLayers > 1) { + for (int layer = 0; layer < temporalLayers; layer++) { + info.fps_allocation[0].push_back(static_cast( + EncoderInfo::kMaxFramerateFraction >> (temporalLayers - 1 - layer))); + } + } + return info; } } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiEncodeSession.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiEncodeSession.cpp index 4f0bd54e..dab2bdb4 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiEncodeSession.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiEncodeSession.cpp @@ -38,7 +38,6 @@ namespace jni name(std::move(name)), config(VA_INVALID_ID), context(VA_INVALID_ID), - surfaces{ VA_INVALID_SURFACE, VA_INVALID_SURFACE, VA_INVALID_SURFACE }, codedBuffer(VA_INVALID_ID) { } @@ -60,7 +59,8 @@ namespace jni } bool VaapiEncodeSession::Create(VAProfile profile, VAEntrypoint entrypoint, uint32_t pictureWidth, - uint32_t pictureHeight, uint32_t surfaceWidth, uint32_t surfaceHeight, uint32_t packedHeaders) + uint32_t pictureHeight, uint32_t surfaceWidth, uint32_t surfaceHeight, uint32_t packedHeaders, + int reconstructedCount) { Destroy(); @@ -83,18 +83,18 @@ namespace jni return false; } - if (!Check(va.CreateSurfaces(display, VA_RT_FORMAT_YUV420, surfaceWidth, surfaceHeight, surfaces, 3, nullptr, 0), - "creating surfaces")) - { - for (VASurfaceID & surface : surfaces) { - surface = VA_INVALID_SURFACE; - } + const int surfaceCount = 1 + reconstructedCount; + surfaces.assign(surfaceCount, VA_INVALID_SURFACE); + if (!Check(va.CreateSurfaces(display, VA_RT_FORMAT_YUV420, surfaceWidth, surfaceHeight, surfaces.data(), + surfaceCount, nullptr, 0), "creating surfaces")) + { + surfaces.clear(); return false; } - if (!Check(va.CreateContext(display, config, pictureWidth, pictureHeight, VA_PROGRESSIVE, surfaces, 3, &context), - "creating the context")) + if (!Check(va.CreateContext(display, config, pictureWidth, pictureHeight, VA_PROGRESSIVE, surfaces.data(), + surfaceCount, &context), "creating the context")) { context = VA_INVALID_ID; return false; @@ -127,12 +127,9 @@ namespace jni va.DestroyContext(display, context); context = VA_INVALID_ID; } - if (surfaces[0] != VA_INVALID_SURFACE) { - va.DestroySurfaces(display, surfaces, 3); - } - - for (VASurfaceID & surface : surfaces) { - surface = VA_INVALID_SURFACE; + if (!surfaces.empty()) { + va.DestroySurfaces(display, surfaces.data(), static_cast(surfaces.size())); + surfaces.clear(); } if (config != VA_INVALID_ID) { @@ -283,6 +280,52 @@ namespace jni && AddMiscParameter(VAEncMiscParameterTypeHRD, &hrd, sizeof(hrd)); } + bool VaapiEncodeSession::AddLayerRateControl(const std::vector & bitratesBps, uint32_t framerate, + const std::vector & pattern, bool keyFrame) + { + const uint32_t layers = static_cast(bitratesBps.size()); + + if (keyFrame) { + VAEncMiscParameterTemporalLayerStructure structure = {}; + structure.number_of_layers = layers; + structure.periodicity = static_cast(pattern.size()); + + for (size_t i = 0; i < pattern.size() && i < 32; i++) { + structure.layer_id[i] = pattern[i]; + } + + if (!AddMiscParameter(VAEncMiscParameterTypeTemporalLayerStructure, &structure, sizeof(structure))) { + return false; + } + } + + for (uint32_t layer = 0; layer < layers; layer++) { + VAEncMiscParameterRateControl rateControl = {}; + rateControl.bits_per_second = bitratesBps[layer]; + rateControl.target_percentage = 100; + rateControl.window_size = 1000; + rateControl.rc_flags.bits.temporal_id = layer; + + // Each layer down has half the frame rate of the one above it. + VAEncMiscParameterFrameRate frameRate = {}; + frameRate.framerate = framerate | ((1u << (layers - 1 - layer)) << 16); + frameRate.framerate_flags.bits.temporal_id = layer; + + if (!AddMiscParameter(VAEncMiscParameterTypeRateControl, &rateControl, sizeof(rateControl)) + || !AddMiscParameter(VAEncMiscParameterTypeFrameRate, &frameRate, sizeof(frameRate))) + { + return false; + } + } + + VAEncMiscParameterHRD hrd = {}; + hrd.buffer_size = static_cast( + static_cast(bitratesBps.back()) * kHrdBufferNumerator / kHrdBufferDenominator); + hrd.initial_buffer_fullness = hrd.buffer_size / 2; + + return AddMiscParameter(VAEncMiscParameterTypeHRD, &hrd, sizeof(hrd)); + } + bool VaapiEncodeSession::Encode() { VADisplay display = library.Display(); diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java index 88d922aa..84c517f0 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java @@ -38,7 +38,8 @@ * other GPUs it is encoded with the Media Foundation encoder of the driver on * Windows, and with its VA-API encoder on Linux. AV1 is encoded on GPUs that * have an AV1 encoder, with NVENC, Media Foundation or VA-API, as a single - * layer; a stream that asks for SVC is encoded in software. H.265 (Main profile) is + * layer, or with VA-API in temporal layers too (L1T2, L1T3); a stream that + * asks for other SVC modes is encoded in software. H.265 (Main profile) is * encoded with NVENC or Media Foundation on GPUs that have an H.265 encoder. * VP9 (profile 0) is encoded on GPUs that have a VP9 encoder, which are * Intel's: with VA-API on Linux, and with Media Foundation on Windows only diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java index c2a7f2f4..5486c161 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java @@ -18,6 +18,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeFalse; @@ -41,6 +42,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -520,6 +522,78 @@ void av1RestartsCleanly() throws Exception { } } + @Test + void av1EncodesTemporalLayers() throws Exception { + // Temporal layers are references between frames of one size, which a + // hardware encoder can make; an SFU can then forward a lower frame + // rate without asking for key frames. + for (String mode : new String[] { "L1T2", "L1T3" }) { + PeerConnectionFactory hardware = hardwareFactory(); + CountDownLatch received = new CountDownLatch(30); + + try (TestMediaCall call = new TestMediaCall(hardware, true, false, AV1)) { + call.setVideoSize(640, 360); + call.negotiate(); + + RTCRtpReceiver receiver = call.getReceiver("video"); + VideoTrack track = (VideoTrack) receiver.getTrack(); + VideoTrackSink sink = frame -> { + frame.release(); + received.countDown(); + }; + track.addSink(sink); + + call.awaitConnected(); + + RTCRtpSendParameters parameters = call.getVideoSender().getParameters(); + parameters.encodings.get(0).scalabilityMode = mode; + call.getVideoSender().setParameters(parameters); + + call.startMedia(); + + assertTrue(received.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), "too few frames received in " + mode); + + Map outbound = awaitOutboundStats(call, "encoderImplementation", "scalabilityMode"); + Map inbound = call.getInboundVideoStats(); + Object pliCount = inbound == null ? null : inbound.get("pliCount"); + + track.removeSink(sink); + receiver.dispose(); + + assertHardware(String.valueOf(outbound.get("encoderImplementation")), HARDWARE_AV1_REQUIRED); + assertEquals(mode, outbound.get("scalabilityMode")); + assertTrue(!(pliCount instanceof Number) || ((Number) pliCount).longValue() <= 1, + "key frames requested: " + pliCount); + } + finally { + hardware.dispose(); + } + } + } + + /** + * Returns the outbound video statistics once they have all the given + * members, which come shortly after the encoder. + */ + private static Map awaitOutboundStats(TestMediaCall call, String... members) + throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + Map outbound = null; + + while (System.nanoTime() < deadline) { + outbound = call.getOutboundVideoStats(); + + if (outbound != null && Arrays.stream(members).allMatch(outbound::containsKey)) { + return outbound; + } + + Thread.sleep(100); + } + + assertNotNull(outbound, "no outbound statistics"); + return outbound; + } + @Test void av1BelowHardwareMinimumStaysAv1() throws Exception { PeerConnectionFactory hardware = hardwareFactory(); From f41d5247f110c54b19c9db1e3fa1c51aee2c7e4d Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Thu, 15 Oct 2026 19:24:24 +0200 Subject: [PATCH 2/2] feat: encode VP9 temporal layers with VA-API The VA-API VP9 encoder takes L1T2 and L1T3 too, which Intel GPUs encode, in error resilient mode and with the VP9 information of flexible mode, as libvpx does. The AV1 and VP9 encoders share the layer handling. --- docs/guide/advanced/video-codecs.md | 2 +- .../video/codec/EncoderOutputProcessor.h | 24 ++- .../media/video/codec/linux/VaapiAv1Encoder.h | 37 +--- .../video/codec/linux/VaapiTemporalLayers.h | 102 ++++++++++ .../media/video/codec/linux/VaapiVp9Encoder.h | 26 +-- .../video/codec/EncoderOutputProcessor.cpp | 77 ++++++- .../video/codec/linux/VaapiAv1Encoder.cpp | 189 +++++------------- .../video/codec/linux/VaapiTemporalLayers.cpp | 173 ++++++++++++++++ .../video/codec/linux/VaapiVp9Encoder.cpp | 110 +++++++--- .../codec/HardwareVideoEncoderFactory.java | 3 +- .../HardwareVideoEncoderIntegrationTest.java | 27 ++- 11 files changed, 542 insertions(+), 228 deletions(-) create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiTemporalLayers.h create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiTemporalLayers.cpp diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index e87bc22e..a8d739bf 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -30,7 +30,7 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder() | Linux | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) where it encodes H.265 | NVENC on NVIDIA RTX 40 and newer, otherwise the VA-API encoder of the GPU driver where it encodes AV1 (e.g. AMD RDNA3 and newer) | | macOS | VideoToolbox, as with `DefaultVideoEncoderFactory` | VideoToolbox, on Macs with a hardware HEVC encoder, such as all with Apple silicon | Software | -Hardware encoders produce a single layer, but for the VA-API AV1 encoder, which encodes temporal layers too (`L1T2` and `L1T3`), with a rate control of each layer, so that an SFU can forward a lower frame rate. A stream that asks for spatial layers, or for temporal layers from another hardware encoder, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom. +Hardware encoders produce a single layer, but for the VA-API encoders of AV1 and VP9, which encode temporal layers too (`L1T2` and `L1T3`), with a rate control of each layer, so that an SFU can forward a lower frame rate. A stream that asks for spatial layers, or for temporal layers from another hardware encoder, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom. VP9 can be encoded in hardware too, with the VP9 encoder of the GPU driver, which Intel GPUs have: through VA-API on Linux, and through Media Foundation on Windows. On Windows it is off by default, since it has not been tested on such a GPU yet; turn it on with a system property before the factory is created: diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h b/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h index 0e158255..fbc761fd 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h @@ -30,6 +30,7 @@ #include "api/video_codecs/scalability_mode.h" #include "modules/video_coding/svc/scalable_video_controller.h" +#include #include #include #include @@ -47,10 +48,11 @@ namespace jni // or the sequence header of AV1. Encoders put them in front of the first // key frame, not necessarily in front of every one, so they are kept and // put back where missing. The frame dependencies WebRTC needs for AV1 and - // H.265 are those of a stream without layers, unless an AV1 encoder that - // makes temporal layers sets its scalability mode and asks for the layer - // of each frame before it encodes it. VP9 frames go out as they come, with - // the VP9 information of a single layer, as the Java encoders get it. + // H.265 are those of a stream without layers, unless an AV1 or VP9 encoder + // that makes temporal layers sets its scalability mode and asks for the + // layer of each frame before it encodes it. VP9 frames go out as they + // come, with the VP9 information of a single layer, as the Java encoders + // get it, or with temporal layers in flexible mode, as libvpx gives it. class EncoderOutputProcessor { public: @@ -59,8 +61,8 @@ namespace jni // Forgets the parameter sets, for a stream that starts over. void Reset(); - // Sets the layers of the stream, temporal ones of AV1 only, as in - // L1T3. Returns false for a mode WebRTC has no structure of. + // Sets the layers of the stream, temporal ones of AV1 and VP9 only, + // as in L1T3. Returns false for a mode WebRTC has no structure of. bool SetScalabilityMode(webrtc::ScalabilityMode mode); // Lets the layers follow the bitrate WebRTC gives each. void OnRatesUpdated(const webrtc::VideoBitrateAllocation & bitrates); @@ -96,6 +98,10 @@ namespace jni void FillCodecSpecificInfo(const webrtc::EncodedImage & image, bool keyFrame, const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::CodecSpecificInfo & info); + // The VP9 information of a picture of a stream with temporal + // layers. + void FillVp9TemporalLayers(const webrtc::EncodedImage & image, bool keyFrame, + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::CodecSpecificInfo & info); private: const webrtc::VideoCodecType codec; @@ -114,6 +120,12 @@ namespace jni // referring to the one before. webrtc::GofInfoVP9 gof; size_t gofIndex = 0; + + // With temporal layers: the pictures since the key frame, and the + // picture each reference buffer holds, which the references of a + // picture are told by. + size_t vp9Picture = 0; + std::array vp9BufferPictures = {}; }; } diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h index e4885707..9197bccc 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h @@ -20,6 +20,7 @@ #include "media/video/codec/EncoderOutputProcessor.h" #include "media/video/codec/linux/VaapiEncodeSession.h" #include "media/video/codec/linux/VaapiLibrary.h" +#include "media/video/codec/linux/VaapiTemporalLayers.h" #include "api/video/video_frame.h" #include "api/video_codecs/sdp_video_format.h" @@ -80,32 +81,11 @@ namespace jni // Reads which headers the driver takes and how it codes them. void QueryCapabilities(); - // What a frame refers to: the slot each of the seven reference - // frames names, the second reference, if any, being golden; the - // slots the frame replaces; and its temporal layer. - struct FrameReferences - { - std::array refFrameIdx = {}; - bool golden = false; - uint8_t refreshFrameFlags = 0; - int temporalId = 0; - }; - - // Turns the buffers of the layer frame into reference slots. - // Returns false if it refers to a slot that holds no frame, or to - // one beyond those kept. - bool PlanReferences(bool keyFrame, const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, - FrameReferences & references) const; - // A reconstructed surface that no reference slot holds. - int FreeSurface() const; - // Adds the rate control of the stream, or of each temporal layer. - bool AddRateControl(bool keyFrame); - - bool Submit(bool keyFrame, const FrameReferences & references, int surface); + bool Submit(bool keyFrame, const VaapiTemporalLayers::Frame & frame); void WriteSequenceHeader(std::vector & obu) const; void WriteFrameHeader(std::vector & obu, FrameHeaderLayout & layout, bool keyFrame, - const FrameReferences & references) const; + const VaapiTemporalLayers::Frame & frame) const; // The seq_level_idx the stream's size, frame rate and bitrate fit. uint8_t Level() const; @@ -128,16 +108,7 @@ namespace jni // The sequence header, kept for the key frames. std::vector sequenceHeader; - // The temporal layers, one without layers, and the bitrate of each, - // counting the layers below it. - int temporalLayers; - std::vector layerBitratesBps; - - // The reconstructed surface each reference slot holds, -1 for - // none: those of the two buffers WebRTC's structures use, up to - // L1T3. A key frame fills the other six slots too, but no frame - // refers to them. - std::array slots; + VaapiTemporalLayers layers; bool referenceValid; // The frames since the last key frame, whose low bits are the // order hint. diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiTemporalLayers.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiTemporalLayers.h new file mode 100644 index 00000000..769fc7ea --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiTemporalLayers.h @@ -0,0 +1,102 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_TEMPORAL_LAYERS_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_TEMPORAL_LAYERS_H_ + +#include "media/video/codec/linux/VaapiEncodeSession.h" + +#include "api/video/video_bitrate_allocation.h" +#include "api/video_codecs/scalability_mode.h" +#include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/svc/scalable_video_controller.h" + +#include +#include +#include +#include + +namespace jni +{ + // The temporal layers of a VA-API encoder of VP9 or AV1, none or those of + // L1T2 and L1T3: which reference slot each frame refers to and which ones + // it replaces, as WebRTC's structure of the mode says, each buffer of the + // structure being the slot of the same number; which reconstructed surface + // each slot holds; and the rate control of each layer, without which + // AMD's driver gives the upper layers next to no bits. + class VaapiTemporalLayers + { + public: + // The reconstructed surfaces a session needs: those of the two + // reference buffers WebRTC's structures use up to L1T3, and that + // of the frame being encoded. + static constexpr int kReconstructedSurfaces = 3; + + // What a frame refers to and replaces. + struct Frame + { + int temporalId = 0; + // The slots of its references, the first being its last frame, + // the second, if any, its golden frame. + std::vector references; + uint8_t refreshFlags = 0; + // The reconstructed surface it is encoded into. + int surface = -1; + }; + + VaapiTemporalLayers(); + + // Starts over in the mode, splitting the bitrate across the layers + // until WebRTC gives its own split. Returns false for a mode other + // than L1T1, L1T2 and L1T3. + bool Configure(webrtc::ScalabilityMode mode, uint32_t bitrateBps); + + // The number of temporal layers, one without layers. + int Count() const; + + // Plans a frame of the layer WebRTC's structure gives. Returns + // nothing if it refers to a slot that holds no frame, or to one + // beyond those kept, or no surface is free. + std::optional Plan(bool keyFrame, + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame) const; + // Takes the frame, once encoded, into the slots it replaces. + void Commit(const Frame & frame); + + // The slots kept, and the reconstructed surface each holds, -1 for + // none. A key frame fills the slots beyond too, but no frame refers + // to them. + int SlotCount() const; + int Surface(int slot) const; + + // Takes the bitrate WebRTC gives each layer. + void OnRatesUpdated(const webrtc::VideoBitrateAllocation & bitrates); + // Adds the rate control of the stream, or of each layer, with the + // layer structure along with that of a key frame. No initial + // quantizer: the rate control picks its own, as FFmpeg leaves it. + bool AddRateControl(VaapiEncodeSession & session, uint32_t bitrateBps, uint32_t framerate, + bool keyFrame) const; + // Fills in the share of the frame rate up to each layer. + void FillFpsAllocation(webrtc::VideoEncoder::EncoderInfo & info) const; + + private: + int count; + std::array slots; + // The bitrate of each layer, counting the layers below it. + std::vector bitratesBps; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Encoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Encoder.h index 9bcfb3aa..54d41d28 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Encoder.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Encoder.h @@ -20,6 +20,7 @@ #include "media/video/codec/EncoderOutputProcessor.h" #include "media/video/codec/linux/VaapiEncodeSession.h" #include "media/video/codec/linux/VaapiLibrary.h" +#include "media/video/codec/linux/VaapiTemporalLayers.h" #include "api/video/video_frame.h" #include "api/video_codecs/sdp_video_format.h" @@ -36,15 +37,16 @@ namespace jni { // Encodes VP9 profile 0 with the GPU driver's VA-API encoder, as Intel - // drivers provide it on Linux. Like hardware VP9 encoding on Windows, it - // has not run on such a GPU yet, and is used only where it is turned on. + // drivers provide it on Linux. // - // It works as the H.265 one does: synchronous, one reference frame, inter - // frames only, the driver writing the uncompressed frame header from the - // parameters it is given, as FFmpeg's VA-API encoder sets them without - // B-frames. Every frame refreshes all eight reference slots, so that the - // next one can refer to it as the last frame whichever slot it names. - // Anything that fails is an error, upon which libvpx takes over. + // It works as the H.265 one does: synchronous, inter frames only, the + // driver writing the uncompressed frame header from the parameters it is + // given, as FFmpeg's VA-API encoder sets them without B-frames. Each frame + // refers to one frame before it: without layers the frame just before; + // with temporal layers (L1T2, L1T3) the one of the same layer or a lower + // one that WebRTC's structure of the mode names, so that a receiver can do + // without the upper layers. Spatial layers, and anything that fails, are + // left to libvpx. class VaapiVp9Encoder : public webrtc::VideoEncoder { public: @@ -59,10 +61,11 @@ namespace jni EncoderInfo GetEncoderInfo() const override; private: - bool Submit(bool keyFrame); + bool Submit(bool keyFrame, const VaapiTemporalLayers::Frame & frame); void FillSequence(VAEncSequenceParameterBufferVP9 & sequence) const; - void FillPicture(VAEncPictureParameterBufferVP9 & picture, bool keyFrame) const; + void FillPicture(VAEncPictureParameterBufferVP9 & picture, bool keyFrame, + const VaapiTemporalLayers::Frame & frame) const; private: VaapiLibrary & library; @@ -76,8 +79,7 @@ namespace jni uint32_t framerate; bool ratesChanged; - // Which of the reconstructed surfaces holds the current frame. - int current; + VaapiTemporalLayers layers; bool referenceValid; webrtc::EncodedImageCallback * callback; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp index 1609dd16..c401dbdd 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp @@ -22,6 +22,7 @@ #include "common_video/h265/h265_common.h" #include "modules/video_coding/codecs/interface/common_constants.h" #include "modules/video_coding/svc/create_scalability_structure.h" +#include "modules/video_coding/svc/scalability_mode_util.h" #include "modules/video_coding/svc/scalable_video_controller_no_layering.h" #include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" #include "rtc_base/logging.h" @@ -409,7 +410,10 @@ namespace jni }; } - if (codec == webrtc::kVideoCodecVP9) { + if (codec == webrtc::kVideoCodecVP9 && scalabilityMode != webrtc::ScalabilityMode::kL1T1) { + FillVp9TemporalLayers(image, keyFrame, layerFrame, info); + } + else if (codec == webrtc::kVideoCodecVP9) { // VP9 has its own RTP header, which WebRTC fills from these: one // spatial and one temporal layer, the group of frames sent with // every key frame. @@ -436,6 +440,77 @@ namespace jni } } + void EncoderOutputProcessor::FillVp9TemporalLayers(const webrtc::EncodedImage & image, bool keyFrame, + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::CodecSpecificInfo & info) + { + // One spatial layer and temporal ones, in flexible mode, as libvpx + // describes them: each picture names the pictures it refers to, by + // how many pictures back they are. + vp9Picture = keyFrame ? 0 : vp9Picture + 1; + + webrtc::CodecSpecificInfoVP9 & vp9 = info.codecSpecific.VP9; + vp9.first_frame_in_picture = true; + vp9.flexible_mode = true; + vp9.ss_data_available = keyFrame; + vp9.non_ref_for_inter_layer_pred = true; + vp9.temporal_idx = static_cast(layerFrame.TemporalId()); + vp9.inter_layer_predicted = false; + vp9.gof_idx = webrtc::kNoGofIdx; + vp9.num_spatial_layers = 1; + vp9.first_active_layer = 0; + vp9.spatial_layer_resolution_present = keyFrame; + vp9.num_ref_pics = 0; + + if (!keyFrame) { + for (const webrtc::CodecBufferUsage & buffer : layerFrame.Buffers()) { + if (!buffer.referenced || buffer.id < 0 || buffer.id >= webrtc::kMaxEncoderBuffers) { + continue; + } + + const size_t pDiff = vp9Picture - vp9BufferPictures[buffer.id]; + bool listed = false; + + for (uint8_t i = 0; i < vp9.num_ref_pics; i++) { + listed |= vp9.p_diff[i] == pDiff; + } + + if (!listed && pDiff > 0 && vp9.num_ref_pics < webrtc::kMaxVp9RefPics) { + vp9.p_diff[vp9.num_ref_pics++] = static_cast(pDiff); + } + } + } + + vp9.inter_pic_predicted = !keyFrame && vp9.num_ref_pics > 0; + + // A switch up to a higher layer is possible where all layers above + // this one are switch points, as libvpx works it out. + const int temporalLayers = webrtc::ScalabilityModeToNumTemporalLayers(scalabilityMode); + vp9.temporal_up_switch = true; + + for (int layer = layerFrame.TemporalId() + 1; layer < temporalLayers && info.generic_frame_info; layer++) { + const auto & indications = info.generic_frame_info->decode_target_indications; + + vp9.temporal_up_switch &= layer < static_cast(indications.size()) + && indications[layer] == webrtc::DecodeTargetIndication::kSwitch; + } + + if (keyFrame) { + vp9.width[0] = image._encodedWidth; + vp9.height[0] = image._encodedHeight; + vp9.gof.num_frames_in_gof = 0; + } + + for (const webrtc::CodecBufferUsage & buffer : layerFrame.Buffers()) { + if ((buffer.updated || keyFrame) && buffer.id >= 0 && buffer.id < webrtc::kMaxEncoderBuffers) { + vp9BufferPictures[buffer.id] = vp9Picture; + } + } + + if (keyFrame) { + vp9BufferPictures.fill(0); + } + } + webrtc::VideoEncoder::ScalingSettings EncoderOutputProcessor::GetScalingSettings() const { // H.265 has no thresholds of its own in WebRTC yet and uses those of diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp index ede55634..dc61ffc7 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp @@ -65,21 +65,6 @@ namespace jni // and other state carry over from. constexpr uint8_t kPrimaryRefLast = 0; - // The slots of the two reference buffers L1T3 has, and the frame being - // encoded. - constexpr int kReconstructedSurfaces = 3; - - // The layers of the frames after a key frame, as - // VAEncMiscParameterTemporalLayerStructure takes them, of L1T2 and - // L1T3 as WebRTC's structures make them: L1T3 is 0, 2, 1, 2. - const std::vector kL1T2Pattern = { 1, 0 }; - const std::vector kL1T3Pattern = { 2, 1, 2, 0 }; - - // The share of the bitrate up to each layer until WebRTC gives its - // own, as WebRTC splits it for temporal layers. - constexpr double kL1T2Shares[] = { 0.6, 1.0 }; - constexpr double kL1T3Shares[] = { 0.4, 0.6, 1.0 }; - // What a frame that cannot be encoded returns: libaom takes over the // stream, which stays in AV1, rather than WebRTC changing the codec. constexpr int32_t kEncodeFailed = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; @@ -129,6 +114,22 @@ namespace jni // The level of a stream that fits none. constexpr uint8_t kMaxParametersLevel = 31; + // ref_frame_idx: every reference frame names the first reference of + // the frame, but golden, which names the second where there is one. + std::array RefFrameIdx(const VaapiTemporalLayers::Frame & frame) + { + std::array refFrameIdx = {}; + + if (!frame.references.empty()) { + refFrameIdx.fill(static_cast(frame.references[0])); + } + if (frame.references.size() > 1) { + refFrameIdx[kGoldenIndex] = static_cast(frame.references[1]); + } + + return refFrameIdx; + } + uint32_t Align(uint32_t value, uint32_t alignment) { return (value + alignment - 1) / alignment * alignment; @@ -253,7 +254,6 @@ namespace jni ratesChanged(false), obuSizeLength(0), txMode(0), - temporalLayers(1), referenceValid(false), frameNumber(0), callback(nullptr), @@ -347,18 +347,10 @@ namespace jni bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; framerate = std::max(1u, codecSettings.maxFramerate); ratesChanged = true; - temporalLayers = mode == webrtc::ScalabilityMode::kL1T3 ? 3 : mode == webrtc::ScalabilityMode::kL1T2 ? 2 : 1; - slots.fill(-1); referenceValid = false; frameNumber = 0; outputProcessor.Reset(); - - layerBitratesBps.clear(); - - for (int layer = 0; layer < temporalLayers && temporalLayers > 1; layer++) { - const double share = temporalLayers == 3 ? kL1T3Shares[layer] : kL1T2Shares[layer]; - layerBitratesBps.push_back(static_cast(bitrateBps * share)); - } + layers.Configure(mode, bitrateBps); if (!outputProcessor.SetScalabilityMode(mode)) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; @@ -368,7 +360,7 @@ namespace jni const uint32_t surfaceHeight = Align(codecSettings.height, kSuperblockSize); if (!session.Create(VAProfileAV1Profile0, entrypoint, codecSettings.width, codecSettings.height, surfaceWidth, - surfaceHeight, kPackedHeaders, kReconstructedSurfaces)) + surfaceHeight, kPackedHeaders, VaapiTemporalLayers::kReconstructedSurfaces)) { Release(); return WEBRTC_VIDEO_CODEC_ERROR; @@ -427,10 +419,9 @@ namespace jni return WEBRTC_VIDEO_CODEC_OK; } - FrameReferences references; - const int surface = FreeSurface(); + const auto layerPlan = layers.Plan(keyFrame, *layerFrame); - if (surface < 0 || !PlanReferences(keyFrame, *layerFrame, references)) { + if (!layerPlan) { RTC_LOG(LS_WARNING) << implementationName << " AV1: a frame refers to a slot that holds no frame"; return kEncodeFailed; } @@ -440,7 +431,7 @@ namespace jni // A frame up to the alignment larger is cropped; one of another size, // which raced with a change of size, is scaled. const bool encoded = session.Upload(frame, static_cast(codecSettings.width), - static_cast(codecSettings.height), kSizeAlignment - 1) && Submit(keyFrame, references, surface) + static_cast(codecSettings.height), kSizeAlignment - 1) && Submit(keyFrame, *layerPlan) && session.ReadOutput(output); session.DropBuffers(); @@ -463,12 +454,7 @@ namespace jni return kEncodeFailed; } - // The slots the frame replaces hold it now. - for (int slot = 0; slot < static_cast(slots.size()); slot++) { - if (references.refreshFrameFlags & (1 << slot)) { - slots[slot] = surface; - } - } + layers.Commit(*layerPlan); referenceValid = true; frameNumber++; @@ -479,81 +465,11 @@ namespace jni return WEBRTC_VIDEO_CODEC_OK; } - bool VaapiAv1Encoder::PlanReferences(bool keyFrame, - const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, FrameReferences & references) const - { - references.temporalId = layerFrame.TemporalId(); - - // A key frame replaces every slot. - if (keyFrame) { - references.refreshFrameFlags = 0xFF; - return true; - } - - std::vector referenced; - - for (const webrtc::CodecBufferUsage & buffer : layerFrame.Buffers()) { - if (buffer.id < 0 || buffer.id >= static_cast(slots.size())) { - return false; - } - if (buffer.referenced) { - if (slots[buffer.id] < 0) { - return false; - } - - referenced.push_back(buffer.id); - } - if (buffer.updated) { - references.refreshFrameFlags |= static_cast(1 << buffer.id); - } - } - - if (referenced.empty()) { - return false; - } - - // Every reference frame names the first reference, but golden, which - // names the second where there is one. - references.refFrameIdx.fill(static_cast(referenced[0])); - - if (referenced.size() > 1) { - references.refFrameIdx[kGoldenIndex] = static_cast(referenced[1]); - references.golden = true; - } - - return true; - } - - bool VaapiAv1Encoder::AddRateControl(bool keyFrame) - { - // No initial quantizer: the rate control picks its own, as FFmpeg - // leaves it. - if (temporalLayers == 1) { - return session.AddRateControl(bitrateBps, framerate, 0); - } - - // Without a rate control of each layer, AMD's driver gives the upper - // layers next to no bits. - return session.AddLayerRateControl(layerBitratesBps, framerate, - temporalLayers == 3 ? kL1T3Pattern : kL1T2Pattern, keyFrame); - } - - int VaapiAv1Encoder::FreeSurface() const - { - for (int surface = 0; surface < kReconstructedSurfaces; surface++) { - if (std::find(slots.begin(), slots.end(), surface) == slots.end()) { - return surface; - } - } - - return -1; - } - - bool VaapiAv1Encoder::Submit(bool keyFrame, const FrameReferences & references, int surface) + bool VaapiAv1Encoder::Submit(bool keyFrame, const VaapiTemporalLayers::Frame & frame) { const uint32_t sbCols = Align(codecSettings.width, kSuperblockSize) / kSuperblockSize; const uint32_t sbRows = Align(codecSettings.height, kSuperblockSize) / kSuperblockSize; - const bool layered = temporalLayers > 1; + const bool layered = layers.Count() > 1; if (keyFrame) { VAEncSequenceParameterBufferAV1 sequence = {}; @@ -574,20 +490,20 @@ namespace jni } } - if ((keyFrame || ratesChanged) && !AddRateControl(keyFrame)) { + if ((keyFrame || ratesChanged) && !layers.AddRateControl(session, bitrateBps, framerate, keyFrame)) { return false; } std::vector frameHeader; FrameHeaderLayout layout; - WriteFrameHeader(frameHeader, layout, keyFrame, references); + WriteFrameHeader(frameHeader, layout, keyFrame, frame); VAEncPictureParameterBufferAV1 picture = {}; picture.frame_width_minus_1 = static_cast(codecSettings.width - 1); picture.frame_height_minus_1 = static_cast(codecSettings.height - 1); - picture.reconstructed_frame = session.Reconstructed(surface); + picture.reconstructed_frame = session.Reconstructed(frame.surface); picture.coded_buf = session.CodedBuffer(); - picture.refresh_frame_flags = references.refreshFrameFlags; + picture.refresh_frame_flags = frame.refreshFlags; for (VASurfaceID & reference : picture.reference_frames) { reference = VA_INVALID_SURFACE; @@ -599,18 +515,19 @@ namespace jni picture.picture_flags.bits.error_resilient_mode = 1; } else { - for (int slot = 0; slot < static_cast(slots.size()); slot++) { - if (slots[slot] >= 0) { - picture.reference_frames[slot] = session.Reconstructed(slots[slot]); + for (int slot = 0; slot < layers.SlotCount(); slot++) { + if (layers.Surface(slot) >= 0) { + picture.reference_frames[slot] = session.Reconstructed(layers.Surface(slot)); } } - std::copy(references.refFrameIdx.begin(), references.refFrameIdx.end(), picture.ref_frame_idx); + const std::array refFrameIdx = RefFrameIdx(frame); + std::copy(refFrameIdx.begin(), refFrameIdx.end(), picture.ref_frame_idx); picture.primary_ref_frame = kPrimaryRefLast; picture.ref_frame_ctrl_l0.fields.search_idx0 = kLastFrame; - if (references.golden) { + if (frame.references.size() > 1) { picture.ref_frame_ctrl_l0.fields.search_idx1 = kGoldenFrame; } @@ -618,10 +535,10 @@ namespace jni } if (layered) { - picture.temporal_id = static_cast(references.temporalId); - picture.hierarchical_level_plus1 = static_cast(references.temporalId + 1); + picture.temporal_id = static_cast(frame.temporalId); + picture.hierarchical_level_plus1 = static_cast(frame.temporalId + 1); picture.tile_group_obu_hdr_info.bits.obu_extension_flag = 1; - picture.tile_group_obu_hdr_info.bits.temporal_id = static_cast(references.temporalId); + picture.tile_group_obu_hdr_info.bits.temporal_id = static_cast(frame.temporalId); } picture.order_hint = static_cast(frameNumber); @@ -678,6 +595,8 @@ namespace jni // point for each, from all layers down to the lowest. Then // operating_point_idc, of spatial layer 0 and those temporal layers, // seq_level_idx, and seq_tier above level 3.3. + const int temporalLayers = layers.Count(); + writer.Put(temporalLayers - 1, 5); for (int point = 0; point < temporalLayers; point++) { @@ -731,11 +650,12 @@ namespace jni } void VaapiAv1Encoder::WriteFrameHeader(std::vector & obu, FrameHeaderLayout & layout, bool keyFrame, - const FrameReferences & references) const + const VaapiTemporalLayers::Frame & frame) const { // uncompressed_header, section 5.9, as FFmpeg's VA-API encoder writes // it, for the sequence header above. - const bool layered = temporalLayers > 1; + const bool layered = layers.Count() > 1; + const std::array refFrameIdx = RefFrameIdx(frame); const uint32_t miCols = 2 * ((codecSettings.width + 7) >> 3); const uint32_t miRows = 2 * ((codecSettings.height + 7) >> 3); const uint32_t sbCols = (miCols + 15) >> 4; @@ -762,12 +682,12 @@ namespace jni if (!keyFrame) { // primary_ref_frame, refresh_frame_flags. writer.Put(kPrimaryRefLast, 3); - writer.Put(references.refreshFrameFlags, 8); + writer.Put(frame.refreshFlags, 8); // frame_refs_short_signaling, then ref_frame_idx. writer.Put(0, 1); for (int i = 0; i < kRefsPerFrame; i++) { - writer.Put(references.refFrameIdx[i], 3); + writer.Put(refFrameIdx[i], 3); } } @@ -849,7 +769,7 @@ namespace jni writer.TrailingBits(); obu = MakeObu(kObuFrameHeader, writer.Bytes(), obuSizeLength, - layered ? std::optional(references.temporalId) : std::nullopt); + layered ? std::optional(frame.temporalId) : std::nullopt); layout.size = static_cast(obu.size() * 8); } @@ -882,16 +802,7 @@ namespace jni bitrateBps = bitrate; outputProcessor.OnRatesUpdated(parameters.bitrate); - - // The bitrate of each temporal layer, counting those below it. - uint32_t sum = 0; - - for (int layer = 0; layer < static_cast(layerBitratesBps.size()); layer++) { - sum += parameters.bitrate.GetBitrate(0, layer); - // A layer WebRTC gives no bitrate is not encoded; it keeps the - // rate of the one below, which the driver takes. - layerBitratesBps[layer] = std::max(sum, 1u); - } + layers.OnRatesUpdated(parameters.bitrate); if (parameters.framerate_fps >= 1.0) { framerate = static_cast(parameters.framerate_fps + 0.5); @@ -911,13 +822,7 @@ namespace jni info.scaling_settings = outputProcessor.GetScalingSettings(); info.requested_resolution_alignment = kSizeAlignment; - // With temporal layers, the share of the frame rate up to each. - if (temporalLayers > 1) { - for (int layer = 0; layer < temporalLayers; layer++) { - info.fps_allocation[0].push_back(static_cast( - EncoderInfo::kMaxFramerateFraction >> (temporalLayers - 1 - layer))); - } - } + layers.FillFpsAllocation(info); return info; } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiTemporalLayers.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiTemporalLayers.cpp new file mode 100644 index 00000000..1e8cfc70 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiTemporalLayers.cpp @@ -0,0 +1,173 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/linux/VaapiTemporalLayers.h" + +#include + +namespace jni +{ + namespace + { + // The layers of the frames after a key frame, as + // VAEncMiscParameterTemporalLayerStructure takes them, of L1T2 and + // L1T3 as WebRTC's structures make them: L1T3 is 0, 2, 1, 2. + const std::vector kL1T2Pattern = { 1, 0 }; + const std::vector kL1T3Pattern = { 2, 1, 2, 0 }; + + // The share of the bitrate up to each layer until WebRTC gives its + // own, as WebRTC splits it for temporal layers. + constexpr double kL1T2Shares[] = { 0.6, 1.0 }; + constexpr double kL1T3Shares[] = { 0.4, 0.6, 1.0 }; + } + + VaapiTemporalLayers::VaapiTemporalLayers() : + count(1) + { + slots.fill(-1); + } + + bool VaapiTemporalLayers::Configure(webrtc::ScalabilityMode mode, uint32_t bitrateBps) + { + switch (mode) { + case webrtc::ScalabilityMode::kL1T1: + count = 1; + break; + case webrtc::ScalabilityMode::kL1T2: + count = 2; + break; + case webrtc::ScalabilityMode::kL1T3: + count = 3; + break; + default: + return false; + } + + slots.fill(-1); + bitratesBps.clear(); + + for (int layer = 0; layer < count && count > 1; layer++) { + const double share = count == 3 ? kL1T3Shares[layer] : kL1T2Shares[layer]; + bitratesBps.push_back(static_cast(bitrateBps * share)); + } + + return true; + } + + int VaapiTemporalLayers::Count() const + { + return count; + } + + std::optional VaapiTemporalLayers::Plan(bool keyFrame, + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame) const + { + Frame frame; + frame.temporalId = layerFrame.TemporalId(); + + for (int surface = 0; surface < kReconstructedSurfaces && frame.surface < 0; surface++) { + if (std::find(slots.begin(), slots.end(), surface) == slots.end()) { + frame.surface = surface; + } + } + + if (frame.surface < 0) { + return std::nullopt; + } + + // A key frame replaces every slot. + if (keyFrame) { + frame.refreshFlags = 0xFF; + return frame; + } + + for (const webrtc::CodecBufferUsage & buffer : layerFrame.Buffers()) { + if (buffer.id < 0 || buffer.id >= static_cast(slots.size())) { + return std::nullopt; + } + if (buffer.referenced) { + if (slots[buffer.id] < 0) { + return std::nullopt; + } + + frame.references.push_back(buffer.id); + } + if (buffer.updated) { + frame.refreshFlags |= static_cast(1 << buffer.id); + } + } + + if (frame.references.empty()) { + return std::nullopt; + } + + return frame; + } + + void VaapiTemporalLayers::Commit(const Frame & frame) + { + for (int slot = 0; slot < static_cast(slots.size()); slot++) { + if (frame.refreshFlags & (1 << slot)) { + slots[slot] = frame.surface; + } + } + } + + int VaapiTemporalLayers::SlotCount() const + { + return static_cast(slots.size()); + } + + int VaapiTemporalLayers::Surface(int slot) const + { + return slots[slot]; + } + + void VaapiTemporalLayers::OnRatesUpdated(const webrtc::VideoBitrateAllocation & bitrates) + { + uint32_t sum = 0; + + for (int layer = 0; layer < static_cast(bitratesBps.size()); layer++) { + sum += bitrates.GetBitrate(0, layer); + // A layer WebRTC gives no bitrate is not encoded; it keeps the + // rate of the one below, which the driver takes. + bitratesBps[layer] = std::max(sum, 1u); + } + } + + bool VaapiTemporalLayers::AddRateControl(VaapiEncodeSession & session, uint32_t bitrateBps, uint32_t framerate, + bool keyFrame) const + { + if (count == 1) { + return session.AddRateControl(bitrateBps, framerate, 0); + } + + return session.AddLayerRateControl(bitratesBps, framerate, count == 3 ? kL1T3Pattern : kL1T2Pattern, + keyFrame); + } + + void VaapiTemporalLayers::FillFpsAllocation(webrtc::VideoEncoder::EncoderInfo & info) const + { + if (count == 1) { + return; + } + + for (int layer = 0; layer < count; layer++) { + info.fps_allocation[0].push_back(static_cast( + webrtc::VideoEncoder::EncoderInfo::kMaxFramerateFraction >> (count - 1 - layer))); + } + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp index c94b0801..ecc507b9 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp @@ -46,8 +46,13 @@ namespace jni constexpr uint8_t kFilterLevel = 16; constexpr uint8_t kSharpnessLevel = 4; - // The reference slot the next frame refers to as its last frame. - constexpr uint32_t kLastSlot = 0; + // The reference frames LAST and GOLDEN in ref_frame_ctrl_l0. + constexpr uint32_t kLastFrame = 1; + constexpr uint32_t kGoldenFrame = 2; + + // What a frame that cannot be encoded returns: libvpx takes over the + // stream, which stays in VP9, rather than WebRTC changing the codec. + constexpr int32_t kEncodeFailed = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; uint32_t Align(uint32_t value, uint32_t alignment) { @@ -66,7 +71,6 @@ namespace jni bitrateBps(0), framerate(30), ratesChanged(false), - current(0), referenceValid(false), callback(nullptr), outputProcessor(webrtc::kVideoCodecVP9, format) @@ -86,8 +90,19 @@ namespace jni if (settings->numberOfSimulcastStreams > 1) { return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED; } - // One spatial and one temporal layer is all this encoder produces. - if (settings->VP9().numberOfSpatialLayers > 1 || settings->VP9().numberOfTemporalLayers > 1) { + // Temporal layers; spatial ones are left to libvpx. + if (settings->VP9().numberOfSpatialLayers > 1) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + const int temporalLayers = settings->VP9().numberOfTemporalLayers; + const webrtc::ScalabilityMode mode = settings->GetScalabilityMode().value_or(temporalLayers == 3 + ? webrtc::ScalabilityMode::kL1T3 + : temporalLayers == 2 ? webrtc::ScalabilityMode::kL1T2 : webrtc::ScalabilityMode::kL1T1); + + if (mode != webrtc::ScalabilityMode::kL1T1 && mode != webrtc::ScalabilityMode::kL1T2 + && mode != webrtc::ScalabilityMode::kL1T3) + { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } @@ -97,14 +112,19 @@ namespace jni bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; framerate = std::max(1u, codecSettings.maxFramerate); ratesChanged = true; - current = 0; referenceValid = false; outputProcessor.Reset(); + layers.Configure(mode, bitrateBps); + + if (!outputProcessor.SetScalabilityMode(mode)) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } const uint32_t surfaceWidth = Align(codecSettings.width, kSurfaceAlignment); const uint32_t surfaceHeight = Align(codecSettings.height, kSurfaceAlignment); - if (!session.Create(VAProfileVP9Profile0, entrypoint, surfaceWidth, surfaceHeight, surfaceWidth, surfaceHeight)) + if (!session.Create(VAProfileVP9Profile0, entrypoint, surfaceWidth, surfaceHeight, surfaceWidth, surfaceHeight, + VA_ENC_PACKED_HEADER_NONE, VaapiTemporalLayers::kReconstructedSurfaces)) { Release(); return WEBRTC_VIDEO_CODEC_ERROR; @@ -113,7 +133,7 @@ namespace jni initialized = true; RTC_LOG(LS_INFO) << implementationName << " VP9 initialized: " << codecSettings.width << "x" - << codecSettings.height << " at " << bitrateBps << " bps"; + << codecSettings.height << " at " << bitrateBps << " bps, " << webrtc::ScalabilityModeToString(mode); return WEBRTC_VIDEO_CODEC_OK; } @@ -150,17 +170,31 @@ namespace jni }); } + const auto layerFrame = outputProcessor.NextFrameConfig(keyFrame); + + // No layer is encoded now, as when WebRTC gives none a bitrate. + if (!layerFrame) { + return WEBRTC_VIDEO_CODEC_OK; + } + + const auto layerPlan = layers.Plan(keyFrame, *layerFrame); + + if (!layerPlan) { + RTC_LOG(LS_WARNING) << implementationName << " VP9: a frame refers to a slot that holds no frame"; + return kEncodeFailed; + } + std::vector output; // VP9 codes any size, so the frame is taken as it is, and scaled only // if it raced with a change of size. const bool encoded = session.Upload(frame, static_cast(codecSettings.width), - static_cast(codecSettings.height), 0) && Submit(keyFrame) && session.ReadOutput(output); + static_cast(codecSettings.height), 0) && Submit(keyFrame, *layerPlan) && session.ReadOutput(output); session.DropBuffers(); if (!encoded) { - return WEBRTC_VIDEO_CODEC_ERROR; + return kEncodeFailed; } webrtc::EncodedImage image; @@ -173,13 +207,12 @@ namespace jni webrtc::CodecSpecificInfo info; - if (!outputProcessor.Process(output, keyFrame, image, info)) { - return WEBRTC_VIDEO_CODEC_ERROR; + if (!outputProcessor.Process(output, keyFrame, *layerFrame, image, info)) { + return kEncodeFailed; } - // The frame just encoded is the reference of the next one. + layers.Commit(*layerPlan); referenceValid = true; - current = 1 - current; ratesChanged = false; callback->OnEncodedImage(image, &info); @@ -187,7 +220,7 @@ namespace jni return WEBRTC_VIDEO_CODEC_OK; } - bool VaapiVp9Encoder::Submit(bool keyFrame) + bool VaapiVp9Encoder::Submit(bool keyFrame, const VaapiTemporalLayers::Frame & frame) { if (keyFrame) { VAEncSequenceParameterBufferVP9 sequence = {}; @@ -198,14 +231,12 @@ namespace jni } } - // No initial quantizer: VP9 rate control picks its own, as FFmpeg - // leaves it. - if ((keyFrame || ratesChanged) && !session.AddRateControl(bitrateBps, framerate, 0)) { + if ((keyFrame || ratesChanged) && !layers.AddRateControl(session, bitrateBps, framerate, keyFrame)) { return false; } VAEncPictureParameterBufferVP9 picture = {}; - FillPicture(picture, keyFrame); + FillPicture(picture, keyFrame, frame); return session.AddBuffer(VAEncPictureParameterBufferType, &picture, sizeof(picture)) && session.Encode(); } @@ -222,13 +253,14 @@ namespace jni sequence.bits_per_second = bitrateBps; } - void VaapiVp9Encoder::FillPicture(VAEncPictureParameterBufferVP9 & picture, bool keyFrame) const + void VaapiVp9Encoder::FillPicture(VAEncPictureParameterBufferVP9 & picture, bool keyFrame, + const VaapiTemporalLayers::Frame & frame) const { picture.frame_width_src = codecSettings.width; picture.frame_height_src = codecSettings.height; picture.frame_width_dst = codecSettings.width; picture.frame_height_dst = codecSettings.height; - picture.reconstructed_frame = session.Reconstructed(current); + picture.reconstructed_frame = session.Reconstructed(frame.surface); picture.coded_buf = session.CodedBuffer(); for (VASurfaceID & reference : picture.reference_frames) { @@ -240,16 +272,36 @@ namespace jni picture.pic_flags.bits.frame_type = kKeyFrameType; } else { - picture.reference_frames[kLastSlot] = session.Reconstructed(1 - current); - // The last frame is the only reference, in list 0. - picture.ref_flags.bits.ref_frame_ctrl_l0 = 1; - picture.ref_flags.bits.ref_last_idx = kLastSlot; + for (int slot = 0; slot < layers.SlotCount(); slot++) { + if (layers.Surface(slot) >= 0) { + picture.reference_frames[slot] = session.Reconstructed(layers.Surface(slot)); + } + } + + // The first reference is the last frame, a second one golden; both + // in list 0. + picture.ref_flags.bits.ref_frame_ctrl_l0 = kLastFrame; + picture.ref_flags.bits.ref_last_idx = static_cast(frame.references[0]); + + if (frame.references.size() > 1) { + picture.ref_flags.bits.ref_frame_ctrl_l0 |= kGoldenFrame; + picture.ref_flags.bits.ref_gf_idx = static_cast(frame.references[1]); + } + picture.pic_flags.bits.frame_type = kInterFrameType; } + picture.ref_flags.bits.temporal_id = static_cast(frame.temporalId); picture.pic_flags.bits.show_frame = 1; - // Every slot holds this frame afterwards. - picture.refresh_frame_flags = 0xFF; + + // VP9 takes motion vectors from the frame decoded before, and carries + // probabilities from frame to frame, neither of which a receiver + // without the upper layers has; error resilience leaves both out, as + // libvpx does with layers. + if (layers.Count() > 1) { + picture.pic_flags.bits.error_resilient_mode = 1; + } + picture.refresh_frame_flags = frame.refreshFlags; picture.luma_ac_qindex = kQIndex; picture.filter_level = kFilterLevel; @@ -266,6 +318,8 @@ namespace jni } bitrateBps = bitrate; + outputProcessor.OnRatesUpdated(parameters.bitrate); + layers.OnRatesUpdated(parameters.bitrate); if (parameters.framerate_fps >= 1.0) { framerate = static_cast(parameters.framerate_fps + 0.5); @@ -284,6 +338,8 @@ namespace jni info.supports_simulcast = false; info.scaling_settings = outputProcessor.GetScalingSettings(); + layers.FillFpsAllocation(info); + return info; } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java index 84c517f0..7b9ea334 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java @@ -45,7 +45,8 @@ * Intel's: with VA-API on Linux, and with Media Foundation on Windows only * with the system property {@code webrtc.hardware.vp9Encoder} set to * {@code true} when the factory is created, since that encoder has not been - * tested on such a GPU yet. Otherwise VP9 is encoded with libvpx. On Linux, H.265 is encoded with NVENC on NVIDIA + * tested on such a GPU yet. With VA-API, VP9 is encoded in temporal layers + * too (L1T2, L1T3). Otherwise VP9 is encoded with libvpx. On Linux, H.265 is encoded with NVENC on NVIDIA * GPUs and with VA-API on the GPUs of Intel and AMD whose driver encodes it. * A hardware encoder that fails to start, for example because the GPU has no * encoder sessions left, or fails while encoding, is replaced by the next one diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java index 5486c161..26e8aa61 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java @@ -524,14 +524,31 @@ void av1RestartsCleanly() throws Exception { @Test void av1EncodesTemporalLayers() throws Exception { - // Temporal layers are references between frames of one size, which a - // hardware encoder can make; an SFU can then forward a lower frame - // rate without asking for key frames. + assertEncodesTemporalLayers(AV1, HARDWARE_AV1_REQUIRED); + } + + @Test + void vp9EncodesTemporalLayers() throws Exception { + // VA-API encodes VP9 in temporal layers, Media Foundation does not. + assumeTrue(OS.contains("linux"), "hardware VP9 encodes temporal layers on Linux only"); + + assertEncodesTemporalLayers(VP9, HARDWARE_VP9_REQUIRED); + } + + /** + * Sends the codec in L1T2 and then L1T3, and checks that the hardware + * encodes the layers, rather than handing them to the software encoder. + * Temporal layers are references between frames of one size, which a + * hardware encoder can make; an SFU can then forward a lower frame rate + * without asking for key frames. + */ + private void assertEncodesTemporalLayers(Predicate codec, boolean required) + throws Exception { for (String mode : new String[] { "L1T2", "L1T3" }) { PeerConnectionFactory hardware = hardwareFactory(); CountDownLatch received = new CountDownLatch(30); - try (TestMediaCall call = new TestMediaCall(hardware, true, false, AV1)) { + try (TestMediaCall call = new TestMediaCall(hardware, true, false, codec)) { call.setVideoSize(640, 360); call.negotiate(); @@ -560,7 +577,7 @@ void av1EncodesTemporalLayers() throws Exception { track.removeSink(sink); receiver.dispose(); - assertHardware(String.valueOf(outbound.get("encoderImplementation")), HARDWARE_AV1_REQUIRED); + assertHardware(String.valueOf(outbound.get("encoderImplementation")), required); assertEquals(mode, outbound.get("scalabilityMode")); assertTrue(!(pliCount instanceof Number) || ((Number) pliCount).longValue() <= 1, "key frames requested: " + pliCount);