diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index e87bc22e..a8d739bf 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -30,7 +30,7 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder() | Linux | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) where it encodes H.265 | NVENC on NVIDIA RTX 40 and newer, otherwise the VA-API encoder of the GPU driver where it encodes AV1 (e.g. AMD RDNA3 and newer) | | macOS | VideoToolbox, as with `DefaultVideoEncoderFactory` | VideoToolbox, on Macs with a hardware HEVC encoder, such as all with Apple silicon | Software | -Hardware encoders produce a single layer, but for the VA-API AV1 encoder, which encodes temporal layers too (`L1T2` and `L1T3`), with a rate control of each layer, so that an SFU can forward a lower frame rate. A stream that asks for spatial layers, or for temporal layers from another hardware encoder, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom. +Hardware encoders produce a single layer, but for the VA-API encoders of AV1 and VP9, which encode temporal layers too (`L1T2` and `L1T3`), with a rate control of each layer, so that an SFU can forward a lower frame rate. A stream that asks for spatial layers, or for temporal layers from another hardware encoder, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom. VP9 can be encoded in hardware too, with the VP9 encoder of the GPU driver, which Intel GPUs have: through VA-API on Linux, and through Media Foundation on Windows. On Windows it is off by default, since it has not been tested on such a GPU yet; turn it on with a system property before the factory is created: diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h b/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h index 0e158255..fbc761fd 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h @@ -30,6 +30,7 @@ #include "api/video_codecs/scalability_mode.h" #include "modules/video_coding/svc/scalable_video_controller.h" +#include #include #include #include @@ -47,10 +48,11 @@ namespace jni // or the sequence header of AV1. Encoders put them in front of the first // key frame, not necessarily in front of every one, so they are kept and // put back where missing. The frame dependencies WebRTC needs for AV1 and - // H.265 are those of a stream without layers, unless an AV1 encoder that - // makes temporal layers sets its scalability mode and asks for the layer - // of each frame before it encodes it. VP9 frames go out as they come, with - // the VP9 information of a single layer, as the Java encoders get it. + // H.265 are those of a stream without layers, unless an AV1 or VP9 encoder + // that makes temporal layers sets its scalability mode and asks for the + // layer of each frame before it encodes it. VP9 frames go out as they + // come, with the VP9 information of a single layer, as the Java encoders + // get it, or with temporal layers in flexible mode, as libvpx gives it. class EncoderOutputProcessor { public: @@ -59,8 +61,8 @@ namespace jni // Forgets the parameter sets, for a stream that starts over. void Reset(); - // Sets the layers of the stream, temporal ones of AV1 only, as in - // L1T3. Returns false for a mode WebRTC has no structure of. + // Sets the layers of the stream, temporal ones of AV1 and VP9 only, + // as in L1T3. Returns false for a mode WebRTC has no structure of. bool SetScalabilityMode(webrtc::ScalabilityMode mode); // Lets the layers follow the bitrate WebRTC gives each. void OnRatesUpdated(const webrtc::VideoBitrateAllocation & bitrates); @@ -96,6 +98,10 @@ namespace jni void FillCodecSpecificInfo(const webrtc::EncodedImage & image, bool keyFrame, const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::CodecSpecificInfo & info); + // The VP9 information of a picture of a stream with temporal + // layers. + void FillVp9TemporalLayers(const webrtc::EncodedImage & image, bool keyFrame, + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::CodecSpecificInfo & info); private: const webrtc::VideoCodecType codec; @@ -114,6 +120,12 @@ namespace jni // referring to the one before. webrtc::GofInfoVP9 gof; size_t gofIndex = 0; + + // With temporal layers: the pictures since the key frame, and the + // picture each reference buffer holds, which the references of a + // picture are told by. + size_t vp9Picture = 0; + std::array vp9BufferPictures = {}; }; } diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h index e4885707..9197bccc 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h @@ -20,6 +20,7 @@ #include "media/video/codec/EncoderOutputProcessor.h" #include "media/video/codec/linux/VaapiEncodeSession.h" #include "media/video/codec/linux/VaapiLibrary.h" +#include "media/video/codec/linux/VaapiTemporalLayers.h" #include "api/video/video_frame.h" #include "api/video_codecs/sdp_video_format.h" @@ -80,32 +81,11 @@ namespace jni // Reads which headers the driver takes and how it codes them. void QueryCapabilities(); - // What a frame refers to: the slot each of the seven reference - // frames names, the second reference, if any, being golden; the - // slots the frame replaces; and its temporal layer. - struct FrameReferences - { - std::array refFrameIdx = {}; - bool golden = false; - uint8_t refreshFrameFlags = 0; - int temporalId = 0; - }; - - // Turns the buffers of the layer frame into reference slots. - // Returns false if it refers to a slot that holds no frame, or to - // one beyond those kept. - bool PlanReferences(bool keyFrame, const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, - FrameReferences & references) const; - // A reconstructed surface that no reference slot holds. - int FreeSurface() const; - // Adds the rate control of the stream, or of each temporal layer. - bool AddRateControl(bool keyFrame); - - bool Submit(bool keyFrame, const FrameReferences & references, int surface); + bool Submit(bool keyFrame, const VaapiTemporalLayers::Frame & frame); void WriteSequenceHeader(std::vector & obu) const; void WriteFrameHeader(std::vector & obu, FrameHeaderLayout & layout, bool keyFrame, - const FrameReferences & references) const; + const VaapiTemporalLayers::Frame & frame) const; // The seq_level_idx the stream's size, frame rate and bitrate fit. uint8_t Level() const; @@ -128,16 +108,7 @@ namespace jni // The sequence header, kept for the key frames. std::vector sequenceHeader; - // The temporal layers, one without layers, and the bitrate of each, - // counting the layers below it. - int temporalLayers; - std::vector layerBitratesBps; - - // The reconstructed surface each reference slot holds, -1 for - // none: those of the two buffers WebRTC's structures use, up to - // L1T3. A key frame fills the other six slots too, but no frame - // refers to them. - std::array slots; + VaapiTemporalLayers layers; bool referenceValid; // The frames since the last key frame, whose low bits are the // order hint. diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiTemporalLayers.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiTemporalLayers.h new file mode 100644 index 00000000..769fc7ea --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiTemporalLayers.h @@ -0,0 +1,102 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_TEMPORAL_LAYERS_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_TEMPORAL_LAYERS_H_ + +#include "media/video/codec/linux/VaapiEncodeSession.h" + +#include "api/video/video_bitrate_allocation.h" +#include "api/video_codecs/scalability_mode.h" +#include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/svc/scalable_video_controller.h" + +#include +#include +#include +#include + +namespace jni +{ + // The temporal layers of a VA-API encoder of VP9 or AV1, none or those of + // L1T2 and L1T3: which reference slot each frame refers to and which ones + // it replaces, as WebRTC's structure of the mode says, each buffer of the + // structure being the slot of the same number; which reconstructed surface + // each slot holds; and the rate control of each layer, without which + // AMD's driver gives the upper layers next to no bits. + class VaapiTemporalLayers + { + public: + // The reconstructed surfaces a session needs: those of the two + // reference buffers WebRTC's structures use up to L1T3, and that + // of the frame being encoded. + static constexpr int kReconstructedSurfaces = 3; + + // What a frame refers to and replaces. + struct Frame + { + int temporalId = 0; + // The slots of its references, the first being its last frame, + // the second, if any, its golden frame. + std::vector references; + uint8_t refreshFlags = 0; + // The reconstructed surface it is encoded into. + int surface = -1; + }; + + VaapiTemporalLayers(); + + // Starts over in the mode, splitting the bitrate across the layers + // until WebRTC gives its own split. Returns false for a mode other + // than L1T1, L1T2 and L1T3. + bool Configure(webrtc::ScalabilityMode mode, uint32_t bitrateBps); + + // The number of temporal layers, one without layers. + int Count() const; + + // Plans a frame of the layer WebRTC's structure gives. Returns + // nothing if it refers to a slot that holds no frame, or to one + // beyond those kept, or no surface is free. + std::optional Plan(bool keyFrame, + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame) const; + // Takes the frame, once encoded, into the slots it replaces. + void Commit(const Frame & frame); + + // The slots kept, and the reconstructed surface each holds, -1 for + // none. A key frame fills the slots beyond too, but no frame refers + // to them. + int SlotCount() const; + int Surface(int slot) const; + + // Takes the bitrate WebRTC gives each layer. + void OnRatesUpdated(const webrtc::VideoBitrateAllocation & bitrates); + // Adds the rate control of the stream, or of each layer, with the + // layer structure along with that of a key frame. No initial + // quantizer: the rate control picks its own, as FFmpeg leaves it. + bool AddRateControl(VaapiEncodeSession & session, uint32_t bitrateBps, uint32_t framerate, + bool keyFrame) const; + // Fills in the share of the frame rate up to each layer. + void FillFpsAllocation(webrtc::VideoEncoder::EncoderInfo & info) const; + + private: + int count; + std::array slots; + // The bitrate of each layer, counting the layers below it. + std::vector bitratesBps; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Encoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Encoder.h index 9bcfb3aa..54d41d28 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Encoder.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Encoder.h @@ -20,6 +20,7 @@ #include "media/video/codec/EncoderOutputProcessor.h" #include "media/video/codec/linux/VaapiEncodeSession.h" #include "media/video/codec/linux/VaapiLibrary.h" +#include "media/video/codec/linux/VaapiTemporalLayers.h" #include "api/video/video_frame.h" #include "api/video_codecs/sdp_video_format.h" @@ -36,15 +37,16 @@ namespace jni { // Encodes VP9 profile 0 with the GPU driver's VA-API encoder, as Intel - // drivers provide it on Linux. Like hardware VP9 encoding on Windows, it - // has not run on such a GPU yet, and is used only where it is turned on. + // drivers provide it on Linux. // - // It works as the H.265 one does: synchronous, one reference frame, inter - // frames only, the driver writing the uncompressed frame header from the - // parameters it is given, as FFmpeg's VA-API encoder sets them without - // B-frames. Every frame refreshes all eight reference slots, so that the - // next one can refer to it as the last frame whichever slot it names. - // Anything that fails is an error, upon which libvpx takes over. + // It works as the H.265 one does: synchronous, inter frames only, the + // driver writing the uncompressed frame header from the parameters it is + // given, as FFmpeg's VA-API encoder sets them without B-frames. Each frame + // refers to one frame before it: without layers the frame just before; + // with temporal layers (L1T2, L1T3) the one of the same layer or a lower + // one that WebRTC's structure of the mode names, so that a receiver can do + // without the upper layers. Spatial layers, and anything that fails, are + // left to libvpx. class VaapiVp9Encoder : public webrtc::VideoEncoder { public: @@ -59,10 +61,11 @@ namespace jni EncoderInfo GetEncoderInfo() const override; private: - bool Submit(bool keyFrame); + bool Submit(bool keyFrame, const VaapiTemporalLayers::Frame & frame); void FillSequence(VAEncSequenceParameterBufferVP9 & sequence) const; - void FillPicture(VAEncPictureParameterBufferVP9 & picture, bool keyFrame) const; + void FillPicture(VAEncPictureParameterBufferVP9 & picture, bool keyFrame, + const VaapiTemporalLayers::Frame & frame) const; private: VaapiLibrary & library; @@ -76,8 +79,7 @@ namespace jni uint32_t framerate; bool ratesChanged; - // Which of the reconstructed surfaces holds the current frame. - int current; + VaapiTemporalLayers layers; bool referenceValid; webrtc::EncodedImageCallback * callback; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp index 1609dd16..c401dbdd 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp @@ -22,6 +22,7 @@ #include "common_video/h265/h265_common.h" #include "modules/video_coding/codecs/interface/common_constants.h" #include "modules/video_coding/svc/create_scalability_structure.h" +#include "modules/video_coding/svc/scalability_mode_util.h" #include "modules/video_coding/svc/scalable_video_controller_no_layering.h" #include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" #include "rtc_base/logging.h" @@ -409,7 +410,10 @@ namespace jni }; } - if (codec == webrtc::kVideoCodecVP9) { + if (codec == webrtc::kVideoCodecVP9 && scalabilityMode != webrtc::ScalabilityMode::kL1T1) { + FillVp9TemporalLayers(image, keyFrame, layerFrame, info); + } + else if (codec == webrtc::kVideoCodecVP9) { // VP9 has its own RTP header, which WebRTC fills from these: one // spatial and one temporal layer, the group of frames sent with // every key frame. @@ -436,6 +440,77 @@ namespace jni } } + void EncoderOutputProcessor::FillVp9TemporalLayers(const webrtc::EncodedImage & image, bool keyFrame, + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::CodecSpecificInfo & info) + { + // One spatial layer and temporal ones, in flexible mode, as libvpx + // describes them: each picture names the pictures it refers to, by + // how many pictures back they are. + vp9Picture = keyFrame ? 0 : vp9Picture + 1; + + webrtc::CodecSpecificInfoVP9 & vp9 = info.codecSpecific.VP9; + vp9.first_frame_in_picture = true; + vp9.flexible_mode = true; + vp9.ss_data_available = keyFrame; + vp9.non_ref_for_inter_layer_pred = true; + vp9.temporal_idx = static_cast(layerFrame.TemporalId()); + vp9.inter_layer_predicted = false; + vp9.gof_idx = webrtc::kNoGofIdx; + vp9.num_spatial_layers = 1; + vp9.first_active_layer = 0; + vp9.spatial_layer_resolution_present = keyFrame; + vp9.num_ref_pics = 0; + + if (!keyFrame) { + for (const webrtc::CodecBufferUsage & buffer : layerFrame.Buffers()) { + if (!buffer.referenced || buffer.id < 0 || buffer.id >= webrtc::kMaxEncoderBuffers) { + continue; + } + + const size_t pDiff = vp9Picture - vp9BufferPictures[buffer.id]; + bool listed = false; + + for (uint8_t i = 0; i < vp9.num_ref_pics; i++) { + listed |= vp9.p_diff[i] == pDiff; + } + + if (!listed && pDiff > 0 && vp9.num_ref_pics < webrtc::kMaxVp9RefPics) { + vp9.p_diff[vp9.num_ref_pics++] = static_cast(pDiff); + } + } + } + + vp9.inter_pic_predicted = !keyFrame && vp9.num_ref_pics > 0; + + // A switch up to a higher layer is possible where all layers above + // this one are switch points, as libvpx works it out. + const int temporalLayers = webrtc::ScalabilityModeToNumTemporalLayers(scalabilityMode); + vp9.temporal_up_switch = true; + + for (int layer = layerFrame.TemporalId() + 1; layer < temporalLayers && info.generic_frame_info; layer++) { + const auto & indications = info.generic_frame_info->decode_target_indications; + + vp9.temporal_up_switch &= layer < static_cast(indications.size()) + && indications[layer] == webrtc::DecodeTargetIndication::kSwitch; + } + + if (keyFrame) { + vp9.width[0] = image._encodedWidth; + vp9.height[0] = image._encodedHeight; + vp9.gof.num_frames_in_gof = 0; + } + + for (const webrtc::CodecBufferUsage & buffer : layerFrame.Buffers()) { + if ((buffer.updated || keyFrame) && buffer.id >= 0 && buffer.id < webrtc::kMaxEncoderBuffers) { + vp9BufferPictures[buffer.id] = vp9Picture; + } + } + + if (keyFrame) { + vp9BufferPictures.fill(0); + } + } + webrtc::VideoEncoder::ScalingSettings EncoderOutputProcessor::GetScalingSettings() const { // H.265 has no thresholds of its own in WebRTC yet and uses those of diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp index ede55634..dc61ffc7 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp @@ -65,21 +65,6 @@ namespace jni // and other state carry over from. constexpr uint8_t kPrimaryRefLast = 0; - // The slots of the two reference buffers L1T3 has, and the frame being - // encoded. - constexpr int kReconstructedSurfaces = 3; - - // The layers of the frames after a key frame, as - // VAEncMiscParameterTemporalLayerStructure takes them, of L1T2 and - // L1T3 as WebRTC's structures make them: L1T3 is 0, 2, 1, 2. - const std::vector kL1T2Pattern = { 1, 0 }; - const std::vector kL1T3Pattern = { 2, 1, 2, 0 }; - - // The share of the bitrate up to each layer until WebRTC gives its - // own, as WebRTC splits it for temporal layers. - constexpr double kL1T2Shares[] = { 0.6, 1.0 }; - constexpr double kL1T3Shares[] = { 0.4, 0.6, 1.0 }; - // What a frame that cannot be encoded returns: libaom takes over the // stream, which stays in AV1, rather than WebRTC changing the codec. constexpr int32_t kEncodeFailed = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; @@ -129,6 +114,22 @@ namespace jni // The level of a stream that fits none. constexpr uint8_t kMaxParametersLevel = 31; + // ref_frame_idx: every reference frame names the first reference of + // the frame, but golden, which names the second where there is one. + std::array RefFrameIdx(const VaapiTemporalLayers::Frame & frame) + { + std::array refFrameIdx = {}; + + if (!frame.references.empty()) { + refFrameIdx.fill(static_cast(frame.references[0])); + } + if (frame.references.size() > 1) { + refFrameIdx[kGoldenIndex] = static_cast(frame.references[1]); + } + + return refFrameIdx; + } + uint32_t Align(uint32_t value, uint32_t alignment) { return (value + alignment - 1) / alignment * alignment; @@ -253,7 +254,6 @@ namespace jni ratesChanged(false), obuSizeLength(0), txMode(0), - temporalLayers(1), referenceValid(false), frameNumber(0), callback(nullptr), @@ -347,18 +347,10 @@ namespace jni bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; framerate = std::max(1u, codecSettings.maxFramerate); ratesChanged = true; - temporalLayers = mode == webrtc::ScalabilityMode::kL1T3 ? 3 : mode == webrtc::ScalabilityMode::kL1T2 ? 2 : 1; - slots.fill(-1); referenceValid = false; frameNumber = 0; outputProcessor.Reset(); - - layerBitratesBps.clear(); - - for (int layer = 0; layer < temporalLayers && temporalLayers > 1; layer++) { - const double share = temporalLayers == 3 ? kL1T3Shares[layer] : kL1T2Shares[layer]; - layerBitratesBps.push_back(static_cast(bitrateBps * share)); - } + layers.Configure(mode, bitrateBps); if (!outputProcessor.SetScalabilityMode(mode)) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; @@ -368,7 +360,7 @@ namespace jni const uint32_t surfaceHeight = Align(codecSettings.height, kSuperblockSize); if (!session.Create(VAProfileAV1Profile0, entrypoint, codecSettings.width, codecSettings.height, surfaceWidth, - surfaceHeight, kPackedHeaders, kReconstructedSurfaces)) + surfaceHeight, kPackedHeaders, VaapiTemporalLayers::kReconstructedSurfaces)) { Release(); return WEBRTC_VIDEO_CODEC_ERROR; @@ -427,10 +419,9 @@ namespace jni return WEBRTC_VIDEO_CODEC_OK; } - FrameReferences references; - const int surface = FreeSurface(); + const auto layerPlan = layers.Plan(keyFrame, *layerFrame); - if (surface < 0 || !PlanReferences(keyFrame, *layerFrame, references)) { + if (!layerPlan) { RTC_LOG(LS_WARNING) << implementationName << " AV1: a frame refers to a slot that holds no frame"; return kEncodeFailed; } @@ -440,7 +431,7 @@ namespace jni // A frame up to the alignment larger is cropped; one of another size, // which raced with a change of size, is scaled. const bool encoded = session.Upload(frame, static_cast(codecSettings.width), - static_cast(codecSettings.height), kSizeAlignment - 1) && Submit(keyFrame, references, surface) + static_cast(codecSettings.height), kSizeAlignment - 1) && Submit(keyFrame, *layerPlan) && session.ReadOutput(output); session.DropBuffers(); @@ -463,12 +454,7 @@ namespace jni return kEncodeFailed; } - // The slots the frame replaces hold it now. - for (int slot = 0; slot < static_cast(slots.size()); slot++) { - if (references.refreshFrameFlags & (1 << slot)) { - slots[slot] = surface; - } - } + layers.Commit(*layerPlan); referenceValid = true; frameNumber++; @@ -479,81 +465,11 @@ namespace jni return WEBRTC_VIDEO_CODEC_OK; } - bool VaapiAv1Encoder::PlanReferences(bool keyFrame, - const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, FrameReferences & references) const - { - references.temporalId = layerFrame.TemporalId(); - - // A key frame replaces every slot. - if (keyFrame) { - references.refreshFrameFlags = 0xFF; - return true; - } - - std::vector referenced; - - for (const webrtc::CodecBufferUsage & buffer : layerFrame.Buffers()) { - if (buffer.id < 0 || buffer.id >= static_cast(slots.size())) { - return false; - } - if (buffer.referenced) { - if (slots[buffer.id] < 0) { - return false; - } - - referenced.push_back(buffer.id); - } - if (buffer.updated) { - references.refreshFrameFlags |= static_cast(1 << buffer.id); - } - } - - if (referenced.empty()) { - return false; - } - - // Every reference frame names the first reference, but golden, which - // names the second where there is one. - references.refFrameIdx.fill(static_cast(referenced[0])); - - if (referenced.size() > 1) { - references.refFrameIdx[kGoldenIndex] = static_cast(referenced[1]); - references.golden = true; - } - - return true; - } - - bool VaapiAv1Encoder::AddRateControl(bool keyFrame) - { - // No initial quantizer: the rate control picks its own, as FFmpeg - // leaves it. - if (temporalLayers == 1) { - return session.AddRateControl(bitrateBps, framerate, 0); - } - - // Without a rate control of each layer, AMD's driver gives the upper - // layers next to no bits. - return session.AddLayerRateControl(layerBitratesBps, framerate, - temporalLayers == 3 ? kL1T3Pattern : kL1T2Pattern, keyFrame); - } - - int VaapiAv1Encoder::FreeSurface() const - { - for (int surface = 0; surface < kReconstructedSurfaces; surface++) { - if (std::find(slots.begin(), slots.end(), surface) == slots.end()) { - return surface; - } - } - - return -1; - } - - bool VaapiAv1Encoder::Submit(bool keyFrame, const FrameReferences & references, int surface) + bool VaapiAv1Encoder::Submit(bool keyFrame, const VaapiTemporalLayers::Frame & frame) { const uint32_t sbCols = Align(codecSettings.width, kSuperblockSize) / kSuperblockSize; const uint32_t sbRows = Align(codecSettings.height, kSuperblockSize) / kSuperblockSize; - const bool layered = temporalLayers > 1; + const bool layered = layers.Count() > 1; if (keyFrame) { VAEncSequenceParameterBufferAV1 sequence = {}; @@ -574,20 +490,20 @@ namespace jni } } - if ((keyFrame || ratesChanged) && !AddRateControl(keyFrame)) { + if ((keyFrame || ratesChanged) && !layers.AddRateControl(session, bitrateBps, framerate, keyFrame)) { return false; } std::vector frameHeader; FrameHeaderLayout layout; - WriteFrameHeader(frameHeader, layout, keyFrame, references); + WriteFrameHeader(frameHeader, layout, keyFrame, frame); VAEncPictureParameterBufferAV1 picture = {}; picture.frame_width_minus_1 = static_cast(codecSettings.width - 1); picture.frame_height_minus_1 = static_cast(codecSettings.height - 1); - picture.reconstructed_frame = session.Reconstructed(surface); + picture.reconstructed_frame = session.Reconstructed(frame.surface); picture.coded_buf = session.CodedBuffer(); - picture.refresh_frame_flags = references.refreshFrameFlags; + picture.refresh_frame_flags = frame.refreshFlags; for (VASurfaceID & reference : picture.reference_frames) { reference = VA_INVALID_SURFACE; @@ -599,18 +515,19 @@ namespace jni picture.picture_flags.bits.error_resilient_mode = 1; } else { - for (int slot = 0; slot < static_cast(slots.size()); slot++) { - if (slots[slot] >= 0) { - picture.reference_frames[slot] = session.Reconstructed(slots[slot]); + for (int slot = 0; slot < layers.SlotCount(); slot++) { + if (layers.Surface(slot) >= 0) { + picture.reference_frames[slot] = session.Reconstructed(layers.Surface(slot)); } } - std::copy(references.refFrameIdx.begin(), references.refFrameIdx.end(), picture.ref_frame_idx); + const std::array refFrameIdx = RefFrameIdx(frame); + std::copy(refFrameIdx.begin(), refFrameIdx.end(), picture.ref_frame_idx); picture.primary_ref_frame = kPrimaryRefLast; picture.ref_frame_ctrl_l0.fields.search_idx0 = kLastFrame; - if (references.golden) { + if (frame.references.size() > 1) { picture.ref_frame_ctrl_l0.fields.search_idx1 = kGoldenFrame; } @@ -618,10 +535,10 @@ namespace jni } if (layered) { - picture.temporal_id = static_cast(references.temporalId); - picture.hierarchical_level_plus1 = static_cast(references.temporalId + 1); + picture.temporal_id = static_cast(frame.temporalId); + picture.hierarchical_level_plus1 = static_cast(frame.temporalId + 1); picture.tile_group_obu_hdr_info.bits.obu_extension_flag = 1; - picture.tile_group_obu_hdr_info.bits.temporal_id = static_cast(references.temporalId); + picture.tile_group_obu_hdr_info.bits.temporal_id = static_cast(frame.temporalId); } picture.order_hint = static_cast(frameNumber); @@ -678,6 +595,8 @@ namespace jni // point for each, from all layers down to the lowest. Then // operating_point_idc, of spatial layer 0 and those temporal layers, // seq_level_idx, and seq_tier above level 3.3. + const int temporalLayers = layers.Count(); + writer.Put(temporalLayers - 1, 5); for (int point = 0; point < temporalLayers; point++) { @@ -731,11 +650,12 @@ namespace jni } void VaapiAv1Encoder::WriteFrameHeader(std::vector & obu, FrameHeaderLayout & layout, bool keyFrame, - const FrameReferences & references) const + const VaapiTemporalLayers::Frame & frame) const { // uncompressed_header, section 5.9, as FFmpeg's VA-API encoder writes // it, for the sequence header above. - const bool layered = temporalLayers > 1; + const bool layered = layers.Count() > 1; + const std::array refFrameIdx = RefFrameIdx(frame); const uint32_t miCols = 2 * ((codecSettings.width + 7) >> 3); const uint32_t miRows = 2 * ((codecSettings.height + 7) >> 3); const uint32_t sbCols = (miCols + 15) >> 4; @@ -762,12 +682,12 @@ namespace jni if (!keyFrame) { // primary_ref_frame, refresh_frame_flags. writer.Put(kPrimaryRefLast, 3); - writer.Put(references.refreshFrameFlags, 8); + writer.Put(frame.refreshFlags, 8); // frame_refs_short_signaling, then ref_frame_idx. writer.Put(0, 1); for (int i = 0; i < kRefsPerFrame; i++) { - writer.Put(references.refFrameIdx[i], 3); + writer.Put(refFrameIdx[i], 3); } } @@ -849,7 +769,7 @@ namespace jni writer.TrailingBits(); obu = MakeObu(kObuFrameHeader, writer.Bytes(), obuSizeLength, - layered ? std::optional(references.temporalId) : std::nullopt); + layered ? std::optional(frame.temporalId) : std::nullopt); layout.size = static_cast(obu.size() * 8); } @@ -882,16 +802,7 @@ namespace jni bitrateBps = bitrate; outputProcessor.OnRatesUpdated(parameters.bitrate); - - // The bitrate of each temporal layer, counting those below it. - uint32_t sum = 0; - - for (int layer = 0; layer < static_cast(layerBitratesBps.size()); layer++) { - sum += parameters.bitrate.GetBitrate(0, layer); - // A layer WebRTC gives no bitrate is not encoded; it keeps the - // rate of the one below, which the driver takes. - layerBitratesBps[layer] = std::max(sum, 1u); - } + layers.OnRatesUpdated(parameters.bitrate); if (parameters.framerate_fps >= 1.0) { framerate = static_cast(parameters.framerate_fps + 0.5); @@ -911,13 +822,7 @@ namespace jni info.scaling_settings = outputProcessor.GetScalingSettings(); info.requested_resolution_alignment = kSizeAlignment; - // With temporal layers, the share of the frame rate up to each. - if (temporalLayers > 1) { - for (int layer = 0; layer < temporalLayers; layer++) { - info.fps_allocation[0].push_back(static_cast( - EncoderInfo::kMaxFramerateFraction >> (temporalLayers - 1 - layer))); - } - } + layers.FillFpsAllocation(info); return info; } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiTemporalLayers.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiTemporalLayers.cpp new file mode 100644 index 00000000..1e8cfc70 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiTemporalLayers.cpp @@ -0,0 +1,173 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/linux/VaapiTemporalLayers.h" + +#include + +namespace jni +{ + namespace + { + // The layers of the frames after a key frame, as + // VAEncMiscParameterTemporalLayerStructure takes them, of L1T2 and + // L1T3 as WebRTC's structures make them: L1T3 is 0, 2, 1, 2. + const std::vector kL1T2Pattern = { 1, 0 }; + const std::vector kL1T3Pattern = { 2, 1, 2, 0 }; + + // The share of the bitrate up to each layer until WebRTC gives its + // own, as WebRTC splits it for temporal layers. + constexpr double kL1T2Shares[] = { 0.6, 1.0 }; + constexpr double kL1T3Shares[] = { 0.4, 0.6, 1.0 }; + } + + VaapiTemporalLayers::VaapiTemporalLayers() : + count(1) + { + slots.fill(-1); + } + + bool VaapiTemporalLayers::Configure(webrtc::ScalabilityMode mode, uint32_t bitrateBps) + { + switch (mode) { + case webrtc::ScalabilityMode::kL1T1: + count = 1; + break; + case webrtc::ScalabilityMode::kL1T2: + count = 2; + break; + case webrtc::ScalabilityMode::kL1T3: + count = 3; + break; + default: + return false; + } + + slots.fill(-1); + bitratesBps.clear(); + + for (int layer = 0; layer < count && count > 1; layer++) { + const double share = count == 3 ? kL1T3Shares[layer] : kL1T2Shares[layer]; + bitratesBps.push_back(static_cast(bitrateBps * share)); + } + + return true; + } + + int VaapiTemporalLayers::Count() const + { + return count; + } + + std::optional VaapiTemporalLayers::Plan(bool keyFrame, + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame) const + { + Frame frame; + frame.temporalId = layerFrame.TemporalId(); + + for (int surface = 0; surface < kReconstructedSurfaces && frame.surface < 0; surface++) { + if (std::find(slots.begin(), slots.end(), surface) == slots.end()) { + frame.surface = surface; + } + } + + if (frame.surface < 0) { + return std::nullopt; + } + + // A key frame replaces every slot. + if (keyFrame) { + frame.refreshFlags = 0xFF; + return frame; + } + + for (const webrtc::CodecBufferUsage & buffer : layerFrame.Buffers()) { + if (buffer.id < 0 || buffer.id >= static_cast(slots.size())) { + return std::nullopt; + } + if (buffer.referenced) { + if (slots[buffer.id] < 0) { + return std::nullopt; + } + + frame.references.push_back(buffer.id); + } + if (buffer.updated) { + frame.refreshFlags |= static_cast(1 << buffer.id); + } + } + + if (frame.references.empty()) { + return std::nullopt; + } + + return frame; + } + + void VaapiTemporalLayers::Commit(const Frame & frame) + { + for (int slot = 0; slot < static_cast(slots.size()); slot++) { + if (frame.refreshFlags & (1 << slot)) { + slots[slot] = frame.surface; + } + } + } + + int VaapiTemporalLayers::SlotCount() const + { + return static_cast(slots.size()); + } + + int VaapiTemporalLayers::Surface(int slot) const + { + return slots[slot]; + } + + void VaapiTemporalLayers::OnRatesUpdated(const webrtc::VideoBitrateAllocation & bitrates) + { + uint32_t sum = 0; + + for (int layer = 0; layer < static_cast(bitratesBps.size()); layer++) { + sum += bitrates.GetBitrate(0, layer); + // A layer WebRTC gives no bitrate is not encoded; it keeps the + // rate of the one below, which the driver takes. + bitratesBps[layer] = std::max(sum, 1u); + } + } + + bool VaapiTemporalLayers::AddRateControl(VaapiEncodeSession & session, uint32_t bitrateBps, uint32_t framerate, + bool keyFrame) const + { + if (count == 1) { + return session.AddRateControl(bitrateBps, framerate, 0); + } + + return session.AddLayerRateControl(bitratesBps, framerate, count == 3 ? kL1T3Pattern : kL1T2Pattern, + keyFrame); + } + + void VaapiTemporalLayers::FillFpsAllocation(webrtc::VideoEncoder::EncoderInfo & info) const + { + if (count == 1) { + return; + } + + for (int layer = 0; layer < count; layer++) { + info.fps_allocation[0].push_back(static_cast( + webrtc::VideoEncoder::EncoderInfo::kMaxFramerateFraction >> (count - 1 - layer))); + } + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp index c94b0801..ecc507b9 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp @@ -46,8 +46,13 @@ namespace jni constexpr uint8_t kFilterLevel = 16; constexpr uint8_t kSharpnessLevel = 4; - // The reference slot the next frame refers to as its last frame. - constexpr uint32_t kLastSlot = 0; + // The reference frames LAST and GOLDEN in ref_frame_ctrl_l0. + constexpr uint32_t kLastFrame = 1; + constexpr uint32_t kGoldenFrame = 2; + + // What a frame that cannot be encoded returns: libvpx takes over the + // stream, which stays in VP9, rather than WebRTC changing the codec. + constexpr int32_t kEncodeFailed = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; uint32_t Align(uint32_t value, uint32_t alignment) { @@ -66,7 +71,6 @@ namespace jni bitrateBps(0), framerate(30), ratesChanged(false), - current(0), referenceValid(false), callback(nullptr), outputProcessor(webrtc::kVideoCodecVP9, format) @@ -86,8 +90,19 @@ namespace jni if (settings->numberOfSimulcastStreams > 1) { return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED; } - // One spatial and one temporal layer is all this encoder produces. - if (settings->VP9().numberOfSpatialLayers > 1 || settings->VP9().numberOfTemporalLayers > 1) { + // Temporal layers; spatial ones are left to libvpx. + if (settings->VP9().numberOfSpatialLayers > 1) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + const int temporalLayers = settings->VP9().numberOfTemporalLayers; + const webrtc::ScalabilityMode mode = settings->GetScalabilityMode().value_or(temporalLayers == 3 + ? webrtc::ScalabilityMode::kL1T3 + : temporalLayers == 2 ? webrtc::ScalabilityMode::kL1T2 : webrtc::ScalabilityMode::kL1T1); + + if (mode != webrtc::ScalabilityMode::kL1T1 && mode != webrtc::ScalabilityMode::kL1T2 + && mode != webrtc::ScalabilityMode::kL1T3) + { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } @@ -97,14 +112,19 @@ namespace jni bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; framerate = std::max(1u, codecSettings.maxFramerate); ratesChanged = true; - current = 0; referenceValid = false; outputProcessor.Reset(); + layers.Configure(mode, bitrateBps); + + if (!outputProcessor.SetScalabilityMode(mode)) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } const uint32_t surfaceWidth = Align(codecSettings.width, kSurfaceAlignment); const uint32_t surfaceHeight = Align(codecSettings.height, kSurfaceAlignment); - if (!session.Create(VAProfileVP9Profile0, entrypoint, surfaceWidth, surfaceHeight, surfaceWidth, surfaceHeight)) + if (!session.Create(VAProfileVP9Profile0, entrypoint, surfaceWidth, surfaceHeight, surfaceWidth, surfaceHeight, + VA_ENC_PACKED_HEADER_NONE, VaapiTemporalLayers::kReconstructedSurfaces)) { Release(); return WEBRTC_VIDEO_CODEC_ERROR; @@ -113,7 +133,7 @@ namespace jni initialized = true; RTC_LOG(LS_INFO) << implementationName << " VP9 initialized: " << codecSettings.width << "x" - << codecSettings.height << " at " << bitrateBps << " bps"; + << codecSettings.height << " at " << bitrateBps << " bps, " << webrtc::ScalabilityModeToString(mode); return WEBRTC_VIDEO_CODEC_OK; } @@ -150,17 +170,31 @@ namespace jni }); } + const auto layerFrame = outputProcessor.NextFrameConfig(keyFrame); + + // No layer is encoded now, as when WebRTC gives none a bitrate. + if (!layerFrame) { + return WEBRTC_VIDEO_CODEC_OK; + } + + const auto layerPlan = layers.Plan(keyFrame, *layerFrame); + + if (!layerPlan) { + RTC_LOG(LS_WARNING) << implementationName << " VP9: a frame refers to a slot that holds no frame"; + return kEncodeFailed; + } + std::vector output; // VP9 codes any size, so the frame is taken as it is, and scaled only // if it raced with a change of size. const bool encoded = session.Upload(frame, static_cast(codecSettings.width), - static_cast(codecSettings.height), 0) && Submit(keyFrame) && session.ReadOutput(output); + static_cast(codecSettings.height), 0) && Submit(keyFrame, *layerPlan) && session.ReadOutput(output); session.DropBuffers(); if (!encoded) { - return WEBRTC_VIDEO_CODEC_ERROR; + return kEncodeFailed; } webrtc::EncodedImage image; @@ -173,13 +207,12 @@ namespace jni webrtc::CodecSpecificInfo info; - if (!outputProcessor.Process(output, keyFrame, image, info)) { - return WEBRTC_VIDEO_CODEC_ERROR; + if (!outputProcessor.Process(output, keyFrame, *layerFrame, image, info)) { + return kEncodeFailed; } - // The frame just encoded is the reference of the next one. + layers.Commit(*layerPlan); referenceValid = true; - current = 1 - current; ratesChanged = false; callback->OnEncodedImage(image, &info); @@ -187,7 +220,7 @@ namespace jni return WEBRTC_VIDEO_CODEC_OK; } - bool VaapiVp9Encoder::Submit(bool keyFrame) + bool VaapiVp9Encoder::Submit(bool keyFrame, const VaapiTemporalLayers::Frame & frame) { if (keyFrame) { VAEncSequenceParameterBufferVP9 sequence = {}; @@ -198,14 +231,12 @@ namespace jni } } - // No initial quantizer: VP9 rate control picks its own, as FFmpeg - // leaves it. - if ((keyFrame || ratesChanged) && !session.AddRateControl(bitrateBps, framerate, 0)) { + if ((keyFrame || ratesChanged) && !layers.AddRateControl(session, bitrateBps, framerate, keyFrame)) { return false; } VAEncPictureParameterBufferVP9 picture = {}; - FillPicture(picture, keyFrame); + FillPicture(picture, keyFrame, frame); return session.AddBuffer(VAEncPictureParameterBufferType, &picture, sizeof(picture)) && session.Encode(); } @@ -222,13 +253,14 @@ namespace jni sequence.bits_per_second = bitrateBps; } - void VaapiVp9Encoder::FillPicture(VAEncPictureParameterBufferVP9 & picture, bool keyFrame) const + void VaapiVp9Encoder::FillPicture(VAEncPictureParameterBufferVP9 & picture, bool keyFrame, + const VaapiTemporalLayers::Frame & frame) const { picture.frame_width_src = codecSettings.width; picture.frame_height_src = codecSettings.height; picture.frame_width_dst = codecSettings.width; picture.frame_height_dst = codecSettings.height; - picture.reconstructed_frame = session.Reconstructed(current); + picture.reconstructed_frame = session.Reconstructed(frame.surface); picture.coded_buf = session.CodedBuffer(); for (VASurfaceID & reference : picture.reference_frames) { @@ -240,16 +272,36 @@ namespace jni picture.pic_flags.bits.frame_type = kKeyFrameType; } else { - picture.reference_frames[kLastSlot] = session.Reconstructed(1 - current); - // The last frame is the only reference, in list 0. - picture.ref_flags.bits.ref_frame_ctrl_l0 = 1; - picture.ref_flags.bits.ref_last_idx = kLastSlot; + for (int slot = 0; slot < layers.SlotCount(); slot++) { + if (layers.Surface(slot) >= 0) { + picture.reference_frames[slot] = session.Reconstructed(layers.Surface(slot)); + } + } + + // The first reference is the last frame, a second one golden; both + // in list 0. + picture.ref_flags.bits.ref_frame_ctrl_l0 = kLastFrame; + picture.ref_flags.bits.ref_last_idx = static_cast(frame.references[0]); + + if (frame.references.size() > 1) { + picture.ref_flags.bits.ref_frame_ctrl_l0 |= kGoldenFrame; + picture.ref_flags.bits.ref_gf_idx = static_cast(frame.references[1]); + } + picture.pic_flags.bits.frame_type = kInterFrameType; } + picture.ref_flags.bits.temporal_id = static_cast(frame.temporalId); picture.pic_flags.bits.show_frame = 1; - // Every slot holds this frame afterwards. - picture.refresh_frame_flags = 0xFF; + + // VP9 takes motion vectors from the frame decoded before, and carries + // probabilities from frame to frame, neither of which a receiver + // without the upper layers has; error resilience leaves both out, as + // libvpx does with layers. + if (layers.Count() > 1) { + picture.pic_flags.bits.error_resilient_mode = 1; + } + picture.refresh_frame_flags = frame.refreshFlags; picture.luma_ac_qindex = kQIndex; picture.filter_level = kFilterLevel; @@ -266,6 +318,8 @@ namespace jni } bitrateBps = bitrate; + outputProcessor.OnRatesUpdated(parameters.bitrate); + layers.OnRatesUpdated(parameters.bitrate); if (parameters.framerate_fps >= 1.0) { framerate = static_cast(parameters.framerate_fps + 0.5); @@ -284,6 +338,8 @@ namespace jni info.supports_simulcast = false; info.scaling_settings = outputProcessor.GetScalingSettings(); + layers.FillFpsAllocation(info); + return info; } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java index 84c517f0..7b9ea334 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java @@ -45,7 +45,8 @@ * Intel's: with VA-API on Linux, and with Media Foundation on Windows only * with the system property {@code webrtc.hardware.vp9Encoder} set to * {@code true} when the factory is created, since that encoder has not been - * tested on such a GPU yet. Otherwise VP9 is encoded with libvpx. On Linux, H.265 is encoded with NVENC on NVIDIA + * tested on such a GPU yet. With VA-API, VP9 is encoded in temporal layers + * too (L1T2, L1T3). Otherwise VP9 is encoded with libvpx. On Linux, H.265 is encoded with NVENC on NVIDIA * GPUs and with VA-API on the GPUs of Intel and AMD whose driver encodes it. * A hardware encoder that fails to start, for example because the GPU has no * encoder sessions left, or fails while encoding, is replaced by the next one diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java index 5486c161..26e8aa61 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java @@ -524,14 +524,31 @@ void av1RestartsCleanly() throws Exception { @Test void av1EncodesTemporalLayers() throws Exception { - // Temporal layers are references between frames of one size, which a - // hardware encoder can make; an SFU can then forward a lower frame - // rate without asking for key frames. + assertEncodesTemporalLayers(AV1, HARDWARE_AV1_REQUIRED); + } + + @Test + void vp9EncodesTemporalLayers() throws Exception { + // VA-API encodes VP9 in temporal layers, Media Foundation does not. + assumeTrue(OS.contains("linux"), "hardware VP9 encodes temporal layers on Linux only"); + + assertEncodesTemporalLayers(VP9, HARDWARE_VP9_REQUIRED); + } + + /** + * Sends the codec in L1T2 and then L1T3, and checks that the hardware + * encodes the layers, rather than handing them to the software encoder. + * Temporal layers are references between frames of one size, which a + * hardware encoder can make; an SFU can then forward a lower frame rate + * without asking for key frames. + */ + private void assertEncodesTemporalLayers(Predicate codec, boolean required) + throws Exception { for (String mode : new String[] { "L1T2", "L1T3" }) { PeerConnectionFactory hardware = hardwareFactory(); CountDownLatch received = new CountDownLatch(30); - try (TestMediaCall call = new TestMediaCall(hardware, true, false, AV1)) { + try (TestMediaCall call = new TestMediaCall(hardware, true, false, codec)) { call.setVideoSize(640, 360); call.negotiate(); @@ -560,7 +577,7 @@ void av1EncodesTemporalLayers() throws Exception { track.removeSink(sink); receiver.dispose(); - assertHardware(String.valueOf(outbound.get("encoderImplementation")), HARDWARE_AV1_REQUIRED); + assertHardware(String.valueOf(outbound.get("encoderImplementation")), required); assertEquals(mode, outbound.get("scalabilityMode")); assertTrue(!(pliCount instanceof Number) || ((Number) pliCount).longValue() <= 1, "key frames requested: " + pliCount);