feat: encode VP9 temporal layers with VA-API

docs/guide/advanced/video-codecs.md

@@ -30,7 +30,7 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder()
 | Linux | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) where it encodes H.265 | NVENC on NVIDIA RTX 40 and newer, otherwise the VA-API encoder of the GPU driver where it encodes AV1 (e.g. AMD RDNA3 and newer) |
 | macOS | VideoToolbox, as with `DefaultVideoEncoderFactory` | VideoToolbox, on Macs with a hardware HEVC encoder, such as all with Apple silicon | Software |
 
-Hardware encoders produce a single layer, but for the VA-API AV1 encoder, which encodes temporal layers too (`L1T2` and `L1T3`), with a rate control of each layer, so that an SFU can forward a lower frame rate. A stream that asks for spatial layers, or for temporal layers from another hardware encoder, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom.
+Hardware encoders produce a single layer, but for the VA-API encoders of AV1 and VP9, which encode temporal layers too (`L1T2` and `L1T3`), with a rate control of each layer, so that an SFU can forward a lower frame rate. A stream that asks for spatial layers, or for temporal layers from another hardware encoder, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom.
 
 VP9 can be encoded in hardware too, with the VP9 encoder of the GPU driver, which Intel GPUs have: through VA-API on Linux, and through Media Foundation on Windows. On Windows it is off by default, since it has not been tested on such a GPU yet; turn it on with a system property before the factory is created:
 

webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h

@@ -30,6 +30,7 @@
 #include "api/video_codecs/scalability_mode.h"
 #include "modules/video_coding/svc/scalable_video_controller.h"
 
+#include <array>
 #include <cstdint>
 #include <memory>
 #include <optional>
@@ -47,10 +48,11 @@ namespace jni
 	// or the sequence header of AV1. Encoders put them in front of the first
 	// key frame, not necessarily in front of every one, so they are kept and
 	// put back where missing. The frame dependencies WebRTC needs for AV1 and
-	// H.265 are those of a stream without layers, unless an AV1 encoder that
-	// makes temporal layers sets its scalability mode and asks for the layer
-	// of each frame before it encodes it. VP9 frames go out as they come, with
-	// the VP9 information of a single layer, as the Java encoders get it.
+	// H.265 are those of a stream without layers, unless an AV1 or VP9 encoder
+	// that makes temporal layers sets its scalability mode and asks for the
+	// layer of each frame before it encodes it. VP9 frames go out as they
+	// come, with the VP9 information of a single layer, as the Java encoders
+	// get it, or with temporal layers in flexible mode, as libvpx gives it.
 	class EncoderOutputProcessor
 	{
 		public:
@@ -59,8 +61,8 @@ namespace jni
 			// Forgets the parameter sets, for a stream that starts over.
 			void Reset();
 
-			// Sets the layers of the stream, temporal ones of AV1 only, as in
-			// L1T3. Returns false for a mode WebRTC has no structure of.
+			// Sets the layers of the stream, temporal ones of AV1 and VP9 only,
+			// as in L1T3. Returns false for a mode WebRTC has no structure of.
 			bool SetScalabilityMode(webrtc::ScalabilityMode mode);
 			// Lets the layers follow the bitrate WebRTC gives each.
 			void OnRatesUpdated(const webrtc::VideoBitrateAllocation & bitrates);
@@ -96,6 +98,10 @@ namespace jni
 
 			void FillCodecSpecificInfo(const webrtc::EncodedImage & image, bool keyFrame,
 				const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::CodecSpecificInfo & info);
+			// The VP9 information of a picture of a stream with temporal
+			// layers.
+			void FillVp9TemporalLayers(const webrtc::EncodedImage & image, bool keyFrame,
+				const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::CodecSpecificInfo & info);
 
 		private:
 			const webrtc::VideoCodecType codec;
@@ -114,6 +120,12 @@ namespace jni
 			// referring to the one before.
 			webrtc::GofInfoVP9 gof;
 			size_t gofIndex = 0;
+
+			// With temporal layers: the pictures since the key frame, and the
+			// picture each reference buffer holds, which the references of a
+			// picture are told by.
+			size_t vp9Picture = 0;
+			std::array<size_t, webrtc::kMaxEncoderBuffers> vp9BufferPictures = {};
 	};
 }
 

webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h

@@ -20,6 +20,7 @@
 #include "media/video/codec/EncoderOutputProcessor.h"
 #include "media/video/codec/linux/VaapiEncodeSession.h"
 #include "media/video/codec/linux/VaapiLibrary.h"
+#include "media/video/codec/linux/VaapiTemporalLayers.h"
 
 #include "api/video/video_frame.h"
 #include "api/video_codecs/sdp_video_format.h"
@@ -80,32 +81,11 @@ namespace jni
 			// Reads which headers the driver takes and how it codes them.
 			void QueryCapabilities();
 
-			// What a frame refers to: the slot each of the seven reference
-			// frames names, the second reference, if any, being golden; the
-			// slots the frame replaces; and its temporal layer.
-			struct FrameReferences
-			{
-				std::array<uint8_t, 7> refFrameIdx = {};
-				bool golden = false;
-				uint8_t refreshFrameFlags = 0;
-				int temporalId = 0;
-			};
-
-			// Turns the buffers of the layer frame into reference slots.
-			// Returns false if it refers to a slot that holds no frame, or to
-			// one beyond those kept.
-			bool PlanReferences(bool keyFrame, const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame,
-				FrameReferences & references) const;
-			// A reconstructed surface that no reference slot holds.
-			int FreeSurface() const;
-			// Adds the rate control of the stream, or of each temporal layer.
-			bool AddRateControl(bool keyFrame);
-
-			bool Submit(bool keyFrame, const FrameReferences & references, int surface);
+			bool Submit(bool keyFrame, const VaapiTemporalLayers::Frame & frame);
 
 			void WriteSequenceHeader(std::vector<uint8_t> & obu) const;
 			void WriteFrameHeader(std::vector<uint8_t> & obu, FrameHeaderLayout & layout, bool keyFrame,
-				const FrameReferences & references) const;
+				const VaapiTemporalLayers::Frame & frame) const;
 			// The seq_level_idx the stream's size, frame rate and bitrate fit.
 			uint8_t Level() const;
 
@@ -128,16 +108,7 @@ namespace jni
 			// The sequence header, kept for the key frames.
 			std::vector<uint8_t> sequenceHeader;
 
-			// The temporal layers, one without layers, and the bitrate of each,
-			// counting the layers below it.
-			int temporalLayers;
-			std::vector<uint32_t> layerBitratesBps;
-
-			// The reconstructed surface each reference slot holds, -1 for
-			// none: those of the two buffers WebRTC's structures use, up to
-			// L1T3. A key frame fills the other six slots too, but no frame
-			// refers to them.
-			std::array<int, 2> slots;
+			VaapiTemporalLayers layers;
 			bool referenceValid;
 			// The frames since the last key frame, whose low bits are the
 			// order hint.

webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiTemporalLayers.h

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2026 Alex Andres
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_TEMPORAL_LAYERS_H_
+#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_TEMPORAL_LAYERS_H_
+
+#include "media/video/codec/linux/VaapiEncodeSession.h"
+
+#include "api/video/video_bitrate_allocation.h"
+#include "api/video_codecs/scalability_mode.h"
+#include "api/video_codecs/video_encoder.h"
+#include "modules/video_coding/svc/scalable_video_controller.h"
+
+#include <array>
+#include <cstdint>
+#include <optional>
+#include <vector>
+
+namespace jni
+{
+	// The temporal layers of a VA-API encoder of VP9 or AV1, none or those of
+	// L1T2 and L1T3: which reference slot each frame refers to and which ones
+	// it replaces, as WebRTC's structure of the mode says, each buffer of the
+	// structure being the slot of the same number; which reconstructed surface
+	// each slot holds; and the rate control of each layer, without which
+	// AMD's driver gives the upper layers next to no bits.
+	class VaapiTemporalLayers
+	{
+		public:
+			// The reconstructed surfaces a session needs: those of the two
+			// reference buffers WebRTC's structures use up to L1T3, and that
+			// of the frame being encoded.
+			static constexpr int kReconstructedSurfaces = 3;
+
+			// What a frame refers to and replaces.
+			struct Frame
+			{
+				int temporalId = 0;
+				// The slots of its references, the first being its last frame,
+				// the second, if any, its golden frame.
+				std::vector<int> references;
+				uint8_t refreshFlags = 0;
+				// The reconstructed surface it is encoded into.
+				int surface = -1;
+			};
+
+			VaapiTemporalLayers();
+
+			// Starts over in the mode, splitting the bitrate across the layers
+			// until WebRTC gives its own split. Returns false for a mode other
+			// than L1T1, L1T2 and L1T3.
+			bool Configure(webrtc::ScalabilityMode mode, uint32_t bitrateBps);
+
+			// The number of temporal layers, one without layers.
+			int Count() const;
+
+			// Plans a frame of the layer WebRTC's structure gives. Returns
+			// nothing if it refers to a slot that holds no frame, or to one
+			// beyond those kept, or no surface is free.
+			std::optional<Frame> Plan(bool keyFrame,
+				const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame) const;
+			// Takes the frame, once encoded, into the slots it replaces.
+			void Commit(const Frame & frame);
+
+			// The slots kept, and the reconstructed surface each holds, -1 for
+			// none. A key frame fills the slots beyond too, but no frame refers
+			// to them.
+			int SlotCount() const;
+			int Surface(int slot) const;
+
+			// Takes the bitrate WebRTC gives each layer.
+			void OnRatesUpdated(const webrtc::VideoBitrateAllocation & bitrates);
+			// Adds the rate control of the stream, or of each layer, with the
+			// layer structure along with that of a key frame. No initial
+			// quantizer: the rate control picks its own, as FFmpeg leaves it.
+			bool AddRateControl(VaapiEncodeSession & session, uint32_t bitrateBps, uint32_t framerate,
+				bool keyFrame) const;
+			// Fills in the share of the frame rate up to each layer.
+			void FillFpsAllocation(webrtc::VideoEncoder::EncoderInfo & info) const;
+
+		private:
+			int count;
+			std::array<int, 2> slots;
+			// The bitrate of each layer, counting the layers below it.
+			std::vector<uint32_t> bitratesBps;
+	};
+}
+
+#endif

webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Encoder.h

@@ -20,6 +20,7 @@
 #include "media/video/codec/EncoderOutputProcessor.h"
 #include "media/video/codec/linux/VaapiEncodeSession.h"
 #include "media/video/codec/linux/VaapiLibrary.h"
+#include "media/video/codec/linux/VaapiTemporalLayers.h"
 
 #include "api/video/video_frame.h"
 #include "api/video_codecs/sdp_video_format.h"
@@ -36,15 +37,16 @@
 namespace jni
 {
 	// Encodes VP9 profile 0 with the GPU driver's VA-API encoder, as Intel
-	// drivers provide it on Linux. Like hardware VP9 encoding on Windows, it
-	// has not run on such a GPU yet, and is used only where it is turned on.
+	// drivers provide it on Linux.
 	//
-	// It works as the H.265 one does: synchronous, one reference frame, inter
-	// frames only, the driver writing the uncompressed frame header from the
-	// parameters it is given, as FFmpeg's VA-API encoder sets them without
-	// B-frames. Every frame refreshes all eight reference slots, so that the
-	// next one can refer to it as the last frame whichever slot it names.
-	// Anything that fails is an error, upon which libvpx takes over.
+	// It works as the H.265 one does: synchronous, inter frames only, the
+	// driver writing the uncompressed frame header from the parameters it is
+	// given, as FFmpeg's VA-API encoder sets them without B-frames. Each frame
+	// refers to one frame before it: without layers the frame just before;
+	// with temporal layers (L1T2, L1T3) the one of the same layer or a lower
+	// one that WebRTC's structure of the mode names, so that a receiver can do
+	// without the upper layers. Spatial layers, and anything that fails, are
+	// left to libvpx.
 	class VaapiVp9Encoder : public webrtc::VideoEncoder
 	{
 		public:
@@ -59,10 +61,11 @@ namespace jni
 			EncoderInfo GetEncoderInfo() const override;
 
 		private:
-			bool Submit(bool keyFrame);
+			bool Submit(bool keyFrame, const VaapiTemporalLayers::Frame & frame);
 
 			void FillSequence(VAEncSequenceParameterBufferVP9 & sequence) const;
-			void FillPicture(VAEncPictureParameterBufferVP9 & picture, bool keyFrame) const;
+			void FillPicture(VAEncPictureParameterBufferVP9 & picture, bool keyFrame,
+				const VaapiTemporalLayers::Frame & frame) const;
 
 		private:
 			VaapiLibrary & library;
@@ -76,8 +79,7 @@ namespace jni
 			uint32_t framerate;
 			bool ratesChanged;
 
-			// Which of the reconstructed surfaces holds the current frame.
-			int current;
+			VaapiTemporalLayers layers;
 			bool referenceValid;
 
 			webrtc::EncodedImageCallback * callback;

webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp

@@ -22,6 +22,7 @@
 #include "common_video/h265/h265_common.h"
 #include "modules/video_coding/codecs/interface/common_constants.h"
 #include "modules/video_coding/svc/create_scalability_structure.h"
+#include "modules/video_coding/svc/scalability_mode_util.h"
 #include "modules/video_coding/svc/scalable_video_controller_no_layering.h"
 #include "modules/video_coding/utility/vp9_uncompressed_header_parser.h"
 #include "rtc_base/logging.h"
@@ -409,7 +410,10 @@ namespace jni
 			};
 		}
 
-		if (codec == webrtc::kVideoCodecVP9) {
+		if (codec == webrtc::kVideoCodecVP9 && scalabilityMode != webrtc::ScalabilityMode::kL1T1) {
+			FillVp9TemporalLayers(image, keyFrame, layerFrame, info);
+		}
+		else if (codec == webrtc::kVideoCodecVP9) {
 			// VP9 has its own RTP header, which WebRTC fills from these: one
 			// spatial and one temporal layer, the group of frames sent with
 			// every key frame.
@@ -436,6 +440,77 @@ namespace jni
 		}
 	}
 
+	void EncoderOutputProcessor::FillVp9TemporalLayers(const webrtc::EncodedImage & image, bool keyFrame,
+		const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::CodecSpecificInfo & info)
+	{
+		// One spatial layer and temporal ones, in flexible mode, as libvpx
+		// describes them: each picture names the pictures it refers to, by
+		// how many pictures back they are.
+		vp9Picture = keyFrame ? 0 : vp9Picture + 1;
+
+		webrtc::CodecSpecificInfoVP9 & vp9 = info.codecSpecific.VP9;
+		vp9.first_frame_in_picture = true;
+		vp9.flexible_mode = true;
+		vp9.ss_data_available = keyFrame;
+		vp9.non_ref_for_inter_layer_pred = true;
+		vp9.temporal_idx = static_cast<uint8_t>(layerFrame.TemporalId());
+		vp9.inter_layer_predicted = false;
+		vp9.gof_idx = webrtc::kNoGofIdx;
+		vp9.num_spatial_layers = 1;
+		vp9.first_active_layer = 0;
+		vp9.spatial_layer_resolution_present = keyFrame;
+		vp9.num_ref_pics = 0;
+
+		if (!keyFrame) {
+			for (const webrtc::CodecBufferUsage & buffer : layerFrame.Buffers()) {
+				if (!buffer.referenced || buffer.id < 0 || buffer.id >= webrtc::kMaxEncoderBuffers) {
+					continue;
+				}
+
+				const size_t pDiff = vp9Picture - vp9BufferPictures[buffer.id];
+				bool listed = false;
+
+				for (uint8_t i = 0; i < vp9.num_ref_pics; i++) {
+					listed |= vp9.p_diff[i] == pDiff;
+				}
+
+				if (!listed && pDiff > 0 && vp9.num_ref_pics < webrtc::kMaxVp9RefPics) {
+					vp9.p_diff[vp9.num_ref_pics++] = static_cast<uint8_t>(pDiff);
+				}
+			}
+		}
+
+		vp9.inter_pic_predicted = !keyFrame && vp9.num_ref_pics > 0;
+
+		// A switch up to a higher layer is possible where all layers above
+		// this one are switch points, as libvpx works it out.
+		const int temporalLayers = webrtc::ScalabilityModeToNumTemporalLayers(scalabilityMode);
+		vp9.temporal_up_switch = true;
+
+		for (int layer = layerFrame.TemporalId() + 1; layer < temporalLayers && info.generic_frame_info; layer++) {
+			const auto & indications = info.generic_frame_info->decode_target_indications;
+
+			vp9.temporal_up_switch &= layer < static_cast<int>(indications.size())
+				&& indications[layer] == webrtc::DecodeTargetIndication::kSwitch;
+		}
+
+		if (keyFrame) {
+			vp9.width[0] = image._encodedWidth;
+			vp9.height[0] = image._encodedHeight;
+			vp9.gof.num_frames_in_gof = 0;
+		}
+
+		for (const webrtc::CodecBufferUsage & buffer : layerFrame.Buffers()) {
+			if ((buffer.updated || keyFrame) && buffer.id >= 0 && buffer.id < webrtc::kMaxEncoderBuffers) {
+				vp9BufferPictures[buffer.id] = vp9Picture;
+			}
+		}
+
+		if (keyFrame) {
+			vp9BufferPictures.fill(0);
+		}
+	}
+
 	webrtc::VideoEncoder::ScalingSettings EncoderOutputProcessor::GetScalingSettings() const
 	{
 		// H.265 has no thresholds of its own in WebRTC yet and uses those of