feat: encode H.264 and H.265 temporal layers with VA-API

docs/guide/advanced/video-codecs.md

@@ -30,7 +30,7 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder()
 | Linux | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) where it encodes H.265 | NVENC on NVIDIA RTX 40 and newer, otherwise the VA-API encoder of the GPU driver where it encodes AV1 (e.g. AMD RDNA3 and newer) |
 | macOS | VideoToolbox, as with `DefaultVideoEncoderFactory` | VideoToolbox, on Macs with a hardware HEVC encoder, such as all with Apple silicon | Software |
 
-Hardware encoders produce a single layer, but for the VA-API encoders of AV1 and VP9, which encode temporal layers too (`L1T2` and `L1T3`), with a rate control of each layer, so that an SFU can forward a lower frame rate. A stream that asks for spatial layers, or for temporal layers from another hardware encoder, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom.
+Hardware encoders produce a single layer, but for the VA-API encoders, which encode temporal layers too (`L1T2` and `L1T3`) in every codec, so that an SFU can forward a lower frame rate. Each layer gets a rate control of its own where the driver has one that works (AMD's for H.265 and AV1, Intel's for VP9); otherwise the layers share one, which treats them alike. A stream that asks for spatial layers, or for temporal layers from another hardware encoder, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom.
 
 VP9 can be encoded in hardware too, with the VP9 encoder of the GPU driver, which Intel GPUs have: through VA-API on Linux, and through Media Foundation on Windows. On Windows it is off by default, since it has not been tested on such a GPU yet; turn it on with a system property before the factory is created:
 

webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h

@@ -48,9 +48,9 @@ namespace jni
 	// or the sequence header of AV1. Encoders put them in front of the first
 	// key frame, not necessarily in front of every one, so they are kept and
 	// put back where missing. The frame dependencies WebRTC needs for AV1 and
-	// H.265 are those of a stream without layers, unless an AV1 or VP9 encoder
-	// that makes temporal layers sets its scalability mode and asks for the
-	// layer of each frame before it encodes it. VP9 frames go out as they
+	// H.265 are those of a stream without layers, unless an encoder that
+	// makes temporal layers sets its scalability mode and asks for the layer
+	// of each frame before it encodes it. VP9 frames go out as they
 	// come, with the VP9 information of a single layer, as the Java encoders
 	// get it, or with temporal layers in flexible mode, as libvpx gives it.
 	class EncoderOutputProcessor
@@ -61,8 +61,8 @@ namespace jni
 			// Forgets the parameter sets, for a stream that starts over.
 			void Reset();
 
-			// Sets the layers of the stream, temporal ones of AV1 and VP9 only,
-			// as in L1T3. Returns false for a mode WebRTC has no structure of.
+			// Sets the layers of the stream, temporal ones only, as in L1T3.
+			// Returns false for a mode WebRTC has no structure of.
 			bool SetScalabilityMode(webrtc::ScalabilityMode mode);
 			// Lets the layers follow the bitrate WebRTC gives each.
 			void OnRatesUpdated(const webrtc::VideoBitrateAllocation & bitrates);

webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h

@@ -35,8 +35,7 @@ namespace jni
 	// parameter buffers of a frame, uploading a frame as NV12, encoding it
 	// and reading what came out. The codec's encoder fills in the parameters.
 	//
-	// Used by the H.265 and VP9 encoders; the H.264 one keeps its own, older
-	// copy of the same.
+	// Used by every VA-API encoder.
 	class VaapiEncodeSession
 	{
 		public:

webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH264Encoder.h

@@ -19,7 +19,9 @@
 
 #include "media/video/codec/EncoderOutputProcessor.h"
 #include "media/video/codec/linux/NalUnitWriter.h"
+#include "media/video/codec/linux/VaapiEncodeSession.h"
 #include "media/video/codec/linux/VaapiLibrary.h"
+#include "media/video/codec/linux/VaapiTemporalLayers.h"
 
 #include "api/video/video_frame.h"
 #include "api/video_codecs/sdp_video_format.h"
@@ -29,6 +31,7 @@
 #include <va/va.h>
 #include <va/va_enc_h264.h>
 
+#include <array>
 #include <cstdint>
 #include <string>
 #include <vector>
@@ -38,15 +41,20 @@ namespace jni
 	// Encodes H.264 Constrained Baseline with the GPU driver's VA-API encoder,
 	// as Intel and AMD drivers provide it on Linux.
 	//
-	// Encoding is synchronous, with one reference frame and P-frames only, so
-	// each frame is encoded before Encode() returns. The encoder writes the
-	// parameter sets and slice headers itself and hands them to the driver as
-	// packed headers, as FFmpeg does, since Intel's iHD driver writes none of
-	// its own; a driver that takes no packed headers writes them from the
-	// parameters it is given. A driver that leaves the parameter sets out of a
-	// key frame, which WebRTC cannot send without them, makes the encoder give
-	// up, as does anything else that fails, so that the software encoder takes
-	// over.
+	// Encoding is synchronous, with P-frames only, so each frame is encoded
+	// before Encode() returns. The encoder writes the parameter sets and slice
+	// headers itself and hands them to the driver as packed headers, as FFmpeg
+	// does, since Intel's iHD driver writes none of its own; a driver that
+	// takes no packed headers writes them from the parameters it is given.
+	// Each frame refers to the frame just before it, or, with temporal layers
+	// (L1T2, L1T3), which need a driver that takes the slice headers, to the
+	// one of its layer or a lower one that WebRTC's structure of the mode
+	// names: the top layer is not used for reference, frame_num may skip the
+	// frames of the layers a receiver goes without, and each slice puts its
+	// reference first in the list. A driver that leaves the parameter sets out
+	// of a key frame, which WebRTC cannot send without them, makes the encoder
+	// give up, as does anything else that fails, so that the software encoder
+	// takes over.
 	class VaapiH264Encoder : public webrtc::VideoEncoder
 	{
 		public:
@@ -61,55 +69,54 @@ namespace jni
 			EncoderInfo GetEncoderInfo() const override;
 
 		private:
-			bool CreateSession();
-			void DestroySession();
-			bool Upload(const webrtc::VideoFrame & frame);
-			bool Submit(bool idr);
-			bool ReadOutput(std::vector<uint8_t> & output);
+			// A frame being encoded: its layer, its frame_num and whether it is
+			// used for reference.
+			struct Picture
+			{
+				VaapiTemporalLayers::Frame frame;
+				uint32_t frameNum = 0;
+				bool reference = true;
+			};
 
-			bool AddBuffer(VABufferType type, void * data, size_t size);
-			bool AddMiscParameter(VAEncMiscParameterType type, const void * data, size_t size);
-			bool AddPackedHeader(VAEncPackedHeaderType type, const std::vector<uint8_t> & data, size_t bits);
-			void DestroyFrameBuffers();
+			bool Submit(bool idr, const Picture & picture);
 
 			void FillSequence(VAEncSequenceParameterBufferH264 & sequence) const;
-			void FillPicture(VAEncPictureParameterBufferH264 & picture, bool idr) const;
-			void FillSlice(VAEncSliceParameterBufferH264 & slice, bool idr) const;
+			void FillPicture(VAEncPictureParameterBufferH264 & parameters, bool idr, const Picture & picture) const;
+			void FillSlice(VAEncSliceParameterBufferH264 & slice, bool idr, const Picture & picture) const;
+
+			// The picture a reference slot holds, as VA-API names it.
+			VAPictureH264 Reference(int slot) const;
+			// The order count of a picture, as pic_order_cnt_type 2 derives it.
+			static int32_t OrderCount(uint32_t frameNum, bool reference);
 
 			// The SPS and PPS, and the header of the slice, matching the
 			// parameters above. Each returns the bits it appended.
 			size_t WriteParameterSets(std::vector<uint8_t> & output) const;
-			size_t WriteSliceHeader(std::vector<uint8_t> & output, bool idr) const;
-
-			bool Check(VAStatus status, const char * operation) const;
+			size_t WriteSliceHeader(std::vector<uint8_t> & output, bool idr, const Picture & picture) const;
 
 		private:
 			VaapiLibrary & library;
-			const VaapiFunctions & va;
 			const std::string implementationName;
+			VaapiEncodeSession session;
 
-			VAConfigID config;
 			// The VA_ENC_PACKED_HEADER_* the driver takes from the encoder.
 			uint32_t packedHeaders;
-			VAContextID context;
-			VASurfaceID inputSurface;
-			// The frame being encoded, and the one it refers to.
-			VASurfaceID reconstructed[2];
-			VABufferID codedBuffer;
-			std::vector<VABufferID> frameBuffers;
 
+			bool initialized;
 			webrtc::VideoCodec codecSettings;
 			uint32_t widthInMbs;
 			uint32_t heightInMbs;
 			uint32_t bitrateBps;
 			uint32_t framerate;
 			bool ratesChanged;
 
-			// The frame number within the current IDR period, and which of
-			// the reconstructed surfaces holds the reference.
-			uint32_t frameNum;
+			// The frame_num of the last reference picture, and the temporal
+			// layers with the frame_num of the picture each reference slot
+			// holds.
+			uint32_t previousReferenceFrameNum;
 			uint32_t idrPicId;
-			int current;
+			VaapiTemporalLayers layers;
+			std::array<uint32_t, 2> slotFrameNums;
 			bool referenceValid;
 
 			webrtc::EncodedImageCallback * callback;

webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Encoder.h

@@ -21,6 +21,7 @@
 #include "media/video/codec/linux/NalUnitWriter.h"
 #include "media/video/codec/linux/VaapiEncodeSession.h"
 #include "media/video/codec/linux/VaapiLibrary.h"
+#include "media/video/codec/linux/VaapiTemporalLayers.h"
 
 #include "api/video/video_frame.h"
 #include "api/video_codecs/sdp_video_format.h"
@@ -30,6 +31,7 @@
 #include <va/va.h>
 #include <va/va_enc_hevc.h>
 
+#include <array>
 #include <cstdint>
 #include <string>
 #include <vector>
@@ -45,7 +47,13 @@ namespace jni
 	// headers for a driver that takes them, as Intel's and AMD's do. A driver
 	// that predicts only with both reference lists filled, as Intel's
 	// low-power encoder, gets each P-frame as a B-frame whose two lists hold
-	// the same previous picture ("low delay B"), as FFmpeg does. The block
+	// the same previous picture ("low delay B"), as FFmpeg does. With
+	// temporal layers (L1T2, L1T3), which need a driver that takes the slice
+	// headers, a picture refers to the one of its layer or a lower one that
+	// WebRTC's structure of the mode names, its slice NAL unit carries its
+	// temporal id, and its reference picture set keeps the pictures of the
+	// reference slots of its layer and below, so that a receiver without the
+	// upper layers misses none of them. The block
 	// sizes and coding tools are those the driver says it takes, as FFmpeg's
 	// VA-API encoder chooses them. The frame size is kept to a multiple of the
 	// smallest coding block; the coded picture is that aligned to 16 and at
@@ -70,21 +78,31 @@ namespace jni
 		private:
 			// What the driver says about the HEVC encoder it has.
 			void QueryCapabilities();
-			bool Submit(bool idr);
+			bool Submit(bool idr, const VaapiTemporalLayers::Frame & frame);
 
 			void FillSequence(VAEncSequenceParameterBufferHEVC & sequence) const;
-			void FillPicture(VAEncPictureParameterBufferHEVC & picture, bool idr) const;
-			void FillSlice(VAEncSliceParameterBufferHEVC & slice, bool idr) const;
+			void FillPicture(VAEncPictureParameterBufferHEVC & picture, bool idr,
+				const VaapiTemporalLayers::Frame & frame) const;
+			void FillSlice(VAEncSliceParameterBufferHEVC & slice, bool idr, const VaapiTemporalLayers::Frame & frame) const;
 
 			// The VPS, SPS and PPS, and the slice segment header, matching
 			// the parameters above. Each returns the bits it appended.
 			size_t WriteParameterSets(std::vector<uint8_t> & output) const;
-			size_t WriteSliceHeader(std::vector<uint8_t> & output, bool idr) const;
+			size_t WriteSliceHeader(std::vector<uint8_t> & output, bool idr, const VaapiTemporalLayers::Frame & frame) const;
 			void WriteProfileTierLevel(NalUnitWriter & writer) const;
 			// diff_cu_qp_delta_depth.
 			uint32_t CuQpDeltaDepth() const;
-			// The picture the current one refers to.
-			VAPictureHEVC Reference() const;
+			// The pictures the reference slots hold, newest first, each once,
+			// and whether the current frame refers to it: its reference
+			// picture set.
+			struct ReferencePicture
+			{
+				int slot;
+				int32_t pictureOrder;
+				bool used;
+			};
+			std::vector<ReferencePicture> ReferencePictureSet(const VaapiTemporalLayers::Frame & frame) const;
+			VAPictureHEVC Picture(const ReferencePicture & reference) const;
 
 		private:
 			VaapiLibrary & library;
@@ -122,10 +140,13 @@ namespace jni
 			uint32_t framerate;
 			bool ratesChanged;
 
-			// The picture order count since the last IDR picture, and which of
-			// the reconstructed surfaces holds the current picture.
+			// The picture order count since the last IDR picture, the temporal
+			// layers, and the picture order count and temporal id of the
+			// picture each of their reference slots holds.
 			int32_t pictureOrder;
-			int current;
+			VaapiTemporalLayers layers;
+			std::array<int32_t, 2> slotOrders;
+			std::array<int, 2> slotTemporalIds;
 			bool referenceValid;
 
 			webrtc::EncodedImageCallback * callback;

webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiTemporalLayers.h

@@ -18,10 +18,12 @@
 #define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_TEMPORAL_LAYERS_H_
 
 #include "media/video/codec/linux/VaapiEncodeSession.h"
+#include "media/video/codec/linux/VaapiLibrary.h"
 
 #include "api/video/video_bitrate_allocation.h"
 #include "api/video_codecs/scalability_mode.h"
 #include "api/video_codecs/video_encoder.h"
+#include "modules/video_coding/svc/scalability_mode_util.h"
 #include "modules/video_coding/svc/scalable_video_controller.h"
 
 #include <array>
@@ -36,7 +38,10 @@ namespace jni
 	// it replaces, as WebRTC's structure of the mode says, each buffer of the
 	// structure being the slot of the same number; which reconstructed surface
 	// each slot holds; and the rate control of each layer, without which
-	// AMD's driver gives the upper layers next to no bits.
+	// AMD's driver gives the upper layers next to no bits, where the driver
+	// says it has one and the encoder wants it. Intel's H.264 and H.265
+	// encoders say not, and refuse the layer structure that goes with it;
+	// their rate control treats every layer alike.
 	class VaapiTemporalLayers
 	{
 		public:
@@ -52,17 +57,28 @@ namespace jni
 				// The slots of its references, the first being its last frame,
 				// the second, if any, its golden frame.
 				std::vector<int> references;
+				// The slots it replaces, all for a key frame, and of those the
+				// ones WebRTC's structure goes on to use: a key frame leaves
+				// the others empty, which nothing refers to then, and which a
+				// codec would otherwise keep as references for good.
 				uint8_t refreshFlags = 0;
+				uint8_t keptFlags = 0;
 				// The reconstructed surface it is encoded into.
 				int surface = -1;
 			};
 
 			VaapiTemporalLayers();
 
+			// Whether the driver's encoder of the profile has a rate control
+			// of each temporal layer, for as many layers as given.
+			static bool LayerRateControl(VaapiLibrary & library, VAProfile profile, VAEntrypoint entrypoint,
+				int layers);
+
 			// Starts over in the mode, splitting the bitrate across the layers
-			// until WebRTC gives its own split. Returns false for a mode other
-			// than L1T1, L1T2 and L1T3.
-			bool Configure(webrtc::ScalabilityMode mode, uint32_t bitrateBps);
+			// until WebRTC gives its own split, for a rate control of each layer
+			// where there is one. Returns false for a mode other than L1T1, L1T2
+			// and L1T3.
+			bool Configure(webrtc::ScalabilityMode mode, uint32_t bitrateBps, bool layerRateControl);
 
 			// The number of temporal layers, one without layers.
 			int Count() const;
@@ -74,6 +90,8 @@ namespace jni
 				const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame) const;
 			// Takes the frame, once encoded, into the slots it replaces.
 			void Commit(const Frame & frame);
+			// Empties a slot whose picture the decoder no longer keeps.
+			void Forget(int slot);
 
 			// The slots kept, and the reconstructed surface each holds, -1 for
 			// none. A key frame fills the slots beyond too, but no frame refers
@@ -93,6 +111,7 @@ namespace jni
 
 		private:
 			int count;
+			bool layerRateControl;
 			std::array<int, 2> slots;
 			// The bitrate of each layer, counting the layers below it.
 			std::vector<uint32_t> bitratesBps;

webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp

@@ -394,12 +394,28 @@ namespace jni
 			info.codecSpecific.H264.temporal_idx = webrtc::kNoTemporalIdx;
 			info.codecSpecific.H264.base_layer_sync = false;
 			info.codecSpecific.H264.idr_frame = keyFrame;
-			return;
+
+			if (scalabilityMode == webrtc::ScalabilityMode::kL1T1) {
+				return;
+			}
+
+			// With temporal layers, the layer, and whether an upper layer frame
+			// refers to the base layer only, as OpenH264 gives them; the frame
+			// dependencies follow below. The base layer is buffer 0 in WebRTC's
+			// structures.
+			bool baseOnly = true;
+
+			for (const webrtc::CodecBufferUsage & buffer : layerFrame.Buffers()) {
+				baseOnly &= !buffer.referenced || buffer.id == 0;
+			}
+
+			info.codecSpecific.H264.temporal_idx = static_cast<uint8_t>(layerFrame.TemporalId());
+			info.codecSpecific.H264.base_layer_sync = layerFrame.TemporalId() > 0 && baseOnly;
 		}
 
 		// AV1 and H.265 have no codec specific information; WebRTC sends
-		// their frame dependencies, and those of VP9, from these, in the
-		// dependency descriptor.
+		// their frame dependencies, and those of VP9 and of H.264 with layers,
+		// from these, in the dependency descriptor.
 		info.generic_frame_info = svcController->OnEncodeDone(layerFrame);
 		info.scalability_mode = scalabilityMode;
 

webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp

@@ -350,7 +350,8 @@ namespace jni
 		referenceValid = false;
 		frameNumber = 0;
 		outputProcessor.Reset();
-		layers.Configure(mode, bitrateBps);
+		layers.Configure(mode, bitrateBps, VaapiTemporalLayers::LayerRateControl(library, VAProfileAV1Profile0,
+			entrypoint, webrtc::ScalabilityModeToNumTemporalLayers(mode)));
 
 		if (!outputProcessor.SetScalabilityMode(mode)) {
 			return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
@@ -436,6 +437,7 @@ namespace jni
 
 		session.DropBuffers();
 
+
 		if (!encoded) {
 			return kEncodeFailed;
 		}