feat: encode AV1 with VA-API on Linux

docs/guide/advanced/video-codecs.md

@@ -27,10 +27,10 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder()
 | Platform | H.264 | H.265 | AV1 |
 |---|---|---|---|
 | Windows | NVENC on NVIDIA GPUs, otherwise the Media Foundation encoder of the GPU driver (AMD, Intel) | NVENC on NVIDIA GPUs, otherwise Media Foundation on GPUs that encode H.265 | NVENC on NVIDIA RTX 40 and newer, otherwise Media Foundation on GPUs that encode AV1 (e.g. AMD RDNA3 and newer, Intel Arc) |
-| Linux | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) where it encodes H.265 | NVENC on NVIDIA RTX 40 and newer |
+| Linux | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) where it encodes H.265 | NVENC on NVIDIA RTX 40 and newer, otherwise the VA-API encoder of the GPU driver where it encodes AV1 (e.g. AMD RDNA3 and newer) |
 | macOS | VideoToolbox, as with `DefaultVideoEncoderFactory` | VideoToolbox, on Macs with a hardware HEVC encoder, such as all with Apple silicon | Software |
 
-Hardware encoders produce a single layer. A stream that asks for SVC, such as an AV1 stream with a scalability mode like `L1T3`, is encoded in software.
+Hardware encoders produce a single layer. A stream that asks for SVC, such as an AV1 stream with a scalability mode like `L1T3`, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom.
 
 VP9 can be encoded in hardware too, with the VP9 encoder of the GPU driver, which Intel GPUs have: through VA-API on Linux, and through Media Foundation on Windows. On Windows it is off by default, since it has not been tested on such a GPU yet; turn it on with a system property before the factory is created:
 

webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h

@@ -0,0 +1,117 @@
+/*
+ * Copyright 2026 Alex Andres
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_AV1_ENCODER_H_
+#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_AV1_ENCODER_H_
+
+#include "media/video/codec/EncoderOutputProcessor.h"
+#include "media/video/codec/linux/VaapiEncodeSession.h"
+#include "media/video/codec/linux/VaapiLibrary.h"
+
+#include "api/video/video_frame.h"
+#include "api/video_codecs/sdp_video_format.h"
+#include "api/video_codecs/video_codec.h"
+#include "api/video_codecs/video_encoder.h"
+
+#include <va/va.h>
+#include <va/va_enc_av1.h>
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+namespace jni
+{
+	// Encodes AV1 profile 0 with the GPU driver's VA-API encoder, as AMD
+	// drivers provide it on Linux.
+	//
+	// It works as FFmpeg's VA-API encoder does without B-frames: synchronous,
+	// one tile, inter frames that refer to the frame before them only, which
+	// alternate between two reference slots. The sequence and frame headers
+	// are written here and handed to the driver as packed headers, with where
+	// the quantizer, loop filter and CDEF parameters are in the frame header,
+	// which the driver's rate control fills in. Anything that fails is an
+	// error, upon which libaom takes over.
+	class VaapiAv1Encoder : public webrtc::VideoEncoder
+	{
+		public:
+			VaapiAv1Encoder(VaapiLibrary & library, VAEntrypoint entrypoint, const webrtc::SdpVideoFormat & format);
+			~VaapiAv1Encoder() override;
+
+			int32_t InitEncode(const webrtc::VideoCodec * codecSettings, const Settings & settings) override;
+			int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback * callback) override;
+			int32_t Release() override;
+			int32_t Encode(const webrtc::VideoFrame & frame, const std::vector<webrtc::VideoFrameType> * frameTypes) override;
+			void SetRates(const RateControlParameters & parameters) override;
+			EncoderInfo GetEncoderInfo() const override;
+
+		private:
+			// Where the driver rewrites the frame header: the bit offsets of
+			// base_q_idx, loop_filter_params() and cdef_params(), the size of
+			// the latter, and the size of the whole frame header OBU.
+			struct FrameHeaderLayout
+			{
+				uint32_t qindexOffset = 0;
+				uint32_t loopFilterOffset = 0;
+				uint32_t cdefOffset = 0;
+				uint32_t cdefSize = 0;
+				uint32_t size = 0;
+			};
+
+			// Reads which headers the driver takes and how it codes them.
+			void QueryCapabilities();
+
+			bool Submit(bool keyFrame);
+
+			void WriteSequenceHeader(std::vector<uint8_t> & obu) const;
+			void WriteFrameHeader(std::vector<uint8_t> & obu, FrameHeaderLayout & layout, bool keyFrame) const;
+			// The seq_level_idx the stream's size, frame rate and bitrate fit.
+			uint8_t Level() const;
+
+		private:
+			VaapiLibrary & library;
+			const VAEntrypoint entrypoint;
+			const std::string implementationName;
+			VaapiEncodeSession session;
+
+			bool initialized;
+			webrtc::VideoCodec codecSettings;
+			uint32_t bitrateBps;
+			uint32_t framerate;
+			bool ratesChanged;
+
+			// The fixed length of obu_size the driver takes, in bytes, zero
+			// where it takes no packed headers; and the TxMode it encodes.
+			uint32_t obuSizeLength;
+			uint32_t txMode;
+			// The sequence header, kept for the key frames.
+			std::vector<uint8_t> sequenceHeader;
+
+			// Which of the reconstructed surfaces, and so which reference slot,
+			// holds the current frame.
+			int current;
+			bool referenceValid;
+			// The frames since the last key frame, whose low bits are the
+			// order hint.
+			uint32_t frameNumber;
+
+			webrtc::EncodedImageCallback * callback;
+
+			EncoderOutputProcessor outputProcessor;
+	};
+}
+
+#endif

webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoEncoderFactory.h

@@ -32,8 +32,8 @@ namespace jni
 	// Creates VA-API encoders on the GPU whose driver encodes H.264, as Intel
 	// and AMD drivers do. It offers H.264 in the profiles WebRTC's software
 	// encoder offers too, and only with packetization mode 1: mode 0 needs
-	// each NAL unit to fit a packet. It offers H.265 Main and VP9 profile 0
-	// as well where the driver encodes them.
+	// each NAL unit to fit a packet. It offers H.265 Main, VP9 profile 0 and
+	// AV1 profile 0 as well where the driver encodes them.
 	class VaapiVideoEncoderFactory : public webrtc::VideoEncoderFactory
 	{
 		public:
@@ -49,14 +49,16 @@ namespace jni
 
 		private:
 			VaapiVideoEncoderFactory(VaapiLibrary & library, bool h265, VAEntrypoint h265Entrypoint, bool vp9,
-				VAEntrypoint vp9Entrypoint);
+				VAEntrypoint vp9Entrypoint, bool av1, VAEntrypoint av1Entrypoint);
 
 		private:
 			VaapiLibrary & library;
 			const bool h265;
 			const VAEntrypoint h265Entrypoint;
 			const bool vp9;
 			const VAEntrypoint vp9Entrypoint;
+			const bool av1;
+			const VAEntrypoint av1Entrypoint;
 	};
 }