fix: write the H.264 and H.265 headers for VA-API drivers that need them

docs/guide/advanced/video-codecs.md

@@ -27,7 +27,7 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder()
 | Platform | H.264 | H.265 | AV1 |
 |---|---|---|---|
 | Windows | NVENC on NVIDIA GPUs, otherwise the Media Foundation encoder of the GPU driver (AMD, Intel) | NVENC on NVIDIA GPUs, otherwise Media Foundation on GPUs that encode H.265 | NVENC on NVIDIA RTX 40 and newer, otherwise Media Foundation on GPUs that encode AV1 (e.g. AMD RDNA3 and newer, Intel Arc) |
-| Linux | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) | NVENC on NVIDIA GPUs; VA-API (Intel, AMD) when turned on, see below | NVENC on NVIDIA RTX 40 and newer |
+| Linux | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) where it encodes H.265 | NVENC on NVIDIA RTX 40 and newer |
 | macOS | VideoToolbox, as with `DefaultVideoEncoderFactory` | VideoToolbox, on Macs with a hardware HEVC encoder, such as all with Apple silicon | Software |
 
 Hardware encoders produce a single layer. A stream that asks for SVC, such as an AV1 stream with a scalability mode like `L1T3`, is encoded in software.
@@ -40,13 +40,7 @@ System.setProperty("webrtc.hardware.vp9Encoder", "true");
 
 Where the GPU has no VP9 encoder, or the stream asks for SVC, VP9 is still encoded with libvpx.
 
-On Linux, H.265 can be encoded with the VA-API encoder of Intel and AMD drivers, also off by default until it has been tested on such GPUs:
-
-```java
-System.setProperty("webrtc.hardware.vaapiExperimental", "true");
-```
-
-It is offered only where the driver encodes H.265 Main, and only for sending: Intel and AMD GPUs do not decode H.265 here yet, so the other end has to.
+On Linux, H.265 is encoded with the VA-API encoder of Intel and AMD drivers where the driver encodes H.265 Main. It is offered only for sending: Intel and AMD GPUs do not decode H.265 here yet, so the other end has to.
 
 NVENC needs an NVIDIA driver of version 522 or newer on Windows, 520 or newer on Linux. VA-API needs libva 2 and a driver that encodes H.264, such as Intel's `intel-media-va-driver` (iHD) or Mesa's `mesa-va-drivers` for AMD, and access to a render node in `/dev/dri`. All of them are loaded at run time, so nothing needs to be installed on machines without them.
 

webrtc-jni/src/main/cpp/include/media/video/codec/HardwareCodecOptions.h

@@ -24,17 +24,16 @@ namespace jni
 	// tested on one.
 	constexpr const char * kHardwareVp9EncoderProperty = "webrtc.hardware.vp9Encoder";
 
-	// The Java system property that turns on the VA-API codecs beyond H.264
-	// encoding, which are off by default: they have not been tested on an
-	// Intel or AMD GPU under Linux yet.
+	// The Java system property that turns on VA-API decoding, which is off by
+	// default: it has not been tested on an Intel or AMD GPU under Linux yet.
 	constexpr const char * kExperimentalVaapiProperty = "webrtc.hardware.vaapiExperimental";
 
 	// Whether hardware VP9 encoding is turned on: the Java system property
 	// above is "true", as Boolean.getBoolean has it. Callable from any thread;
 	// false where Java cannot be asked, for one with an exception pending.
 	bool IsHardwareVp9EncoderEnabled();
 
-	// Whether the experimental VA-API codecs are turned on, the same way.
+	// Whether VA-API decoding is turned on, the same way.
 	bool IsExperimentalVaapiEnabled();
 }
 

webrtc-jni/src/main/cpp/include/media/video/codec/linux/NalUnitWriter.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2026 Alex Andres
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_NAL_UNIT_WRITER_H_
+#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_NAL_UNIT_WRITER_H_
+
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+
+namespace jni
+{
+	// Writes one H.264 or H.265 NAL unit, header included, bit by bit, as the
+	// VA-API encoders hand the parameter sets and slice headers to drivers
+	// that do not write them themselves.
+	class NalUnitWriter
+	{
+		public:
+			NalUnitWriter();
+
+			// The low count bits of value, the most significant first.
+			void Bits(uint32_t value, int count);
+			void Flag(bool value);
+			// Exponential-Golomb codes, ue(v) and se(v).
+			void Ue(uint32_t value);
+			void Se(int32_t value);
+			// A one bit and zero bits up to the next byte: rbsp_trailing_bits()
+			// at the end of a parameter set, byte_alignment() at the end of an
+			// H.265 slice segment header.
+			void AlignWithStopBit();
+
+			// Appends the NAL unit with a four byte start code and emulation
+			// prevention bytes, and returns how many bits that took. A last
+			// partial byte, the end of an H.264 slice header the driver goes on
+			// writing, is appended padded with zeros and counted without them.
+			size_t AppendTo(std::vector<uint8_t> & output) const;
+
+		private:
+			std::vector<uint8_t> bytes;
+			// The bits used in the last byte, 0 when it is full.
+			int bitsInLastByte;
+	};
+}
+
+#endif

webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h

@@ -47,10 +47,14 @@ namespace jni
 			VaapiEncodeSession(const VaapiEncodeSession &) = delete;
 			VaapiEncodeSession & operator=(const VaapiEncodeSession &) = delete;
 
-			// Creates the session for 4:2:0 at constant bitrate, with surfaces
-			// of the given size, which is that of the frame aligned as the
-			// codec needs.
-			bool Create(VAProfile profile, VAEntrypoint entrypoint, uint32_t surfaceWidth, uint32_t surfaceHeight);
+			// Creates the session for 4:2:0 at constant bitrate, for pictures
+			// of the given size, in surfaces of the given size, which is that
+			// of the picture aligned as the codec needs, and with the
+			// VA_ENC_PACKED_HEADER_* the encoder hands the driver. Mesa takes
+			// the size of the coded picture from the context, so the context
+			// has the picture's.
+			bool Create(VAProfile profile, VAEntrypoint entrypoint, uint32_t pictureWidth, uint32_t pictureHeight,
+				uint32_t surfaceWidth, uint32_t surfaceHeight, uint32_t packedHeaders = VA_ENC_PACKED_HEADER_NONE);
 			void Destroy();
 
 			VASurfaceID InputSurface() const;
@@ -65,6 +69,9 @@ namespace jni
 
 			bool AddBuffer(VABufferType type, void * data, size_t size);
 			bool AddMiscParameter(VAEncMiscParameterType type, const void * data, size_t size);
+			// A header the encoder wrote, of the given length in bits, with
+			// start codes and emulation prevention bytes.
+			bool AddPackedHeader(VAEncPackedHeaderType type, const std::vector<uint8_t> & data, size_t bits);
 			// The rate control, frame rate and HRD parameters, for constant
 			// bitrate with the given initial quantizer.
 			bool AddRateControl(uint32_t bitrateBps, uint32_t framerate, uint32_t initialQp);

webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH264Encoder.h

@@ -18,6 +18,7 @@
 #define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_H264_ENCODER_H_
 
 #include "media/video/codec/EncoderOutputProcessor.h"
+#include "media/video/codec/linux/NalUnitWriter.h"
 #include "media/video/codec/linux/VaapiLibrary.h"
 
 #include "api/video/video_frame.h"
@@ -38,11 +39,14 @@ namespace jni
 	// as Intel and AMD drivers provide it on Linux.
 	//
 	// Encoding is synchronous, with one reference frame and P-frames only, so
-	// each frame is encoded before Encode() returns. The driver writes the
-	// parameter sets and slice headers itself from the parameters it is given.
-	// A driver that leaves the parameter sets out of a key frame, which WebRTC
-	// cannot send without them, makes the encoder give up, as does anything
-	// else that fails, so that the software encoder takes over.
+	// each frame is encoded before Encode() returns. The encoder writes the
+	// parameter sets and slice headers itself and hands them to the driver as
+	// packed headers, as FFmpeg does, since Intel's iHD driver writes none of
+	// its own; a driver that takes no packed headers writes them from the
+	// parameters it is given. A driver that leaves the parameter sets out of a
+	// key frame, which WebRTC cannot send without them, makes the encoder give
+	// up, as does anything else that fails, so that the software encoder takes
+	// over.
 	class VaapiH264Encoder : public webrtc::VideoEncoder
 	{
 		public:
@@ -65,12 +69,18 @@ namespace jni
 
 			bool AddBuffer(VABufferType type, void * data, size_t size);
 			bool AddMiscParameter(VAEncMiscParameterType type, const void * data, size_t size);
+			bool AddPackedHeader(VAEncPackedHeaderType type, const std::vector<uint8_t> & data, size_t bits);
 			void DestroyFrameBuffers();
 
 			void FillSequence(VAEncSequenceParameterBufferH264 & sequence) const;
 			void FillPicture(VAEncPictureParameterBufferH264 & picture, bool idr) const;
 			void FillSlice(VAEncSliceParameterBufferH264 & slice, bool idr) const;
 
+			// The SPS and PPS, and the header of the slice, matching the
+			// parameters above. Each returns the bits it appended.
+			size_t WriteParameterSets(std::vector<uint8_t> & output) const;
+			size_t WriteSliceHeader(std::vector<uint8_t> & output, bool idr) const;
+
 			bool Check(VAStatus status, const char * operation) const;
 
 		private:
@@ -79,6 +89,8 @@ namespace jni
 			const std::string implementationName;
 
 			VAConfigID config;
+			// The VA_ENC_PACKED_HEADER_* the driver takes from the encoder.
+			uint32_t packedHeaders;
 			VAContextID context;
 			VASurfaceID inputSurface;
 			// The frame being encoded, and the one it refers to.

webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Encoder.h

@@ -18,6 +18,7 @@
 #define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_H265_ENCODER_H_
 
 #include "media/video/codec/EncoderOutputProcessor.h"
+#include "media/video/codec/linux/NalUnitWriter.h"
 #include "media/video/codec/linux/VaapiEncodeSession.h"
 #include "media/video/codec/linux/VaapiLibrary.h"
 
@@ -36,18 +37,23 @@
 namespace jni
 {
 	// Encodes H.265 Main with the GPU driver's VA-API encoder, as Intel and
-	// AMD drivers provide it on Linux. Experimental: it has not run on such a
-	// GPU yet, and is used only where it is turned on.
+	// AMD drivers provide it on Linux. Tested with Intel's iHD and AMD's
+	// radeonsi drivers.
 	//
 	// It works as VaapiH264Encoder does: synchronous, one reference frame,
-	// P-frames only, the driver writing the parameter sets and slice headers
-	// from the parameters it is given. The block sizes and coding tools are
-	// those the driver says it takes, as FFmpeg's VA-API encoder chooses them.
-	// The frame size is kept to a multiple of the smallest coding block, so
-	// the coded picture is the frame and needs no cropping. There is no
-	// software H.265 to fall back to: anything that fails, including a driver
-	// that writes no parameter sets into key frames, is an error, upon which
-	// WebRTC switches to the next codec it negotiated.
+	// P-frames only, and the encoder writing the parameter sets and slice
+	// headers for a driver that takes them, as Intel's and AMD's do. A driver
+	// that predicts only with both reference lists filled, as Intel's
+	// low-power encoder, gets each P-frame as a B-frame whose two lists hold
+	// the same previous picture ("low delay B"), as FFmpeg does. The block
+	// sizes and coding tools are those the driver says it takes, as FFmpeg's
+	// VA-API encoder chooses them. The frame size is kept to a multiple of the
+	// smallest coding block; the coded picture is that aligned to 16, which
+	// AMD's encoder needs, and cropped to the frame by the conformance window
+	// of the SPS. There is no software H.265 to fall back to: anything that
+	// fails, including a driver that writes no parameter sets into key
+	// frames, is an error, upon which WebRTC switches to the next codec it
+	// negotiated.
 	class VaapiH265Encoder : public webrtc::VideoEncoder
 	{
 		public:
@@ -69,6 +75,14 @@ namespace jni
 			void FillSequence(VAEncSequenceParameterBufferHEVC & sequence) const;
 			void FillPicture(VAEncPictureParameterBufferHEVC & picture, bool idr) const;
 			void FillSlice(VAEncSliceParameterBufferHEVC & slice, bool idr) const;
+
+			// The VPS, SPS and PPS, and the slice segment header, matching
+			// the parameters above. Each returns the bits it appended.
+			size_t WriteParameterSets(std::vector<uint8_t> & output) const;
+			size_t WriteSliceHeader(std::vector<uint8_t> & output, bool idr) const;
+			void WriteProfileTierLevel(NalUnitWriter & writer) const;
+			// diff_cu_qp_delta_depth.
+			uint32_t CuQpDeltaDepth() const;
 			// The picture the current one refers to.
 			VAPictureHEVC Reference() const;
 
@@ -91,9 +105,17 @@ namespace jni
 			bool signDataHiding;
 			bool transformSkip;
 			bool cuQpDelta;
+			// The VA_ENC_PACKED_HEADER_* the driver takes from the encoder.
+			uint32_t packedHeaders;
+			// Whether P-frames are sent as B-frames with the same reference in
+			// both lists.
+			bool lowDelayB;
 
 			bool initialized;
 			webrtc::VideoCodec codecSettings;
+			// The size of the coded picture.
+			uint32_t codedWidth;
+			uint32_t codedHeight;
 			uint32_t widthInCtbs;
 			uint32_t heightInCtbs;
 			uint32_t bitrateBps;

webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoEncoderFactory.h

@@ -32,15 +32,15 @@ namespace jni
 	// Creates VA-API encoders on the GPU whose driver encodes H.264, as Intel
 	// and AMD drivers do. It offers H.264 in the profiles WebRTC's software
 	// encoder offers too, and only with packetization mode 1: mode 0 needs
-	// each NAL unit to fit a packet. With the experimental VA-API codecs
-	// turned on, it offers H.265 Main as well, and with hardware VP9 encoding
-	// turned on VP9 profile 0, where the driver encodes them.
+	// each NAL unit to fit a packet. It offers H.265 Main as well where the
+	// driver encodes it, and VP9 profile 0 where the driver encodes it and
+	// hardware VP9 encoding is turned on.
 	class VaapiVideoEncoderFactory : public webrtc::VideoEncoderFactory
 	{
 		public:
 			// Returns a factory, or null if no GPU driver encodes H.264
 			// through VA-API.
-			static std::unique_ptr<VaapiVideoEncoderFactory> Create(bool experimental, bool vp9Enabled);
+			static std::unique_ptr<VaapiVideoEncoderFactory> Create(bool vp9Enabled);
 
 			~VaapiVideoEncoderFactory() override = default;
 

webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp

@@ -33,10 +33,8 @@ namespace jni
 		if (auto nvenc = NvencVideoEncoderFactory::Create()) {
 			factories.push_back(std::move(nvenc));
 		}
-		// The VA-API codecs beyond H.264 only if asked for, H.265 and VP9
-		// each with its own switch: they have not been tested on an Intel or
-		// AMD GPU yet.
-		auto vaapi = VaapiVideoEncoderFactory::Create(IsExperimentalVaapiEnabled(), IsHardwareVp9EncoderEnabled());
+		// VP9 only if asked for: it has not been tested on an Intel GPU yet.
+		auto vaapi = VaapiVideoEncoderFactory::Create(IsHardwareVp9EncoderEnabled());
 
 		if (vaapi) {
 			factories.push_back(std::move(vaapi));

webrtc-jni/src/main/cpp/src/media/video/codec/linux/NalUnitWriter.cpp

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2026 Alex Andres
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "media/video/codec/linux/NalUnitWriter.h"
+
+namespace jni
+{
+	NalUnitWriter::NalUnitWriter() :
+		bitsInLastByte(0)
+	{
+	}
+
+	void NalUnitWriter::Bits(uint32_t value, int count)
+	{
+		for (int bit = count - 1; bit >= 0; bit--) {
+			if (bitsInLastByte == 0) {
+				bytes.push_back(0);
+			}
+
+			bytes.back() |= static_cast<uint8_t>(((value >> bit) & 1) << (7 - bitsInLastByte));
+			bitsInLastByte = (bitsInLastByte + 1) % 8;
+		}
+	}
+
+	void NalUnitWriter::Flag(bool value)
+	{
+		Bits(value ? 1 : 0, 1);
+	}
+
+	void NalUnitWriter::Ue(uint32_t value)
+	{
+		// value + 1 in binary, after as many zeros as it has bits less one.
+		const uint64_t coded = static_cast<uint64_t>(value) + 1;
+		int length = 0;
+
+		while ((coded >> length) > 1) {
+			length++;
+		}
+
+		Bits(0, length);
+
+		for (int bit = length; bit >= 0; bit--) {
+			Bits(static_cast<uint32_t>((coded >> bit) & 1), 1);
+		}
+	}
+
+	void NalUnitWriter::Se(int32_t value)
+	{
+		// Positive values to odd codes, the others to even ones.
+		const int64_t wide = value;
+		Ue(static_cast<uint32_t>(wide > 0 ? 2 * wide - 1 : -2 * wide));
+	}
+
+	void NalUnitWriter::AlignWithStopBit()
+	{
+		Bits(1, 1);
+
+		if (bitsInLastByte != 0) {
+			Bits(0, 8 - bitsInLastByte);
+		}
+	}
+
+	size_t NalUnitWriter::AppendTo(std::vector<uint8_t> & output) const
+	{
+		const size_t start = output.size();
+
+		output.insert(output.end(), { 0, 0, 0, 1 });
+
+		int zeros = 0;
+
+		for (uint8_t byte : bytes) {
+			// No three bytes 00 00 0x with x up to 3 inside the unit.
+			if (zeros >= 2 && byte <= 3) {
+				output.push_back(3);
+				zeros = 0;
+			}
+
+			output.push_back(byte);
+			zeros = byte == 0 ? zeros + 1 : 0;
+		}
+
+		const size_t padding = bitsInLastByte == 0 ? 0 : 8 - bitsInLastByte;
+
+		return (output.size() - start) * 8 - padding;
+	}
+}