From 1fbffe160452ff920dad7c9eaa755eb824e1391b Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 15 Oct 2026 00:59:03 +0200 Subject: [PATCH 1/2] fix: write the H.264 and H.265 headers for VA-API drivers that need them Intel's iHD driver writes no parameter sets or slice headers, so the encoders now write them and pass them as packed headers. H.265 also uses low-delay B frames on Intel, the QP depth iHD expects, and a 16-aligned coded size cropped through the SPS for Mesa. --- .../media/video/codec/linux/NalUnitWriter.h | 58 ++++ .../video/codec/linux/VaapiEncodeSession.h | 15 +- .../video/codec/linux/VaapiH264Encoder.h | 22 +- .../video/codec/linux/VaapiH265Encoder.h | 42 ++- .../media/video/codec/linux/NalUnitWriter.cpp | 99 ++++++ .../video/codec/linux/VaapiEncodeSession.cpp | 28 +- .../video/codec/linux/VaapiH264Encoder.cpp | 200 ++++++++++- .../video/codec/linux/VaapiH265Encoder.cpp | 319 +++++++++++++++++- .../video/codec/linux/VaapiVp9Encoder.cpp | 6 +- 9 files changed, 739 insertions(+), 50 deletions(-) create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/linux/NalUnitWriter.h create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/linux/NalUnitWriter.cpp diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/NalUnitWriter.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/NalUnitWriter.h new file mode 100644 index 000000000..f11c02d9a --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/NalUnitWriter.h @@ -0,0 +1,58 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_NAL_UNIT_WRITER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_NAL_UNIT_WRITER_H_ + +#include +#include +#include + +namespace jni +{ + // Writes one H.264 or H.265 NAL unit, header included, bit by bit, as the + // VA-API encoders hand the parameter sets and slice headers to drivers + // that do not write them themselves. + class NalUnitWriter + { + public: + NalUnitWriter(); + + // The low count bits of value, the most significant first. + void Bits(uint32_t value, int count); + void Flag(bool value); + // Exponential-Golomb codes, ue(v) and se(v). + void Ue(uint32_t value); + void Se(int32_t value); + // A one bit and zero bits up to the next byte: rbsp_trailing_bits() + // at the end of a parameter set, byte_alignment() at the end of an + // H.265 slice segment header. + void AlignWithStopBit(); + + // Appends the NAL unit with a four byte start code and emulation + // prevention bytes, and returns how many bits that took. A last + // partial byte, the end of an H.264 slice header the driver goes on + // writing, is appended padded with zeros and counted without them. + size_t AppendTo(std::vector & output) const; + + private: + std::vector bytes; + // The bits used in the last byte, 0 when it is full. + int bitsInLastByte; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h index 32cd6408b..42dd66214 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h @@ -47,10 +47,14 @@ namespace jni VaapiEncodeSession(const VaapiEncodeSession &) = delete; VaapiEncodeSession & operator=(const VaapiEncodeSession &) = delete; - // Creates the session for 4:2:0 at constant bitrate, with surfaces - // of the given size, which is that of the frame aligned as the - // codec needs. - bool Create(VAProfile profile, VAEntrypoint entrypoint, uint32_t surfaceWidth, uint32_t surfaceHeight); + // Creates the session for 4:2:0 at constant bitrate, for pictures + // of the given size, in surfaces of the given size, which is that + // of the picture aligned as the codec needs, and with the + // VA_ENC_PACKED_HEADER_* the encoder hands the driver. Mesa takes + // the size of the coded picture from the context, so the context + // has the picture's. + bool Create(VAProfile profile, VAEntrypoint entrypoint, uint32_t pictureWidth, uint32_t pictureHeight, + uint32_t surfaceWidth, uint32_t surfaceHeight, uint32_t packedHeaders = VA_ENC_PACKED_HEADER_NONE); void Destroy(); VASurfaceID InputSurface() const; @@ -65,6 +69,9 @@ namespace jni bool AddBuffer(VABufferType type, void * data, size_t size); bool AddMiscParameter(VAEncMiscParameterType type, const void * data, size_t size); + // A header the encoder wrote, of the given length in bits, with + // start codes and emulation prevention bytes. + bool AddPackedHeader(VAEncPackedHeaderType type, const std::vector & data, size_t bits); // The rate control, frame rate and HRD parameters, for constant // bitrate with the given initial quantizer. bool AddRateControl(uint32_t bitrateBps, uint32_t framerate, uint32_t initialQp); diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH264Encoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH264Encoder.h index e2d5cc036..a94be3a5a 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH264Encoder.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH264Encoder.h @@ -18,6 +18,7 @@ #define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_H264_ENCODER_H_ #include "media/video/codec/EncoderOutputProcessor.h" +#include "media/video/codec/linux/NalUnitWriter.h" #include "media/video/codec/linux/VaapiLibrary.h" #include "api/video/video_frame.h" @@ -38,11 +39,14 @@ namespace jni // as Intel and AMD drivers provide it on Linux. // // Encoding is synchronous, with one reference frame and P-frames only, so - // each frame is encoded before Encode() returns. The driver writes the - // parameter sets and slice headers itself from the parameters it is given. - // A driver that leaves the parameter sets out of a key frame, which WebRTC - // cannot send without them, makes the encoder give up, as does anything - // else that fails, so that the software encoder takes over. + // each frame is encoded before Encode() returns. The encoder writes the + // parameter sets and slice headers itself and hands them to the driver as + // packed headers, as FFmpeg does, since Intel's iHD driver writes none of + // its own; a driver that takes no packed headers writes them from the + // parameters it is given. A driver that leaves the parameter sets out of a + // key frame, which WebRTC cannot send without them, makes the encoder give + // up, as does anything else that fails, so that the software encoder takes + // over. class VaapiH264Encoder : public webrtc::VideoEncoder { public: @@ -65,12 +69,18 @@ namespace jni bool AddBuffer(VABufferType type, void * data, size_t size); bool AddMiscParameter(VAEncMiscParameterType type, const void * data, size_t size); + bool AddPackedHeader(VAEncPackedHeaderType type, const std::vector & data, size_t bits); void DestroyFrameBuffers(); void FillSequence(VAEncSequenceParameterBufferH264 & sequence) const; void FillPicture(VAEncPictureParameterBufferH264 & picture, bool idr) const; void FillSlice(VAEncSliceParameterBufferH264 & slice, bool idr) const; + // The SPS and PPS, and the header of the slice, matching the + // parameters above. Each returns the bits it appended. + size_t WriteParameterSets(std::vector & output) const; + size_t WriteSliceHeader(std::vector & output, bool idr) const; + bool Check(VAStatus status, const char * operation) const; private: @@ -79,6 +89,8 @@ namespace jni const std::string implementationName; VAConfigID config; + // The VA_ENC_PACKED_HEADER_* the driver takes from the encoder. + uint32_t packedHeaders; VAContextID context; VASurfaceID inputSurface; // The frame being encoded, and the one it refers to. diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Encoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Encoder.h index 0e06ec5d1..c6a820264 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Encoder.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Encoder.h @@ -18,6 +18,7 @@ #define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_H265_ENCODER_H_ #include "media/video/codec/EncoderOutputProcessor.h" +#include "media/video/codec/linux/NalUnitWriter.h" #include "media/video/codec/linux/VaapiEncodeSession.h" #include "media/video/codec/linux/VaapiLibrary.h" @@ -36,18 +37,23 @@ namespace jni { // Encodes H.265 Main with the GPU driver's VA-API encoder, as Intel and - // AMD drivers provide it on Linux. Experimental: it has not run on such a - // GPU yet, and is used only where it is turned on. + // AMD drivers provide it on Linux. Tested with Intel's iHD and AMD's + // radeonsi drivers. // // It works as VaapiH264Encoder does: synchronous, one reference frame, - // P-frames only, the driver writing the parameter sets and slice headers - // from the parameters it is given. The block sizes and coding tools are - // those the driver says it takes, as FFmpeg's VA-API encoder chooses them. - // The frame size is kept to a multiple of the smallest coding block, so - // the coded picture is the frame and needs no cropping. There is no - // software H.265 to fall back to: anything that fails, including a driver - // that writes no parameter sets into key frames, is an error, upon which - // WebRTC switches to the next codec it negotiated. + // P-frames only, and the encoder writing the parameter sets and slice + // headers for a driver that takes them, as Intel's and AMD's do. A driver + // that predicts only with both reference lists filled, as Intel's + // low-power encoder, gets each P-frame as a B-frame whose two lists hold + // the same previous picture ("low delay B"), as FFmpeg does. The block + // sizes and coding tools are those the driver says it takes, as FFmpeg's + // VA-API encoder chooses them. The frame size is kept to a multiple of the + // smallest coding block; the coded picture is that aligned to 16, which + // AMD's encoder needs, and cropped to the frame by the conformance window + // of the SPS. There is no software H.265 to fall back to: anything that + // fails, including a driver that writes no parameter sets into key + // frames, is an error, upon which WebRTC switches to the next codec it + // negotiated. class VaapiH265Encoder : public webrtc::VideoEncoder { public: @@ -69,6 +75,14 @@ namespace jni void FillSequence(VAEncSequenceParameterBufferHEVC & sequence) const; void FillPicture(VAEncPictureParameterBufferHEVC & picture, bool idr) const; void FillSlice(VAEncSliceParameterBufferHEVC & slice, bool idr) const; + + // The VPS, SPS and PPS, and the slice segment header, matching + // the parameters above. Each returns the bits it appended. + size_t WriteParameterSets(std::vector & output) const; + size_t WriteSliceHeader(std::vector & output, bool idr) const; + void WriteProfileTierLevel(NalUnitWriter & writer) const; + // diff_cu_qp_delta_depth. + uint32_t CuQpDeltaDepth() const; // The picture the current one refers to. VAPictureHEVC Reference() const; @@ -91,9 +105,17 @@ namespace jni bool signDataHiding; bool transformSkip; bool cuQpDelta; + // The VA_ENC_PACKED_HEADER_* the driver takes from the encoder. + uint32_t packedHeaders; + // Whether P-frames are sent as B-frames with the same reference in + // both lists. + bool lowDelayB; bool initialized; webrtc::VideoCodec codecSettings; + // The size of the coded picture. + uint32_t codedWidth; + uint32_t codedHeight; uint32_t widthInCtbs; uint32_t heightInCtbs; uint32_t bitrateBps; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/NalUnitWriter.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/NalUnitWriter.cpp new file mode 100644 index 000000000..0187bd027 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/NalUnitWriter.cpp @@ -0,0 +1,99 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/linux/NalUnitWriter.h" + +namespace jni +{ + NalUnitWriter::NalUnitWriter() : + bitsInLastByte(0) + { + } + + void NalUnitWriter::Bits(uint32_t value, int count) + { + for (int bit = count - 1; bit >= 0; bit--) { + if (bitsInLastByte == 0) { + bytes.push_back(0); + } + + bytes.back() |= static_cast(((value >> bit) & 1) << (7 - bitsInLastByte)); + bitsInLastByte = (bitsInLastByte + 1) % 8; + } + } + + void NalUnitWriter::Flag(bool value) + { + Bits(value ? 1 : 0, 1); + } + + void NalUnitWriter::Ue(uint32_t value) + { + // value + 1 in binary, after as many zeros as it has bits less one. + const uint64_t coded = static_cast(value) + 1; + int length = 0; + + while ((coded >> length) > 1) { + length++; + } + + Bits(0, length); + + for (int bit = length; bit >= 0; bit--) { + Bits(static_cast((coded >> bit) & 1), 1); + } + } + + void NalUnitWriter::Se(int32_t value) + { + // Positive values to odd codes, the others to even ones. + const int64_t wide = value; + Ue(static_cast(wide > 0 ? 2 * wide - 1 : -2 * wide)); + } + + void NalUnitWriter::AlignWithStopBit() + { + Bits(1, 1); + + if (bitsInLastByte != 0) { + Bits(0, 8 - bitsInLastByte); + } + } + + size_t NalUnitWriter::AppendTo(std::vector & output) const + { + const size_t start = output.size(); + + output.insert(output.end(), { 0, 0, 0, 1 }); + + int zeros = 0; + + for (uint8_t byte : bytes) { + // No three bytes 00 00 0x with x up to 3 inside the unit. + if (zeros >= 2 && byte <= 3) { + output.push_back(3); + zeros = 0; + } + + output.push_back(byte); + zeros = byte == 0 ? zeros + 1 : 0; + } + + const size_t padding = bitsInLastByte == 0 ? 0 : 8 - bitsInLastByte; + + return (output.size() - start) * 8 - padding; + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiEncodeSession.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiEncodeSession.cpp index adda0ba2a..4f0bd54ea 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiEncodeSession.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiEncodeSession.cpp @@ -59,20 +59,26 @@ namespace jni return false; } - bool VaapiEncodeSession::Create(VAProfile profile, VAEntrypoint entrypoint, uint32_t surfaceWidth, - uint32_t surfaceHeight) + bool VaapiEncodeSession::Create(VAProfile profile, VAEntrypoint entrypoint, uint32_t pictureWidth, + uint32_t pictureHeight, uint32_t surfaceWidth, uint32_t surfaceHeight, uint32_t packedHeaders) { Destroy(); VADisplay display = library.Display(); - VAConfigAttrib attributes[2] = {}; + VAConfigAttrib attributes[3] = {}; attributes[0].type = VAConfigAttribRTFormat; attributes[0].value = VA_RT_FORMAT_YUV420; attributes[1].type = VAConfigAttribRateControl; attributes[1].value = VA_RC_CBR; + attributes[2].type = VAConfigAttribEncPackedHeaders; + attributes[2].value = packedHeaders; - if (!Check(va.CreateConfig(display, profile, entrypoint, attributes, 2, &config), "creating the configuration")) { + const int attributeCount = packedHeaders != VA_ENC_PACKED_HEADER_NONE ? 3 : 2; + + if (!Check(va.CreateConfig(display, profile, entrypoint, attributes, attributeCount, &config), + "creating the configuration")) + { config = VA_INVALID_ID; return false; } @@ -87,7 +93,7 @@ namespace jni return false; } - if (!Check(va.CreateContext(display, config, surfaceWidth, surfaceHeight, VA_PROGRESSIVE, surfaces, 3, &context), + if (!Check(va.CreateContext(display, config, pictureWidth, pictureHeight, VA_PROGRESSIVE, surfaces, 3, &context), "creating the context")) { context = VA_INVALID_ID; @@ -243,6 +249,18 @@ namespace jni return AddBuffer(VAEncMiscParameterBufferType, buffer.data(), buffer.size()); } + bool VaapiEncodeSession::AddPackedHeader(VAEncPackedHeaderType type, const std::vector & data, + size_t bits) + { + VAEncPackedHeaderParameterBuffer parameters = {}; + parameters.type = type; + parameters.bit_length = static_cast(bits); + parameters.has_emulation_bytes = 1; + + return AddBuffer(VAEncPackedHeaderParameterBufferType, ¶meters, sizeof(parameters)) + && AddBuffer(VAEncPackedHeaderDataBufferType, const_cast(data.data()), data.size()); + } + bool VaapiEncodeSession::AddRateControl(uint32_t bitrateBps, uint32_t framerate, uint32_t initialQp) { VAEncMiscParameterRateControl rateControl = {}; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Encoder.cpp index 734f3577c..97f9840c7 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Encoder.cpp @@ -43,6 +43,21 @@ namespace jni constexpr uint8_t kSliceTypeP = 0; constexpr uint8_t kSliceTypeI = 2; + // H.264 NAL unit types (table 7-1), and the nal_ref_idc of everything + // the encoder writes, all of which is used for reference. + constexpr uint32_t kNalSlice = 1; + constexpr uint32_t kNalIdrSlice = 5; + constexpr uint32_t kNalSps = 7; + constexpr uint32_t kNalPps = 8; + constexpr uint32_t kNalRefIdc = 3; + + // Constrained Baseline: Baseline with constraint_set1_flag. + constexpr uint32_t kProfileIdcBaseline = 66; + + // The packed headers the encoder writes where the driver takes both: + // its slice headers go with its parameter sets only. + constexpr uint32_t kPackedHeaders = VA_ENC_PACKED_HEADER_SEQUENCE | VA_ENC_PACKED_HEADER_SLICE; + constexpr uint32_t kInitialQp = 26; // How many seconds of video the rate control may buffer, as a @@ -95,6 +110,7 @@ namespace jni va(library.Api()), implementationName("VA-API (" + library.Vendor() + ")"), config(VA_INVALID_ID), + packedHeaders(0), context(VA_INVALID_ID), inputSurface(VA_INVALID_SURFACE), reconstructed{ VA_INVALID_SURFACE, VA_INVALID_SURFACE }, @@ -179,14 +195,30 @@ namespace jni { VADisplay display = library.Display(); - VAConfigAttrib attributes[2] = {}; + VAConfigAttrib packed = {}; + packed.type = VAConfigAttribEncPackedHeaders; + + packedHeaders = 0; + + if (va.GetConfigAttributes(display, VAProfileH264ConstrainedBaseline, library.Entrypoint(), &packed, 1) == + VA_STATUS_SUCCESS && packed.value != VA_ATTRIB_NOT_SUPPORTED + && (packed.value & kPackedHeaders) == kPackedHeaders) + { + packedHeaders = kPackedHeaders; + } + + VAConfigAttrib attributes[3] = {}; attributes[0].type = VAConfigAttribRTFormat; attributes[0].value = VA_RT_FORMAT_YUV420; attributes[1].type = VAConfigAttribRateControl; attributes[1].value = VA_RC_CBR; + attributes[2].type = VAConfigAttribEncPackedHeaders; + attributes[2].value = packedHeaders; + + const int attributeCount = packedHeaders != 0 ? 3 : 2; - if (!Check(va.CreateConfig(display, VAProfileH264ConstrainedBaseline, library.Entrypoint(), attributes, 2, - &config), "creating the configuration")) + if (!Check(va.CreateConfig(display, VAProfileH264ConstrainedBaseline, library.Entrypoint(), attributes, + attributeCount, &config), "creating the configuration")) { config = VA_INVALID_ID; return false; @@ -447,9 +479,29 @@ namespace jni VAEncSliceParameterBufferH264 slice = {}; FillSlice(slice, idr); - if (!AddBuffer(VAEncPictureParameterBufferType, &picture, sizeof(picture)) || - !AddBuffer(VAEncSliceParameterBufferType, &slice, sizeof(slice))) - { + if (!AddBuffer(VAEncPictureParameterBufferType, &picture, sizeof(picture))) { + return false; + } + + if (idr && (packedHeaders & VA_ENC_PACKED_HEADER_SEQUENCE)) { + std::vector parameterSets; + const size_t bits = WriteParameterSets(parameterSets); + + if (!AddPackedHeader(VAEncPackedHeaderSequence, parameterSets, bits)) { + return false; + } + } + + if (packedHeaders & VA_ENC_PACKED_HEADER_SLICE) { + std::vector sliceHeader; + const size_t bits = WriteSliceHeader(sliceHeader, idr); + + if (!AddPackedHeader(VAEncPackedHeaderSlice, sliceHeader, bits)) { + return false; + } + } + + if (!AddBuffer(VAEncSliceParameterBufferType, &slice, sizeof(slice))) { return false; } @@ -527,6 +579,18 @@ namespace jni return AddBuffer(VAEncMiscParameterBufferType, buffer.data(), buffer.size()); } + bool VaapiH264Encoder::AddPackedHeader(VAEncPackedHeaderType type, const std::vector & data, + size_t bits) + { + VAEncPackedHeaderParameterBuffer parameters = {}; + parameters.type = type; + parameters.bit_length = static_cast(bits); + parameters.has_emulation_bytes = 1; + + return AddBuffer(VAEncPackedHeaderParameterBufferType, ¶meters, sizeof(parameters)) + && AddBuffer(VAEncPackedHeaderDataBufferType, const_cast(data.data()), data.size()); + } + void VaapiH264Encoder::DestroyFrameBuffers() { for (VABufferID buffer : frameBuffers) { @@ -646,6 +710,130 @@ namespace jni slice.disable_deblocking_filter_idc = 0; } + size_t VaapiH264Encoder::WriteParameterSets(std::vector & output) const + { + // The SPS (7.3.2.1.1) of FillSequence(). + NalUnitWriter sps; + sps.Bits(0, 1); + sps.Bits(kNalRefIdc, 2); + sps.Bits(kNalSps, 5); + + sps.Bits(kProfileIdcBaseline, 8); + // constraint_set0_flag and constraint_set1_flag, the other four and + // reserved_zero_2bits not. + sps.Bits(0b11000000, 8); + sps.Bits(LevelIdc(widthInMbs * heightInMbs, framerate), 8); + sps.Ue(0); + sps.Ue(kLog2MaxFrameNum - 4); + sps.Ue(2); + sps.Ue(1); + sps.Flag(false); + sps.Ue(widthInMbs - 1); + sps.Ue(heightInMbs - 1); + sps.Flag(true); + sps.Flag(true); + + const uint32_t croppedRight = widthInMbs * 16 - codecSettings.width; + const uint32_t croppedBottom = heightInMbs * 16 - codecSettings.height; + const bool cropped = croppedRight > 0 || croppedBottom > 0; + + sps.Flag(cropped); + + if (cropped) { + sps.Ue(0); + sps.Ue(croppedRight / 2); + sps.Ue(0); + sps.Ue(croppedBottom / 2); + } + + // VUI (E.1.1) with the timing only. + sps.Flag(true); + sps.Flag(false); + sps.Flag(false); + sps.Flag(false); + sps.Flag(false); + sps.Flag(true); + sps.Bits(1, 32); + sps.Bits(framerate * 2, 32); + sps.Flag(false); + sps.Flag(false); + sps.Flag(false); + sps.Flag(false); + sps.Flag(false); + sps.AlignWithStopBit(); + + // The PPS (7.3.2.2) of FillPicture(). + NalUnitWriter pps; + pps.Bits(0, 1); + pps.Bits(kNalRefIdc, 2); + pps.Bits(kNalPps, 5); + + pps.Ue(0); + pps.Ue(0); + // CAVLC. + pps.Flag(false); + pps.Flag(false); + pps.Ue(0); + pps.Ue(0); + pps.Ue(0); + pps.Flag(false); + pps.Bits(0, 2); + pps.Se(static_cast(kInitialQp) - 26); + pps.Se(0); + pps.Se(0); + // deblocking_filter_control_present_flag. + pps.Flag(true); + pps.Flag(false); + pps.Flag(false); + pps.AlignWithStopBit(); + + return sps.AppendTo(output) + pps.AppendTo(output); + } + + size_t VaapiH264Encoder::WriteSliceHeader(std::vector & output, bool idr) const + { + // The slice header (7.3.3) of FillSlice(), which the driver goes on + // from with the slice data. + NalUnitWriter header; + header.Bits(0, 1); + header.Bits(kNalRefIdc, 2); + header.Bits(idr ? kNalIdrSlice : kNalSlice, 5); + + header.Ue(0); + header.Ue(idr ? kSliceTypeI : kSliceTypeP); + header.Ue(0); + header.Bits(frameNum, kLog2MaxFrameNum); + + if (idr) { + header.Ue(idrPicId); + } + else { + // num_ref_idx_active_override_flag and + // ref_pic_list_modification_flag_l0. + header.Flag(false); + header.Flag(false); + } + + // dec_ref_pic_marking(): no_output_of_prior_pics_flag and + // long_term_reference_flag, or adaptive_ref_pic_marking_mode_flag. + if (idr) { + header.Flag(false); + header.Flag(false); + } + else { + header.Flag(false); + } + + header.Se(0); + // disable_deblocking_filter_idc, slice_alpha_c0_offset_div2 and + // slice_beta_offset_div2. + header.Ue(0); + header.Se(0); + header.Se(0); + + return header.AppendTo(output); + } + void VaapiH264Encoder::SetRates(const RateControlParameters & parameters) { const uint32_t bitrate = parameters.bitrate.get_sum_bps(); diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Encoder.cpp index b341e6960..e2a3171f2 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Encoder.cpp @@ -32,10 +32,25 @@ namespace jni constexpr uint32_t kIntraPeriod = 3000; // H.265 slice types (7.4.7.1) and NAL unit types (table 7-1). + constexpr uint8_t kSliceTypeB = 0; constexpr uint8_t kSliceTypeP = 1; constexpr uint8_t kSliceTypeI = 2; constexpr uint8_t kNalTrailR = 1; constexpr uint8_t kNalIdrWRadl = 19; + constexpr uint8_t kNalVps = 32; + constexpr uint8_t kNalSps = 33; + constexpr uint8_t kNalPps = 34; + + // The coded picture is whole blocks of 16x16: AMD's encoder codes such + // blocks and leaves the conformance window to the SPS it is given. + constexpr uint32_t kPictureAlignment = 16; + + // slice_pic_order_cnt_lsb has 8 bits. + constexpr uint32_t kLog2MaxPicOrderCntLsb = 8; + + // The packed headers the encoder writes where the driver takes both: + // its slice headers go with its parameter sets only. + constexpr uint32_t kPackedHeaders = VA_ENC_PACKED_HEADER_SEQUENCE | VA_ENC_PACKED_HEADER_SLICE; // VA-API's picture coding types. constexpr uint32_t kCodingTypeI = 1; @@ -109,8 +124,12 @@ namespace jni signDataHiding(false), transformSkip(false), cuQpDelta(true), + packedHeaders(VA_ENC_PACKED_HEADER_NONE), + lowDelayB(false), initialized(false), codecSettings(), + codedWidth(0), + codedHeight(0), widthInCtbs(0), heightInCtbs(0), bitrateBps(0), @@ -132,11 +151,13 @@ namespace jni void VaapiH265Encoder::QueryCapabilities() { - VAConfigAttrib attributes[2] = {}; + VAConfigAttrib attributes[4] = {}; attributes[0].type = VAConfigAttribEncHEVCBlockSizes; attributes[1].type = VAConfigAttribEncHEVCFeatures; + attributes[2].type = VAConfigAttribEncPackedHeaders; + attributes[3].type = VAConfigAttribPredictionDirection; - if (library.Api().GetConfigAttributes(library.Display(), VAProfileHEVCMain, entrypoint, attributes, 2) != + if (library.Api().GetConfigAttributes(library.Display(), VAProfileHEVCMain, entrypoint, attributes, 4) != VA_STATUS_SUCCESS) { return; @@ -164,6 +185,14 @@ namespace jni transformSkip = Allowed(features.bits.transform_skip); cuQpDelta = Allowed(features.bits.cu_qp_delta); } + + if (attributes[2].value != VA_ATTRIB_NOT_SUPPORTED && (attributes[2].value & kPackedHeaders) == kPackedHeaders) { + packedHeaders = kPackedHeaders; + } + + if (attributes[3].value != VA_ATTRIB_NOT_SUPPORTED) { + lowDelayB = (attributes[3].value & VA_PREDICTION_DIRECTION_BI_NOT_EMPTY) != 0; + } } int32_t VaapiH265Encoder::InitEncode(const webrtc::VideoCodec * settings, const Settings & encoderSettings) @@ -175,10 +204,9 @@ namespace jni return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED; } - // The coded picture is kept to whole coding blocks, so that it is the - // frame and needs no cropping. WebRTC is asked for frames of that - // alignment, but the first frames of a source can come before it - // knows, so the encoder takes the aligned size below and drops the + // The frame is kept to whole coding blocks. WebRTC is asked for frames + // of that alignment, but the first frames of a source can come before + // it knows, so the encoder takes the aligned size below and drops the // rows and columns beyond it. const uint32_t alignment = 1u << log2MinCbSize; @@ -193,8 +221,11 @@ namespace jni codecSettings = *settings; codecSettings.width -= codecSettings.width % alignment; codecSettings.height -= codecSettings.height % alignment; - widthInCtbs = (codecSettings.width + ctbSize - 1) / ctbSize; - heightInCtbs = (codecSettings.height + ctbSize - 1) / ctbSize; + const uint32_t pictureAlignment = std::max(alignment, kPictureAlignment); + codedWidth = (codecSettings.width + pictureAlignment - 1) / pictureAlignment * pictureAlignment; + codedHeight = (codecSettings.height + pictureAlignment - 1) / pictureAlignment * pictureAlignment; + widthInCtbs = (codedWidth + ctbSize - 1) / ctbSize; + heightInCtbs = (codedHeight + ctbSize - 1) / ctbSize; bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; framerate = std::max(1u, codecSettings.maxFramerate); ratesChanged = true; @@ -203,8 +234,11 @@ namespace jni referenceValid = false; outputProcessor.Reset(); - // Whole coding tree blocks, which the driver may write all of. - if (!session.Create(VAProfileHEVCMain, entrypoint, widthInCtbs << log2CtbSize, heightInCtbs << log2CtbSize)) { + // Surfaces of whole coding tree blocks, which the driver may write all + // of. + if (!session.Create(VAProfileHEVCMain, entrypoint, codedWidth, codedHeight, widthInCtbs << log2CtbSize, + heightInCtbs << log2CtbSize, packedHeaders)) + { Release(); return WEBRTC_VIDEO_CODEC_ERROR; } @@ -314,23 +348,43 @@ namespace jni VAEncSliceParameterBufferHEVC slice = {}; FillSlice(slice, idr); - return session.AddBuffer(VAEncPictureParameterBufferType, &picture, sizeof(picture)) - && session.AddBuffer(VAEncSliceParameterBufferType, &slice, sizeof(slice)) - && session.Encode(); + if (!session.AddBuffer(VAEncPictureParameterBufferType, &picture, sizeof(picture))) { + return false; + } + + if (idr && (packedHeaders & VA_ENC_PACKED_HEADER_SEQUENCE)) { + std::vector parameterSets; + const size_t bits = WriteParameterSets(parameterSets); + + if (!session.AddPackedHeader(VAEncPackedHeaderSequence, parameterSets, bits)) { + return false; + } + } + + if (packedHeaders & VA_ENC_PACKED_HEADER_SLICE) { + std::vector sliceHeader; + const size_t bits = WriteSliceHeader(sliceHeader, idr); + + if (!session.AddPackedHeader(VAEncPackedHeaderSlice, sliceHeader, bits)) { + return false; + } + } + + return session.AddBuffer(VAEncSliceParameterBufferType, &slice, sizeof(slice)) && session.Encode(); } void VaapiH265Encoder::FillSequence(VAEncSequenceParameterBufferHEVC & sequence) const { // Main profile, main tier. sequence.general_profile_idc = 1; - sequence.general_level_idc = LevelIdc(codecSettings.width * codecSettings.height, framerate); + sequence.general_level_idc = LevelIdc(codedWidth * codedHeight, framerate); sequence.general_tier_flag = 0; sequence.intra_period = kIntraPeriod; sequence.intra_idr_period = kIntraPeriod; sequence.ip_period = 1; sequence.bits_per_second = bitrateBps; - sequence.pic_width_in_luma_samples = static_cast(codecSettings.width); - sequence.pic_height_in_luma_samples = static_cast(codecSettings.height); + sequence.pic_width_in_luma_samples = static_cast(codedWidth); + sequence.pic_height_in_luma_samples = static_cast(codedHeight); sequence.seq_fields.bits.chroma_format_idc = 1; sequence.seq_fields.bits.amp_enabled_flag = amp ? 1 : 0; @@ -384,7 +438,10 @@ namespace jni picture.collocated_ref_pic_index = 0xFF; picture.last_picture = 0; picture.pic_init_qp = kInitialQp; - picture.diff_cu_qp_delta_depth = 0; + // The QP may change for every coding block of the smallest size: Intel's + // low-power encoder changes it that often whatever it is told, and a + // stream that says otherwise decodes with the wrong QP. + picture.diff_cu_qp_delta_depth = static_cast(CuQpDeltaDepth()); picture.num_ref_idx_l0_default_active_minus1 = 0; picture.num_ref_idx_l1_default_active_minus1 = 0; picture.slice_pic_parameter_set_id = 0; @@ -403,7 +460,7 @@ namespace jni { slice.slice_segment_address = 0; slice.num_ctu_in_slice = widthInCtbs * heightInCtbs; - slice.slice_type = idr ? kSliceTypeI : kSliceTypeP; + slice.slice_type = idr ? kSliceTypeI : (lowDelayB ? kSliceTypeB : kSliceTypeP); slice.slice_pic_parameter_set_id = 0; slice.num_ref_idx_l0_active_minus1 = 0; slice.num_ref_idx_l1_active_minus1 = 0; @@ -417,16 +474,242 @@ namespace jni if (!idr) { slice.ref_pic_list0[0] = Reference(); + + if (lowDelayB) { + slice.ref_pic_list1[0] = Reference(); + } } slice.max_num_merge_cand = 5; slice.slice_qp_delta = 0; slice.slice_fields.bits.last_slice_of_pic_flag = 1; + slice.slice_fields.bits.slice_sao_luma_flag = sao ? 1 : 0; slice.slice_fields.bits.slice_sao_chroma_flag = sao ? 1 : 0; } + uint32_t VaapiH265Encoder::CuQpDeltaDepth() const + { + return cuQpDelta ? log2CtbSize - log2MinCbSize : 0; + } + + void VaapiH265Encoder::WriteProfileTierLevel(NalUnitWriter & writer) const + { + // profile_tier_level(1, 0) (7.3.3): Main profile, main tier. + writer.Bits(0, 2); + writer.Flag(false); + writer.Bits(1, 5); + // Compatible with Main and Main 10. + writer.Bits(0x60000000, 32); + // Progressive frames only. + writer.Flag(true); + writer.Flag(false); + writer.Flag(false); + writer.Flag(true); + // general_reserved_zero_43bits and general_inbld_flag. + writer.Bits(0, 32); + writer.Bits(0, 12); + writer.Bits(LevelIdc(codedWidth * codedHeight, framerate), 8); + } + + size_t VaapiH265Encoder::WriteParameterSets(std::vector & output) const + { + const auto nalHeader = [](NalUnitWriter & writer, uint8_t type) { + writer.Bits(0, 1); + writer.Bits(type, 6); + writer.Bits(0, 6); + writer.Bits(1, 3); + }; + + // The VPS (7.3.2.1). + NalUnitWriter vps; + nalHeader(vps, kNalVps); + vps.Bits(0, 4); + vps.Flag(true); + vps.Flag(true); + vps.Bits(0, 6); + vps.Bits(0, 3); + vps.Flag(true); + vps.Bits(0xFFFF, 16); + WriteProfileTierLevel(vps); + // One sub-layer, decoding with the current and one reference + // picture, without reordering. + vps.Flag(false); + vps.Ue(1); + vps.Ue(0); + vps.Ue(0); + vps.Bits(0, 6); + vps.Ue(0); + vps.Flag(false); + vps.Flag(false); + vps.AlignWithStopBit(); + + // The SPS (7.3.2.2) of FillSequence(). + NalUnitWriter sps; + nalHeader(sps, kNalSps); + sps.Bits(0, 4); + sps.Bits(0, 3); + sps.Flag(true); + WriteProfileTierLevel(sps); + sps.Ue(0); + sps.Ue(1); + sps.Ue(codedWidth); + sps.Ue(codedHeight); + + // The conformance window, in units of two luma samples for 4:2:0. + const uint32_t croppedRight = codedWidth - codecSettings.width; + const uint32_t croppedBottom = codedHeight - codecSettings.height; + const bool cropped = croppedRight > 0 || croppedBottom > 0; + + sps.Flag(cropped); + + if (cropped) { + sps.Ue(0); + sps.Ue(croppedRight / 2); + sps.Ue(0); + sps.Ue(croppedBottom / 2); + } + + sps.Ue(0); + sps.Ue(0); + sps.Ue(kLog2MaxPicOrderCntLsb - 4); + sps.Flag(false); + sps.Ue(1); + sps.Ue(0); + sps.Ue(0); + sps.Ue(log2MinCbSize - 3); + sps.Ue(log2CtbSize - log2MinCbSize); + sps.Ue(log2MinTbSize - 2); + sps.Ue(log2MaxTbSize - log2MinTbSize); + sps.Ue(maxTransformDepthInter); + sps.Ue(maxTransformDepthIntra); + sps.Flag(false); + sps.Flag(amp); + sps.Flag(sao); + sps.Flag(false); + // The reference picture sets are in the slice headers. + sps.Ue(0); + sps.Flag(false); + sps.Flag(false); + sps.Flag(false); + + // VUI (E.2.1) with the timing only. + sps.Flag(true); + sps.Flag(false); + sps.Flag(false); + sps.Flag(false); + sps.Flag(false); + sps.Flag(false); + sps.Flag(false); + sps.Flag(false); + sps.Flag(false); + sps.Flag(true); + sps.Bits(1, 32); + sps.Bits(framerate, 32); + sps.Flag(false); + sps.Flag(false); + sps.Flag(false); + + sps.Flag(false); + sps.AlignWithStopBit(); + + // The PPS (7.3.2.3) of FillPicture(). + NalUnitWriter pps; + nalHeader(pps, kNalPps); + pps.Ue(0); + pps.Ue(0); + pps.Flag(false); + pps.Flag(false); + pps.Bits(0, 3); + pps.Flag(signDataHiding); + pps.Flag(false); + pps.Ue(0); + pps.Ue(0); + pps.Se(static_cast(kInitialQp) - 26); + pps.Flag(false); + pps.Flag(transformSkip); + pps.Flag(cuQpDelta); + + if (cuQpDelta) { + pps.Ue(CuQpDeltaDepth()); + } + + pps.Se(0); + pps.Se(0); + // pps_slice_chroma_qp_offsets_present_flag, weighted_pred_flag, + // weighted_bipred_flag, transquant_bypass_enabled_flag, + // tiles_enabled_flag, entropy_coding_sync_enabled_flag, + // pps_loop_filter_across_slices_enabled_flag, + // deblocking_filter_control_present_flag, + // pps_scaling_list_data_present_flag and + // lists_modification_present_flag. + for (int flag = 0; flag < 10; flag++) { + pps.Flag(false); + } + + pps.Ue(0); + pps.Flag(false); + pps.Flag(false); + pps.AlignWithStopBit(); + + return vps.AppendTo(output) + sps.AppendTo(output) + pps.AppendTo(output); + } + + size_t VaapiH265Encoder::WriteSliceHeader(std::vector & output, bool idr) const + { + // The slice segment header (7.3.6.1) of FillSlice(), up to the slice + // data, which the driver writes. + NalUnitWriter header; + header.Bits(0, 1); + header.Bits(idr ? kNalIdrWRadl : kNalTrailR, 6); + header.Bits(0, 6); + header.Bits(1, 3); + + header.Flag(true); + + if (idr) { + header.Flag(false); + } + + header.Ue(0); + header.Ue(idr ? kSliceTypeI : (lowDelayB ? kSliceTypeB : kSliceTypeP)); + + if (!idr) { + header.Bits(static_cast(pictureOrder) & ((1u << kLog2MaxPicOrderCntLsb) - 1), + kLog2MaxPicOrderCntLsb); + // st_ref_pic_set(0) here, not in the SPS: the previous picture, + // used by this one. + header.Flag(false); + header.Ue(1); + header.Ue(0); + header.Ue(0); + header.Flag(true); + } + + if (sao) { + header.Flag(sao); + header.Flag(sao); + } + + if (!idr) { + // num_ref_idx_active_override_flag, then mvd_l1_zero_flag of a B + // slice, and five_minus_max_num_merge_cand. + header.Flag(false); + + if (lowDelayB) { + header.Flag(false); + } + + header.Ue(0); + } + + header.Se(0); + header.AlignWithStopBit(); + + return header.AppendTo(output); + } + void VaapiH265Encoder::SetRates(const RateControlParameters & parameters) { const uint32_t bitrate = parameters.bitrate.get_sum_bps(); diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp index 139a59afa..c94b0801d 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp @@ -101,8 +101,10 @@ namespace jni referenceValid = false; outputProcessor.Reset(); - if (!session.Create(VAProfileVP9Profile0, entrypoint, Align(codecSettings.width, kSurfaceAlignment), - Align(codecSettings.height, kSurfaceAlignment))) + const uint32_t surfaceWidth = Align(codecSettings.width, kSurfaceAlignment); + const uint32_t surfaceHeight = Align(codecSettings.height, kSurfaceAlignment); + + if (!session.Create(VAProfileVP9Profile0, entrypoint, surfaceWidth, surfaceHeight, surfaceWidth, surfaceHeight)) { Release(); return WEBRTC_VIDEO_CODEC_ERROR; From 689bdc0a3e008dd15196dd92b6e19af416420375 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 15 Oct 2026 00:59:03 +0200 Subject: [PATCH 2/2] feat: encode H.265 with VA-API on Linux without turning it on webrtc.hardware.vaapiExperimental now only turns on VA-API decoding. --- docs/guide/advanced/video-codecs.md | 10 +-- .../media/video/codec/HardwareCodecOptions.h | 7 +- .../codec/linux/VaapiVideoEncoderFactory.h | 8 +-- .../LinuxHardwareVideoCodecFactories.cpp | 6 +- .../codec/linux/VaapiVideoEncoderFactory.cpp | 9 ++- .../codec/HardwareVideoEncoderFactory.java | 6 +- .../HardwareVideoEncoderIntegrationTest.java | 64 +++++++------------ 7 files changed, 41 insertions(+), 69 deletions(-) diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index 962a9de62..b7a235c54 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -27,7 +27,7 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder() | Platform | H.264 | H.265 | AV1 | |---|---|---|---| | Windows | NVENC on NVIDIA GPUs, otherwise the Media Foundation encoder of the GPU driver (AMD, Intel) | NVENC on NVIDIA GPUs, otherwise Media Foundation on GPUs that encode H.265 | NVENC on NVIDIA RTX 40 and newer, otherwise Media Foundation on GPUs that encode AV1 (e.g. AMD RDNA3 and newer, Intel Arc) | -| Linux | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) | NVENC on NVIDIA GPUs; VA-API (Intel, AMD) when turned on, see below | NVENC on NVIDIA RTX 40 and newer | +| Linux | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) where it encodes H.265 | NVENC on NVIDIA RTX 40 and newer | | macOS | VideoToolbox, as with `DefaultVideoEncoderFactory` | VideoToolbox, on Macs with a hardware HEVC encoder, such as all with Apple silicon | Software | Hardware encoders produce a single layer. A stream that asks for SVC, such as an AV1 stream with a scalability mode like `L1T3`, is encoded in software. @@ -40,13 +40,7 @@ System.setProperty("webrtc.hardware.vp9Encoder", "true"); Where the GPU has no VP9 encoder, or the stream asks for SVC, VP9 is still encoded with libvpx. -On Linux, H.265 can be encoded with the VA-API encoder of Intel and AMD drivers, also off by default until it has been tested on such GPUs: - -```java -System.setProperty("webrtc.hardware.vaapiExperimental", "true"); -``` - -It is offered only where the driver encodes H.265 Main, and only for sending: Intel and AMD GPUs do not decode H.265 here yet, so the other end has to. +On Linux, H.265 is encoded with the VA-API encoder of Intel and AMD drivers where the driver encodes H.265 Main. It is offered only for sending: Intel and AMD GPUs do not decode H.265 here yet, so the other end has to. NVENC needs an NVIDIA driver of version 522 or newer on Windows, 520 or newer on Linux. VA-API needs libva 2 and a driver that encodes H.264, such as Intel's `intel-media-va-driver` (iHD) or Mesa's `mesa-va-drivers` for AMD, and access to a render node in `/dev/dri`. All of them are loaded at run time, so nothing needs to be installed on machines without them. diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareCodecOptions.h b/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareCodecOptions.h index 81c151d85..33726a641 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareCodecOptions.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareCodecOptions.h @@ -24,9 +24,8 @@ namespace jni // tested on one. constexpr const char * kHardwareVp9EncoderProperty = "webrtc.hardware.vp9Encoder"; - // The Java system property that turns on the VA-API codecs beyond H.264 - // encoding, which are off by default: they have not been tested on an - // Intel or AMD GPU under Linux yet. + // The Java system property that turns on VA-API decoding, which is off by + // default: it has not been tested on an Intel or AMD GPU under Linux yet. constexpr const char * kExperimentalVaapiProperty = "webrtc.hardware.vaapiExperimental"; // Whether hardware VP9 encoding is turned on: the Java system property @@ -34,7 +33,7 @@ namespace jni // false where Java cannot be asked, for one with an exception pending. bool IsHardwareVp9EncoderEnabled(); - // Whether the experimental VA-API codecs are turned on, the same way. + // Whether VA-API decoding is turned on, the same way. bool IsExperimentalVaapiEnabled(); } diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoEncoderFactory.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoEncoderFactory.h index 972214e15..101afd510 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoEncoderFactory.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoEncoderFactory.h @@ -32,15 +32,15 @@ namespace jni // Creates VA-API encoders on the GPU whose driver encodes H.264, as Intel // and AMD drivers do. It offers H.264 in the profiles WebRTC's software // encoder offers too, and only with packetization mode 1: mode 0 needs - // each NAL unit to fit a packet. With the experimental VA-API codecs - // turned on, it offers H.265 Main as well, and with hardware VP9 encoding - // turned on VP9 profile 0, where the driver encodes them. + // each NAL unit to fit a packet. It offers H.265 Main as well where the + // driver encodes it, and VP9 profile 0 where the driver encodes it and + // hardware VP9 encoding is turned on. class VaapiVideoEncoderFactory : public webrtc::VideoEncoderFactory { public: // Returns a factory, or null if no GPU driver encodes H.264 // through VA-API. - static std::unique_ptr Create(bool experimental, bool vp9Enabled); + static std::unique_ptr Create(bool vp9Enabled); ~VaapiVideoEncoderFactory() override = default; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp index ff33028cf..48c0cf627 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp @@ -33,10 +33,8 @@ namespace jni if (auto nvenc = NvencVideoEncoderFactory::Create()) { factories.push_back(std::move(nvenc)); } - // The VA-API codecs beyond H.264 only if asked for, H.265 and VP9 - // each with its own switch: they have not been tested on an Intel or - // AMD GPU yet. - auto vaapi = VaapiVideoEncoderFactory::Create(IsExperimentalVaapiEnabled(), IsHardwareVp9EncoderEnabled()); + // VP9 only if asked for: it has not been tested on an Intel GPU yet. + auto vaapi = VaapiVideoEncoderFactory::Create(IsHardwareVp9EncoderEnabled()); if (vaapi) { factories.push_back(std::move(vaapi)); diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp index 47b37bcfa..d4bd2df0c 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp @@ -27,7 +27,7 @@ namespace jni { - std::unique_ptr VaapiVideoEncoderFactory::Create(bool experimental, bool vp9Enabled) + std::unique_ptr VaapiVideoEncoderFactory::Create(bool vp9Enabled) { VaapiLibrary * library = VaapiLibrary::Get(); @@ -36,14 +36,13 @@ namespace jni } VAEntrypoint h265Entrypoint = VAEntrypointEncSlice; - const bool h265 = experimental && library->EncodeEntrypoint(VAProfileHEVCMain, &h265Entrypoint); + const bool h265 = library->EncodeEntrypoint(VAProfileHEVCMain, &h265Entrypoint); VAEntrypoint vp9Entrypoint = VAEntrypointEncSlice; const bool vp9 = vp9Enabled && library->EncodeEntrypoint(VAProfileVP9Profile0, &vp9Entrypoint); - if (experimental) { - RTC_LOG(LS_INFO) << "VA-API (" << library->Vendor() << ") H.265 encoding: " << h265; - } + RTC_LOG(LS_INFO) << "VA-API (" << library->Vendor() << ") H.265 encoding: " << h265; + if (vp9Enabled) { RTC_LOG(LS_INFO) << "VA-API (" << library->Vendor() << ") VP9 encoding: " << vp9; } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java index 77bb58655..638191ace 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java @@ -45,10 +45,8 @@ * with the system property * {@code webrtc.hardware.vp9Encoder} set to {@code true} when the factory is * created: the encoder has not been tested on such a GPU yet. Without it, VP9 - * is encoded with libvpx. On Linux, H.265 is encoded with VA-API on the GPUs - * of Intel and AMD only with the system property - * {@code webrtc.hardware.vaapiExperimental} set to {@code true}, for the same - * reason; NVENC encodes it regardless. + * is encoded with libvpx. On Linux, H.265 is encoded with NVENC on NVIDIA + * GPUs and with VA-API on the GPUs of Intel and AMD whose driver encodes it. * A hardware encoder that fails to start, for example because the GPU has no * encoder sessions left, or fails while encoding, is replaced by the next one * in line, and finally by the software encoder of the same codec, so a stream diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java index dc447ffa4..4c2bd3e51 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java @@ -85,9 +85,6 @@ class HardwareVideoEncoderIntegrationTest extends TestBase { /** The property that turns on hardware VP9 encoding, which is off by default. */ private static final String VP9_ENCODER_PROPERTY = "webrtc.hardware.vp9Encoder"; - /** The property that turns on VA-API's codecs beyond H.264, off by default. */ - private static final String VAAPI_EXPERIMENTAL_PROPERTY = "webrtc.hardware.vaapiExperimental"; - private static final String OS = System.getProperty("os.name").toLowerCase(Locale.ROOT); /** @@ -329,50 +326,37 @@ void hardwareH265EncoderSendsWholeAccessUnits() throws Exception { assumeTrue(OS.contains("win") || OS.contains("linux"), "hardware encoders are implemented on Windows and Linux only"); - // On Linux, VA-API's H.265 encoder is there only when turned on. - String previous = System.setProperty(VAAPI_EXPERIMENTAL_PROPERTY, "true"); - - try { - // The receiver needs no H.265 decoder of its own: a Java decoder - // checks what arrives, so that a GPU that encodes but does not - // decode H.265 is tested as well. - boolean available = offersH265(new HardwareVideoEncoderFactory().getSupportedCodecs()); - - if (HARDWARE_H265_REQUIRED) { - assertTrue(available, "no hardware H.265 encoder"); - } - else { - assumeTrue(available, "no hardware H.265 encoder"); - } + // The receiver needs no H.265 decoder of its own: a Java decoder + // checks what arrives, so that a GPU that encodes but does not + // decode H.265 is tested as well. + boolean available = offersH265(new HardwareVideoEncoderFactory().getSupportedCodecs()); - H265CheckingDecoderFactory decoders = new H265CheckingDecoderFactory(); - PeerConnectionFactory hardware = PeerConnectionFactory.builder() - .setAudioDeviceModule(audioDevModule) - .setVideoEncoderFactory(new HardwareVideoEncoderFactory()) - .setVideoDecoderFactory(decoders) - .build(); + if (HARDWARE_H265_REQUIRED) { + assertTrue(available, "no hardware H.265 encoder"); + } + else { + assumeTrue(available, "no hardware H.265 encoder"); + } - // 640x360, which hardware H.265 encoders take (see above). - try (Run run = new Run(hardware, H265, 640, 360)) { - run.awaitFrames(10); + H265CheckingDecoderFactory decoders = new H265CheckingDecoderFactory(); + PeerConnectionFactory hardware = PeerConnectionFactory.builder() + .setAudioDeviceModule(audioDevModule) + .setVideoEncoderFactory(new HardwareVideoEncoderFactory()) + .setVideoDecoderFactory(decoders) + .build(); - assertHardware(run.implementation(), true); - } - finally { - hardware.dispose(); - } + // 640x360, which hardware H.265 encoders take (see above). + try (Run run = new Run(hardware, H265, 640, 360)) { + run.awaitFrames(10); - assertNull(decoders.problem.get(), decoders.problem.get()); - assertTrue(decoders.keyFrames.get() > 0, "no key frame arrived"); + assertHardware(run.implementation(), true); } finally { - if (previous == null) { - System.clearProperty(VAAPI_EXPERIMENTAL_PROPERTY); - } - else { - System.setProperty(VAAPI_EXPERIMENTAL_PROPERTY, previous); - } + hardware.dispose(); } + + assertNull(decoders.problem.get(), decoders.problem.get()); + assertTrue(decoders.keyFrames.get() > 0, "no key frame arrived"); } private static boolean offersH265(List codecs) {