From 2dc345d1bcb60018c318c55915a9b960a5dddcee Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Thu, 15 Oct 2026 17:28:25 +0200 Subject: [PATCH] feat: encode AV1 with VA-API on Linux Adds an AV1 encoder for GPUs whose VA-API driver encodes AV1 profile 0, such as AMD's, writing the sequence and frame headers as packed headers the way FFmpeg does. Frames are kept to multiples of 8, since AMD's driver pads the width to that. --- docs/guide/advanced/video-codecs.md | 4 +- .../media/video/codec/linux/VaapiAv1Encoder.h | 117 +++ .../codec/linux/VaapiVideoEncoderFactory.h | 8 +- .../video/codec/linux/VaapiAv1Encoder.cpp | 748 ++++++++++++++++++ .../codec/linux/VaapiVideoEncoderFactory.cpp | 22 +- .../codec/HardwareVideoEncoderFactory.java | 4 +- .../HardwareVideoEncoderIntegrationTest.java | 109 +++ 7 files changed, 1000 insertions(+), 12 deletions(-) create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index 91eafb0c..5f57f9c7 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -27,10 +27,10 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder() | Platform | H.264 | H.265 | AV1 | |---|---|---|---| | Windows | NVENC on NVIDIA GPUs, otherwise the Media Foundation encoder of the GPU driver (AMD, Intel) | NVENC on NVIDIA GPUs, otherwise Media Foundation on GPUs that encode H.265 | NVENC on NVIDIA RTX 40 and newer, otherwise Media Foundation on GPUs that encode AV1 (e.g. AMD RDNA3 and newer, Intel Arc) | -| Linux | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) where it encodes H.265 | NVENC on NVIDIA RTX 40 and newer | +| Linux | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) where it encodes H.265 | NVENC on NVIDIA RTX 40 and newer, otherwise the VA-API encoder of the GPU driver where it encodes AV1 (e.g. AMD RDNA3 and newer) | | macOS | VideoToolbox, as with `DefaultVideoEncoderFactory` | VideoToolbox, on Macs with a hardware HEVC encoder, such as all with Apple silicon | Software | -Hardware encoders produce a single layer. A stream that asks for SVC, such as an AV1 stream with a scalability mode like `L1T3`, is encoded in software. +Hardware encoders produce a single layer. A stream that asks for SVC, such as an AV1 stream with a scalability mode like `L1T3`, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom. VP9 can be encoded in hardware too, with the VP9 encoder of the GPU driver, which Intel GPUs have: through VA-API on Linux, and through Media Foundation on Windows. On Windows it is off by default, since it has not been tested on such a GPU yet; turn it on with a system property before the factory is created: diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h new file mode 100644 index 00000000..9e5dbbad --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h @@ -0,0 +1,117 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_AV1_ENCODER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_AV1_ENCODER_H_ + +#include "media/video/codec/EncoderOutputProcessor.h" +#include "media/video/codec/linux/VaapiEncodeSession.h" +#include "media/video/codec/linux/VaapiLibrary.h" + +#include "api/video/video_frame.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" + +#include +#include + +#include +#include +#include + +namespace jni +{ + // Encodes AV1 profile 0 with the GPU driver's VA-API encoder, as AMD + // drivers provide it on Linux. + // + // It works as FFmpeg's VA-API encoder does without B-frames: synchronous, + // one tile, inter frames that refer to the frame before them only, which + // alternate between two reference slots. The sequence and frame headers + // are written here and handed to the driver as packed headers, with where + // the quantizer, loop filter and CDEF parameters are in the frame header, + // which the driver's rate control fills in. Anything that fails is an + // error, upon which libaom takes over. + class VaapiAv1Encoder : public webrtc::VideoEncoder + { + public: + VaapiAv1Encoder(VaapiLibrary & library, VAEntrypoint entrypoint, const webrtc::SdpVideoFormat & format); + ~VaapiAv1Encoder() override; + + int32_t InitEncode(const webrtc::VideoCodec * codecSettings, const Settings & settings) override; + int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback * callback) override; + int32_t Release() override; + int32_t Encode(const webrtc::VideoFrame & frame, const std::vector * frameTypes) override; + void SetRates(const RateControlParameters & parameters) override; + EncoderInfo GetEncoderInfo() const override; + + private: + // Where the driver rewrites the frame header: the bit offsets of + // base_q_idx, loop_filter_params() and cdef_params(), the size of + // the latter, and the size of the whole frame header OBU. + struct FrameHeaderLayout + { + uint32_t qindexOffset = 0; + uint32_t loopFilterOffset = 0; + uint32_t cdefOffset = 0; + uint32_t cdefSize = 0; + uint32_t size = 0; + }; + + // Reads which headers the driver takes and how it codes them. + void QueryCapabilities(); + + bool Submit(bool keyFrame); + + void WriteSequenceHeader(std::vector & obu) const; + void WriteFrameHeader(std::vector & obu, FrameHeaderLayout & layout, bool keyFrame) const; + // The seq_level_idx the stream's size, frame rate and bitrate fit. + uint8_t Level() const; + + private: + VaapiLibrary & library; + const VAEntrypoint entrypoint; + const std::string implementationName; + VaapiEncodeSession session; + + bool initialized; + webrtc::VideoCodec codecSettings; + uint32_t bitrateBps; + uint32_t framerate; + bool ratesChanged; + + // The fixed length of obu_size the driver takes, in bytes, zero + // where it takes no packed headers; and the TxMode it encodes. + uint32_t obuSizeLength; + uint32_t txMode; + // The sequence header, kept for the key frames. + std::vector sequenceHeader; + + // Which of the reconstructed surfaces, and so which reference slot, + // holds the current frame. + int current; + bool referenceValid; + // The frames since the last key frame, whose low bits are the + // order hint. + uint32_t frameNumber; + + webrtc::EncodedImageCallback * callback; + + EncoderOutputProcessor outputProcessor; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoEncoderFactory.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoEncoderFactory.h index 29919bf9..1e1cc76e 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoEncoderFactory.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoEncoderFactory.h @@ -32,8 +32,8 @@ namespace jni // Creates VA-API encoders on the GPU whose driver encodes H.264, as Intel // and AMD drivers do. It offers H.264 in the profiles WebRTC's software // encoder offers too, and only with packetization mode 1: mode 0 needs - // each NAL unit to fit a packet. It offers H.265 Main and VP9 profile 0 - // as well where the driver encodes them. + // each NAL unit to fit a packet. It offers H.265 Main, VP9 profile 0 and + // AV1 profile 0 as well where the driver encodes them. class VaapiVideoEncoderFactory : public webrtc::VideoEncoderFactory { public: @@ -49,7 +49,7 @@ namespace jni private: VaapiVideoEncoderFactory(VaapiLibrary & library, bool h265, VAEntrypoint h265Entrypoint, bool vp9, - VAEntrypoint vp9Entrypoint); + VAEntrypoint vp9Entrypoint, bool av1, VAEntrypoint av1Entrypoint); private: VaapiLibrary & library; @@ -57,6 +57,8 @@ namespace jni const VAEntrypoint h265Entrypoint; const bool vp9; const VAEntrypoint vp9Entrypoint; + const bool av1; + const VAEntrypoint av1Entrypoint; }; } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp new file mode 100644 index 00000000..277bc3b2 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp @@ -0,0 +1,748 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/linux/VaapiAv1Encoder.h" + +#include "api/video/encoded_image.h" +#include "api/video_codecs/scalability_mode.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" + +#include + +namespace jni +{ + namespace + { + // Key frames come when WebRTC asks for them; this only bounds how + // long a stream may go without one if it never does. + constexpr uint32_t kKeyFrameInterval = 3000; + + // Superblocks of 64x64; surfaces are kept to whole superblocks, as + // FFmpeg's VA-API encoder does. + constexpr uint32_t kSuperblockSize = 64; + + // AMD's driver codes the width in multiples of 8 and the height in + // multiples of 2, and pads the picture to that without a render size, + // so that a decoder would show the padding. The encoder asks WebRTC + // for frames of that size. + constexpr uint32_t kSizeAlignment = 8; + + // The largest tile, section A.3 of the AV1 specification. A frame + // that needs more than one is left to libaom. + constexpr uint32_t kMaxTileWidth = 4096; + constexpr uint32_t kMaxTileArea = 4096 * 2304; + + // OBU types and frame types, sections 6.2.2 and 6.8.2. + constexpr uint8_t kObuSequenceHeader = 1; + constexpr uint8_t kObuFrameHeader = 3; + constexpr uint32_t kKeyFrameType = 0; + constexpr uint32_t kInterFrameType = 1; + + constexpr int kRefsPerFrame = 7; + constexpr uint8_t kPrimaryRefNone = 7; + constexpr uint32_t kLastFrame = 1; + constexpr uint32_t kOrderHintBits = 8; + + // TxMode: largest, or chosen per block. + constexpr uint32_t kTxModeLargest = 1; + constexpr uint32_t kTxModeSelect = 2; + // The bits of VAConfigAttribValEncAV1Ext2::tx_mode_support. + constexpr uint32_t kTxModeLargestSupported = 0x02; + constexpr uint32_t kTxModeSelectSupported = 0x04; + + // The quantizer index the frame header starts from, FFmpeg's; the + // rate control sets that of each frame, within these bounds. + constexpr uint8_t kQIndex = 128; + constexpr uint8_t kMinQIndex = 1; + constexpr uint8_t kMaxQIndex = 255; + + constexpr uint32_t kPackedHeaders = VA_ENC_PACKED_HEADER_SEQUENCE | VA_ENC_PACKED_HEADER_PICTURE; + + // The levels of annex A.3: seq_level_idx, the largest picture, the + // largest width and height, the largest display rate in samples per + // second and the largest bitrate of the main tier. + struct Level + { + uint8_t index; + uint64_t maxPictureSize; + uint32_t maxWidth; + uint32_t maxHeight; + uint64_t maxDisplayRate; + uint64_t maxBitrate; + }; + + constexpr Level kLevels[] = { + { 0, 147456, 2048, 1152, 4423680, 1500000 }, + { 1, 278784, 2816, 1584, 8363520, 3000000 }, + { 4, 665856, 4352, 2448, 19975680, 6000000 }, + { 5, 1065024, 5504, 3096, 31950720, 10000000 }, + { 8, 2359296, 6144, 3456, 70778880, 12000000 }, + { 9, 2359296, 6144, 3456, 141557760, 20000000 }, + { 12, 8912896, 8192, 4352, 267386880, 30000000 }, + { 13, 8912896, 8192, 4352, 534773760, 40000000 }, + { 14, 8912896, 8192, 4352, 1069547520, 60000000 }, + { 16, 35651584, 16384, 8704, 1069547520, 60000000 }, + }; + + // The level of a stream that fits none. + constexpr uint8_t kMaxParametersLevel = 31; + + uint32_t Align(uint32_t value, uint32_t alignment) + { + return (value + alignment - 1) / alignment * alignment; + } + + int FloorLog2(uint32_t value) + { + int log = 0; + + while (value > 1) { + value >>= 1; + log++; + } + + return log; + } + + // tile_log2 of section 5.9.15. + int TileLog2(uint32_t blockSize, uint32_t target) + { + int k = 0; + + while ((blockSize << k) < target) { + k++; + } + + return k; + } + + // Writes the bits of an OBU's payload, most significant first. + class BitWriter + { + public: + void Put(uint32_t value, int bits) + { + for (int i = bits - 1; i >= 0; i--) { + PutBit((value >> i) & 1); + } + } + + void PutBit(uint32_t bit) + { + if (count % 8 == 0) { + bytes.push_back(0); + } + if (bit) { + bytes.back() |= static_cast(0x80 >> (count % 8)); + } + + count++; + } + + // trailing_bits, section 5.3.4: a one, then zeros to the next + // byte. + void TrailingBits() + { + PutBit(1); + + while (count % 8 != 0) { + PutBit(0); + } + } + + uint32_t Position() const + { + return count; + } + + const std::vector & Bytes() const + { + return bytes; + } + + private: + std::vector bytes; + uint32_t count = 0; + }; + + // An OBU of the payload: the header, without an extension, and + // obu_size as a leb128 of the given fixed length, which is what the + // driver rewrites the size of the frame header OBU as. + std::vector MakeObu(uint8_t type, const std::vector & payload, uint32_t sizeLength) + { + std::vector obu; + obu.push_back(static_cast((type << 3) | 0x02)); + + const uint32_t size = static_cast(payload.size()); + + for (uint32_t i = 0; i < sizeLength; i++) { + uint8_t byte = static_cast((size >> (7 * i)) & 0x7F); + + if (i + 1 < sizeLength) { + byte |= 0x80; + } + + obu.push_back(byte); + } + + obu.insert(obu.end(), payload.begin(), payload.end()); + + return obu; + } + } + + VaapiAv1Encoder::VaapiAv1Encoder(VaapiLibrary & library, VAEntrypoint entrypoint, + const webrtc::SdpVideoFormat & format) : + library(library), + entrypoint(entrypoint), + implementationName("VA-API (" + library.Vendor() + ")"), + session(library, "VA-API AV1"), + initialized(false), + codecSettings(), + bitrateBps(0), + framerate(30), + ratesChanged(false), + obuSizeLength(0), + txMode(0), + current(0), + referenceValid(false), + frameNumber(0), + callback(nullptr), + outputProcessor(webrtc::kVideoCodecAV1, format) + { + QueryCapabilities(); + } + + VaapiAv1Encoder::~VaapiAv1Encoder() + { + Release(); + } + + void VaapiAv1Encoder::QueryCapabilities() + { + VAConfigAttrib attributes[2] = {}; + attributes[0].type = VAConfigAttribEncPackedHeaders; + attributes[1].type = VAConfigAttribEncAV1Ext2; + + if (library.Api().GetConfigAttributes(library.Display(), VAProfileAV1Profile0, entrypoint, attributes, 2) != + VA_STATUS_SUCCESS) + { + return; + } + + // The headers are written here; a driver that does not take them is + // not used. + if (attributes[0].value == VA_ATTRIB_NOT_SUPPORTED || (attributes[0].value & kPackedHeaders) != kPackedHeaders) { + return; + } + if (attributes[1].value == VA_ATTRIB_NOT_SUPPORTED) { + return; + } + + VAConfigAttribValEncAV1Ext2 ext2; + ext2.value = attributes[1].value; + + // Coded lossless frames, which ONLY_4X4 is for, are not made. + if (ext2.bits.tx_mode_support & kTxModeSelectSupported) { + txMode = kTxModeSelect; + } + else if (ext2.bits.tx_mode_support & kTxModeLargestSupported) { + txMode = kTxModeLargest; + } + else { + return; + } + + obuSizeLength = ext2.bits.obu_size_bytes_minus1 + 1; + } + + int32_t VaapiAv1Encoder::InitEncode(const webrtc::VideoCodec * settings, const Settings & encoderSettings) + { + if (settings == nullptr || settings->width == 0 || settings->height == 0) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (settings->numberOfSimulcastStreams > 1) { + return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED; + } + // A single layer; spatial or temporal layers are left to libaom. + if (settings->GetScalabilityMode().value_or(webrtc::ScalabilityMode::kL1T1) != webrtc::ScalabilityMode::kL1T1) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (obuSizeLength == 0) { + RTC_LOG(LS_WARNING) << implementationName << " AV1: the driver takes no packed headers, or does not say " + "how it codes them"; + return WEBRTC_VIDEO_CODEC_ERROR; + } + if (settings->width < kSizeAlignment || settings->height < kSizeAlignment) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + // One tile, which is all this encoder writes. + if (settings->width > kMaxTileWidth || uint64_t(Align(settings->width, kSuperblockSize)) + * Align(settings->height, kSuperblockSize) > kMaxTileArea) + { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + Release(); + + // The first frames of a source can come before WebRTC knows the + // alignment the encoder asks for, so the encoder takes the aligned + // size below and drops the last rows and columns of such frames. + codecSettings = *settings; + codecSettings.width -= codecSettings.width % kSizeAlignment; + codecSettings.height -= codecSettings.height % kSizeAlignment; + bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; + framerate = std::max(1u, codecSettings.maxFramerate); + ratesChanged = true; + current = 0; + referenceValid = false; + frameNumber = 0; + outputProcessor.Reset(); + + const uint32_t surfaceWidth = Align(codecSettings.width, kSuperblockSize); + const uint32_t surfaceHeight = Align(codecSettings.height, kSuperblockSize); + + if (!session.Create(VAProfileAV1Profile0, entrypoint, codecSettings.width, codecSettings.height, surfaceWidth, + surfaceHeight, kPackedHeaders)) + { + Release(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + + WriteSequenceHeader(sequenceHeader); + + initialized = true; + + RTC_LOG(LS_INFO) << implementationName << " AV1 initialized: " << codecSettings.width << "x" + << codecSettings.height << " at " << bitrateBps << " bps"; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VaapiAv1Encoder::RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback * encodeCallback) + { + callback = encodeCallback; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VaapiAv1Encoder::Release() + { + session.Destroy(); + + initialized = false; + referenceValid = false; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VaapiAv1Encoder::Encode(const webrtc::VideoFrame & frame, + const std::vector * frameTypes) + { + if (!initialized || callback == nullptr) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + + bool keyFrame = !referenceValid || frameNumber >= kKeyFrameInterval; + + if (frameTypes != nullptr) { + keyFrame |= std::any_of(frameTypes->begin(), frameTypes->end(), [](webrtc::VideoFrameType type) { + return type == webrtc::VideoFrameType::kVideoFrameKey; + }); + } + + if (keyFrame) { + frameNumber = 0; + } + + std::vector output; + + // A frame up to the alignment larger is cropped; one of another size, + // which raced with a change of size, is scaled. + const bool encoded = session.Upload(frame, static_cast(codecSettings.width), + static_cast(codecSettings.height), kSizeAlignment - 1) && Submit(keyFrame) + && session.ReadOutput(output); + + session.DropBuffers(); + + if (!encoded) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + webrtc::EncodedImage image; + image._encodedWidth = codecSettings.width; + image._encodedHeight = codecSettings.height; + image.SetRtpTimestamp(frame.rtp_timestamp()); + image.capture_time_ms_ = frame.render_time_ms(); + image.ntp_time_ms_ = frame.ntp_time_ms(); + image.rotation_ = frame.rotation(); + + webrtc::CodecSpecificInfo info; + + if (!outputProcessor.Process(output, keyFrame, image, info)) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // The frame just encoded is the reference of the next one. + referenceValid = true; + current = 1 - current; + frameNumber++; + ratesChanged = false; + + callback->OnEncodedImage(image, &info); + + return WEBRTC_VIDEO_CODEC_OK; + } + + bool VaapiAv1Encoder::Submit(bool keyFrame) + { + const uint32_t sbCols = Align(codecSettings.width, kSuperblockSize) / kSuperblockSize; + const uint32_t sbRows = Align(codecSettings.height, kSuperblockSize) / kSuperblockSize; + // The slot of the frame this one refers to, which the reconstructed + // surfaces stand for. + const int referenceSlot = 1 - current; + + if (keyFrame) { + VAEncSequenceParameterBufferAV1 sequence = {}; + sequence.seq_profile = 0; + sequence.seq_level_idx = Level(); + sequence.seq_tier = 0; + // Key frames only where asked for. + sequence.intra_period = kKeyFrameInterval; + sequence.ip_period = 1; + sequence.bits_per_second = bitrateBps; + sequence.seq_fields.bits.enable_order_hint = 1; + sequence.seq_fields.bits.enable_cdef = 1; + sequence.order_hint_bits_minus_1 = kOrderHintBits - 1; + + if (!session.AddBuffer(VAEncSequenceParameterBufferType, &sequence, sizeof(sequence))) { + return false; + } + } + + // No initial quantizer: the rate control picks its own, as FFmpeg + // leaves it. + if ((keyFrame || ratesChanged) && !session.AddRateControl(bitrateBps, framerate, 0)) { + return false; + } + + std::vector frameHeader; + FrameHeaderLayout layout; + WriteFrameHeader(frameHeader, layout, keyFrame); + + VAEncPictureParameterBufferAV1 picture = {}; + picture.frame_width_minus_1 = static_cast(codecSettings.width - 1); + picture.frame_height_minus_1 = static_cast(codecSettings.height - 1); + picture.reconstructed_frame = session.Reconstructed(current); + picture.coded_buf = session.CodedBuffer(); + + for (VASurfaceID & reference : picture.reference_frames) { + reference = VA_INVALID_SURFACE; + } + + if (keyFrame) { + picture.primary_ref_frame = kPrimaryRefNone; + picture.refresh_frame_flags = 0xFF; + picture.picture_flags.bits.frame_type = kKeyFrameType; + picture.picture_flags.bits.error_resilient_mode = 1; + } + else { + picture.reference_frames[referenceSlot] = session.Reconstructed(referenceSlot); + + for (int i = 0; i < kRefsPerFrame; i++) { + picture.ref_frame_idx[i] = static_cast(referenceSlot); + } + + // Every reference frame names the same slot, so any of them is + // the primary one; FFmpeg takes the index of the slot. + picture.primary_ref_frame = static_cast(referenceSlot); + picture.refresh_frame_flags = static_cast(1 << current); + picture.ref_frame_ctrl_l0.fields.search_idx0 = kLastFrame; + picture.picture_flags.bits.frame_type = kInterFrameType; + } + + picture.order_hint = static_cast(frameNumber); + picture.base_qindex = kQIndex; + picture.min_base_qindex = kMinQIndex; + picture.max_base_qindex = kMaxQIndex; + picture.mode_control_flags.bits.tx_mode = txMode; + + picture.tile_cols = 1; + picture.tile_rows = 1; + picture.width_in_sbs_minus_1[0] = static_cast(sbCols - 1); + picture.height_in_sbs_minus_1[0] = static_cast(sbRows - 1); + picture.tile_group_obu_hdr_info.bits.obu_has_size_field = 1; + + // Where the driver writes what its rate control decides, and the size + // of the frame header OBU, which comes after the sequence header in + // a key frame. + picture.bit_offset_qindex = layout.qindexOffset; + picture.bit_offset_loopfilter_params = layout.loopFilterOffset; + picture.bit_offset_cdef_params = layout.cdefOffset; + picture.size_in_bits_cdef_params = layout.cdefSize; + picture.size_in_bits_frame_hdr_obu = layout.size; + picture.byte_offset_frame_hdr_obu_size = (keyFrame ? static_cast(sequenceHeader.size()) : 0) + 1; + + VAEncTileGroupBufferAV1 tileGroup = {}; + tileGroup.tg_start = 0; + tileGroup.tg_end = 0; + + return session.AddBuffer(VAEncPictureParameterBufferType, &picture, sizeof(picture)) + && (!keyFrame || session.AddPackedHeader(VAEncPackedHeaderSequence, sequenceHeader, sequenceHeader.size() * 8)) + && session.AddPackedHeader(VAEncPackedHeaderPicture, frameHeader, layout.size) + && session.AddBuffer(VAEncSliceParameterBufferType, &tileGroup, sizeof(tileGroup)) + && session.Encode(); + } + + void VaapiAv1Encoder::WriteSequenceHeader(std::vector & obu) const + { + // sequence_header_obu, section 5.5, with the tools FFmpeg's VA-API + // encoder turns on: order hints and CDEF. + const int widthBits = FloorLog2(codecSettings.width) + 1; + const int heightBits = FloorLog2(codecSettings.height) + 1; + const uint8_t level = Level(); + + BitWriter writer; + // seq_profile, still_picture, reduced_still_picture_header. + writer.Put(0, 3); + writer.Put(0, 1); + writer.Put(0, 1); + // timing_info_present_flag, initial_display_delay_present_flag, + // operating_points_cnt_minus_1, operating_point_idc[0]. + writer.Put(0, 1); + writer.Put(0, 1); + writer.Put(0, 5); + writer.Put(0, 12); + // seq_level_idx[0], and seq_tier[0] above level 3.3. + writer.Put(level, 5); + + if (level > 7) { + writer.Put(0, 1); + } + + writer.Put(widthBits - 1, 4); + writer.Put(heightBits - 1, 4); + writer.Put(codecSettings.width - 1, widthBits); + writer.Put(codecSettings.height - 1, heightBits); + // frame_id_numbers_present_flag, use_128x128_superblock, + // enable_filter_intra, enable_intra_edge_filter, + // enable_interintra_compound, enable_masked_compound, + // enable_warped_motion, enable_dual_filter. + writer.Put(0, 8); + // enable_order_hint, enable_jnt_comp, enable_ref_frame_mvs. + writer.Put(1, 1); + writer.Put(0, 1); + writer.Put(0, 1); + // seq_choose_screen_content_tools and seq_force_screen_content_tools: + // no screen content tools, so seq_force_integer_mv is not coded. + writer.Put(0, 1); + writer.Put(0, 1); + writer.Put(kOrderHintBits - 1, 3); + // enable_superres, enable_cdef, enable_restoration. + writer.Put(0, 1); + writer.Put(1, 1); + writer.Put(0, 1); + // color_config, section 5.5.2: 8 bit 4:2:0 without a color + // description: high_bitdepth, mono_chrome, + // color_description_present_flag, color_range, + // chroma_sample_position, separate_uv_delta_q. + writer.Put(0, 1); + writer.Put(0, 1); + writer.Put(0, 1); + writer.Put(0, 1); + writer.Put(0, 2); + writer.Put(0, 1); + // film_grain_params_present. + writer.Put(0, 1); + writer.TrailingBits(); + + obu = MakeObu(kObuSequenceHeader, writer.Bytes(), obuSizeLength); + } + + void VaapiAv1Encoder::WriteFrameHeader(std::vector & obu, FrameHeaderLayout & layout, bool keyFrame) const + { + // uncompressed_header, section 5.9, as FFmpeg's VA-API encoder writes + // it, for the sequence header above. + const int referenceSlot = 1 - current; + const uint32_t miCols = 2 * ((codecSettings.width + 7) >> 3); + const uint32_t miRows = 2 * ((codecSettings.height + 7) >> 3); + const uint32_t sbCols = (miCols + 15) >> 4; + const uint32_t sbRows = (miRows + 15) >> 4; + // The OBU header and obu_size come before the payload. + const uint32_t payloadOffset = (1 + obuSizeLength) * 8; + + BitWriter writer; + // show_existing_frame, frame_type, show_frame. + writer.Put(0, 1); + writer.Put(keyFrame ? kKeyFrameType : kInterFrameType, 2); + writer.Put(1, 1); + + if (!keyFrame) { + // error_resilient_mode; a key frame that is shown has it set. + writer.Put(0, 1); + } + + // disable_cdf_update, frame_size_override_flag, order_hint. + writer.Put(0, 1); + writer.Put(0, 1); + writer.Put(frameNumber & ((1u << kOrderHintBits) - 1), kOrderHintBits); + + if (!keyFrame) { + // primary_ref_frame, refresh_frame_flags. + writer.Put(referenceSlot, 3); + writer.Put(1u << current, 8); + // frame_refs_short_signaling, then ref_frame_idx: every reference + // is the frame before. + writer.Put(0, 1); + + for (int i = 0; i < kRefsPerFrame; i++) { + writer.Put(referenceSlot, 3); + } + } + + // frame_size() takes the size of the sequence header; render_size(): + // render_and_frame_size_different. + writer.Put(0, 1); + + if (!keyFrame) { + // allow_high_precision_mv, is_filter_switchable, + // interpolation_filter (EIGHTTAP), is_motion_mode_switchable. + writer.Put(0, 1); + writer.Put(0, 1); + writer.Put(0, 2); + writer.Put(0, 1); + } + + // disable_frame_end_update_cdf. + writer.Put(0, 1); + + // tile_info, section 5.9.15: one uniform tile, which takes no more + // tile columns or rows where it could. + writer.Put(1, 1); + + if (TileLog2(1, std::min(sbCols, 64u)) > 0) { + writer.Put(0, 1); + } + if (TileLog2(1, std::min(sbRows, 64u)) > 0) { + writer.Put(0, 1); + } + + // quantization_params: base_q_idx, no delta coded for DeltaQYDc, + // DeltaQUDc and DeltaQUAc, using_qmatrix. + layout.qindexOffset = payloadOffset + writer.Position(); + writer.Put(kQIndex, 8); + writer.Put(0, 1); + writer.Put(0, 1); + writer.Put(0, 1); + writer.Put(0, 1); + // segmentation_enabled; delta_q_present, base_q_idx being above 0. + writer.Put(0, 1); + writer.Put(0, 1); + + // loop_filter_params: loop_filter_level[0] and [1] of 0, which leave + // out those of U and V, loop_filter_sharpness, + // loop_filter_delta_enabled. + layout.loopFilterOffset = payloadOffset + writer.Position(); + writer.Put(0, 6); + writer.Put(0, 6); + writer.Put(0, 3); + writer.Put(0, 1); + + // cdef_params: cdef_damping_minus_3, cdef_bits, and the strengths of + // the one filter. + layout.cdefOffset = payloadOffset + writer.Position(); + writer.Put(0, 2); + writer.Put(0, 2); + writer.Put(0, 4); + writer.Put(0, 2); + writer.Put(0, 4); + writer.Put(0, 2); + layout.cdefSize = payloadOffset + writer.Position() - layout.cdefOffset; + + // read_tx_mode: tx_mode_select. + writer.Put(txMode == kTxModeSelect ? 1 : 0, 1); + + if (!keyFrame) { + // reference_select. + writer.Put(0, 1); + } + + // reduced_tx_set. + writer.Put(0, 1); + + if (!keyFrame) { + // global_motion_params: is_global of each reference frame. + writer.Put(0, kRefsPerFrame); + } + + writer.TrailingBits(); + + obu = MakeObu(kObuFrameHeader, writer.Bytes(), obuSizeLength); + layout.size = static_cast(obu.size() * 8); + } + + uint8_t VaapiAv1Encoder::Level() const + { + const uint64_t pictureSize = uint64_t(codecSettings.width) * codecSettings.height; + const uint64_t displayRate = pictureSize * std::max(1u, codecSettings.maxFramerate); + const uint64_t bitrate = uint64_t(std::max(codecSettings.maxBitrate, codecSettings.startBitrate)) * 1000; + + for (const auto & level : kLevels) { + if (pictureSize <= level.maxPictureSize && codecSettings.width <= level.maxWidth + && codecSettings.height <= level.maxHeight && displayRate <= level.maxDisplayRate + && bitrate <= level.maxBitrate) + { + return level.index; + } + } + + return kMaxParametersLevel; + } + + void VaapiAv1Encoder::SetRates(const RateControlParameters & parameters) + { + const uint32_t bitrate = parameters.bitrate.get_sum_bps(); + + // A zero bitrate pauses the stream; the encoder gets no frames then. + if (bitrate == 0) { + return; + } + + bitrateBps = bitrate; + + if (parameters.framerate_fps >= 1.0) { + framerate = static_cast(parameters.framerate_fps + 0.5); + } + + // Applied with the next frame. + ratesChanged = true; + } + + webrtc::VideoEncoder::EncoderInfo VaapiAv1Encoder::GetEncoderInfo() const + { + EncoderInfo info; + info.implementation_name = implementationName; + info.is_hardware_accelerated = true; + info.supports_native_handle = false; + info.supports_simulcast = false; + info.scaling_settings = outputProcessor.GetScalingSettings(); + info.requested_resolution_alignment = kSizeAlignment; + + return info; + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp index 9e3ca2ed..4bc6a9c1 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp @@ -15,6 +15,7 @@ */ #include "media/video/codec/linux/VaapiVideoEncoderFactory.h" +#include "media/video/codec/linux/VaapiAv1Encoder.h" #include "media/video/codec/linux/VaapiH264Encoder.h" #include "media/video/codec/linux/VaapiH265Encoder.h" #include "media/video/codec/linux/VaapiVp9Encoder.h" @@ -41,20 +42,25 @@ namespace jni VAEntrypoint vp9Entrypoint = VAEntrypointEncSlice; const bool vp9 = library->EncodeEntrypoint(VAProfileVP9Profile0, &vp9Entrypoint); + VAEntrypoint av1Entrypoint = VAEntrypointEncSlice; + const bool av1 = library->EncodeEntrypoint(VAProfileAV1Profile0, &av1Entrypoint); + RTC_LOG(LS_INFO) << "VA-API (" << library->Vendor() << ") H.265 encoding: " << h265 << ", VP9 encoding: " - << vp9; + << vp9 << ", AV1 encoding: " << av1; - return std::unique_ptr( - new VaapiVideoEncoderFactory(*library, h265, h265Entrypoint, vp9, vp9Entrypoint)); + return std::unique_ptr(new VaapiVideoEncoderFactory(*library, h265, h265Entrypoint, + vp9, vp9Entrypoint, av1, av1Entrypoint)); } VaapiVideoEncoderFactory::VaapiVideoEncoderFactory(VaapiLibrary & library, bool h265, VAEntrypoint h265Entrypoint, - bool vp9, VAEntrypoint vp9Entrypoint) : + bool vp9, VAEntrypoint vp9Entrypoint, bool av1, VAEntrypoint av1Entrypoint) : library(library), h265(h265), h265Entrypoint(h265Entrypoint), vp9(vp9), - vp9Entrypoint(vp9Entrypoint) + vp9Entrypoint(vp9Entrypoint), + av1(av1), + av1Entrypoint(av1Entrypoint) { } @@ -71,6 +77,9 @@ namespace jni if (vp9) { formats.push_back(webrtc::SdpVideoFormat::VP9Profile0()); } + if (av1) { + formats.push_back(webrtc::SdpVideoFormat::AV1Profile0()); + } return formats; } @@ -86,6 +95,9 @@ namespace jni if (codec == webrtc::kVideoCodecVP9) { return std::make_unique(library, vp9Entrypoint, format); } + if (codec == webrtc::kVideoCodecAV1) { + return std::make_unique(library, av1Entrypoint, format); + } return std::make_unique(library, format); } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java index 50adfc14..88d922aa 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java @@ -37,8 +37,8 @@ * H.264 is encoded on an NVIDIA GPU with NVENC, on Windows and Linux. On * other GPUs it is encoded with the Media Foundation encoder of the driver on * Windows, and with its VA-API encoder on Linux. AV1 is encoded on GPUs that - * have an AV1 encoder, with NVENC or Media Foundation, as a single layer; a - * stream that asks for SVC is encoded in software. H.265 (Main profile) is + * have an AV1 encoder, with NVENC, Media Foundation or VA-API, as a single + * layer; a stream that asks for SVC is encoded in software. H.265 (Main profile) is * encoded with NVENC or Media Foundation on GPUs that have an H.265 encoder. * VP9 (profile 0) is encoded on GPUs that have a VP9 encoder, which are * Intel's: with VA-API on Linux, and with Media Foundation on Windows only diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java index 9d73eea6..c2a7f2f4 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java @@ -438,6 +438,115 @@ void hardwareEncodesAv1() throws Exception { } } + @Test + void av1EncodesOddSize() throws Exception { + PeerConnectionFactory hardware = hardwareFactory(); + + // AMD's encoder pads the width to a multiple of 8, so the encoder asks + // for frames of that alignment, rather than sending the padding. + try (Run run = new Run(hardware, AV1, 641, 361)) { + run.awaitFrames(10); + + assertAv1(run); + assertEquals(0, run.width.get() % 8, "width " + run.width.get()); + } + finally { + hardware.dispose(); + } + } + + @Test + void av1FollowsResolutionChange() throws Exception { + PeerConnectionFactory hardware = hardwareFactory(); + + try (Run run = new Run(hardware, AV1, 640, 360)) { + run.awaitFrames(10); + + // A new size starts the encoder over, with a new session and a + // key frame with a new sequence header. + run.call.setVideoSize(960, 540); + run.awaitWidth(960); + + run.call.setVideoSize(640, 360); + run.awaitWidth(640); + + assertAv1(run); + } + finally { + hardware.dispose(); + } + } + + @Test + void av1FollowsBitrateCap() throws Exception { + PeerConnectionFactory hardware = hardwareFactory(); + + try (Run run = new Run(hardware, AV1, 640, 360)) { + run.awaitFrames(10); + + // The encoder is reconfigured in place, without a new session or + // a lost stream. + run.setMaxBitrate(150_000); + run.awaitTargetBitrate(150_000); + run.awaitMoreFrames(10); + + run.setMaxBitrate(2_000_000); + run.awaitMoreFrames(10); + + assertAv1(run); + } + finally { + hardware.dispose(); + } + } + + @Test + void av1RestartsCleanly() throws Exception { + PeerConnectionFactory hardware = hardwareFactory(); + + try { + // A session left behind by each call would run the hardware out of + // sessions, and the later calls would be encoded by libaom. + for (int i = 0; i < 10; i++) { + try (Run run = new Run(hardware, AV1, 640, 360)) { + run.awaitFrames(5); + + assertAv1(run); + } + } + } + finally { + hardware.dispose(); + } + } + + @Test + void av1BelowHardwareMinimumStaysAv1() throws Exception { + PeerConnectionFactory hardware = hardwareFactory(); + + // Smaller than AMD's AV1 encoder takes (320x128); libaom encodes it + // there, and the stream goes on in AV1 either way. + try (Run run = new Run(hardware, AV1, 160, 90)) { + run.awaitFrames(10); + + assertEquals("video/AV1", run.codec()); + assertEquals(160, run.width.get()); + } + finally { + hardware.dispose(); + } + } + + /** + * Checks that the call is still in AV1 and, where the GPU encodes AV1, is + * encoded in hardware, rather than handed over to libaom; skips the test + * elsewhere, unless hardware is required. + */ + private static void assertAv1(Run run) throws InterruptedException { + assertEquals("video/AV1", run.codec()); + assertHardware(run.implementation(), HARDWARE_AV1_REQUIRED); + } + @Test void hardwareEncodes1080p() throws Exception { assumeNvenc();