diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index 91eafb0c..5f57f9c7 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -27,10 +27,10 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder() | Platform | H.264 | H.265 | AV1 | |---|---|---|---| | Windows | NVENC on NVIDIA GPUs, otherwise the Media Foundation encoder of the GPU driver (AMD, Intel) | NVENC on NVIDIA GPUs, otherwise Media Foundation on GPUs that encode H.265 | NVENC on NVIDIA RTX 40 and newer, otherwise Media Foundation on GPUs that encode AV1 (e.g. AMD RDNA3 and newer, Intel Arc) | -| Linux | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) where it encodes H.265 | NVENC on NVIDIA RTX 40 and newer | +| Linux | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) where it encodes H.265 | NVENC on NVIDIA RTX 40 and newer, otherwise the VA-API encoder of the GPU driver where it encodes AV1 (e.g. AMD RDNA3 and newer) | | macOS | VideoToolbox, as with `DefaultVideoEncoderFactory` | VideoToolbox, on Macs with a hardware HEVC encoder, such as all with Apple silicon | Software | -Hardware encoders produce a single layer. A stream that asks for SVC, such as an AV1 stream with a scalability mode like `L1T3`, is encoded in software. +Hardware encoders produce a single layer. A stream that asks for SVC, such as an AV1 stream with a scalability mode like `L1T3`, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom. VP9 can be encoded in hardware too, with the VP9 encoder of the GPU driver, which Intel GPUs have: through VA-API on Linux, and through Media Foundation on Windows. On Windows it is off by default, since it has not been tested on such a GPU yet; turn it on with a system property before the factory is created: diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h new file mode 100644 index 00000000..9e5dbbad --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h @@ -0,0 +1,117 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_AV1_ENCODER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_AV1_ENCODER_H_ + +#include "media/video/codec/EncoderOutputProcessor.h" +#include "media/video/codec/linux/VaapiEncodeSession.h" +#include "media/video/codec/linux/VaapiLibrary.h" + +#include "api/video/video_frame.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" + +#include +#include + +#include +#include +#include + +namespace jni +{ + // Encodes AV1 profile 0 with the GPU driver's VA-API encoder, as AMD + // drivers provide it on Linux. + // + // It works as FFmpeg's VA-API encoder does without B-frames: synchronous, + // one tile, inter frames that refer to the frame before them only, which + // alternate between two reference slots. The sequence and frame headers + // are written here and handed to the driver as packed headers, with where + // the quantizer, loop filter and CDEF parameters are in the frame header, + // which the driver's rate control fills in. Anything that fails is an + // error, upon which libaom takes over. + class VaapiAv1Encoder : public webrtc::VideoEncoder + { + public: + VaapiAv1Encoder(VaapiLibrary & library, VAEntrypoint entrypoint, const webrtc::SdpVideoFormat & format); + ~VaapiAv1Encoder() override; + + int32_t InitEncode(const webrtc::VideoCodec * codecSettings, const Settings & settings) override; + int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback * callback) override; + int32_t Release() override; + int32_t Encode(const webrtc::VideoFrame & frame, const std::vector * frameTypes) override; + void SetRates(const RateControlParameters & parameters) override; + EncoderInfo GetEncoderInfo() const override; + + private: + // Where the driver rewrites the frame header: the bit offsets of + // base_q_idx, loop_filter_params() and cdef_params(), the size of + // the latter, and the size of the whole frame header OBU. + struct FrameHeaderLayout + { + uint32_t qindexOffset = 0; + uint32_t loopFilterOffset = 0; + uint32_t cdefOffset = 0; + uint32_t cdefSize = 0; + uint32_t size = 0; + }; + + // Reads which headers the driver takes and how it codes them. + void QueryCapabilities(); + + bool Submit(bool keyFrame); + + void WriteSequenceHeader(std::vector & obu) const; + void WriteFrameHeader(std::vector & obu, FrameHeaderLayout & layout, bool keyFrame) const; + // The seq_level_idx the stream's size, frame rate and bitrate fit. + uint8_t Level() const; + + private: + VaapiLibrary & library; + const VAEntrypoint entrypoint; + const std::string implementationName; + VaapiEncodeSession session; + + bool initialized; + webrtc::VideoCodec codecSettings; + uint32_t bitrateBps; + uint32_t framerate; + bool ratesChanged; + + // The fixed length of obu_size the driver takes, in bytes, zero + // where it takes no packed headers; and the TxMode it encodes. + uint32_t obuSizeLength; + uint32_t txMode; + // The sequence header, kept for the key frames. + std::vector sequenceHeader; + + // Which of the reconstructed surfaces, and so which reference slot, + // holds the current frame. + int current; + bool referenceValid; + // The frames since the last key frame, whose low bits are the + // order hint. + uint32_t frameNumber; + + webrtc::EncodedImageCallback * callback; + + EncoderOutputProcessor outputProcessor; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoEncoderFactory.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoEncoderFactory.h index 29919bf9..1e1cc76e 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoEncoderFactory.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoEncoderFactory.h @@ -32,8 +32,8 @@ namespace jni // Creates VA-API encoders on the GPU whose driver encodes H.264, as Intel // and AMD drivers do. It offers H.264 in the profiles WebRTC's software // encoder offers too, and only with packetization mode 1: mode 0 needs - // each NAL unit to fit a packet. It offers H.265 Main and VP9 profile 0 - // as well where the driver encodes them. + // each NAL unit to fit a packet. It offers H.265 Main, VP9 profile 0 and + // AV1 profile 0 as well where the driver encodes them. class VaapiVideoEncoderFactory : public webrtc::VideoEncoderFactory { public: @@ -49,7 +49,7 @@ namespace jni private: VaapiVideoEncoderFactory(VaapiLibrary & library, bool h265, VAEntrypoint h265Entrypoint, bool vp9, - VAEntrypoint vp9Entrypoint); + VAEntrypoint vp9Entrypoint, bool av1, VAEntrypoint av1Entrypoint); private: VaapiLibrary & library; @@ -57,6 +57,8 @@ namespace jni const VAEntrypoint h265Entrypoint; const bool vp9; const VAEntrypoint vp9Entrypoint; + const bool av1; + const VAEntrypoint av1Entrypoint; }; } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp new file mode 100644 index 00000000..277bc3b2 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp @@ -0,0 +1,748 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/linux/VaapiAv1Encoder.h" + +#include "api/video/encoded_image.h" +#include "api/video_codecs/scalability_mode.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" + +#include + +namespace jni +{ + namespace + { + // Key frames come when WebRTC asks for them; this only bounds how + // long a stream may go without one if it never does. + constexpr uint32_t kKeyFrameInterval = 3000; + + // Superblocks of 64x64; surfaces are kept to whole superblocks, as + // FFmpeg's VA-API encoder does. + constexpr uint32_t kSuperblockSize = 64; + + // AMD's driver codes the width in multiples of 8 and the height in + // multiples of 2, and pads the picture to that without a render size, + // so that a decoder would show the padding. The encoder asks WebRTC + // for frames of that size. + constexpr uint32_t kSizeAlignment = 8; + + // The largest tile, section A.3 of the AV1 specification. A frame + // that needs more than one is left to libaom. + constexpr uint32_t kMaxTileWidth = 4096; + constexpr uint32_t kMaxTileArea = 4096 * 2304; + + // OBU types and frame types, sections 6.2.2 and 6.8.2. + constexpr uint8_t kObuSequenceHeader = 1; + constexpr uint8_t kObuFrameHeader = 3; + constexpr uint32_t kKeyFrameType = 0; + constexpr uint32_t kInterFrameType = 1; + + constexpr int kRefsPerFrame = 7; + constexpr uint8_t kPrimaryRefNone = 7; + constexpr uint32_t kLastFrame = 1; + constexpr uint32_t kOrderHintBits = 8; + + // TxMode: largest, or chosen per block. + constexpr uint32_t kTxModeLargest = 1; + constexpr uint32_t kTxModeSelect = 2; + // The bits of VAConfigAttribValEncAV1Ext2::tx_mode_support. + constexpr uint32_t kTxModeLargestSupported = 0x02; + constexpr uint32_t kTxModeSelectSupported = 0x04; + + // The quantizer index the frame header starts from, FFmpeg's; the + // rate control sets that of each frame, within these bounds. + constexpr uint8_t kQIndex = 128; + constexpr uint8_t kMinQIndex = 1; + constexpr uint8_t kMaxQIndex = 255; + + constexpr uint32_t kPackedHeaders = VA_ENC_PACKED_HEADER_SEQUENCE | VA_ENC_PACKED_HEADER_PICTURE; + + // The levels of annex A.3: seq_level_idx, the largest picture, the + // largest width and height, the largest display rate in samples per + // second and the largest bitrate of the main tier. + struct Level + { + uint8_t index; + uint64_t maxPictureSize; + uint32_t maxWidth; + uint32_t maxHeight; + uint64_t maxDisplayRate; + uint64_t maxBitrate; + }; + + constexpr Level kLevels[] = { + { 0, 147456, 2048, 1152, 4423680, 1500000 }, + { 1, 278784, 2816, 1584, 8363520, 3000000 }, + { 4, 665856, 4352, 2448, 19975680, 6000000 }, + { 5, 1065024, 5504, 3096, 31950720, 10000000 }, + { 8, 2359296, 6144, 3456, 70778880, 12000000 }, + { 9, 2359296, 6144, 3456, 141557760, 20000000 }, + { 12, 8912896, 8192, 4352, 267386880, 30000000 }, + { 13, 8912896, 8192, 4352, 534773760, 40000000 }, + { 14, 8912896, 8192, 4352, 1069547520, 60000000 }, + { 16, 35651584, 16384, 8704, 1069547520, 60000000 }, + }; + + // The level of a stream that fits none. + constexpr uint8_t kMaxParametersLevel = 31; + + uint32_t Align(uint32_t value, uint32_t alignment) + { + return (value + alignment - 1) / alignment * alignment; + } + + int FloorLog2(uint32_t value) + { + int log = 0; + + while (value > 1) { + value >>= 1; + log++; + } + + return log; + } + + // tile_log2 of section 5.9.15. + int TileLog2(uint32_t blockSize, uint32_t target) + { + int k = 0; + + while ((blockSize << k) < target) { + k++; + } + + return k; + } + + // Writes the bits of an OBU's payload, most significant first. + class BitWriter + { + public: + void Put(uint32_t value, int bits) + { + for (int i = bits - 1; i >= 0; i--) { + PutBit((value >> i) & 1); + } + } + + void PutBit(uint32_t bit) + { + if (count % 8 == 0) { + bytes.push_back(0); + } + if (bit) { + bytes.back() |= static_cast(0x80 >> (count % 8)); + } + + count++; + } + + // trailing_bits, section 5.3.4: a one, then zeros to the next + // byte. + void TrailingBits() + { + PutBit(1); + + while (count % 8 != 0) { + PutBit(0); + } + } + + uint32_t Position() const + { + return count; + } + + const std::vector & Bytes() const + { + return bytes; + } + + private: + std::vector bytes; + uint32_t count = 0; + }; + + // An OBU of the payload: the header, without an extension, and + // obu_size as a leb128 of the given fixed length, which is what the + // driver rewrites the size of the frame header OBU as. + std::vector MakeObu(uint8_t type, const std::vector & payload, uint32_t sizeLength) + { + std::vector obu; + obu.push_back(static_cast((type << 3) | 0x02)); + + const uint32_t size = static_cast(payload.size()); + + for (uint32_t i = 0; i < sizeLength; i++) { + uint8_t byte = static_cast((size >> (7 * i)) & 0x7F); + + if (i + 1 < sizeLength) { + byte |= 0x80; + } + + obu.push_back(byte); + } + + obu.insert(obu.end(), payload.begin(), payload.end()); + + return obu; + } + } + + VaapiAv1Encoder::VaapiAv1Encoder(VaapiLibrary & library, VAEntrypoint entrypoint, + const webrtc::SdpVideoFormat & format) : + library(library), + entrypoint(entrypoint), + implementationName("VA-API (" + library.Vendor() + ")"), + session(library, "VA-API AV1"), + initialized(false), + codecSettings(), + bitrateBps(0), + framerate(30), + ratesChanged(false), + obuSizeLength(0), + txMode(0), + current(0), + referenceValid(false), + frameNumber(0), + callback(nullptr), + outputProcessor(webrtc::kVideoCodecAV1, format) + { + QueryCapabilities(); + } + + VaapiAv1Encoder::~VaapiAv1Encoder() + { + Release(); + } + + void VaapiAv1Encoder::QueryCapabilities() + { + VAConfigAttrib attributes[2] = {}; + attributes[0].type = VAConfigAttribEncPackedHeaders; + attributes[1].type = VAConfigAttribEncAV1Ext2; + + if (library.Api().GetConfigAttributes(library.Display(), VAProfileAV1Profile0, entrypoint, attributes, 2) != + VA_STATUS_SUCCESS) + { + return; + } + + // The headers are written here; a driver that does not take them is + // not used. + if (attributes[0].value == VA_ATTRIB_NOT_SUPPORTED || (attributes[0].value & kPackedHeaders) != kPackedHeaders) { + return; + } + if (attributes[1].value == VA_ATTRIB_NOT_SUPPORTED) { + return; + } + + VAConfigAttribValEncAV1Ext2 ext2; + ext2.value = attributes[1].value; + + // Coded lossless frames, which ONLY_4X4 is for, are not made. + if (ext2.bits.tx_mode_support & kTxModeSelectSupported) { + txMode = kTxModeSelect; + } + else if (ext2.bits.tx_mode_support & kTxModeLargestSupported) { + txMode = kTxModeLargest; + } + else { + return; + } + + obuSizeLength = ext2.bits.obu_size_bytes_minus1 + 1; + } + + int32_t VaapiAv1Encoder::InitEncode(const webrtc::VideoCodec * settings, const Settings & encoderSettings) + { + if (settings == nullptr || settings->width == 0 || settings->height == 0) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (settings->numberOfSimulcastStreams > 1) { + return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED; + } + // A single layer; spatial or temporal layers are left to libaom. + if (settings->GetScalabilityMode().value_or(webrtc::ScalabilityMode::kL1T1) != webrtc::ScalabilityMode::kL1T1) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (obuSizeLength == 0) { + RTC_LOG(LS_WARNING) << implementationName << " AV1: the driver takes no packed headers, or does not say " + "how it codes them"; + return WEBRTC_VIDEO_CODEC_ERROR; + } + if (settings->width < kSizeAlignment || settings->height < kSizeAlignment) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + // One tile, which is all this encoder writes. + if (settings->width > kMaxTileWidth || uint64_t(Align(settings->width, kSuperblockSize)) + * Align(settings->height, kSuperblockSize) > kMaxTileArea) + { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + Release(); + + // The first frames of a source can come before WebRTC knows the + // alignment the encoder asks for, so the encoder takes the aligned + // size below and drops the last rows and columns of such frames. + codecSettings = *settings; + codecSettings.width -= codecSettings.width % kSizeAlignment; + codecSettings.height -= codecSettings.height % kSizeAlignment; + bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; + framerate = std::max(1u, codecSettings.maxFramerate); + ratesChanged = true; + current = 0; + referenceValid = false; + frameNumber = 0; + outputProcessor.Reset(); + + const uint32_t surfaceWidth = Align(codecSettings.width, kSuperblockSize); + const uint32_t surfaceHeight = Align(codecSettings.height, kSuperblockSize); + + if (!session.Create(VAProfileAV1Profile0, entrypoint, codecSettings.width, codecSettings.height, surfaceWidth, + surfaceHeight, kPackedHeaders)) + { + Release(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + + WriteSequenceHeader(sequenceHeader); + + initialized = true; + + RTC_LOG(LS_INFO) << implementationName << " AV1 initialized: " << codecSettings.width << "x" + << codecSettings.height << " at " << bitrateBps << " bps"; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VaapiAv1Encoder::RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback * encodeCallback) + { + callback = encodeCallback; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VaapiAv1Encoder::Release() + { + session.Destroy(); + + initialized = false; + referenceValid = false; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VaapiAv1Encoder::Encode(const webrtc::VideoFrame & frame, + const std::vector * frameTypes) + { + if (!initialized || callback == nullptr) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + + bool keyFrame = !referenceValid || frameNumber >= kKeyFrameInterval; + + if (frameTypes != nullptr) { + keyFrame |= std::any_of(frameTypes->begin(), frameTypes->end(), [](webrtc::VideoFrameType type) { + return type == webrtc::VideoFrameType::kVideoFrameKey; + }); + } + + if (keyFrame) { + frameNumber = 0; + } + + std::vector output; + + // A frame up to the alignment larger is cropped; one of another size, + // which raced with a change of size, is scaled. + const bool encoded = session.Upload(frame, static_cast(codecSettings.width), + static_cast(codecSettings.height), kSizeAlignment - 1) && Submit(keyFrame) + && session.ReadOutput(output); + + session.DropBuffers(); + + if (!encoded) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + webrtc::EncodedImage image; + image._encodedWidth = codecSettings.width; + image._encodedHeight = codecSettings.height; + image.SetRtpTimestamp(frame.rtp_timestamp()); + image.capture_time_ms_ = frame.render_time_ms(); + image.ntp_time_ms_ = frame.ntp_time_ms(); + image.rotation_ = frame.rotation(); + + webrtc::CodecSpecificInfo info; + + if (!outputProcessor.Process(output, keyFrame, image, info)) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // The frame just encoded is the reference of the next one. + referenceValid = true; + current = 1 - current; + frameNumber++; + ratesChanged = false; + + callback->OnEncodedImage(image, &info); + + return WEBRTC_VIDEO_CODEC_OK; + } + + bool VaapiAv1Encoder::Submit(bool keyFrame) + { + const uint32_t sbCols = Align(codecSettings.width, kSuperblockSize) / kSuperblockSize; + const uint32_t sbRows = Align(codecSettings.height, kSuperblockSize) / kSuperblockSize; + // The slot of the frame this one refers to, which the reconstructed + // surfaces stand for. + const int referenceSlot = 1 - current; + + if (keyFrame) { + VAEncSequenceParameterBufferAV1 sequence = {}; + sequence.seq_profile = 0; + sequence.seq_level_idx = Level(); + sequence.seq_tier = 0; + // Key frames only where asked for. + sequence.intra_period = kKeyFrameInterval; + sequence.ip_period = 1; + sequence.bits_per_second = bitrateBps; + sequence.seq_fields.bits.enable_order_hint = 1; + sequence.seq_fields.bits.enable_cdef = 1; + sequence.order_hint_bits_minus_1 = kOrderHintBits - 1; + + if (!session.AddBuffer(VAEncSequenceParameterBufferType, &sequence, sizeof(sequence))) { + return false; + } + } + + // No initial quantizer: the rate control picks its own, as FFmpeg + // leaves it. + if ((keyFrame || ratesChanged) && !session.AddRateControl(bitrateBps, framerate, 0)) { + return false; + } + + std::vector frameHeader; + FrameHeaderLayout layout; + WriteFrameHeader(frameHeader, layout, keyFrame); + + VAEncPictureParameterBufferAV1 picture = {}; + picture.frame_width_minus_1 = static_cast(codecSettings.width - 1); + picture.frame_height_minus_1 = static_cast(codecSettings.height - 1); + picture.reconstructed_frame = session.Reconstructed(current); + picture.coded_buf = session.CodedBuffer(); + + for (VASurfaceID & reference : picture.reference_frames) { + reference = VA_INVALID_SURFACE; + } + + if (keyFrame) { + picture.primary_ref_frame = kPrimaryRefNone; + picture.refresh_frame_flags = 0xFF; + picture.picture_flags.bits.frame_type = kKeyFrameType; + picture.picture_flags.bits.error_resilient_mode = 1; + } + else { + picture.reference_frames[referenceSlot] = session.Reconstructed(referenceSlot); + + for (int i = 0; i < kRefsPerFrame; i++) { + picture.ref_frame_idx[i] = static_cast(referenceSlot); + } + + // Every reference frame names the same slot, so any of them is + // the primary one; FFmpeg takes the index of the slot. + picture.primary_ref_frame = static_cast(referenceSlot); + picture.refresh_frame_flags = static_cast(1 << current); + picture.ref_frame_ctrl_l0.fields.search_idx0 = kLastFrame; + picture.picture_flags.bits.frame_type = kInterFrameType; + } + + picture.order_hint = static_cast(frameNumber); + picture.base_qindex = kQIndex; + picture.min_base_qindex = kMinQIndex; + picture.max_base_qindex = kMaxQIndex; + picture.mode_control_flags.bits.tx_mode = txMode; + + picture.tile_cols = 1; + picture.tile_rows = 1; + picture.width_in_sbs_minus_1[0] = static_cast(sbCols - 1); + picture.height_in_sbs_minus_1[0] = static_cast(sbRows - 1); + picture.tile_group_obu_hdr_info.bits.obu_has_size_field = 1; + + // Where the driver writes what its rate control decides, and the size + // of the frame header OBU, which comes after the sequence header in + // a key frame. + picture.bit_offset_qindex = layout.qindexOffset; + picture.bit_offset_loopfilter_params = layout.loopFilterOffset; + picture.bit_offset_cdef_params = layout.cdefOffset; + picture.size_in_bits_cdef_params = layout.cdefSize; + picture.size_in_bits_frame_hdr_obu = layout.size; + picture.byte_offset_frame_hdr_obu_size = (keyFrame ? static_cast(sequenceHeader.size()) : 0) + 1; + + VAEncTileGroupBufferAV1 tileGroup = {}; + tileGroup.tg_start = 0; + tileGroup.tg_end = 0; + + return session.AddBuffer(VAEncPictureParameterBufferType, &picture, sizeof(picture)) + && (!keyFrame || session.AddPackedHeader(VAEncPackedHeaderSequence, sequenceHeader, sequenceHeader.size() * 8)) + && session.AddPackedHeader(VAEncPackedHeaderPicture, frameHeader, layout.size) + && session.AddBuffer(VAEncSliceParameterBufferType, &tileGroup, sizeof(tileGroup)) + && session.Encode(); + } + + void VaapiAv1Encoder::WriteSequenceHeader(std::vector & obu) const + { + // sequence_header_obu, section 5.5, with the tools FFmpeg's VA-API + // encoder turns on: order hints and CDEF. + const int widthBits = FloorLog2(codecSettings.width) + 1; + const int heightBits = FloorLog2(codecSettings.height) + 1; + const uint8_t level = Level(); + + BitWriter writer; + // seq_profile, still_picture, reduced_still_picture_header. + writer.Put(0, 3); + writer.Put(0, 1); + writer.Put(0, 1); + // timing_info_present_flag, initial_display_delay_present_flag, + // operating_points_cnt_minus_1, operating_point_idc[0]. + writer.Put(0, 1); + writer.Put(0, 1); + writer.Put(0, 5); + writer.Put(0, 12); + // seq_level_idx[0], and seq_tier[0] above level 3.3. + writer.Put(level, 5); + + if (level > 7) { + writer.Put(0, 1); + } + + writer.Put(widthBits - 1, 4); + writer.Put(heightBits - 1, 4); + writer.Put(codecSettings.width - 1, widthBits); + writer.Put(codecSettings.height - 1, heightBits); + // frame_id_numbers_present_flag, use_128x128_superblock, + // enable_filter_intra, enable_intra_edge_filter, + // enable_interintra_compound, enable_masked_compound, + // enable_warped_motion, enable_dual_filter. + writer.Put(0, 8); + // enable_order_hint, enable_jnt_comp, enable_ref_frame_mvs. + writer.Put(1, 1); + writer.Put(0, 1); + writer.Put(0, 1); + // seq_choose_screen_content_tools and seq_force_screen_content_tools: + // no screen content tools, so seq_force_integer_mv is not coded. + writer.Put(0, 1); + writer.Put(0, 1); + writer.Put(kOrderHintBits - 1, 3); + // enable_superres, enable_cdef, enable_restoration. + writer.Put(0, 1); + writer.Put(1, 1); + writer.Put(0, 1); + // color_config, section 5.5.2: 8 bit 4:2:0 without a color + // description: high_bitdepth, mono_chrome, + // color_description_present_flag, color_range, + // chroma_sample_position, separate_uv_delta_q. + writer.Put(0, 1); + writer.Put(0, 1); + writer.Put(0, 1); + writer.Put(0, 1); + writer.Put(0, 2); + writer.Put(0, 1); + // film_grain_params_present. + writer.Put(0, 1); + writer.TrailingBits(); + + obu = MakeObu(kObuSequenceHeader, writer.Bytes(), obuSizeLength); + } + + void VaapiAv1Encoder::WriteFrameHeader(std::vector & obu, FrameHeaderLayout & layout, bool keyFrame) const + { + // uncompressed_header, section 5.9, as FFmpeg's VA-API encoder writes + // it, for the sequence header above. + const int referenceSlot = 1 - current; + const uint32_t miCols = 2 * ((codecSettings.width + 7) >> 3); + const uint32_t miRows = 2 * ((codecSettings.height + 7) >> 3); + const uint32_t sbCols = (miCols + 15) >> 4; + const uint32_t sbRows = (miRows + 15) >> 4; + // The OBU header and obu_size come before the payload. + const uint32_t payloadOffset = (1 + obuSizeLength) * 8; + + BitWriter writer; + // show_existing_frame, frame_type, show_frame. + writer.Put(0, 1); + writer.Put(keyFrame ? kKeyFrameType : kInterFrameType, 2); + writer.Put(1, 1); + + if (!keyFrame) { + // error_resilient_mode; a key frame that is shown has it set. + writer.Put(0, 1); + } + + // disable_cdf_update, frame_size_override_flag, order_hint. + writer.Put(0, 1); + writer.Put(0, 1); + writer.Put(frameNumber & ((1u << kOrderHintBits) - 1), kOrderHintBits); + + if (!keyFrame) { + // primary_ref_frame, refresh_frame_flags. + writer.Put(referenceSlot, 3); + writer.Put(1u << current, 8); + // frame_refs_short_signaling, then ref_frame_idx: every reference + // is the frame before. + writer.Put(0, 1); + + for (int i = 0; i < kRefsPerFrame; i++) { + writer.Put(referenceSlot, 3); + } + } + + // frame_size() takes the size of the sequence header; render_size(): + // render_and_frame_size_different. + writer.Put(0, 1); + + if (!keyFrame) { + // allow_high_precision_mv, is_filter_switchable, + // interpolation_filter (EIGHTTAP), is_motion_mode_switchable. + writer.Put(0, 1); + writer.Put(0, 1); + writer.Put(0, 2); + writer.Put(0, 1); + } + + // disable_frame_end_update_cdf. + writer.Put(0, 1); + + // tile_info, section 5.9.15: one uniform tile, which takes no more + // tile columns or rows where it could. + writer.Put(1, 1); + + if (TileLog2(1, std::min(sbCols, 64u)) > 0) { + writer.Put(0, 1); + } + if (TileLog2(1, std::min(sbRows, 64u)) > 0) { + writer.Put(0, 1); + } + + // quantization_params: base_q_idx, no delta coded for DeltaQYDc, + // DeltaQUDc and DeltaQUAc, using_qmatrix. + layout.qindexOffset = payloadOffset + writer.Position(); + writer.Put(kQIndex, 8); + writer.Put(0, 1); + writer.Put(0, 1); + writer.Put(0, 1); + writer.Put(0, 1); + // segmentation_enabled; delta_q_present, base_q_idx being above 0. + writer.Put(0, 1); + writer.Put(0, 1); + + // loop_filter_params: loop_filter_level[0] and [1] of 0, which leave + // out those of U and V, loop_filter_sharpness, + // loop_filter_delta_enabled. + layout.loopFilterOffset = payloadOffset + writer.Position(); + writer.Put(0, 6); + writer.Put(0, 6); + writer.Put(0, 3); + writer.Put(0, 1); + + // cdef_params: cdef_damping_minus_3, cdef_bits, and the strengths of + // the one filter. + layout.cdefOffset = payloadOffset + writer.Position(); + writer.Put(0, 2); + writer.Put(0, 2); + writer.Put(0, 4); + writer.Put(0, 2); + writer.Put(0, 4); + writer.Put(0, 2); + layout.cdefSize = payloadOffset + writer.Position() - layout.cdefOffset; + + // read_tx_mode: tx_mode_select. + writer.Put(txMode == kTxModeSelect ? 1 : 0, 1); + + if (!keyFrame) { + // reference_select. + writer.Put(0, 1); + } + + // reduced_tx_set. + writer.Put(0, 1); + + if (!keyFrame) { + // global_motion_params: is_global of each reference frame. + writer.Put(0, kRefsPerFrame); + } + + writer.TrailingBits(); + + obu = MakeObu(kObuFrameHeader, writer.Bytes(), obuSizeLength); + layout.size = static_cast(obu.size() * 8); + } + + uint8_t VaapiAv1Encoder::Level() const + { + const uint64_t pictureSize = uint64_t(codecSettings.width) * codecSettings.height; + const uint64_t displayRate = pictureSize * std::max(1u, codecSettings.maxFramerate); + const uint64_t bitrate = uint64_t(std::max(codecSettings.maxBitrate, codecSettings.startBitrate)) * 1000; + + for (const auto & level : kLevels) { + if (pictureSize <= level.maxPictureSize && codecSettings.width <= level.maxWidth + && codecSettings.height <= level.maxHeight && displayRate <= level.maxDisplayRate + && bitrate <= level.maxBitrate) + { + return level.index; + } + } + + return kMaxParametersLevel; + } + + void VaapiAv1Encoder::SetRates(const RateControlParameters & parameters) + { + const uint32_t bitrate = parameters.bitrate.get_sum_bps(); + + // A zero bitrate pauses the stream; the encoder gets no frames then. + if (bitrate == 0) { + return; + } + + bitrateBps = bitrate; + + if (parameters.framerate_fps >= 1.0) { + framerate = static_cast(parameters.framerate_fps + 0.5); + } + + // Applied with the next frame. + ratesChanged = true; + } + + webrtc::VideoEncoder::EncoderInfo VaapiAv1Encoder::GetEncoderInfo() const + { + EncoderInfo info; + info.implementation_name = implementationName; + info.is_hardware_accelerated = true; + info.supports_native_handle = false; + info.supports_simulcast = false; + info.scaling_settings = outputProcessor.GetScalingSettings(); + info.requested_resolution_alignment = kSizeAlignment; + + return info; + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp index 9e3ca2ed..4bc6a9c1 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp @@ -15,6 +15,7 @@ */ #include "media/video/codec/linux/VaapiVideoEncoderFactory.h" +#include "media/video/codec/linux/VaapiAv1Encoder.h" #include "media/video/codec/linux/VaapiH264Encoder.h" #include "media/video/codec/linux/VaapiH265Encoder.h" #include "media/video/codec/linux/VaapiVp9Encoder.h" @@ -41,20 +42,25 @@ namespace jni VAEntrypoint vp9Entrypoint = VAEntrypointEncSlice; const bool vp9 = library->EncodeEntrypoint(VAProfileVP9Profile0, &vp9Entrypoint); + VAEntrypoint av1Entrypoint = VAEntrypointEncSlice; + const bool av1 = library->EncodeEntrypoint(VAProfileAV1Profile0, &av1Entrypoint); + RTC_LOG(LS_INFO) << "VA-API (" << library->Vendor() << ") H.265 encoding: " << h265 << ", VP9 encoding: " - << vp9; + << vp9 << ", AV1 encoding: " << av1; - return std::unique_ptr( - new VaapiVideoEncoderFactory(*library, h265, h265Entrypoint, vp9, vp9Entrypoint)); + return std::unique_ptr(new VaapiVideoEncoderFactory(*library, h265, h265Entrypoint, + vp9, vp9Entrypoint, av1, av1Entrypoint)); } VaapiVideoEncoderFactory::VaapiVideoEncoderFactory(VaapiLibrary & library, bool h265, VAEntrypoint h265Entrypoint, - bool vp9, VAEntrypoint vp9Entrypoint) : + bool vp9, VAEntrypoint vp9Entrypoint, bool av1, VAEntrypoint av1Entrypoint) : library(library), h265(h265), h265Entrypoint(h265Entrypoint), vp9(vp9), - vp9Entrypoint(vp9Entrypoint) + vp9Entrypoint(vp9Entrypoint), + av1(av1), + av1Entrypoint(av1Entrypoint) { } @@ -71,6 +77,9 @@ namespace jni if (vp9) { formats.push_back(webrtc::SdpVideoFormat::VP9Profile0()); } + if (av1) { + formats.push_back(webrtc::SdpVideoFormat::AV1Profile0()); + } return formats; } @@ -86,6 +95,9 @@ namespace jni if (codec == webrtc::kVideoCodecVP9) { return std::make_unique(library, vp9Entrypoint, format); } + if (codec == webrtc::kVideoCodecAV1) { + return std::make_unique(library, av1Entrypoint, format); + } return std::make_unique(library, format); } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java index 50adfc14..88d922aa 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java @@ -37,8 +37,8 @@ * H.264 is encoded on an NVIDIA GPU with NVENC, on Windows and Linux. On * other GPUs it is encoded with the Media Foundation encoder of the driver on * Windows, and with its VA-API encoder on Linux. AV1 is encoded on GPUs that - * have an AV1 encoder, with NVENC or Media Foundation, as a single layer; a - * stream that asks for SVC is encoded in software. H.265 (Main profile) is + * have an AV1 encoder, with NVENC, Media Foundation or VA-API, as a single + * layer; a stream that asks for SVC is encoded in software. H.265 (Main profile) is * encoded with NVENC or Media Foundation on GPUs that have an H.265 encoder. * VP9 (profile 0) is encoded on GPUs that have a VP9 encoder, which are * Intel's: with VA-API on Linux, and with Media Foundation on Windows only diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java index 9d73eea6..c2a7f2f4 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java @@ -438,6 +438,115 @@ void hardwareEncodesAv1() throws Exception { } } + @Test + void av1EncodesOddSize() throws Exception { + PeerConnectionFactory hardware = hardwareFactory(); + + // AMD's encoder pads the width to a multiple of 8, so the encoder asks + // for frames of that alignment, rather than sending the padding. + try (Run run = new Run(hardware, AV1, 641, 361)) { + run.awaitFrames(10); + + assertAv1(run); + assertEquals(0, run.width.get() % 8, "width " + run.width.get()); + } + finally { + hardware.dispose(); + } + } + + @Test + void av1FollowsResolutionChange() throws Exception { + PeerConnectionFactory hardware = hardwareFactory(); + + try (Run run = new Run(hardware, AV1, 640, 360)) { + run.awaitFrames(10); + + // A new size starts the encoder over, with a new session and a + // key frame with a new sequence header. + run.call.setVideoSize(960, 540); + run.awaitWidth(960); + + run.call.setVideoSize(640, 360); + run.awaitWidth(640); + + assertAv1(run); + } + finally { + hardware.dispose(); + } + } + + @Test + void av1FollowsBitrateCap() throws Exception { + PeerConnectionFactory hardware = hardwareFactory(); + + try (Run run = new Run(hardware, AV1, 640, 360)) { + run.awaitFrames(10); + + // The encoder is reconfigured in place, without a new session or + // a lost stream. + run.setMaxBitrate(150_000); + run.awaitTargetBitrate(150_000); + run.awaitMoreFrames(10); + + run.setMaxBitrate(2_000_000); + run.awaitMoreFrames(10); + + assertAv1(run); + } + finally { + hardware.dispose(); + } + } + + @Test + void av1RestartsCleanly() throws Exception { + PeerConnectionFactory hardware = hardwareFactory(); + + try { + // A session left behind by each call would run the hardware out of + // sessions, and the later calls would be encoded by libaom. + for (int i = 0; i < 10; i++) { + try (Run run = new Run(hardware, AV1, 640, 360)) { + run.awaitFrames(5); + + assertAv1(run); + } + } + } + finally { + hardware.dispose(); + } + } + + @Test + void av1BelowHardwareMinimumStaysAv1() throws Exception { + PeerConnectionFactory hardware = hardwareFactory(); + + // Smaller than AMD's AV1 encoder takes (320x128); libaom encodes it + // there, and the stream goes on in AV1 either way. + try (Run run = new Run(hardware, AV1, 160, 90)) { + run.awaitFrames(10); + + assertEquals("video/AV1", run.codec()); + assertEquals(160, run.width.get()); + } + finally { + hardware.dispose(); + } + } + + /** + * Checks that the call is still in AV1 and, where the GPU encodes AV1, is + * encoded in hardware, rather than handed over to libaom; skips the test + * elsewhere, unless hardware is required. + */ + private static void assertAv1(Run run) throws InterruptedException { + assertEquals("video/AV1", run.codec()); + assertHardware(run.implementation(), HARDWARE_AV1_REQUIRED); + } + @Test void hardwareEncodes1080p() throws Exception { assumeNvenc();