From 9a9ac78e8a380e6d4c37ee42add481d907f1ad30 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Thu, 15 Oct 2026 16:33:01 +0200 Subject: [PATCH] feat: decode AV1 with VA-API on Linux Adds an AV1 decoder for GPUs whose VA-API driver decodes AV1 profile 0, such as AMD's, with its own parser for the sequence and frame headers. Film grain, superres, spatial layers and other bit depths go to dav1d. --- docs/guide/advanced/video-codecs.md | 4 +- .../media/video/codec/linux/Av1Parser.h | 327 ++++ .../media/video/codec/linux/VaapiAv1Decoder.h | 130 ++ .../codec/linux/VaapiVideoDecoderFactory.h | 7 +- .../src/media/video/codec/linux/Av1Parser.cpp | 1619 +++++++++++++++++ .../LinuxHardwareVideoCodecFactories.cpp | 2 +- .../video/codec/linux/VaapiAv1Decoder.cpp | 716 ++++++++ .../codec/linux/VaapiVideoDecoderFactory.cpp | 19 +- .../codec/HardwareVideoDecoderFactory.java | 6 +- .../HardwareVideoDecoderIntegrationTest.java | 41 +- 10 files changed, 2856 insertions(+), 15 deletions(-) create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/linux/Av1Parser.h create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Decoder.h create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/linux/Av1Parser.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Decoder.cpp diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index 0338fcb8..57146bd1 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -62,10 +62,10 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder() | Platform | Hardware decoding | |---|---| | Windows | H.264, H.265 (Main profile), AV1 and VP9 (profile 0), on GPUs that decode them, through the Media Foundation decoders of Windows on Direct3D 11 (DXVA) | -| Linux | H.264, H.265 (Main profile), VP9 and AV1 (profile 0) on NVIDIA GPUs that decode them, through NVDEC; VP9 (profile 0) and H.265 (Main profile) on Intel and AMD GPUs through VA-API | +| Linux | H.264, H.265 (Main profile), VP9 and AV1 (profile 0) on NVIDIA GPUs that decode them, through NVDEC; VP9 (profile 0), H.265 (Main profile) and AV1 (profile 0) on Intel and AMD GPUs through VA-API, where the driver decodes them | | macOS | H.264 through VideoToolbox, as with `DefaultVideoDecoderFactory`; VP9 (profile 0) and H.265 (Main profile) through the decoders of VideoToolbox, on Macs that have them, and AV1 (profile 0) on Apple M3 and newer | -On Linux, NVDEC needs an NVIDIA driver, which provides `libcuda.so.1` and `libnvcuvid.so.1`; both are loaded when the factory is made, so a machine without the driver is not affected. Streams with spatial layers are decoded in software. +On Linux, NVDEC needs an NVIDIA driver, which provides `libcuda.so.1` and `libnvcuvid.so.1`; both are loaded when the factory is made, so a machine without the driver is not affected. Streams with spatial layers are decoded in software. Through VA-API, AV1 is decoded in 8 bit 4:2:0; streams with film grain, superres, a higher bit depth or a single plane are decoded by dav1d. On Windows and Linux, H.264 is decoded in hardware in its 4:2:0 profiles. High 4:4:4 Predictive, which GPU decoders do not take, is decoded in software. diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/Av1Parser.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/Av1Parser.h new file mode 100644 index 00000000..eb84bfac --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/Av1Parser.h @@ -0,0 +1,327 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_AV1_PARSER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_AV1_PARSER_H_ + +#include +#include +#include +#include +#include + +namespace jni +{ + // The OBU types of section 6.2.2 of the AV1 specification. + enum class Av1ObuType : uint8_t + { + SequenceHeader = 1, + TemporalDelimiter = 2, + FrameHeader = 3, + TileGroup = 4, + Metadata = 5, + Frame = 6, + RedundantFrameHeader = 7, + TileList = 8, + Padding = 15 + }; + + // The frame types of section 6.8.2. + enum Av1FrameType : uint8_t + { + kAv1KeyFrame = 0, + kAv1InterFrame = 1, + kAv1IntraOnlyFrame = 2, + kAv1SwitchFrame = 3 + }; + + // One OBU of a temporal unit, its payload a span of the data it came in. + struct Av1Obu + { + Av1ObuType type = Av1ObuType::Padding; + bool hasExtension = false; + uint8_t temporalId = 0; + uint8_t spatialId = 0; + std::span payload; + }; + + // The sequence header OBU, as section 5.5 defines it, with what a decoder + // of operating point 0 needs. + struct Av1SequenceHeader + { + uint8_t profile = 0; + bool stillPicture = false; + bool reducedStillPictureHeader = false; + + bool decoderModelInfoPresent = false; + bool equalPictureInterval = false; + uint8_t bufferRemovalTimeLength = 0; + uint8_t framePresentationTimeLength = 0; + uint8_t operatingPointCount = 1; + std::array operatingPointIdc = {}; + std::array decoderModelPresentForOperatingPoint = {}; + + uint8_t frameWidthBits = 0; + uint8_t frameHeightBits = 0; + uint32_t maxFrameWidth = 0; + uint32_t maxFrameHeight = 0; + + bool frameIdNumbersPresent = false; + uint8_t deltaFrameIdLength = 0; + uint8_t frameIdLength = 0; + + bool use128x128Superblock = false; + bool enableFilterIntra = false; + bool enableIntraEdgeFilter = false; + bool enableInterintraCompound = false; + bool enableMaskedCompound = false; + bool enableWarpedMotion = false; + bool enableDualFilter = false; + bool enableOrderHint = false; + bool enableJntComp = false; + bool enableRefFrameMvs = false; + // 0, 1, or 2 for a choice made by each frame. + uint8_t seqForceScreenContentTools = 2; + uint8_t seqForceIntegerMv = 2; + uint8_t orderHintBits = 0; + + bool enableSuperres = false; + bool enableCdef = false; + bool enableRestoration = false; + + uint8_t bitDepth = 8; + bool monochrome = false; + uint8_t colorPrimaries = 2; + uint8_t transferCharacteristics = 2; + uint8_t matrixCoefficients = 2; + bool colorRange = false; + bool subsamplingX = true; + bool subsamplingY = true; + bool separateUvDeltaQ = false; + + bool filmGrainParamsPresent = false; + }; + + // The frame header OBU, the uncompressed header of section 5.9, with what + // earlier frames left that it depends on resolved: the loop filter deltas, + // the segmentation and the global motion that carry over from the primary + // reference frame, and the size taken from a reference frame. + struct Av1FrameHeader + { + bool showExistingFrame = false; + uint8_t frameToShow = 0; + + uint8_t frameType = kAv1KeyFrame; + bool frameIsIntra = true; + bool showFrame = true; + bool showableFrame = false; + bool errorResilientMode = false; + bool disableCdfUpdate = false; + bool allowScreenContentTools = false; + bool forceIntegerMv = false; + uint32_t currentFrameId = 0; + uint8_t orderHint = 0; + // 7 for none. + uint8_t primaryRefFrame = 7; + uint8_t refreshFrameFlags = 0; + + // FrameWidth is the coded width, UpscaledWidth the width with + // superres; they differ only with superres. + uint32_t frameWidth = 0; + uint32_t frameHeight = 0; + uint32_t upscaledWidth = 0; + uint32_t renderWidth = 0; + uint32_t renderHeight = 0; + bool useSuperres = false; + uint8_t superresDenom = 8; + uint32_t miCols = 0; + uint32_t miRows = 0; + + bool allowIntrabc = false; + std::array refFrameIdx = {}; + bool allowHighPrecisionMv = false; + // 0 to 3, or 4 for switchable. + uint8_t interpolationFilter = 0; + bool isMotionModeSwitchable = false; + bool useRefFrameMvs = false; + bool disableFrameEndUpdateCdf = true; + + bool uniformTileSpacing = true; + uint8_t tileCols = 1; + uint8_t tileRows = 1; + uint8_t tileColsLog2 = 0; + uint8_t tileRowsLog2 = 0; + // The width and height of each tile column and row in superblocks. + std::array tileWidthSbs = {}; + std::array tileHeightSbs = {}; + uint16_t contextUpdateTileId = 0; + uint8_t tileSizeBytes = 4; + + uint8_t baseQIndex = 0; + int8_t deltaQYDc = 0; + int8_t deltaQUDc = 0; + int8_t deltaQUAc = 0; + int8_t deltaQVDc = 0; + int8_t deltaQVAc = 0; + bool usingQmatrix = false; + uint8_t qmY = 0; + uint8_t qmU = 0; + uint8_t qmV = 0; + + bool segmentationEnabled = false; + bool segmentationUpdateMap = false; + bool segmentationTemporalUpdate = false; + bool segmentationUpdateData = false; + bool featureEnabled[8][8] = {}; + int16_t featureData[8][8] = {}; + + bool deltaQPresent = false; + uint8_t deltaQRes = 0; + bool deltaLfPresent = false; + uint8_t deltaLfRes = 0; + bool deltaLfMulti = false; + + bool codedLossless = false; + bool allLossless = false; + + // Luma vertical and horizontal, then U and V. + std::array loopFilterLevel = {}; + uint8_t loopFilterSharpness = 0; + bool loopFilterDeltaEnabled = false; + bool loopFilterDeltaUpdate = false; + std::array loopFilterRefDeltas = {}; + std::array loopFilterModeDeltas = {}; + + uint8_t cdefDampingMinus3 = 0; + uint8_t cdefBits = 0; + // The secondary strengths as coded, 3 standing for 4. + std::array cdefYPriStrength = {}; + std::array cdefYSecStrength = {}; + std::array cdefUvPriStrength = {}; + std::array cdefUvSecStrength = {}; + + // FrameRestorationType: none, Wiener, self-guided, switchable. + std::array frameRestorationType = {}; + uint8_t lrUnitShift = 0; + uint8_t lrUvShift = 0; + + // Only 4x4, largest, or select. + uint8_t txMode = 0; + bool referenceSelect = false; + bool skipModePresent = false; + bool allowWarpedMotion = false; + bool reducedTxSet = false; + + // By reference frame, LAST_FRAME (1) to ALTREF_FRAME (7): identity, + // translation, rotation and zoom, or affine; and whether the shear + // of the warp is valid. + std::array gmType = {}; + std::array, 8> gmParams = {}; + std::array gmValid = {}; + + // Film grain applied to the output, which is not parsed further. + bool applyGrain = false; + + // The size of the uncompressed header in bytes, after which the tile + // group of a frame OBU starts. + uint32_t headerBytes = 0; + }; + + // One tile of a frame, where its data is in the tile group it came in. + struct Av1Tile + { + uint16_t row = 0; + uint16_t column = 0; + uint32_t offset = 0; + uint32_t size = 0; + }; + + // The tiles of one tile group OBU. + struct Av1TileGroup + { + uint16_t start = 0; + uint16_t end = 0; + std::vector tiles; + }; + + // Parses the sequence and frame headers of an AV1 stream, in the order + // its frames are decoded, keeping what carries over from one frame to the + // next in the eight reference slots. + class Av1Parser + { + public: + Av1Parser(); + + // Forgets the sequence header and the reference frames, as before a + // new stream. + void Reset(); + + // The OBUs of a temporal unit, or nothing if they do not fit into + // the data. Takes the OBUs of operating point 0 only, where the + // sequence header is known. + std::optional> SplitObus(std::span data) const; + + // Parses a sequence header OBU. Returns false if it is not valid. + bool ParseSequenceHeader(std::span payload); + + const std::optional & SequenceHeader() const; + + // Parses a frame header OBU, or the header of a frame OBU. Returns + // nothing if it is not valid, or refers to a reference frame that + // is not there. + std::optional ParseFrameHeader(std::span payload, const Av1Obu & obu); + + // Parses a tile group OBU of the frame. Returns nothing if it is + // not valid. + std::optional ParseTileGroup(std::span payload, + const Av1FrameHeader & header) const; + + // Takes the frame into the reference slots its refresh flags name, + // once it is decoded, or, for a key frame shown again, all of them; + // section 7.20 and 7.21 of the specification. + void UpdateReferences(const Av1FrameHeader & header); + + private: + // What a reference slot holds of the frame in it. + struct Reference + { + bool valid = false; + uint32_t frameId = 0; + uint32_t upscaledWidth = 0; + uint32_t frameWidth = 0; + uint32_t frameHeight = 0; + uint32_t renderWidth = 0; + uint32_t renderHeight = 0; + uint8_t frameType = kAv1KeyFrame; + uint8_t orderHint = 0; + std::array, 8> gmParams = {}; + std::array loopFilterRefDeltas = {}; + std::array loopFilterModeDeltas = {}; + bool featureEnabled[8][8] = {}; + int16_t featureData[8][8] = {}; + }; + + int RelativeDistance(int a, int b) const; + // What the specification calls set_frame_refs. + bool SetFrameRefs(Av1FrameHeader & header, uint8_t lastFrameIdx, uint8_t goldFrameIdx) const; + + private: + std::optional sequence; + std::array references; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Decoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Decoder.h new file mode 100644 index 00000000..e74b095d --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Decoder.h @@ -0,0 +1,130 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_AV1_DECODER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_AV1_DECODER_H_ + +#include "media/video/codec/linux/Av1Parser.h" +#include "media/video/codec/linux/VaapiLibrary.h" + +#include "api/video/encoded_image.h" +#include "api/video_codecs/video_decoder.h" + +#include +#include + +#include +#include +#include +#include +#include + +namespace jni +{ + // Decodes AV1 profile 0 in 8 bit 4:2:0 with the GPU driver's VA-API + // decoder, as AMD drivers provide it on Linux. + // + // VA-API leaves the parsing of the stream to the application: the + // sequence and frame headers are parsed here, and where each tile is; the + // driver decodes the tiles. Decoding is synchronous. The decoded frame is + // copied from its surface in NV12 and converted to I420, which is what + // WebRTC's frames hold. + // + // Whatever it cannot decode, such as another profile or bit depth, spatial + // layers, film grain, superres or an inter frame that changes the size, + // and whatever fails, returns WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE, so + // that dav1d takes over. + class VaapiAv1Decoder : public webrtc::VideoDecoder + { + public: + explicit VaapiAv1Decoder(VaapiLibrary & library); + ~VaapiAv1Decoder() override; + + using webrtc::VideoDecoder::Decode; + + bool Configure(const Settings & settings) override; + int32_t Decode(const webrtc::EncodedImage & image, int64_t renderTimeMs) override; + int32_t RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback * callback) override; + int32_t Release() override; + DecoderInfo GetDecoderInfo() const override; + const char * ImplementationName() const override; + + private: + // A frame of a temporal unit: its header, and, unless it shows a + // frame again, the data of its tile groups one after another and + // its tiles, with their offsets into that data. + struct Frame + { + Av1FrameHeader header; + std::vector tileData; + std::vector tiles; + }; + + // Parses the OBUs of one temporal unit into its frames, checking + // that all of them can be decoded before any is; false where the + // software decoder has to take over, which decodes the temporal + // unit again, so that nothing may have been output by then. + bool ParseTemporalUnit(std::span data, std::vector & frames); + // Takes a tile group of the frame in. + bool AddTileGroup(std::span payload, Frame & frame); + // Decodes and shows the frames. + bool DecodeFrames(const std::vector & frames, const webrtc::EncodedImage & image, + int64_t renderTimeMs); + + // Creates the configuration, the context and the surfaces for + // frames of the size. + bool CreateSession(uint32_t width, uint32_t height); + void DestroySession(); + + // Decodes the frame into the surface. + bool DecodeFrame(const Frame & frame, int surface); + // Hands the frame in the surface to WebRTC. + bool Output(int surface, const webrtc::EncodedImage & image, int64_t renderTimeMs); + // A surface that no reference slot holds. + int FreeSurface() const; + + bool Check(VAStatus status, const char * operation) const; + + private: + VaapiLibrary & library; + const VaapiFunctions & va; + const std::string implementationName; + + webrtc::DecodedImageCallback * callback; + + Av1Parser parser; + + VAConfigID config; + VAContextID context; + std::vector surfaces; + uint32_t width; + uint32_t height; + + // The image a decoded surface is copied into, where the driver + // copies surfaces at all. + VAImage downloadImage; + + // The surface each of the eight reference slots holds, -1 for none. + std::array slots; + + // Whether a key frame has been decoded, and whether anything since + // then failed for good. + bool started; + bool failed; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoDecoderFactory.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoDecoderFactory.h index 98fff8bf..1e266e96 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoDecoderFactory.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoDecoderFactory.h @@ -29,8 +29,8 @@ namespace jni { - // Creates VA-API decoders on the GPUs of Intel and AMD, for VP9 profile 0 - // and H.265 Main where the driver decodes them. + // Creates VA-API decoders on the GPUs of Intel and AMD, for VP9 profile 0, + // H.265 Main and AV1 profile 0 where the driver decodes them. class VaapiVideoDecoderFactory : public webrtc::VideoDecoderFactory { public: @@ -45,12 +45,13 @@ namespace jni const webrtc::SdpVideoFormat & format) override; private: - VaapiVideoDecoderFactory(VaapiLibrary & library, bool vp9, bool h265); + VaapiVideoDecoderFactory(VaapiLibrary & library, bool vp9, bool h265, bool av1); private: VaapiLibrary & library; const bool vp9; const bool h265; + const bool av1; }; } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/Av1Parser.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/Av1Parser.cpp new file mode 100644 index 00000000..8eeffd73 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/Av1Parser.cpp @@ -0,0 +1,1619 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/linux/Av1Parser.h" + +#include "rtc_base/bitstream_reader.h" + +#include +#include + +namespace jni +{ + namespace + { + constexpr int kRefsPerFrame = 7; + constexpr int kNumRefFrames = 8; + constexpr uint8_t kPrimaryRefNone = 7; + constexpr uint8_t kAllFrames = 0xff; + + // The reference frames of section 6.10.24. + constexpr int kLastFrame = 1; + constexpr int kLast2Frame = 2; + constexpr int kLast3Frame = 3; + constexpr int kGoldenFrame = 4; + constexpr int kBwdrefFrame = 5; + constexpr int kAltref2Frame = 6; + constexpr int kAltrefFrame = 7; + + constexpr uint8_t kSelectScreenContentTools = 2; + constexpr uint8_t kSelectIntegerMv = 2; + + constexpr uint8_t kSuperresNum = 8; + constexpr uint8_t kSuperresDenomMin = 9; + constexpr int kSuperresDenomBits = 3; + + constexpr int kMaxTileWidth = 4096; + constexpr int kMaxTileArea = 4096 * 2304; + constexpr int kMaxTileRows = 64; + constexpr int kMaxTileCols = 64; + + constexpr int kMaxSegments = 8; + constexpr int kSegLvlMax = 8; + constexpr int kSegLvlAltQ = 0; + constexpr int kSegmentationFeatureBits[kSegLvlMax] = { 8, 6, 6, 6, 6, 3, 0, 0 }; + constexpr bool kSegmentationFeatureSigned[kSegLvlMax] = { true, true, true, true, true, false, false, false }; + constexpr int kSegmentationFeatureMax[kSegLvlMax] = { 255, 63, 63, 63, 63, 7, 0, 0 }; + + // Remap_Lr_Type: none, switchable, Wiener, self-guided as coded, to + // FrameRestorationType. + constexpr uint8_t kRemapLrType[4] = { 0, 3, 1, 2 }; + + // The global motion types and parameters of section 5.9.24. + constexpr uint8_t kIdentity = 0; + constexpr uint8_t kTranslation = 1; + constexpr uint8_t kRotzoom = 2; + constexpr uint8_t kAffine = 3; + constexpr int kWarpedModelPrecBits = 16; + constexpr int kGmAbsTransOnlyBits = 9; + constexpr int kGmTransOnlyPrecBits = 3; + constexpr int kGmAbsTransBits = 12; + constexpr int kGmTransPrecBits = 6; + constexpr int kGmAbsAlphaBits = 12; + constexpr int kGmAlphaPrecBits = 15; + constexpr int kWarpParamReduceBits = 6; + constexpr int kDivLutBits = 8; + constexpr int kDivLutPrecBits = 14; + + constexpr int kTotalRefsPerFrame = 8; + + // f(n) of the specification for widths that fit. + uint32_t ReadBits(webrtc::BitstreamReader & reader, int bits) + { + return bits == 0 ? 0 : static_cast(reader.ReadBits(bits)); + } + + // su(n): an n bit two's complement number. + int32_t ReadSu(webrtc::BitstreamReader & reader, int bits) + { + int32_t value = static_cast(ReadBits(reader, bits)); + const int32_t signMask = 1 << (bits - 1); + + if (value & signMask) { + value -= 2 * signMask; + } + + return value; + } + + // read_delta_q. + int8_t ReadDeltaQ(webrtc::BitstreamReader & reader) + { + return reader.Read() ? static_cast(ReadSu(reader, 7)) : 0; + } + + // uvlc(). + uint32_t ReadUvlc(webrtc::BitstreamReader & reader) + { + int leadingZeros = 0; + + while (reader.Ok() && !reader.Read()) { + leadingZeros++; + + if (leadingZeros >= 32) { + reader.Invalidate(); + return 0; + } + } + + return static_cast((1ull << leadingZeros) - 1 + ReadBits(reader, leadingZeros)); + } + + // tile_log2. + int TileLog2(int blockSize, int target) + { + int k = 0; + + while ((blockSize << k) < target) { + k++; + } + + return k; + } + + int FloorLog2(uint32_t x) + { + int s = 0; + + while (x > 1) { + x >>= 1; + s++; + } + + return s; + } + + int64_t Round2Signed(int64_t x, int n) + { + if (n == 0) { + return x; + } + + return x >= 0 ? (x + (int64_t(1) << (n - 1))) >> n : -((-x + (int64_t(1) << (n - 1))) >> n); + } + + int32_t InverseRecenter(int32_t r, int32_t v) + { + if (v > 2 * r) { + return v; + } + if (v & 1) { + return r - ((v + 1) >> 1); + } + + return r + (v >> 1); + } + + // decode_subexp. + int32_t DecodeSubexp(webrtc::BitstreamReader & reader, int32_t numSyms) + { + int i = 0; + int32_t mk = 0; + const int k = 3; + + while (reader.Ok()) { + const int b2 = i ? k + i - 1 : k; + const int32_t a = 1 << b2; + + if (numSyms <= mk + 3 * a) { + return static_cast(reader.ReadNonSymmetric(static_cast(numSyms - mk))) + mk; + } + if (!reader.Read()) { + return static_cast(ReadBits(reader, b2)) + mk; + } + + i++; + mk += a; + } + + return 0; + } + + // decode_signed_subexp_with_ref. + int32_t DecodeSignedSubexpWithRef(webrtc::BitstreamReader & reader, int32_t low, int32_t high, int32_t r) + { + const int32_t mx = high - low; + r -= low; + + const int32_t v = DecodeSubexp(reader, mx); + const int32_t x = (r << 1) <= mx ? InverseRecenter(r, v) : mx - 1 - InverseRecenter(mx - 1 - r, v); + + return x + low; + } + + // The divisor lookup of section 7.11.3.7, Div_Lut[i] being + // 2^14 * 256 / (256 + i) rounded. + int32_t DivLut(int i) + { + const int32_t d = 256 + i; + + return ((1 << (kDivLutPrecBits + kDivLutBits)) + d / 2) / d; + } + + // Whether the shear of a warp is valid, section 7.11.3.6. + bool ShearValid(const std::array & params) + { + // resolve_divisor. + const int32_t d = params[2]; + + if (d == 0) { + return false; + } + + const uint32_t absD = static_cast(std::abs(d)); + const int n = FloorLog2(absD); + const int64_t e = absD - (int64_t(1) << n); + const int64_t f = n > kDivLutBits ? Round2Signed(e, n - kDivLutBits) : e << (kDivLutBits - n); + const int divShift = n + kDivLutPrecBits; + const int64_t divFactor = d < 0 ? -DivLut(static_cast(f)) : DivLut(static_cast(f)); + + auto clip = [](int64_t value) { + return std::clamp(value, -32768, 32767); + }; + + const int64_t alpha0 = clip(params[2] - (int64_t(1) << kWarpedModelPrecBits)); + const int64_t beta0 = clip(params[3]); + const int64_t v = int64_t(params[4]) << kWarpedModelPrecBits; + const int64_t gamma0 = clip(Round2Signed(v * divFactor, divShift)); + const int64_t w = int64_t(params[3]) * params[4]; + const int64_t delta0 = clip(params[5] - Round2Signed(w * divFactor, divShift) + - (int64_t(1) << kWarpedModelPrecBits)); + + auto reduce = [](int64_t value) { + return Round2Signed(value, kWarpParamReduceBits) * (int64_t(1) << kWarpParamReduceBits); + }; + + const int64_t alpha = reduce(alpha0); + const int64_t beta = reduce(beta0); + const int64_t gamma = reduce(gamma0); + const int64_t delta = reduce(delta0); + + if (4 * std::llabs(alpha) + 7 * std::llabs(beta) >= (int64_t(1) << kWarpedModelPrecBits)) { + return false; + } + if (4 * std::llabs(gamma) + 4 * std::llabs(delta) >= (int64_t(1) << kWarpedModelPrecBits)) { + return false; + } + + return true; + } + + std::array, 8> DefaultGmParams() + { + std::array, 8> params = {}; + + for (auto & ref : params) { + for (int i = 0; i < 6; i++) { + ref[i] = (i % 3 == 2) ? 1 << kWarpedModelPrecBits : 0; + } + } + + return params; + } + + constexpr std::array kDefaultLoopFilterRefDeltas = { 1, 0, 0, 0, -1, 0, -1, -1 }; + + // Reads a leb128 value from the bytes, section 4.10.5. + bool ReadLeb128(std::span data, size_t & position, uint64_t & value) + { + value = 0; + + for (int i = 0; i < 8; i++) { + if (position >= data.size()) { + return false; + } + + const uint8_t byte = data[position++]; + value |= uint64_t(byte & 0x7f) << (i * 7); + + if (!(byte & 0x80)) { + return true; + } + } + + return false; + } + } + + Av1Parser::Av1Parser() + { + Reset(); + } + + void Av1Parser::Reset() + { + sequence.reset(); + references.fill(Reference()); + } + + const std::optional & Av1Parser::SequenceHeader() const + { + return sequence; + } + + std::optional> Av1Parser::SplitObus(std::span data) const + { + std::vector obus; + size_t position = 0; + + while (position < data.size()) { + const uint8_t header = data[position++]; + + if (header & 0x80) { + // The forbidden bit. + return std::nullopt; + } + + Av1Obu obu; + obu.type = static_cast((header >> 3) & 0x0f); + obu.hasExtension = (header & 0x04) != 0; + + const bool hasSize = (header & 0x02) != 0; + + if (obu.hasExtension) { + if (position >= data.size()) { + return std::nullopt; + } + + const uint8_t extension = data[position++]; + obu.temporalId = extension >> 5; + obu.spatialId = (extension >> 3) & 0x03; + } + + uint64_t size = data.size() - position; + + if (hasSize && !ReadLeb128(data, position, size)) { + return std::nullopt; + } + if (size > data.size() - position) { + return std::nullopt; + } + + obu.payload = data.subspan(position, static_cast(size)); + position += static_cast(size); + + // Section 7.5: the OBUs of layers outside the operating point are + // dropped. + const uint16_t idc = sequence ? sequence->operatingPointIdc[0] : 0; + + if (obu.type != Av1ObuType::SequenceHeader && obu.type != Av1ObuType::TemporalDelimiter + && obu.hasExtension && idc != 0) + { + const bool inTemporalLayer = (idc >> obu.temporalId) & 1; + const bool inSpatialLayer = (idc >> (obu.spatialId + 8)) & 1; + + if (!inTemporalLayer || !inSpatialLayer) { + continue; + } + } + + obus.push_back(obu); + } + + return obus; + } + + bool Av1Parser::ParseSequenceHeader(std::span payload) + { + webrtc::BitstreamReader reader(payload); + Av1SequenceHeader header; + + header.profile = static_cast(ReadBits(reader, 3)); + header.stillPicture = reader.Read(); + header.reducedStillPictureHeader = reader.Read(); + + if (header.reducedStillPictureHeader) { + header.operatingPointCount = 1; + header.operatingPointIdc[0] = 0; + // seq_level_idx[0]. + reader.ConsumeBits(5); + } + else { + const bool timingInfoPresent = reader.Read(); + uint8_t bufferDelayLength = 0; + + if (timingInfoPresent) { + // num_units_in_display_tick, time_scale. + reader.ConsumeBits(64); + header.equalPictureInterval = reader.Read(); + + if (header.equalPictureInterval) { + ReadUvlc(reader); + } + + header.decoderModelInfoPresent = reader.Read(); + + if (header.decoderModelInfoPresent) { + bufferDelayLength = static_cast(ReadBits(reader, 5) + 1); + // num_units_in_decoding_tick. + reader.ConsumeBits(32); + header.bufferRemovalTimeLength = static_cast(ReadBits(reader, 5) + 1); + header.framePresentationTimeLength = static_cast(ReadBits(reader, 5) + 1); + } + } + + const bool initialDisplayDelayPresent = reader.Read(); + header.operatingPointCount = static_cast(ReadBits(reader, 5) + 1); + + for (int i = 0; i < header.operatingPointCount; i++) { + header.operatingPointIdc[i] = static_cast(ReadBits(reader, 12)); + + const uint32_t seqLevelIdx = ReadBits(reader, 5); + + if (seqLevelIdx > 7) { + // seq_tier. + reader.ConsumeBits(1); + } + + if (header.decoderModelInfoPresent) { + header.decoderModelPresentForOperatingPoint[i] = reader.Read(); + + if (header.decoderModelPresentForOperatingPoint[i]) { + // decoder_buffer_delay, encoder_buffer_delay, + // low_delay_mode_flag. + reader.ConsumeBits(2 * bufferDelayLength + 1); + } + } + if (initialDisplayDelayPresent && reader.Read()) { + // initial_display_delay_minus_1. + reader.ConsumeBits(4); + } + } + } + + header.frameWidthBits = static_cast(ReadBits(reader, 4) + 1); + header.frameHeightBits = static_cast(ReadBits(reader, 4) + 1); + header.maxFrameWidth = ReadBits(reader, header.frameWidthBits) + 1; + header.maxFrameHeight = ReadBits(reader, header.frameHeightBits) + 1; + + header.frameIdNumbersPresent = header.reducedStillPictureHeader ? false : reader.Read(); + + if (header.frameIdNumbersPresent) { + header.deltaFrameIdLength = static_cast(ReadBits(reader, 4) + 2); + header.frameIdLength = static_cast(ReadBits(reader, 3) + 1 + header.deltaFrameIdLength); + } + + header.use128x128Superblock = reader.Read(); + header.enableFilterIntra = reader.Read(); + header.enableIntraEdgeFilter = reader.Read(); + + if (!header.reducedStillPictureHeader) { + header.enableInterintraCompound = reader.Read(); + header.enableMaskedCompound = reader.Read(); + header.enableWarpedMotion = reader.Read(); + header.enableDualFilter = reader.Read(); + header.enableOrderHint = reader.Read(); + + if (header.enableOrderHint) { + header.enableJntComp = reader.Read(); + header.enableRefFrameMvs = reader.Read(); + } + + const bool seqChooseScreenContentTools = reader.Read(); + header.seqForceScreenContentTools = seqChooseScreenContentTools + ? kSelectScreenContentTools + : static_cast(reader.ReadBit()); + + if (header.seqForceScreenContentTools > 0) { + const bool seqChooseIntegerMv = reader.Read(); + header.seqForceIntegerMv = seqChooseIntegerMv + ? kSelectIntegerMv + : static_cast(reader.ReadBit()); + } + else { + header.seqForceIntegerMv = kSelectIntegerMv; + } + + if (header.enableOrderHint) { + header.orderHintBits = static_cast(ReadBits(reader, 3) + 1); + } + } + + header.enableSuperres = reader.Read(); + header.enableCdef = reader.Read(); + header.enableRestoration = reader.Read(); + + // color_config, section 5.5.2. + const bool highBitdepth = reader.Read(); + + if (header.profile == 2 && highBitdepth) { + header.bitDepth = reader.Read() ? 12 : 10; + } + else { + header.bitDepth = highBitdepth ? 10 : 8; + } + + header.monochrome = header.profile == 1 ? false : reader.Read(); + + if (reader.Read()) { + header.colorPrimaries = static_cast(ReadBits(reader, 8)); + header.transferCharacteristics = static_cast(ReadBits(reader, 8)); + header.matrixCoefficients = static_cast(ReadBits(reader, 8)); + } + + if (header.monochrome) { + header.colorRange = reader.Read(); + header.subsamplingX = true; + header.subsamplingY = true; + header.separateUvDeltaQ = false; + } + else { + // BT.709 primaries with the sRGB transfer and the identity matrix. + if (header.colorPrimaries == 1 && header.transferCharacteristics == 13 && header.matrixCoefficients == 0) { + header.colorRange = true; + header.subsamplingX = false; + header.subsamplingY = false; + } + else { + header.colorRange = reader.Read(); + + if (header.profile == 0) { + header.subsamplingX = true; + header.subsamplingY = true; + } + else if (header.profile == 1) { + header.subsamplingX = false; + header.subsamplingY = false; + } + else if (header.bitDepth == 12) { + header.subsamplingX = reader.Read(); + header.subsamplingY = header.subsamplingX ? reader.Read() : false; + } + else { + header.subsamplingX = true; + header.subsamplingY = false; + } + + if (header.subsamplingX && header.subsamplingY) { + // chroma_sample_position. + reader.ConsumeBits(2); + } + } + + header.separateUvDeltaQ = reader.Read(); + } + + header.filmGrainParamsPresent = reader.Read(); + + if (!reader.Ok()) { + return false; + } + + sequence = header; + + return true; + } + + int Av1Parser::RelativeDistance(int a, int b) const + { + if (!sequence->enableOrderHint) { + return 0; + } + + const int bits = sequence->orderHintBits; + int diff = a - b; + const int m = 1 << (bits - 1); + + diff = (diff & (m - 1)) - (diff & m); + + return diff; + } + + bool Av1Parser::SetFrameRefs(Av1FrameHeader & header, uint8_t lastFrameIdx, uint8_t goldFrameIdx) const + { + int refFrameIdx[kRefsPerFrame]; + bool usedFrame[kNumRefFrames] = {}; + int shiftedOrderHints[kNumRefFrames]; + + std::fill(std::begin(refFrameIdx), std::end(refFrameIdx), -1); + + refFrameIdx[kLastFrame - kLastFrame] = lastFrameIdx; + refFrameIdx[kGoldenFrame - kLastFrame] = goldFrameIdx; + usedFrame[lastFrameIdx] = true; + usedFrame[goldFrameIdx] = true; + + const int curFrameHint = 1 << (sequence->orderHintBits - 1); + + for (int i = 0; i < kNumRefFrames; i++) { + shiftedOrderHints[i] = curFrameHint + RelativeDistance(references[i].orderHint, header.orderHint); + } + + if (shiftedOrderHints[lastFrameIdx] >= curFrameHint || shiftedOrderHints[goldFrameIdx] >= curFrameHint) { + return false; + } + + auto findLatestBackward = [&]() { + int ref = -1; + int latestOrderHint = 0; + + for (int i = 0; i < kNumRefFrames; i++) { + const int hint = shiftedOrderHints[i]; + + if (!usedFrame[i] && hint >= curFrameHint && (ref < 0 || hint >= latestOrderHint)) { + ref = i; + latestOrderHint = hint; + } + } + + return ref; + }; + auto findEarliestBackward = [&]() { + int ref = -1; + int earliestOrderHint = 0; + + for (int i = 0; i < kNumRefFrames; i++) { + const int hint = shiftedOrderHints[i]; + + if (!usedFrame[i] && hint >= curFrameHint && (ref < 0 || hint < earliestOrderHint)) { + ref = i; + earliestOrderHint = hint; + } + } + + return ref; + }; + auto findLatestForward = [&]() { + int ref = -1; + int latestOrderHint = 0; + + for (int i = 0; i < kNumRefFrames; i++) { + const int hint = shiftedOrderHints[i]; + + if (!usedFrame[i] && hint < curFrameHint && (ref < 0 || hint >= latestOrderHint)) { + ref = i; + latestOrderHint = hint; + } + } + + return ref; + }; + + int ref = findLatestBackward(); + + if (ref >= 0) { + refFrameIdx[kAltrefFrame - kLastFrame] = ref; + usedFrame[ref] = true; + } + + ref = findEarliestBackward(); + + if (ref >= 0) { + refFrameIdx[kBwdrefFrame - kLastFrame] = ref; + usedFrame[ref] = true; + } + + ref = findEarliestBackward(); + + if (ref >= 0) { + refFrameIdx[kAltref2Frame - kLastFrame] = ref; + usedFrame[ref] = true; + } + + constexpr int kRefFrameList[kRefsPerFrame - 2] = { + kLast2Frame, kLast3Frame, kBwdrefFrame, kAltref2Frame, kAltrefFrame + }; + + for (int refFrame : kRefFrameList) { + if (refFrameIdx[refFrame - kLastFrame] < 0) { + ref = findLatestForward(); + + if (ref >= 0) { + refFrameIdx[refFrame - kLastFrame] = ref; + usedFrame[ref] = true; + } + } + } + + ref = -1; + int earliestOrderHint = 0; + + for (int i = 0; i < kNumRefFrames; i++) { + const int hint = shiftedOrderHints[i]; + + if (ref < 0 || hint < earliestOrderHint) { + ref = i; + earliestOrderHint = hint; + } + } + + for (int i = 0; i < kRefsPerFrame; i++) { + header.refFrameIdx[i] = static_cast(refFrameIdx[i] < 0 ? ref : refFrameIdx[i]); + } + + return true; + } + + std::optional Av1Parser::ParseFrameHeader(std::span payload, const Av1Obu & obu) + { + if (!sequence) { + return std::nullopt; + } + + const Av1SequenceHeader & seq = *sequence; + webrtc::BitstreamReader reader(payload); + Av1FrameHeader header; + + auto temporalPointInfo = [&]() { + // frame_presentation_time. + reader.ConsumeBits(seq.framePresentationTimeLength); + }; + + if (seq.reducedStillPictureHeader) { + header.showExistingFrame = false; + header.frameType = kAv1KeyFrame; + header.frameIsIntra = true; + header.showFrame = true; + header.showableFrame = false; + } + else { + header.showExistingFrame = reader.Read(); + + if (header.showExistingFrame) { + header.frameToShow = static_cast(ReadBits(reader, 3)); + + if (seq.decoderModelInfoPresent && !seq.equalPictureInterval) { + temporalPointInfo(); + } + if (seq.frameIdNumbersPresent) { + // display_frame_id. + reader.ConsumeBits(seq.frameIdLength); + } + + const Reference & shown = references[header.frameToShow]; + + if (!reader.Ok() || !shown.valid) { + return std::nullopt; + } + + header.frameType = shown.frameType; + header.refreshFrameFlags = header.frameType == kAv1KeyFrame ? kAllFrames : 0; + header.upscaledWidth = shown.upscaledWidth; + header.frameWidth = shown.frameWidth; + header.frameHeight = shown.frameHeight; + header.renderWidth = shown.renderWidth; + header.renderHeight = shown.renderHeight; + header.showFrame = true; + + return header; + } + + header.frameType = static_cast(ReadBits(reader, 2)); + header.frameIsIntra = header.frameType == kAv1IntraOnlyFrame || header.frameType == kAv1KeyFrame; + header.showFrame = reader.Read(); + + if (header.showFrame && seq.decoderModelInfoPresent && !seq.equalPictureInterval) { + temporalPointInfo(); + } + + header.showableFrame = header.showFrame + ? header.frameType != kAv1KeyFrame + : reader.Read(); + + if (header.frameType == kAv1SwitchFrame || (header.frameType == kAv1KeyFrame && header.showFrame)) { + header.errorResilientMode = true; + } + else { + header.errorResilientMode = reader.Read(); + } + } + + if (header.frameType == kAv1KeyFrame && header.showFrame) { + for (Reference & reference : references) { + reference.valid = false; + reference.orderHint = 0; + } + } + + header.disableCdfUpdate = reader.Read(); + + header.allowScreenContentTools = seq.seqForceScreenContentTools == kSelectScreenContentTools + ? reader.Read() + : seq.seqForceScreenContentTools != 0; + + if (header.allowScreenContentTools) { + header.forceIntegerMv = seq.seqForceIntegerMv == kSelectIntegerMv + ? reader.Read() + : seq.seqForceIntegerMv != 0; + } + else { + header.forceIntegerMv = false; + } + + if (header.frameIsIntra) { + header.forceIntegerMv = true; + } + + if (seq.frameIdNumbersPresent) { + header.currentFrameId = ReadBits(reader, seq.frameIdLength); + } + + bool frameSizeOverride; + + if (header.frameType == kAv1SwitchFrame) { + frameSizeOverride = true; + } + else if (seq.reducedStillPictureHeader) { + frameSizeOverride = false; + } + else { + frameSizeOverride = reader.Read(); + } + + header.orderHint = static_cast(ReadBits(reader, seq.orderHintBits)); + + if (header.frameIsIntra || header.errorResilientMode) { + header.primaryRefFrame = kPrimaryRefNone; + } + else { + header.primaryRefFrame = static_cast(ReadBits(reader, 3)); + } + + if (seq.decoderModelInfoPresent) { + const bool bufferRemovalTimePresent = reader.Read(); + + if (bufferRemovalTimePresent) { + for (int op = 0; op < seq.operatingPointCount; op++) { + if (!seq.decoderModelPresentForOperatingPoint[op]) { + continue; + } + + const uint16_t idc = seq.operatingPointIdc[op]; + const bool inTemporalLayer = (idc >> obu.temporalId) & 1; + const bool inSpatialLayer = (idc >> (obu.spatialId + 8)) & 1; + + if (idc == 0 || (inTemporalLayer && inSpatialLayer)) { + // buffer_removal_time. + reader.ConsumeBits(seq.bufferRemovalTimeLength); + } + } + } + } + + if (header.frameType == kAv1SwitchFrame || (header.frameType == kAv1KeyFrame && header.showFrame)) { + header.refreshFrameFlags = kAllFrames; + } + else { + header.refreshFrameFlags = static_cast(ReadBits(reader, 8)); + } + + if ((!header.frameIsIntra || header.refreshFrameFlags != kAllFrames) + && header.errorResilientMode && seq.enableOrderHint) + { + for (Reference & reference : references) { + const uint8_t refOrderHint = static_cast(ReadBits(reader, seq.orderHintBits)); + + if (refOrderHint != reference.orderHint) { + reference.valid = false; + reference.orderHint = refOrderHint; + } + } + } + + // superres_params and compute_image_size. + auto superresParams = [&]() { + header.useSuperres = seq.enableSuperres ? reader.Read() : false; + header.superresDenom = header.useSuperres + ? static_cast(ReadBits(reader, kSuperresDenomBits) + kSuperresDenomMin) + : kSuperresNum; + header.upscaledWidth = header.frameWidth; + header.frameWidth = (header.upscaledWidth * kSuperresNum + header.superresDenom / 2) / header.superresDenom; + header.miCols = 2 * ((header.frameWidth + 7) >> 3); + header.miRows = 2 * ((header.frameHeight + 7) >> 3); + }; + auto frameSize = [&]() { + if (frameSizeOverride) { + header.frameWidth = ReadBits(reader, seq.frameWidthBits) + 1; + header.frameHeight = ReadBits(reader, seq.frameHeightBits) + 1; + } + else { + header.frameWidth = seq.maxFrameWidth; + header.frameHeight = seq.maxFrameHeight; + } + + superresParams(); + }; + auto renderSize = [&]() { + if (reader.Read()) { + header.renderWidth = ReadBits(reader, 16) + 1; + header.renderHeight = ReadBits(reader, 16) + 1; + } + else { + header.renderWidth = header.upscaledWidth; + header.renderHeight = header.frameHeight; + } + }; + + if (header.frameIsIntra) { + frameSize(); + renderSize(); + + if (header.allowScreenContentTools && header.upscaledWidth == header.frameWidth) { + header.allowIntrabc = reader.Read(); + } + } + else { + bool frameRefsShortSignaling = false; + + if (seq.enableOrderHint) { + frameRefsShortSignaling = reader.Read(); + + if (frameRefsShortSignaling) { + const uint8_t lastFrameIdx = static_cast(ReadBits(reader, 3)); + const uint8_t goldFrameIdx = static_cast(ReadBits(reader, 3)); + + if (!reader.Ok() || !SetFrameRefs(header, lastFrameIdx, goldFrameIdx)) { + return std::nullopt; + } + } + } + + for (int i = 0; i < kRefsPerFrame; i++) { + if (!frameRefsShortSignaling) { + header.refFrameIdx[i] = static_cast(ReadBits(reader, 3)); + } + if (seq.frameIdNumbersPresent) { + // delta_frame_id_minus_1. + reader.ConsumeBits(seq.deltaFrameIdLength); + } + } + + for (int i = 0; i < kRefsPerFrame; i++) { + if (!references[header.refFrameIdx[i]].valid) { + return std::nullopt; + } + } + + if (frameSizeOverride && !header.errorResilientMode) { + // frame_size_with_refs. + bool foundRef = false; + + for (int i = 0; i < kRefsPerFrame; i++) { + foundRef = reader.Read(); + + if (foundRef) { + const Reference & reference = references[header.refFrameIdx[i]]; + + header.upscaledWidth = reference.upscaledWidth; + header.frameWidth = header.upscaledWidth; + header.frameHeight = reference.frameHeight; + header.renderWidth = reference.renderWidth; + header.renderHeight = reference.renderHeight; + break; + } + } + + if (foundRef) { + superresParams(); + } + else { + frameSize(); + renderSize(); + } + } + else { + frameSize(); + renderSize(); + } + + header.allowHighPrecisionMv = header.forceIntegerMv ? false : reader.Read(); + + // read_interpolation_filter. + const bool isFilterSwitchable = reader.Read(); + header.interpolationFilter = isFilterSwitchable ? 4 : static_cast(ReadBits(reader, 2)); + + header.isMotionModeSwitchable = reader.Read(); + + if (header.errorResilientMode || !seq.enableRefFrameMvs) { + header.useRefFrameMvs = false; + } + else { + header.useRefFrameMvs = reader.Read(); + } + } + + if (seq.reducedStillPictureHeader || header.disableCdfUpdate) { + header.disableFrameEndUpdateCdf = true; + } + else { + header.disableFrameEndUpdateCdf = reader.Read(); + } + + // setup_past_independence or load_previous: what carries over. + std::array, 8> prevGmParams; + + if (header.primaryRefFrame == kPrimaryRefNone) { + prevGmParams = DefaultGmParams(); + header.loopFilterRefDeltas = kDefaultLoopFilterRefDeltas; + header.loopFilterModeDeltas = { 0, 0 }; + } + else { + const Reference & previous = references[header.refFrameIdx[header.primaryRefFrame]]; + + prevGmParams = previous.gmParams; + header.loopFilterRefDeltas = previous.loopFilterRefDeltas; + header.loopFilterModeDeltas = previous.loopFilterModeDeltas; + + std::copy(&previous.featureEnabled[0][0], &previous.featureEnabled[0][0] + kMaxSegments * kSegLvlMax, + &header.featureEnabled[0][0]); + std::copy(&previous.featureData[0][0], &previous.featureData[0][0] + kMaxSegments * kSegLvlMax, + &header.featureData[0][0]); + } + + // tile_info, section 5.9.15. + { + const int sbCols = seq.use128x128Superblock ? ((header.miCols + 31) >> 5) : ((header.miCols + 15) >> 4); + const int sbRows = seq.use128x128Superblock ? ((header.miRows + 31) >> 5) : ((header.miRows + 15) >> 4); + const int sbShift = seq.use128x128Superblock ? 5 : 4; + const int sbSize = sbShift + 2; + const int maxTileWidthSb = kMaxTileWidth >> sbSize; + int maxTileAreaSb = kMaxTileArea >> (2 * sbSize); + const int minLog2TileCols = TileLog2(maxTileWidthSb, sbCols); + const int maxLog2TileCols = TileLog2(1, std::min(sbCols, kMaxTileCols)); + const int maxLog2TileRows = TileLog2(1, std::min(sbRows, kMaxTileRows)); + const int minLog2Tiles = std::max(minLog2TileCols, TileLog2(maxTileAreaSb, sbRows * sbCols)); + + header.uniformTileSpacing = reader.Read(); + + if (header.uniformTileSpacing) { + int tileColsLog2 = minLog2TileCols; + + while (tileColsLog2 < maxLog2TileCols && reader.Ok()) { + if (!reader.Read()) { + break; + } + + tileColsLog2++; + } + + const int tileWidthSb = (sbCols + (1 << tileColsLog2) - 1) >> tileColsLog2; + int i = 0; + + for (int startSb = 0; startSb < sbCols; startSb += tileWidthSb) { + header.tileWidthSbs[i++] = static_cast(std::min(tileWidthSb, sbCols - startSb)); + } + + header.tileCols = static_cast(i); + header.tileColsLog2 = static_cast(tileColsLog2); + + const int minLog2TileRows = std::max(minLog2Tiles - tileColsLog2, 0); + int tileRowsLog2 = minLog2TileRows; + + while (tileRowsLog2 < maxLog2TileRows && reader.Ok()) { + if (!reader.Read()) { + break; + } + + tileRowsLog2++; + } + + const int tileHeightSb = (sbRows + (1 << tileRowsLog2) - 1) >> tileRowsLog2; + i = 0; + + for (int startSb = 0; startSb < sbRows; startSb += tileHeightSb) { + header.tileHeightSbs[i++] = static_cast(std::min(tileHeightSb, sbRows - startSb)); + } + + header.tileRows = static_cast(i); + header.tileRowsLog2 = static_cast(tileRowsLog2); + } + else { + int widestTileSb = 0; + int startSb = 0; + int i = 0; + + for (; startSb < sbCols && reader.Ok(); i++) { + if (i >= kMaxTileCols) { + return std::nullopt; + } + + const int maxWidth = std::min(sbCols - startSb, maxTileWidthSb); + const int sizeSb = static_cast(reader.ReadNonSymmetric(static_cast(maxWidth))) + 1; + + header.tileWidthSbs[i] = static_cast(sizeSb); + widestTileSb = std::max(sizeSb, widestTileSb); + startSb += sizeSb; + } + + header.tileCols = static_cast(i); + header.tileColsLog2 = static_cast(TileLog2(1, i)); + + if (minLog2Tiles > 0) { + maxTileAreaSb = (sbRows * sbCols) >> (minLog2Tiles + 1); + } + else { + maxTileAreaSb = sbRows * sbCols; + } + + const int maxTileHeightSb = std::max(maxTileAreaSb / std::max(widestTileSb, 1), 1); + + startSb = 0; + i = 0; + + for (; startSb < sbRows && reader.Ok(); i++) { + if (i >= kMaxTileRows) { + return std::nullopt; + } + + const int maxHeight = std::min(sbRows - startSb, maxTileHeightSb); + const int sizeSb = static_cast(reader.ReadNonSymmetric(static_cast(maxHeight))) + 1; + + header.tileHeightSbs[i] = static_cast(sizeSb); + startSb += sizeSb; + } + + header.tileRows = static_cast(i); + header.tileRowsLog2 = static_cast(TileLog2(1, i)); + } + + if (header.tileColsLog2 > 0 || header.tileRowsLog2 > 0) { + header.contextUpdateTileId = static_cast( + ReadBits(reader, header.tileRowsLog2 + header.tileColsLog2)); + header.tileSizeBytes = static_cast(ReadBits(reader, 2) + 1); + } + else { + header.contextUpdateTileId = 0; + } + } + + // quantization_params, section 5.9.12. + header.baseQIndex = static_cast(ReadBits(reader, 8)); + header.deltaQYDc = ReadDeltaQ(reader); + + if (!seq.monochrome) { + const bool diffUvDelta = seq.separateUvDeltaQ ? reader.Read() : false; + + header.deltaQUDc = ReadDeltaQ(reader); + header.deltaQUAc = ReadDeltaQ(reader); + + if (diffUvDelta) { + header.deltaQVDc = ReadDeltaQ(reader); + header.deltaQVAc = ReadDeltaQ(reader); + } + else { + header.deltaQVDc = header.deltaQUDc; + header.deltaQVAc = header.deltaQUAc; + } + } + + header.usingQmatrix = reader.Read(); + + if (header.usingQmatrix) { + header.qmY = static_cast(ReadBits(reader, 4)); + header.qmU = static_cast(ReadBits(reader, 4)); + header.qmV = seq.separateUvDeltaQ ? static_cast(ReadBits(reader, 4)) : header.qmU; + } + + // segmentation_params, section 5.9.14. + header.segmentationEnabled = reader.Read(); + + if (header.segmentationEnabled) { + if (header.primaryRefFrame == kPrimaryRefNone) { + header.segmentationUpdateMap = true; + header.segmentationTemporalUpdate = false; + header.segmentationUpdateData = true; + } + else { + header.segmentationUpdateMap = reader.Read(); + header.segmentationTemporalUpdate = header.segmentationUpdateMap ? reader.Read() : false; + header.segmentationUpdateData = reader.Read(); + } + + if (header.segmentationUpdateData) { + for (int i = 0; i < kMaxSegments; i++) { + for (int j = 0; j < kSegLvlMax; j++) { + const bool enabled = reader.Read(); + int clippedValue = 0; + + if (enabled) { + const int bits = kSegmentationFeatureBits[j]; + const int limit = kSegmentationFeatureMax[j]; + + if (kSegmentationFeatureSigned[j]) { + clippedValue = std::clamp(static_cast(ReadSu(reader, 1 + bits)), -limit, limit); + } + else { + clippedValue = std::clamp(static_cast(ReadBits(reader, bits)), 0, limit); + } + } + + header.featureEnabled[i][j] = enabled; + header.featureData[i][j] = static_cast(clippedValue); + } + } + } + } + else { + for (int i = 0; i < kMaxSegments; i++) { + for (int j = 0; j < kSegLvlMax; j++) { + header.featureEnabled[i][j] = false; + header.featureData[i][j] = 0; + } + } + } + + // delta_q_params and delta_lf_params. + if (header.baseQIndex > 0) { + header.deltaQPresent = reader.Read(); + } + if (header.deltaQPresent) { + header.deltaQRes = static_cast(ReadBits(reader, 2)); + + if (!header.allowIntrabc) { + header.deltaLfPresent = reader.Read(); + } + if (header.deltaLfPresent) { + header.deltaLfRes = static_cast(ReadBits(reader, 2)); + header.deltaLfMulti = reader.Read(); + } + } + + header.codedLossless = true; + + for (int segment = 0; segment < kMaxSegments; segment++) { + int qindex = header.baseQIndex; + + if (header.segmentationEnabled && header.featureEnabled[segment][kSegLvlAltQ]) { + qindex = std::clamp(qindex + header.featureData[segment][kSegLvlAltQ], 0, 255); + } + + const bool lossless = qindex == 0 && header.deltaQYDc == 0 && header.deltaQUAc == 0 + && header.deltaQUDc == 0 && header.deltaQVAc == 0 && header.deltaQVDc == 0; + + if (!lossless) { + header.codedLossless = false; + } + } + + header.allLossless = header.codedLossless && header.frameWidth == header.upscaledWidth; + + // loop_filter_params, section 5.9.11. + if (header.codedLossless || header.allowIntrabc) { + header.loopFilterLevel = {}; + header.loopFilterRefDeltas = kDefaultLoopFilterRefDeltas; + header.loopFilterModeDeltas = { 0, 0 }; + } + else { + header.loopFilterLevel[0] = static_cast(ReadBits(reader, 6)); + header.loopFilterLevel[1] = static_cast(ReadBits(reader, 6)); + + if (!seq.monochrome && (header.loopFilterLevel[0] || header.loopFilterLevel[1])) { + header.loopFilterLevel[2] = static_cast(ReadBits(reader, 6)); + header.loopFilterLevel[3] = static_cast(ReadBits(reader, 6)); + } + + header.loopFilterSharpness = static_cast(ReadBits(reader, 3)); + header.loopFilterDeltaEnabled = reader.Read(); + + if (header.loopFilterDeltaEnabled) { + header.loopFilterDeltaUpdate = reader.Read(); + + if (header.loopFilterDeltaUpdate) { + for (int i = 0; i < kTotalRefsPerFrame; i++) { + if (reader.Read()) { + header.loopFilterRefDeltas[i] = static_cast(ReadSu(reader, 7)); + } + } + for (int i = 0; i < 2; i++) { + if (reader.Read()) { + header.loopFilterModeDeltas[i] = static_cast(ReadSu(reader, 7)); + } + } + } + } + } + + // cdef_params, section 5.9.19. + if (header.codedLossless || header.allowIntrabc || !seq.enableCdef) { + header.cdefBits = 0; + header.cdefDampingMinus3 = 0; + } + else { + header.cdefDampingMinus3 = static_cast(ReadBits(reader, 2)); + header.cdefBits = static_cast(ReadBits(reader, 2)); + + for (int i = 0; i < (1 << header.cdefBits); i++) { + header.cdefYPriStrength[i] = static_cast(ReadBits(reader, 4)); + header.cdefYSecStrength[i] = static_cast(ReadBits(reader, 2)); + + if (!seq.monochrome) { + header.cdefUvPriStrength[i] = static_cast(ReadBits(reader, 4)); + header.cdefUvSecStrength[i] = static_cast(ReadBits(reader, 2)); + } + } + } + + // lr_params, section 5.9.20. + if (!header.allLossless && !header.allowIntrabc && seq.enableRestoration) { + bool usesLr = false; + bool usesChromaLr = false; + const int planes = seq.monochrome ? 1 : 3; + + for (int i = 0; i < planes; i++) { + header.frameRestorationType[i] = kRemapLrType[ReadBits(reader, 2)]; + + if (header.frameRestorationType[i] != 0) { + usesLr = true; + + if (i > 0) { + usesChromaLr = true; + } + } + } + + if (usesLr) { + if (seq.use128x128Superblock) { + header.lrUnitShift = static_cast(reader.ReadBit() + 1); + } + else { + header.lrUnitShift = static_cast(reader.ReadBit()); + + if (header.lrUnitShift) { + header.lrUnitShift += static_cast(reader.ReadBit()); + } + } + + if (seq.subsamplingX && seq.subsamplingY && usesChromaLr) { + header.lrUvShift = static_cast(reader.ReadBit()); + } + } + } + + // read_tx_mode. + if (header.codedLossless) { + header.txMode = 0; + } + else { + header.txMode = reader.Read() ? 2 : 1; + } + + // frame_reference_mode. + header.referenceSelect = header.frameIsIntra ? false : reader.Read(); + + // skip_mode_params, section 5.9.22. + bool skipModeAllowed = false; + + if (!header.frameIsIntra && header.referenceSelect && seq.enableOrderHint) { + int forwardIdx = -1; + int backwardIdx = -1; + int forwardHint = 0; + int backwardHint = 0; + + for (int i = 0; i < kRefsPerFrame; i++) { + const int refHint = references[header.refFrameIdx[i]].orderHint; + + if (RelativeDistance(refHint, header.orderHint) < 0) { + if (forwardIdx < 0 || RelativeDistance(refHint, forwardHint) > 0) { + forwardIdx = i; + forwardHint = refHint; + } + } + else if (RelativeDistance(refHint, header.orderHint) > 0) { + if (backwardIdx < 0 || RelativeDistance(refHint, backwardHint) < 0) { + backwardIdx = i; + backwardHint = refHint; + } + } + } + + if (forwardIdx < 0) { + skipModeAllowed = false; + } + else if (backwardIdx >= 0) { + skipModeAllowed = true; + } + else { + int secondForwardIdx = -1; + int secondForwardHint = 0; + + for (int i = 0; i < kRefsPerFrame; i++) { + const int refHint = references[header.refFrameIdx[i]].orderHint; + + if (RelativeDistance(refHint, forwardHint) < 0) { + if (secondForwardIdx < 0 || RelativeDistance(refHint, secondForwardHint) > 0) { + secondForwardIdx = i; + secondForwardHint = refHint; + } + } + } + + skipModeAllowed = secondForwardIdx >= 0; + } + } + + header.skipModePresent = skipModeAllowed ? reader.Read() : false; + + if (header.frameIsIntra || header.errorResilientMode || !seq.enableWarpedMotion) { + header.allowWarpedMotion = false; + } + else { + header.allowWarpedMotion = reader.Read(); + } + + header.reducedTxSet = reader.Read(); + + // global_motion_params, section 5.9.24. + header.gmParams = DefaultGmParams(); + header.gmType.fill(kIdentity); + header.gmValid.fill(true); + + if (!header.frameIsIntra) { + for (int ref = kLastFrame; ref <= kAltrefFrame; ref++) { + uint8_t type = kIdentity; + + if (reader.Read()) { + if (reader.Read()) { + type = kRotzoom; + } + else { + type = reader.Read() ? kTranslation : kAffine; + } + } + + header.gmType[ref] = type; + + auto readGlobalParam = [&](int idx) { + int absBits = kGmAbsAlphaBits; + int precBits = kGmAlphaPrecBits; + + if (idx < 2) { + if (type == kTranslation) { + absBits = kGmAbsTransOnlyBits - !header.allowHighPrecisionMv; + precBits = kGmTransOnlyPrecBits - !header.allowHighPrecisionMv; + } + else { + absBits = kGmAbsTransBits; + precBits = kGmTransPrecBits; + } + } + + const int precDiff = kWarpedModelPrecBits - precBits; + const int32_t round = (idx % 3) == 2 ? (1 << kWarpedModelPrecBits) : 0; + const int32_t sub = (idx % 3) == 2 ? (1 << precBits) : 0; + const int32_t mx = 1 << absBits; + const int32_t r = (prevGmParams[ref][idx] >> precDiff) - sub; + + header.gmParams[ref][idx] = (DecodeSignedSubexpWithRef(reader, -mx, mx + 1, r) * (1 << precDiff)) + + round; + }; + + if (type >= kRotzoom) { + readGlobalParam(2); + readGlobalParam(3); + + if (type == kAffine) { + readGlobalParam(4); + readGlobalParam(5); + } + else { + header.gmParams[ref][4] = -header.gmParams[ref][3]; + header.gmParams[ref][5] = header.gmParams[ref][2]; + } + } + if (type >= kTranslation) { + readGlobalParam(0); + readGlobalParam(1); + } + + header.gmValid[ref] = type <= kTranslation || ShearValid(header.gmParams[ref]); + } + } + + // film_grain_params, section 5.9.30. + if (seq.filmGrainParamsPresent && (header.showFrame || header.showableFrame)) { + header.applyGrain = reader.Read(); + + if (header.applyGrain) { + // Not parsed further: a frame with film grain is left to + // the software decoder. + return reader.Ok() ? std::optional(header) : std::nullopt; + } + } + + if (!reader.Ok()) { + return std::nullopt; + } + + const int bitsRead = static_cast(payload.size() * 8) - reader.RemainingBitCount(); + header.headerBytes = static_cast((bitsRead + 7) / 8); + + return header; + } + + std::optional Av1Parser::ParseTileGroup(std::span payload, + const Av1FrameHeader & header) const + { + webrtc::BitstreamReader reader(payload); + Av1TileGroup group; + + const int numTiles = header.tileCols * header.tileRows; + bool tileStartAndEndPresent = false; + + if (numTiles > 1) { + tileStartAndEndPresent = reader.Read(); + } + + if (numTiles == 1 || !tileStartAndEndPresent) { + group.start = 0; + group.end = static_cast(numTiles - 1); + } + else { + const int tileBits = header.tileColsLog2 + header.tileRowsLog2; + + group.start = static_cast(ReadBits(reader, tileBits)); + group.end = static_cast(ReadBits(reader, tileBits)); + } + + if (!reader.Ok() || group.end < group.start || group.end >= numTiles) { + return std::nullopt; + } + + // byte_alignment. + const int bitsRead = static_cast(payload.size() * 8) - reader.RemainingBitCount(); + size_t position = static_cast((bitsRead + 7) / 8); + + for (int tileNum = group.start; tileNum <= group.end; tileNum++) { + Av1Tile tile; + tile.row = static_cast(tileNum / header.tileCols); + tile.column = static_cast(tileNum % header.tileCols); + + size_t tileSize; + + if (tileNum == group.end) { + if (position >= payload.size()) { + return std::nullopt; + } + + tileSize = payload.size() - position; + } + else { + if (payload.size() - position < header.tileSizeBytes) { + return std::nullopt; + } + + // tile_size_minus_1, le(TileSizeBytes). + tileSize = 0; + + for (int i = 0; i < header.tileSizeBytes; i++) { + tileSize |= size_t(payload[position + i]) << (8 * i); + } + + tileSize += 1; + position += header.tileSizeBytes; + + if (tileSize > payload.size() - position) { + return std::nullopt; + } + } + + tile.offset = static_cast(position); + tile.size = static_cast(tileSize); + group.tiles.push_back(tile); + + position += tileSize; + } + + return group; + } + + void Av1Parser::UpdateReferences(const Av1FrameHeader & header) + { + if (header.showExistingFrame) { + // A key frame shown again is loaded, and then refreshes all slots. + if (header.frameType == kAv1KeyFrame) { + const Reference shown = references[header.frameToShow]; + + references.fill(shown); + } + + return; + } + + Reference reference; + reference.valid = true; + reference.frameId = header.currentFrameId; + reference.upscaledWidth = header.upscaledWidth; + reference.frameWidth = header.frameWidth; + reference.frameHeight = header.frameHeight; + reference.renderWidth = header.renderWidth; + reference.renderHeight = header.renderHeight; + reference.frameType = header.frameType; + reference.orderHint = header.orderHint; + reference.gmParams = header.gmParams; + reference.loopFilterRefDeltas = header.loopFilterRefDeltas; + reference.loopFilterModeDeltas = header.loopFilterModeDeltas; + + std::copy(&header.featureEnabled[0][0], &header.featureEnabled[0][0] + kMaxSegments * kSegLvlMax, + &reference.featureEnabled[0][0]); + std::copy(&header.featureData[0][0], &header.featureData[0][0] + kMaxSegments * kSegLvlMax, + &reference.featureData[0][0]); + + for (int slot = 0; slot < kNumRefFrames; slot++) { + if (header.refreshFrameFlags & (1 << slot)) { + references[slot] = reference; + } + } + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp index 3b05a400..4a1324d4 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp @@ -48,7 +48,7 @@ namespace jni std::vector> factories; // NVDEC on NVIDIA GPUs, for H.264, H.265, VP9 and AV1; VA-API for the - // GPUs of Intel and AMD, for VP9 and H.265. + // GPUs of Intel and AMD, for VP9, H.265 and AV1. if (auto nvdec = NvdecVideoDecoderFactory::Create()) { factories.push_back(std::move(nvdec)); } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Decoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Decoder.cpp new file mode 100644 index 00000000..ac48fd1a --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Decoder.cpp @@ -0,0 +1,716 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/linux/VaapiAv1Decoder.h" + +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" +#include "third_party/libyuv/include/libyuv/convert.h" + +#include + +namespace jni +{ + namespace + { + // The eight reference frames, the frame being decoded, and one more, + // so that a free surface is always there. + constexpr int kSurfaceCount = 10; + } + + VaapiAv1Decoder::VaapiAv1Decoder(VaapiLibrary & library) : + library(library), + va(library.Api()), + implementationName("VA-API (" + library.Vendor() + ")"), + callback(nullptr), + config(VA_INVALID_ID), + context(VA_INVALID_ID), + width(0), + height(0), + downloadImage(), + started(false), + failed(false) + { + downloadImage.image_id = VA_INVALID_ID; + slots.fill(-1); + } + + VaapiAv1Decoder::~VaapiAv1Decoder() + { + Release(); + } + + bool VaapiAv1Decoder::Check(VAStatus status, const char * operation) const + { + if (status == VA_STATUS_SUCCESS) { + return true; + } + + RTC_LOG(LS_WARNING) << implementationName << " AV1: " << operation << " failed: " + << library.ErrorString(status); + + return false; + } + + bool VaapiAv1Decoder::Configure(const Settings & settings) + { + if (settings.codec_type() != webrtc::kVideoCodecAV1) { + return false; + } + + Release(); + + RTC_LOG(LS_INFO) << implementationName << " AV1 decoder configured"; + + return true; + } + + int32_t VaapiAv1Decoder::Decode(const webrtc::EncodedImage & image, int64_t renderTimeMs) + { + if (callback == nullptr) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (image.size() == 0) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (failed) { + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + // Decoding starts with a key frame. Until one has come, WebRTC is + // asked for it. + if (!started && image.FrameType() != webrtc::VideoFrameType::kVideoFrameKey) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + std::vector frames; + + if (!ParseTemporalUnit(std::span(image.data(), image.size()), frames) + || !DecodeFrames(frames, image, renderTimeMs)) + { + failed = true; + + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VaapiAv1Decoder::RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback * decodeCallback) + { + callback = decodeCallback; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VaapiAv1Decoder::Release() + { + DestroySession(); + + parser.Reset(); + started = false; + failed = false; + + return WEBRTC_VIDEO_CODEC_OK; + } + + webrtc::VideoDecoder::DecoderInfo VaapiAv1Decoder::GetDecoderInfo() const + { + DecoderInfo info; + info.implementation_name = implementationName; + info.is_hardware_accelerated = true; + + return info; + } + + const char * VaapiAv1Decoder::ImplementationName() const + { + return implementationName.c_str(); + } + + bool VaapiAv1Decoder::ParseTemporalUnit(std::span data, std::vector & frames) + { + const auto obus = parser.SplitObus(data); + + if (!obus) { + RTC_LOG(LS_WARNING) << implementationName << " AV1: OBUs that do not fit into the frame"; + return false; + } + + for (const Av1Obu & obu : *obus) { + if (obu.spatialId > 0) { + // Spatial layers predict from frames of another size. + RTC_LOG(LS_INFO) << implementationName << " AV1: a stream with spatial layers"; + return false; + } + } + + // The size the frames are decoded at, which a key frame may change. + uint32_t frameWidth = context != VA_INVALID_ID ? width : 0; + uint32_t frameHeight = context != VA_INVALID_ID ? height : 0; + + // The frame whose tiles are coming. + std::optional pending; + + // Takes a tile group of the pending frame in, and the frame into the + // reference slots once it has all of its tiles. + auto addTileGroup = [&](std::span payload) { + if (!pending || !AddTileGroup(payload, *pending)) { + return false; + } + + const Av1FrameHeader & header = pending->header; + + if (pending->tiles.size() == size_t(header.tileCols) * header.tileRows) { + parser.UpdateReferences(header); + frames.push_back(std::move(*pending)); + pending.reset(); + } + + return true; + }; + + for (const Av1Obu & obu : *obus) { + switch (obu.type) { + case Av1ObuType::SequenceHeader: { + if (!parser.ParseSequenceHeader(obu.payload)) { + RTC_LOG(LS_WARNING) << implementationName << " AV1: a broken sequence header"; + return false; + } + + const Av1SequenceHeader & sequence = *parser.SequenceHeader(); + + if (sequence.profile != 0 || sequence.bitDepth != 8 || sequence.monochrome) { + RTC_LOG(LS_INFO) << implementationName << " AV1: a stream that is not 8 bit 4:2:0 profile 0"; + return false; + } + break; + } + + case Av1ObuType::FrameHeader: + case Av1ObuType::RedundantFrameHeader: + case Av1ObuType::Frame: { + if (pending) { + // Copies of the header of the frame whose tiles are + // coming; a frame OBU has to start a frame. + if (obu.type == Av1ObuType::Frame) { + return false; + } + break; + } + + const std::optional header = parser.ParseFrameHeader(obu.payload, obu); + + if (!header) { + RTC_LOG(LS_INFO) << implementationName << " AV1: a broken frame header, or one that " + "refers to a frame that is not there"; + return false; + } + + if (header->showExistingFrame) { + if (obu.type == Av1ObuType::Frame) { + return false; + } + + parser.UpdateReferences(*header); + frames.push_back(Frame { *header }); + break; + } + + if (header->applyGrain) { + RTC_LOG(LS_INFO) << implementationName << " AV1: a frame with film grain"; + return false; + } + if (header->useSuperres) { + RTC_LOG(LS_INFO) << implementationName << " AV1: a frame with superres"; + return false; + } + + if (header->upscaledWidth != frameWidth || header->frameHeight != frameHeight) { + // A key frame that is shown starts a new session, as it + // replaces all references; frames that scale their + // references are left to dav1d. + if (header->frameType != kAv1KeyFrame || !header->showFrame) { + RTC_LOG(LS_INFO) << implementationName << " AV1: a frame changes the size to " + << header->upscaledWidth << "x" << header->frameHeight; + return false; + } + + frameWidth = header->upscaledWidth; + frameHeight = header->frameHeight; + } + + pending = Frame { *header }; + + if (obu.type == Av1ObuType::Frame) { + if (header->headerBytes > obu.payload.size() + || !addTileGroup(obu.payload.subspan(header->headerBytes))) + { + return false; + } + } + break; + } + + case Av1ObuType::TileGroup: + if (!addTileGroup(obu.payload)) { + return false; + } + break; + + default: + // Temporal delimiters, metadata and padding. + break; + } + } + + if (pending) { + RTC_LOG(LS_WARNING) << implementationName << " AV1: a frame without all of its tiles"; + return false; + } + + return true; + } + + bool VaapiAv1Decoder::AddTileGroup(std::span payload, Frame & frame) + { + const std::optional group = parser.ParseTileGroup(payload, frame.header); + + if (!group || group->start != frame.tiles.size()) { + RTC_LOG(LS_WARNING) << implementationName << " AV1: a broken tile group, or one out of order"; + return false; + } + + const uint32_t base = static_cast(frame.tileData.size()); + + frame.tileData.insert(frame.tileData.end(), payload.begin(), payload.end()); + + for (const Av1Tile & tile : group->tiles) { + VASliceParameterBufferAV1 slice = {}; + slice.slice_data_size = tile.size; + slice.slice_data_offset = base + tile.offset; + slice.slice_data_flag = VA_SLICE_DATA_FLAG_ALL; + slice.tile_row = tile.row; + slice.tile_column = tile.column; + + frame.tiles.push_back(slice); + } + + return true; + } + + bool VaapiAv1Decoder::DecodeFrames(const std::vector & frames, const webrtc::EncodedImage & image, + int64_t renderTimeMs) + { + for (const Frame & frame : frames) { + const Av1FrameHeader & header = frame.header; + + if (header.showExistingFrame) { + const int surface = slots[header.frameToShow]; + + if (surface < 0 || !Output(surface, image, renderTimeMs)) { + return false; + } + + // A key frame shown again takes all reference slots. + if (header.frameType == kAv1KeyFrame) { + slots.fill(surface); + } + + continue; + } + + if (context == VA_INVALID_ID || header.upscaledWidth != width || header.frameHeight != height) { + if (!CreateSession(header.upscaledWidth, header.frameHeight)) { + return false; + } + } + + const int surface = FreeSurface(); + + if (surface < 0 || !DecodeFrame(frame, surface)) { + return false; + } + + for (int slot = 0; slot < 8; slot++) { + if (header.refreshFrameFlags & (1 << slot)) { + slots[slot] = surface; + } + } + + started = true; + + if (header.showFrame && !Output(surface, image, renderTimeMs)) { + return false; + } + } + + return true; + } + + bool VaapiAv1Decoder::CreateSession(uint32_t frameWidth, uint32_t frameHeight) + { + DestroySession(); + + VADisplay display = library.Display(); + + VAConfigAttrib attribute = {}; + attribute.type = VAConfigAttribRTFormat; + attribute.value = VA_RT_FORMAT_YUV420; + + if (!Check(va.CreateConfig(display, VAProfileAV1Profile0, VAEntrypointVLD, &attribute, 1, &config), + "creating the configuration")) + { + config = VA_INVALID_ID; + return false; + } + + surfaces.assign(kSurfaceCount, VA_INVALID_SURFACE); + + if (!Check(va.CreateSurfaces(display, VA_RT_FORMAT_YUV420, frameWidth, frameHeight, surfaces.data(), + kSurfaceCount, nullptr, 0), "creating surfaces")) + { + surfaces.clear(); + DestroySession(); + return false; + } + + if (!Check(va.CreateContext(display, config, static_cast(frameWidth), static_cast(frameHeight), + VA_PROGRESSIVE, surfaces.data(), kSurfaceCount, &context), "creating the context")) + { + context = VA_INVALID_ID; + DestroySession(); + return false; + } + + // Copying a surface into an image reads it from cached memory, which + // mapping it directly does not. Where the driver does not copy, the + // surface is mapped. + VAImageFormat format = {}; + format.fourcc = VA_FOURCC_NV12; + format.byte_order = VA_LSB_FIRST; + format.bits_per_pixel = 12; + + if (va.CreateImage(display, &format, static_cast(frameWidth), static_cast(frameHeight), + &downloadImage) != VA_STATUS_SUCCESS) + { + downloadImage.image_id = VA_INVALID_ID; + } + + width = frameWidth; + height = frameHeight; + + RTC_LOG(LS_INFO) << implementationName << " AV1 decoding " << width << "x" << height; + + return true; + } + + void VaapiAv1Decoder::DestroySession() + { + VADisplay display = library.Display(); + + if (downloadImage.image_id != VA_INVALID_ID) { + va.DestroyImage(display, downloadImage.image_id); + downloadImage.image_id = VA_INVALID_ID; + } + if (context != VA_INVALID_ID) { + va.DestroyContext(display, context); + context = VA_INVALID_ID; + } + if (!surfaces.empty()) { + va.DestroySurfaces(display, surfaces.data(), static_cast(surfaces.size())); + surfaces.clear(); + } + if (config != VA_INVALID_ID) { + va.DestroyConfig(display, config); + config = VA_INVALID_ID; + } + + width = 0; + height = 0; + slots.fill(-1); + } + + int VaapiAv1Decoder::FreeSurface() const + { + for (int surface = 0; surface < static_cast(surfaces.size()); surface++) { + if (std::find(slots.begin(), slots.end(), surface) == slots.end()) { + return surface; + } + } + + return -1; + } + + bool VaapiAv1Decoder::DecodeFrame(const Frame & frame, int surface) + { + const Av1FrameHeader & header = frame.header; + const Av1SequenceHeader & sequence = *parser.SequenceHeader(); + + VADecPictureParameterBufferAV1 picture = {}; + picture.profile = sequence.profile; + picture.order_hint_bits_minus_1 = static_cast(std::max(sequence.orderHintBits - 1, 0)); + picture.bit_depth_idx = 0; + picture.matrix_coefficients = sequence.matrixCoefficients; + + auto & seqFields = picture.seq_info_fields.fields; + seqFields.still_picture = sequence.stillPicture ? 1 : 0; + seqFields.use_128x128_superblock = sequence.use128x128Superblock ? 1 : 0; + seqFields.enable_filter_intra = sequence.enableFilterIntra ? 1 : 0; + seqFields.enable_intra_edge_filter = sequence.enableIntraEdgeFilter ? 1 : 0; + seqFields.enable_interintra_compound = sequence.enableInterintraCompound ? 1 : 0; + seqFields.enable_masked_compound = sequence.enableMaskedCompound ? 1 : 0; + seqFields.enable_dual_filter = sequence.enableDualFilter ? 1 : 0; + seqFields.enable_order_hint = sequence.enableOrderHint ? 1 : 0; + seqFields.enable_jnt_comp = sequence.enableJntComp ? 1 : 0; + seqFields.enable_cdef = sequence.enableCdef ? 1 : 0; + seqFields.mono_chrome = sequence.monochrome ? 1 : 0; + seqFields.color_range = sequence.colorRange ? 1 : 0; + seqFields.subsampling_x = sequence.subsamplingX ? 1 : 0; + seqFields.subsampling_y = sequence.subsamplingY ? 1 : 0; + seqFields.film_grain_params_present = sequence.filmGrainParamsPresent ? 1 : 0; + + picture.current_frame = surfaces[surface]; + picture.current_display_picture = surfaces[surface]; + picture.frame_width_minus1 = static_cast(header.upscaledWidth - 1); + picture.frame_height_minus1 = static_cast(header.frameHeight - 1); + + // A key frame that is shown refers to nothing. + const bool resets = header.frameType == kAv1KeyFrame && header.showFrame; + + for (int slot = 0; slot < 8; slot++) { + picture.ref_frame_map[slot] = !resets && slots[slot] >= 0 ? surfaces[slots[slot]] : VA_INVALID_SURFACE; + } + for (int i = 0; i < 7; i++) { + picture.ref_frame_idx[i] = header.refFrameIdx[i]; + } + + picture.primary_ref_frame = header.primaryRefFrame; + picture.order_hint = header.orderHint; + + auto & segmentation = picture.seg_info; + segmentation.segment_info_fields.bits.enabled = header.segmentationEnabled ? 1 : 0; + segmentation.segment_info_fields.bits.update_map = header.segmentationUpdateMap ? 1 : 0; + segmentation.segment_info_fields.bits.temporal_update = header.segmentationTemporalUpdate ? 1 : 0; + segmentation.segment_info_fields.bits.update_data = header.segmentationUpdateData ? 1 : 0; + + for (int segment = 0; segment < 8; segment++) { + for (int feature = 0; feature < 8; feature++) { + segmentation.feature_mask[segment] |= (header.featureEnabled[segment][feature] ? 1 : 0) << feature; + segmentation.feature_data[segment][feature] = header.featureData[segment][feature]; + } + } + + picture.tile_cols = header.tileCols; + picture.tile_rows = header.tileRows; + + for (int i = 0; i < header.tileCols && i < 63; i++) { + picture.width_in_sbs_minus_1[i] = static_cast(header.tileWidthSbs[i] - 1); + } + for (int i = 0; i < header.tileRows && i < 63; i++) { + picture.height_in_sbs_minus_1[i] = static_cast(header.tileHeightSbs[i] - 1); + } + + picture.context_update_tile_id = header.contextUpdateTileId; + + auto & picFields = picture.pic_info_fields.bits; + picFields.frame_type = header.frameType; + picFields.show_frame = header.showFrame ? 1 : 0; + picFields.showable_frame = header.showableFrame ? 1 : 0; + picFields.error_resilient_mode = header.errorResilientMode ? 1 : 0; + picFields.disable_cdf_update = header.disableCdfUpdate ? 1 : 0; + picFields.allow_screen_content_tools = header.allowScreenContentTools ? 1 : 0; + picFields.force_integer_mv = header.forceIntegerMv ? 1 : 0; + picFields.allow_intrabc = header.allowIntrabc ? 1 : 0; + picFields.use_superres = header.useSuperres ? 1 : 0; + picFields.allow_high_precision_mv = header.allowHighPrecisionMv ? 1 : 0; + picFields.is_motion_mode_switchable = header.isMotionModeSwitchable ? 1 : 0; + picFields.use_ref_frame_mvs = header.useRefFrameMvs ? 1 : 0; + picFields.disable_frame_end_update_cdf = header.disableFrameEndUpdateCdf ? 1 : 0; + picFields.uniform_tile_spacing_flag = header.uniformTileSpacing ? 1 : 0; + picFields.allow_warped_motion = header.allowWarpedMotion ? 1 : 0; + picFields.large_scale_tile = 0; + + picture.superres_scale_denominator = header.superresDenom; + picture.interp_filter = header.interpolationFilter; + picture.filter_level[0] = header.loopFilterLevel[0]; + picture.filter_level[1] = header.loopFilterLevel[1]; + picture.filter_level_u = header.loopFilterLevel[2]; + picture.filter_level_v = header.loopFilterLevel[3]; + + auto & loopFilterFields = picture.loop_filter_info_fields.bits; + loopFilterFields.sharpness_level = header.loopFilterSharpness; + loopFilterFields.mode_ref_delta_enabled = header.loopFilterDeltaEnabled ? 1 : 0; + loopFilterFields.mode_ref_delta_update = header.loopFilterDeltaUpdate ? 1 : 0; + + std::copy(header.loopFilterRefDeltas.begin(), header.loopFilterRefDeltas.end(), picture.ref_deltas); + std::copy(header.loopFilterModeDeltas.begin(), header.loopFilterModeDeltas.end(), picture.mode_deltas); + + picture.base_qindex = header.baseQIndex; + picture.y_dc_delta_q = header.deltaQYDc; + picture.u_dc_delta_q = header.deltaQUDc; + picture.u_ac_delta_q = header.deltaQUAc; + picture.v_dc_delta_q = header.deltaQVDc; + picture.v_ac_delta_q = header.deltaQVAc; + + auto & qmatrixFields = picture.qmatrix_fields.bits; + qmatrixFields.using_qmatrix = header.usingQmatrix ? 1 : 0; + qmatrixFields.qm_y = header.qmY; + qmatrixFields.qm_u = header.qmU; + qmatrixFields.qm_v = header.qmV; + + auto & modeFields = picture.mode_control_fields.bits; + modeFields.delta_q_present_flag = header.deltaQPresent ? 1 : 0; + modeFields.log2_delta_q_res = header.deltaQRes; + modeFields.delta_lf_present_flag = header.deltaLfPresent ? 1 : 0; + modeFields.log2_delta_lf_res = header.deltaLfRes; + modeFields.delta_lf_multi = header.deltaLfMulti ? 1 : 0; + modeFields.tx_mode = header.txMode; + modeFields.reference_select = header.referenceSelect ? 1 : 0; + modeFields.reduced_tx_set_used = header.reducedTxSet ? 1 : 0; + modeFields.skip_mode_present = header.skipModePresent ? 1 : 0; + + picture.cdef_damping_minus_3 = header.cdefDampingMinus3; + picture.cdef_bits = header.cdefBits; + + for (int i = 0; i < 8; i++) { + picture.cdef_y_strengths[i] = static_cast((header.cdefYPriStrength[i] << 2) + | header.cdefYSecStrength[i]); + picture.cdef_uv_strengths[i] = static_cast((header.cdefUvPriStrength[i] << 2) + | header.cdefUvSecStrength[i]); + } + + auto & restorationFields = picture.loop_restoration_fields.bits; + restorationFields.yframe_restoration_type = header.frameRestorationType[0]; + restorationFields.cbframe_restoration_type = header.frameRestorationType[1]; + restorationFields.crframe_restoration_type = header.frameRestorationType[2]; + restorationFields.lr_unit_shift = header.lrUnitShift; + restorationFields.lr_uv_shift = header.lrUvShift; + + for (int i = 0; i < 7; i++) { + const int ref = i + 1; + VAWarpedMotionParamsAV1 & warp = picture.wm[i]; + + warp.wmtype = static_cast(header.gmType[ref]); + + for (int j = 0; j < 6; j++) { + warp.wmmat[j] = header.gmParams[ref][j]; + } + + warp.invalid = header.gmValid[ref] ? 0 : 1; + } + + VADisplay display = library.Display(); + VABufferID buffers[3] = { VA_INVALID_ID, VA_INVALID_ID, VA_INVALID_ID }; + + // vaCreateBuffer copies the data, though it takes it as not const. + auto * tiles = const_cast(frame.tiles.data()); + auto * tileData = const_cast(frame.tileData.data()); + + bool created = Check(va.CreateBuffer(display, context, VAPictureParameterBufferType, sizeof(picture), 1, + &picture, &buffers[0]), "creating the picture parameters") + && Check(va.CreateBuffer(display, context, VASliceParameterBufferType, sizeof(VASliceParameterBufferAV1), + static_cast(frame.tiles.size()), tiles, &buffers[1]), "creating the tile parameters") + && Check(va.CreateBuffer(display, context, VASliceDataBufferType, + static_cast(frame.tileData.size()), 1, tileData, &buffers[2]), "creating the tile data"); + + bool decoded = false; + + if (created && Check(va.BeginPicture(display, context, surfaces[surface]), "beginning a picture")) { + const bool rendered = Check(va.RenderPicture(display, context, buffers, 3), "rendering a picture"); + // A picture that was begun has to be ended. + const bool ended = Check(va.EndPicture(display, context), "ending a picture"); + + decoded = rendered && ended && Check(va.SyncSurface(display, surfaces[surface]), "waiting for a picture"); + } + + for (VABufferID buffer : buffers) { + if (buffer != VA_INVALID_ID) { + va.DestroyBuffer(display, buffer); + } + } + + return decoded; + } + + bool VaapiAv1Decoder::Output(int surface, const webrtc::EncodedImage & image, int64_t renderTimeMs) + { + VADisplay display = library.Display(); + const int frameWidth = static_cast(width); + const int frameHeight = static_cast(height); + + VAImage mapped = {}; + mapped.image_id = VA_INVALID_ID; + bool derived = false; + + if (downloadImage.image_id != VA_INVALID_ID + && va.GetImage(display, surfaces[surface], 0, 0, width, height, downloadImage.image_id) + == VA_STATUS_SUCCESS) + { + mapped = downloadImage; + } + else if (va.DeriveImage(display, surfaces[surface], &mapped) == VA_STATUS_SUCCESS) { + derived = true; + + if (mapped.format.fourcc != VA_FOURCC_NV12) { + RTC_LOG(LS_WARNING) << implementationName << " AV1: the decoded surface is not NV12"; + va.DestroyImage(display, mapped.image_id); + return false; + } + } + else { + RTC_LOG(LS_WARNING) << implementationName << " AV1: the decoded surface cannot be read"; + return false; + } + + void * data = nullptr; + bool converted = false; + webrtc::scoped_refptr i420 = webrtc::I420Buffer::Create(frameWidth, frameHeight); + + if (Check(va.MapBuffer(display, mapped.buf, &data), "mapping a decoded picture")) { + const uint8_t * base = static_cast(data); + + converted = libyuv::NV12ToI420(base + mapped.offsets[0], static_cast(mapped.pitches[0]), + base + mapped.offsets[1], static_cast(mapped.pitches[1]), + i420->MutableDataY(), i420->StrideY(), i420->MutableDataU(), i420->StrideU(), + i420->MutableDataV(), i420->StrideV(), frameWidth, frameHeight) == 0; + + va.UnmapBuffer(display, mapped.buf); + } + + if (derived) { + va.DestroyImage(display, mapped.image_id); + } + + if (!converted) { + return false; + } + + webrtc::VideoFrame frame = webrtc::VideoFrame::Builder() + .set_video_frame_buffer(i420) + .set_rtp_timestamp(image.RtpTimestamp()) + .set_timestamp_ms(renderTimeMs) + .set_ntp_time_ms(image.ntp_time_ms_) + .set_rotation(image.rotation_) + .build(); + + callback->Decoded(frame, std::nullopt, std::nullopt); + + return true; + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoDecoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoDecoderFactory.cpp index 1870d950..ce3488be 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoDecoderFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoDecoderFactory.cpp @@ -17,6 +17,7 @@ #include "media/video/codec/linux/VaapiVideoDecoderFactory.h" #include "media/video/codec/H265Format.h" +#include "media/video/codec/linux/VaapiAv1Decoder.h" #include "media/video/codec/linux/VaapiH265Decoder.h" #include "media/video/codec/linux/VaapiVp9Decoder.h" @@ -34,21 +35,23 @@ namespace jni const bool vp9 = library->Decodes(VAProfileVP9Profile0); const bool h265 = library->Decodes(VAProfileHEVCMain); + const bool av1 = library->Decodes(VAProfileAV1Profile0); RTC_LOG(LS_INFO) << "VA-API (" << library->Vendor() << ") VP9 decoding: " << vp9 << ", H.265 decoding: " - << h265; + << h265 << ", AV1 decoding: " << av1; - if (!vp9 && !h265) { + if (!vp9 && !h265 && !av1) { return nullptr; } - return std::unique_ptr(new VaapiVideoDecoderFactory(*library, vp9, h265)); + return std::unique_ptr(new VaapiVideoDecoderFactory(*library, vp9, h265, av1)); } - VaapiVideoDecoderFactory::VaapiVideoDecoderFactory(VaapiLibrary & library, bool vp9, bool h265) : + VaapiVideoDecoderFactory::VaapiVideoDecoderFactory(VaapiLibrary & library, bool vp9, bool h265, bool av1) : library(library), vp9(vp9), - h265(h265) + h265(h265), + av1(av1) { } @@ -62,6 +65,9 @@ namespace jni if (h265) { formats.push_back(CreateH265MainFormat()); } + if (av1) { + formats.push_back(webrtc::SdpVideoFormat::AV1Profile0()); + } return formats; } @@ -75,6 +81,9 @@ namespace jni if (vp9 && format.IsSameCodec(webrtc::SdpVideoFormat::VP9Profile0())) { return std::make_unique(library); } + if (av1 && format.IsSameCodec(webrtc::SdpVideoFormat::AV1Profile0())) { + return std::make_unique(library); + } return nullptr; } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java index 814e0dd6..a98c93cb 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java @@ -55,8 +55,10 @@ * are decoded on NVIDIA GPUs with NVDEC, each where the GPU decodes it (AV1 * since Ampere), loaded from the NVIDIA driver when the factory is made; a * machine without the driver is not affected. Streams with spatial layers go - * to libvpx and dav1d. On the GPUs of Intel and AMD, VP9 (profile 0) and - * H.265 (Main profile) are decoded with VA-API, where the driver decodes them. + * to libvpx and dav1d. On the GPUs of Intel and AMD, VP9 (profile 0), H.265 + * (Main profile) and AV1 (profile 0) are decoded with VA-API, where the driver + * decodes them; AV1 streams with film grain, superres, more than 8 bits or a + * single plane go to dav1d. *

* The supported codecs are those of a {@link DefaultVideoDecoderFactory}, in * the same order. On Windows and Linux, H.264 High 4:4:4 Predictive, diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java index 61f9738b..bd1cd244 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java @@ -424,6 +424,21 @@ void defaultDecodesVp9InSoftware() throws Exception { void hardwareFollowsResolutionChange() throws Exception { assumeVp9Platform(); + assertFollowsResolutionChange(VP9, HardwareVideoDecoderIntegrationTest::expectVp9Hardware); + } + + @Test + void hardwareFollowsAv1ResolutionChange() throws Exception { + assertFollowsResolutionChange(AV1, implementation -> expectHardware(implementation, HARDWARE_AV1_REQUIRED)); + } + + /** + * Receives video that halves its size during the call, and checks that the + * hardware decoder follows it, rather than handing over to the software + * decoder. + */ + private void assertFollowsResolutionChange(Predicate codec, + Consumer expectHardware) throws Exception { PeerConnectionFactory hardware = PeerConnectionFactory.builder() .setAudioDeviceModule(audioDevModule) .setVideoDecoderFactory(new HardwareVideoDecoderFactory()) @@ -432,7 +447,7 @@ void hardwareFollowsResolutionChange() throws Exception { CountDownLatch full = new CountDownLatch(10); CountDownLatch half = new CountDownLatch(10); - try (TestMediaCall call = new TestMediaCall(hardware, true, false, VP9)) { + try (TestMediaCall call = new TestMediaCall(hardware, true, false, codec)) { // Large enough that half the size is still one every hardware // decoder takes: NVDEC does not decode VP9 below 128 pixels on the // shorter side, and falls back to libvpx for such a stream. @@ -462,7 +477,7 @@ else if (width == 320 && frame.buffer.getHeight() == 240) { String implementation = decoderImplementationOf(call); - expectVp9Hardware(implementation); + expectHardware.accept(implementation); // The sender restarts at half the size, with a key frame; the // decoder has to start a session for it. @@ -531,6 +546,28 @@ void vp9SpatialLayers() throws Exception { assertFalse(isHardware(result.decoder), result.decoder); } + @Test + void av1NeedsNoKeyFrames() throws Exception { + CallResult result = receiveVideo(hardwareFactory(), AV1, 320, 240, null, 60); + + expectHardware(result.decoder, HARDWARE_AV1_REQUIRED); + + // As with VP9: a stream that is decoded has the one key frame it + // started with. + assertTrue(result.pliCount <= 1, "key frames requested: " + result.pliCount); + assertTrue(result.keyFrames <= 2, "key frames decoded: " + result.keyFrames); + } + + @Test + void av1TemporalLayers() throws Exception { + // Temporal layers are one spatial layer, which the hardware decodes. + CallResult result = receiveVideo(hardwareFactory(), AV1, 320, 240, "L1T3", 60); + + expectHardware(result.decoder, HARDWARE_AV1_REQUIRED); + + assertTrue(result.pliCount <= 1, "key frames requested: " + result.pliCount); + } + private PeerConnectionFactory hardwareFactory() { return PeerConnectionFactory.builder() .setAudioDeviceModule(audioDevModule)