diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index 91eafb0c..9f36e4ce 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -62,10 +62,10 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder() | Platform | Hardware decoding | |---|---| | Windows | H.264, H.265 (Main profile), AV1 and VP9 (profile 0), on GPUs that decode them, through the Media Foundation decoders of Windows on Direct3D 11 (DXVA) | -| Linux | H.264, H.265 (Main profile), VP9 and AV1 (profile 0) on NVIDIA GPUs that decode them, through NVDEC; H.264, VP9 (profile 0) and H.265 (Main profile) on Intel and AMD GPUs through VA-API | +| Linux | H.264, H.265 (Main profile), VP9 and AV1 (profile 0) on NVIDIA GPUs that decode them, through NVDEC; H.264, VP9 (profile 0), H.265 (Main profile) and AV1 (profile 0) on Intel and AMD GPUs through VA-API, where the driver decodes them | | macOS | H.264 through VideoToolbox, as with `DefaultVideoDecoderFactory`; VP9 (profile 0) and H.265 (Main profile) through the decoders of VideoToolbox, on Macs that have them, and AV1 (profile 0) on Apple M3 and newer | -On Linux, NVDEC needs an NVIDIA driver, which provides `libcuda.so.1` and `libnvcuvid.so.1`; both are loaded when the factory is made, so a machine without the driver is not affected. Streams with spatial layers are decoded in software. +On Linux, NVDEC needs an NVIDIA driver, which provides `libcuda.so.1` and `libnvcuvid.so.1`; both are loaded when the factory is made, so a machine without the driver is not affected. Streams with spatial layers are decoded in software. Through VA-API, AV1 is decoded in 8 bit 4:2:0; streams with film grain, superres, a higher bit depth or a single plane are decoded by dav1d. On Windows and Linux, H.264 is decoded in hardware in its 4:2:0 profiles. High 4:4:4 Predictive, which GPU decoders do not take, is decoded in software. diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/Av1Parser.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/Av1Parser.h new file mode 100644 index 00000000..eb84bfac --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/Av1Parser.h @@ -0,0 +1,327 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_AV1_PARSER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_AV1_PARSER_H_ + +#include +#include +#include +#include +#include + +namespace jni +{ + // The OBU types of section 6.2.2 of the AV1 specification. + enum class Av1ObuType : uint8_t + { + SequenceHeader = 1, + TemporalDelimiter = 2, + FrameHeader = 3, + TileGroup = 4, + Metadata = 5, + Frame = 6, + RedundantFrameHeader = 7, + TileList = 8, + Padding = 15 + }; + + // The frame types of section 6.8.2. + enum Av1FrameType : uint8_t + { + kAv1KeyFrame = 0, + kAv1InterFrame = 1, + kAv1IntraOnlyFrame = 2, + kAv1SwitchFrame = 3 + }; + + // One OBU of a temporal unit, its payload a span of the data it came in. + struct Av1Obu + { + Av1ObuType type = Av1ObuType::Padding; + bool hasExtension = false; + uint8_t temporalId = 0; + uint8_t spatialId = 0; + std::span payload; + }; + + // The sequence header OBU, as section 5.5 defines it, with what a decoder + // of operating point 0 needs. + struct Av1SequenceHeader + { + uint8_t profile = 0; + bool stillPicture = false; + bool reducedStillPictureHeader = false; + + bool decoderModelInfoPresent = false; + bool equalPictureInterval = false; + uint8_t bufferRemovalTimeLength = 0; + uint8_t framePresentationTimeLength = 0; + uint8_t operatingPointCount = 1; + std::array operatingPointIdc = {}; + std::array decoderModelPresentForOperatingPoint = {}; + + uint8_t frameWidthBits = 0; + uint8_t frameHeightBits = 0; + uint32_t maxFrameWidth = 0; + uint32_t maxFrameHeight = 0; + + bool frameIdNumbersPresent = false; + uint8_t deltaFrameIdLength = 0; + uint8_t frameIdLength = 0; + + bool use128x128Superblock = false; + bool enableFilterIntra = false; + bool enableIntraEdgeFilter = false; + bool enableInterintraCompound = false; + bool enableMaskedCompound = false; + bool enableWarpedMotion = false; + bool enableDualFilter = false; + bool enableOrderHint = false; + bool enableJntComp = false; + bool enableRefFrameMvs = false; + // 0, 1, or 2 for a choice made by each frame. + uint8_t seqForceScreenContentTools = 2; + uint8_t seqForceIntegerMv = 2; + uint8_t orderHintBits = 0; + + bool enableSuperres = false; + bool enableCdef = false; + bool enableRestoration = false; + + uint8_t bitDepth = 8; + bool monochrome = false; + uint8_t colorPrimaries = 2; + uint8_t transferCharacteristics = 2; + uint8_t matrixCoefficients = 2; + bool colorRange = false; + bool subsamplingX = true; + bool subsamplingY = true; + bool separateUvDeltaQ = false; + + bool filmGrainParamsPresent = false; + }; + + // The frame header OBU, the uncompressed header of section 5.9, with what + // earlier frames left that it depends on resolved: the loop filter deltas, + // the segmentation and the global motion that carry over from the primary + // reference frame, and the size taken from a reference frame. + struct Av1FrameHeader + { + bool showExistingFrame = false; + uint8_t frameToShow = 0; + + uint8_t frameType = kAv1KeyFrame; + bool frameIsIntra = true; + bool showFrame = true; + bool showableFrame = false; + bool errorResilientMode = false; + bool disableCdfUpdate = false; + bool allowScreenContentTools = false; + bool forceIntegerMv = false; + uint32_t currentFrameId = 0; + uint8_t orderHint = 0; + // 7 for none. + uint8_t primaryRefFrame = 7; + uint8_t refreshFrameFlags = 0; + + // FrameWidth is the coded width, UpscaledWidth the width with + // superres; they differ only with superres. + uint32_t frameWidth = 0; + uint32_t frameHeight = 0; + uint32_t upscaledWidth = 0; + uint32_t renderWidth = 0; + uint32_t renderHeight = 0; + bool useSuperres = false; + uint8_t superresDenom = 8; + uint32_t miCols = 0; + uint32_t miRows = 0; + + bool allowIntrabc = false; + std::array refFrameIdx = {}; + bool allowHighPrecisionMv = false; + // 0 to 3, or 4 for switchable. + uint8_t interpolationFilter = 0; + bool isMotionModeSwitchable = false; + bool useRefFrameMvs = false; + bool disableFrameEndUpdateCdf = true; + + bool uniformTileSpacing = true; + uint8_t tileCols = 1; + uint8_t tileRows = 1; + uint8_t tileColsLog2 = 0; + uint8_t tileRowsLog2 = 0; + // The width and height of each tile column and row in superblocks. + std::array tileWidthSbs = {}; + std::array tileHeightSbs = {}; + uint16_t contextUpdateTileId = 0; + uint8_t tileSizeBytes = 4; + + uint8_t baseQIndex = 0; + int8_t deltaQYDc = 0; + int8_t deltaQUDc = 0; + int8_t deltaQUAc = 0; + int8_t deltaQVDc = 0; + int8_t deltaQVAc = 0; + bool usingQmatrix = false; + uint8_t qmY = 0; + uint8_t qmU = 0; + uint8_t qmV = 0; + + bool segmentationEnabled = false; + bool segmentationUpdateMap = false; + bool segmentationTemporalUpdate = false; + bool segmentationUpdateData = false; + bool featureEnabled[8][8] = {}; + int16_t featureData[8][8] = {}; + + bool deltaQPresent = false; + uint8_t deltaQRes = 0; + bool deltaLfPresent = false; + uint8_t deltaLfRes = 0; + bool deltaLfMulti = false; + + bool codedLossless = false; + bool allLossless = false; + + // Luma vertical and horizontal, then U and V. + std::array loopFilterLevel = {}; + uint8_t loopFilterSharpness = 0; + bool loopFilterDeltaEnabled = false; + bool loopFilterDeltaUpdate = false; + std::array loopFilterRefDeltas = {}; + std::array loopFilterModeDeltas = {}; + + uint8_t cdefDampingMinus3 = 0; + uint8_t cdefBits = 0; + // The secondary strengths as coded, 3 standing for 4. + std::array cdefYPriStrength = {}; + std::array cdefYSecStrength = {}; + std::array cdefUvPriStrength = {}; + std::array cdefUvSecStrength = {}; + + // FrameRestorationType: none, Wiener, self-guided, switchable. + std::array frameRestorationType = {}; + uint8_t lrUnitShift = 0; + uint8_t lrUvShift = 0; + + // Only 4x4, largest, or select. + uint8_t txMode = 0; + bool referenceSelect = false; + bool skipModePresent = false; + bool allowWarpedMotion = false; + bool reducedTxSet = false; + + // By reference frame, LAST_FRAME (1) to ALTREF_FRAME (7): identity, + // translation, rotation and zoom, or affine; and whether the shear + // of the warp is valid. + std::array gmType = {}; + std::array, 8> gmParams = {}; + std::array gmValid = {}; + + // Film grain applied to the output, which is not parsed further. + bool applyGrain = false; + + // The size of the uncompressed header in bytes, after which the tile + // group of a frame OBU starts. + uint32_t headerBytes = 0; + }; + + // One tile of a frame, where its data is in the tile group it came in. + struct Av1Tile + { + uint16_t row = 0; + uint16_t column = 0; + uint32_t offset = 0; + uint32_t size = 0; + }; + + // The tiles of one tile group OBU. + struct Av1TileGroup + { + uint16_t start = 0; + uint16_t end = 0; + std::vector tiles; + }; + + // Parses the sequence and frame headers of an AV1 stream, in the order + // its frames are decoded, keeping what carries over from one frame to the + // next in the eight reference slots. + class Av1Parser + { + public: + Av1Parser(); + + // Forgets the sequence header and the reference frames, as before a + // new stream. + void Reset(); + + // The OBUs of a temporal unit, or nothing if they do not fit into + // the data. Takes the OBUs of operating point 0 only, where the + // sequence header is known. + std::optional> SplitObus(std::span data) const; + + // Parses a sequence header OBU. Returns false if it is not valid. + bool ParseSequenceHeader(std::span payload); + + const std::optional & SequenceHeader() const; + + // Parses a frame header OBU, or the header of a frame OBU. Returns + // nothing if it is not valid, or refers to a reference frame that + // is not there. + std::optional ParseFrameHeader(std::span payload, const Av1Obu & obu); + + // Parses a tile group OBU of the frame. Returns nothing if it is + // not valid. + std::optional ParseTileGroup(std::span payload, + const Av1FrameHeader & header) const; + + // Takes the frame into the reference slots its refresh flags name, + // once it is decoded, or, for a key frame shown again, all of them; + // section 7.20 and 7.21 of the specification. + void UpdateReferences(const Av1FrameHeader & header); + + private: + // What a reference slot holds of the frame in it. + struct Reference + { + bool valid = false; + uint32_t frameId = 0; + uint32_t upscaledWidth = 0; + uint32_t frameWidth = 0; + uint32_t frameHeight = 0; + uint32_t renderWidth = 0; + uint32_t renderHeight = 0; + uint8_t frameType = kAv1KeyFrame; + uint8_t orderHint = 0; + std::array, 8> gmParams = {}; + std::array loopFilterRefDeltas = {}; + std::array loopFilterModeDeltas = {}; + bool featureEnabled[8][8] = {}; + int16_t featureData[8][8] = {}; + }; + + int RelativeDistance(int a, int b) const; + // What the specification calls set_frame_refs. + bool SetFrameRefs(Av1FrameHeader & header, uint8_t lastFrameIdx, uint8_t goldFrameIdx) const; + + private: + std::optional sequence; + std::array references; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Decoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Decoder.h new file mode 100644 index 00000000..e74b095d --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Decoder.h @@ -0,0 +1,130 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_AV1_DECODER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_AV1_DECODER_H_ + +#include "media/video/codec/linux/Av1Parser.h" +#include "media/video/codec/linux/VaapiLibrary.h" + +#include "api/video/encoded_image.h" +#include "api/video_codecs/video_decoder.h" + +#include +#include + +#include +#include +#include +#include +#include + +namespace jni +{ + // Decodes AV1 profile 0 in 8 bit 4:2:0 with the GPU driver's VA-API + // decoder, as AMD drivers provide it on Linux. + // + // VA-API leaves the parsing of the stream to the application: the + // sequence and frame headers are parsed here, and where each tile is; the + // driver decodes the tiles. Decoding is synchronous. The decoded frame is + // copied from its surface in NV12 and converted to I420, which is what + // WebRTC's frames hold. + // + // Whatever it cannot decode, such as another profile or bit depth, spatial + // layers, film grain, superres or an inter frame that changes the size, + // and whatever fails, returns WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE, so + // that dav1d takes over. + class VaapiAv1Decoder : public webrtc::VideoDecoder + { + public: + explicit VaapiAv1Decoder(VaapiLibrary & library); + ~VaapiAv1Decoder() override; + + using webrtc::VideoDecoder::Decode; + + bool Configure(const Settings & settings) override; + int32_t Decode(const webrtc::EncodedImage & image, int64_t renderTimeMs) override; + int32_t RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback * callback) override; + int32_t Release() override; + DecoderInfo GetDecoderInfo() const override; + const char * ImplementationName() const override; + + private: + // A frame of a temporal unit: its header, and, unless it shows a + // frame again, the data of its tile groups one after another and + // its tiles, with their offsets into that data. + struct Frame + { + Av1FrameHeader header; + std::vector tileData; + std::vector tiles; + }; + + // Parses the OBUs of one temporal unit into its frames, checking + // that all of them can be decoded before any is; false where the + // software decoder has to take over, which decodes the temporal + // unit again, so that nothing may have been output by then. + bool ParseTemporalUnit(std::span data, std::vector & frames); + // Takes a tile group of the frame in. + bool AddTileGroup(std::span payload, Frame & frame); + // Decodes and shows the frames. + bool DecodeFrames(const std::vector & frames, const webrtc::EncodedImage & image, + int64_t renderTimeMs); + + // Creates the configuration, the context and the surfaces for + // frames of the size. + bool CreateSession(uint32_t width, uint32_t height); + void DestroySession(); + + // Decodes the frame into the surface. + bool DecodeFrame(const Frame & frame, int surface); + // Hands the frame in the surface to WebRTC. + bool Output(int surface, const webrtc::EncodedImage & image, int64_t renderTimeMs); + // A surface that no reference slot holds. + int FreeSurface() const; + + bool Check(VAStatus status, const char * operation) const; + + private: + VaapiLibrary & library; + const VaapiFunctions & va; + const std::string implementationName; + + webrtc::DecodedImageCallback * callback; + + Av1Parser parser; + + VAConfigID config; + VAContextID context; + std::vector surfaces; + uint32_t width; + uint32_t height; + + // The image a decoded surface is copied into, where the driver + // copies surfaces at all. + VAImage downloadImage; + + // The surface each of the eight reference slots holds, -1 for none. + std::array slots; + + // Whether a key frame has been decoded, and whether anything since + // then failed for good. + bool started; + bool failed; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoDecoderFactory.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoDecoderFactory.h index 3c83aedc..2a984e3c 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoDecoderFactory.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoDecoderFactory.h @@ -30,8 +30,8 @@ namespace jni { // Creates VA-API decoders on the GPUs of Intel and AMD, for H.264 (but - // High 4:4:4 Predictive), VP9 profile 0 and H.265 Main where the driver - // decodes them. + // High 4:4:4 Predictive), VP9 profile 0, H.265 Main and AV1 profile 0 + // where the driver decodes them. class VaapiVideoDecoderFactory : public webrtc::VideoDecoderFactory { public: @@ -46,13 +46,14 @@ namespace jni const webrtc::SdpVideoFormat & format) override; private: - VaapiVideoDecoderFactory(VaapiLibrary & library, bool h264, bool vp9, bool h265); + VaapiVideoDecoderFactory(VaapiLibrary & library, bool h264, bool vp9, bool h265, bool av1); private: VaapiLibrary & library; const bool h264; const bool vp9; const bool h265; + const bool av1; }; } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/Av1Parser.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/Av1Parser.cpp new file mode 100644 index 00000000..8eeffd73 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/Av1Parser.cpp @@ -0,0 +1,1619 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/linux/Av1Parser.h" + +#include "rtc_base/bitstream_reader.h" + +#include +#include + +namespace jni +{ + namespace + { + constexpr int kRefsPerFrame = 7; + constexpr int kNumRefFrames = 8; + constexpr uint8_t kPrimaryRefNone = 7; + constexpr uint8_t kAllFrames = 0xff; + + // The reference frames of section 6.10.24. + constexpr int kLastFrame = 1; + constexpr int kLast2Frame = 2; + constexpr int kLast3Frame = 3; + constexpr int kGoldenFrame = 4; + constexpr int kBwdrefFrame = 5; + constexpr int kAltref2Frame = 6; + constexpr int kAltrefFrame = 7; + + constexpr uint8_t kSelectScreenContentTools = 2; + constexpr uint8_t kSelectIntegerMv = 2; + + constexpr uint8_t kSuperresNum = 8; + constexpr uint8_t kSuperresDenomMin = 9; + constexpr int kSuperresDenomBits = 3; + + constexpr int kMaxTileWidth = 4096; + constexpr int kMaxTileArea = 4096 * 2304; + constexpr int kMaxTileRows = 64; + constexpr int kMaxTileCols = 64; + + constexpr int kMaxSegments = 8; + constexpr int kSegLvlMax = 8; + constexpr int kSegLvlAltQ = 0; + constexpr int kSegmentationFeatureBits[kSegLvlMax] = { 8, 6, 6, 6, 6, 3, 0, 0 }; + constexpr bool kSegmentationFeatureSigned[kSegLvlMax] = { true, true, true, true, true, false, false, false }; + constexpr int kSegmentationFeatureMax[kSegLvlMax] = { 255, 63, 63, 63, 63, 7, 0, 0 }; + + // Remap_Lr_Type: none, switchable, Wiener, self-guided as coded, to + // FrameRestorationType. + constexpr uint8_t kRemapLrType[4] = { 0, 3, 1, 2 }; + + // The global motion types and parameters of section 5.9.24. + constexpr uint8_t kIdentity = 0; + constexpr uint8_t kTranslation = 1; + constexpr uint8_t kRotzoom = 2; + constexpr uint8_t kAffine = 3; + constexpr int kWarpedModelPrecBits = 16; + constexpr int kGmAbsTransOnlyBits = 9; + constexpr int kGmTransOnlyPrecBits = 3; + constexpr int kGmAbsTransBits = 12; + constexpr int kGmTransPrecBits = 6; + constexpr int kGmAbsAlphaBits = 12; + constexpr int kGmAlphaPrecBits = 15; + constexpr int kWarpParamReduceBits = 6; + constexpr int kDivLutBits = 8; + constexpr int kDivLutPrecBits = 14; + + constexpr int kTotalRefsPerFrame = 8; + + // f(n) of the specification for widths that fit. + uint32_t ReadBits(webrtc::BitstreamReader & reader, int bits) + { + return bits == 0 ? 0 : static_cast(reader.ReadBits(bits)); + } + + // su(n): an n bit two's complement number. + int32_t ReadSu(webrtc::BitstreamReader & reader, int bits) + { + int32_t value = static_cast(ReadBits(reader, bits)); + const int32_t signMask = 1 << (bits - 1); + + if (value & signMask) { + value -= 2 * signMask; + } + + return value; + } + + // read_delta_q. + int8_t ReadDeltaQ(webrtc::BitstreamReader & reader) + { + return reader.Read() ? static_cast(ReadSu(reader, 7)) : 0; + } + + // uvlc(). + uint32_t ReadUvlc(webrtc::BitstreamReader & reader) + { + int leadingZeros = 0; + + while (reader.Ok() && !reader.Read()) { + leadingZeros++; + + if (leadingZeros >= 32) { + reader.Invalidate(); + return 0; + } + } + + return static_cast((1ull << leadingZeros) - 1 + ReadBits(reader, leadingZeros)); + } + + // tile_log2. + int TileLog2(int blockSize, int target) + { + int k = 0; + + while ((blockSize << k) < target) { + k++; + } + + return k; + } + + int FloorLog2(uint32_t x) + { + int s = 0; + + while (x > 1) { + x >>= 1; + s++; + } + + return s; + } + + int64_t Round2Signed(int64_t x, int n) + { + if (n == 0) { + return x; + } + + return x >= 0 ? (x + (int64_t(1) << (n - 1))) >> n : -((-x + (int64_t(1) << (n - 1))) >> n); + } + + int32_t InverseRecenter(int32_t r, int32_t v) + { + if (v > 2 * r) { + return v; + } + if (v & 1) { + return r - ((v + 1) >> 1); + } + + return r + (v >> 1); + } + + // decode_subexp. + int32_t DecodeSubexp(webrtc::BitstreamReader & reader, int32_t numSyms) + { + int i = 0; + int32_t mk = 0; + const int k = 3; + + while (reader.Ok()) { + const int b2 = i ? k + i - 1 : k; + const int32_t a = 1 << b2; + + if (numSyms <= mk + 3 * a) { + return static_cast(reader.ReadNonSymmetric(static_cast(numSyms - mk))) + mk; + } + if (!reader.Read()) { + return static_cast(ReadBits(reader, b2)) + mk; + } + + i++; + mk += a; + } + + return 0; + } + + // decode_signed_subexp_with_ref. + int32_t DecodeSignedSubexpWithRef(webrtc::BitstreamReader & reader, int32_t low, int32_t high, int32_t r) + { + const int32_t mx = high - low; + r -= low; + + const int32_t v = DecodeSubexp(reader, mx); + const int32_t x = (r << 1) <= mx ? InverseRecenter(r, v) : mx - 1 - InverseRecenter(mx - 1 - r, v); + + return x + low; + } + + // The divisor lookup of section 7.11.3.7, Div_Lut[i] being + // 2^14 * 256 / (256 + i) rounded. + int32_t DivLut(int i) + { + const int32_t d = 256 + i; + + return ((1 << (kDivLutPrecBits + kDivLutBits)) + d / 2) / d; + } + + // Whether the shear of a warp is valid, section 7.11.3.6. + bool ShearValid(const std::array & params) + { + // resolve_divisor. + const int32_t d = params[2]; + + if (d == 0) { + return false; + } + + const uint32_t absD = static_cast(std::abs(d)); + const int n = FloorLog2(absD); + const int64_t e = absD - (int64_t(1) << n); + const int64_t f = n > kDivLutBits ? Round2Signed(e, n - kDivLutBits) : e << (kDivLutBits - n); + const int divShift = n + kDivLutPrecBits; + const int64_t divFactor = d < 0 ? -DivLut(static_cast(f)) : DivLut(static_cast(f)); + + auto clip = [](int64_t value) { + return std::clamp(value, -32768, 32767); + }; + + const int64_t alpha0 = clip(params[2] - (int64_t(1) << kWarpedModelPrecBits)); + const int64_t beta0 = clip(params[3]); + const int64_t v = int64_t(params[4]) << kWarpedModelPrecBits; + const int64_t gamma0 = clip(Round2Signed(v * divFactor, divShift)); + const int64_t w = int64_t(params[3]) * params[4]; + const int64_t delta0 = clip(params[5] - Round2Signed(w * divFactor, divShift) + - (int64_t(1) << kWarpedModelPrecBits)); + + auto reduce = [](int64_t value) { + return Round2Signed(value, kWarpParamReduceBits) * (int64_t(1) << kWarpParamReduceBits); + }; + + const int64_t alpha = reduce(alpha0); + const int64_t beta = reduce(beta0); + const int64_t gamma = reduce(gamma0); + const int64_t delta = reduce(delta0); + + if (4 * std::llabs(alpha) + 7 * std::llabs(beta) >= (int64_t(1) << kWarpedModelPrecBits)) { + return false; + } + if (4 * std::llabs(gamma) + 4 * std::llabs(delta) >= (int64_t(1) << kWarpedModelPrecBits)) { + return false; + } + + return true; + } + + std::array, 8> DefaultGmParams() + { + std::array, 8> params = {}; + + for (auto & ref : params) { + for (int i = 0; i < 6; i++) { + ref[i] = (i % 3 == 2) ? 1 << kWarpedModelPrecBits : 0; + } + } + + return params; + } + + constexpr std::array kDefaultLoopFilterRefDeltas = { 1, 0, 0, 0, -1, 0, -1, -1 }; + + // Reads a leb128 value from the bytes, section 4.10.5. + bool ReadLeb128(std::span data, size_t & position, uint64_t & value) + { + value = 0; + + for (int i = 0; i < 8; i++) { + if (position >= data.size()) { + return false; + } + + const uint8_t byte = data[position++]; + value |= uint64_t(byte & 0x7f) << (i * 7); + + if (!(byte & 0x80)) { + return true; + } + } + + return false; + } + } + + Av1Parser::Av1Parser() + { + Reset(); + } + + void Av1Parser::Reset() + { + sequence.reset(); + references.fill(Reference()); + } + + const std::optional & Av1Parser::SequenceHeader() const + { + return sequence; + } + + std::optional> Av1Parser::SplitObus(std::span data) const + { + std::vector obus; + size_t position = 0; + + while (position < data.size()) { + const uint8_t header = data[position++]; + + if (header & 0x80) { + // The forbidden bit. + return std::nullopt; + } + + Av1Obu obu; + obu.type = static_cast((header >> 3) & 0x0f); + obu.hasExtension = (header & 0x04) != 0; + + const bool hasSize = (header & 0x02) != 0; + + if (obu.hasExtension) { + if (position >= data.size()) { + return std::nullopt; + } + + const uint8_t extension = data[position++]; + obu.temporalId = extension >> 5; + obu.spatialId = (extension >> 3) & 0x03; + } + + uint64_t size = data.size() - position; + + if (hasSize && !ReadLeb128(data, position, size)) { + return std::nullopt; + } + if (size > data.size() - position) { + return std::nullopt; + } + + obu.payload = data.subspan(position, static_cast(size)); + position += static_cast(size); + + // Section 7.5: the OBUs of layers outside the operating point are + // dropped. + const uint16_t idc = sequence ? sequence->operatingPointIdc[0] : 0; + + if (obu.type != Av1ObuType::SequenceHeader && obu.type != Av1ObuType::TemporalDelimiter + && obu.hasExtension && idc != 0) + { + const bool inTemporalLayer = (idc >> obu.temporalId) & 1; + const bool inSpatialLayer = (idc >> (obu.spatialId + 8)) & 1; + + if (!inTemporalLayer || !inSpatialLayer) { + continue; + } + } + + obus.push_back(obu); + } + + return obus; + } + + bool Av1Parser::ParseSequenceHeader(std::span payload) + { + webrtc::BitstreamReader reader(payload); + Av1SequenceHeader header; + + header.profile = static_cast(ReadBits(reader, 3)); + header.stillPicture = reader.Read(); + header.reducedStillPictureHeader = reader.Read(); + + if (header.reducedStillPictureHeader) { + header.operatingPointCount = 1; + header.operatingPointIdc[0] = 0; + // seq_level_idx[0]. + reader.ConsumeBits(5); + } + else { + const bool timingInfoPresent = reader.Read(); + uint8_t bufferDelayLength = 0; + + if (timingInfoPresent) { + // num_units_in_display_tick, time_scale. + reader.ConsumeBits(64); + header.equalPictureInterval = reader.Read(); + + if (header.equalPictureInterval) { + ReadUvlc(reader); + } + + header.decoderModelInfoPresent = reader.Read(); + + if (header.decoderModelInfoPresent) { + bufferDelayLength = static_cast(ReadBits(reader, 5) + 1); + // num_units_in_decoding_tick. + reader.ConsumeBits(32); + header.bufferRemovalTimeLength = static_cast(ReadBits(reader, 5) + 1); + header.framePresentationTimeLength = static_cast(ReadBits(reader, 5) + 1); + } + } + + const bool initialDisplayDelayPresent = reader.Read(); + header.operatingPointCount = static_cast(ReadBits(reader, 5) + 1); + + for (int i = 0; i < header.operatingPointCount; i++) { + header.operatingPointIdc[i] = static_cast(ReadBits(reader, 12)); + + const uint32_t seqLevelIdx = ReadBits(reader, 5); + + if (seqLevelIdx > 7) { + // seq_tier. + reader.ConsumeBits(1); + } + + if (header.decoderModelInfoPresent) { + header.decoderModelPresentForOperatingPoint[i] = reader.Read(); + + if (header.decoderModelPresentForOperatingPoint[i]) { + // decoder_buffer_delay, encoder_buffer_delay, + // low_delay_mode_flag. + reader.ConsumeBits(2 * bufferDelayLength + 1); + } + } + if (initialDisplayDelayPresent && reader.Read()) { + // initial_display_delay_minus_1. + reader.ConsumeBits(4); + } + } + } + + header.frameWidthBits = static_cast(ReadBits(reader, 4) + 1); + header.frameHeightBits = static_cast(ReadBits(reader, 4) + 1); + header.maxFrameWidth = ReadBits(reader, header.frameWidthBits) + 1; + header.maxFrameHeight = ReadBits(reader, header.frameHeightBits) + 1; + + header.frameIdNumbersPresent = header.reducedStillPictureHeader ? false : reader.Read(); + + if (header.frameIdNumbersPresent) { + header.deltaFrameIdLength = static_cast(ReadBits(reader, 4) + 2); + header.frameIdLength = static_cast(ReadBits(reader, 3) + 1 + header.deltaFrameIdLength); + } + + header.use128x128Superblock = reader.Read(); + header.enableFilterIntra = reader.Read(); + header.enableIntraEdgeFilter = reader.Read(); + + if (!header.reducedStillPictureHeader) { + header.enableInterintraCompound = reader.Read(); + header.enableMaskedCompound = reader.Read(); + header.enableWarpedMotion = reader.Read(); + header.enableDualFilter = reader.Read(); + header.enableOrderHint = reader.Read(); + + if (header.enableOrderHint) { + header.enableJntComp = reader.Read(); + header.enableRefFrameMvs = reader.Read(); + } + + const bool seqChooseScreenContentTools = reader.Read(); + header.seqForceScreenContentTools = seqChooseScreenContentTools + ? kSelectScreenContentTools + : static_cast(reader.ReadBit()); + + if (header.seqForceScreenContentTools > 0) { + const bool seqChooseIntegerMv = reader.Read(); + header.seqForceIntegerMv = seqChooseIntegerMv + ? kSelectIntegerMv + : static_cast(reader.ReadBit()); + } + else { + header.seqForceIntegerMv = kSelectIntegerMv; + } + + if (header.enableOrderHint) { + header.orderHintBits = static_cast(ReadBits(reader, 3) + 1); + } + } + + header.enableSuperres = reader.Read(); + header.enableCdef = reader.Read(); + header.enableRestoration = reader.Read(); + + // color_config, section 5.5.2. + const bool highBitdepth = reader.Read(); + + if (header.profile == 2 && highBitdepth) { + header.bitDepth = reader.Read() ? 12 : 10; + } + else { + header.bitDepth = highBitdepth ? 10 : 8; + } + + header.monochrome = header.profile == 1 ? false : reader.Read(); + + if (reader.Read()) { + header.colorPrimaries = static_cast(ReadBits(reader, 8)); + header.transferCharacteristics = static_cast(ReadBits(reader, 8)); + header.matrixCoefficients = static_cast(ReadBits(reader, 8)); + } + + if (header.monochrome) { + header.colorRange = reader.Read(); + header.subsamplingX = true; + header.subsamplingY = true; + header.separateUvDeltaQ = false; + } + else { + // BT.709 primaries with the sRGB transfer and the identity matrix. + if (header.colorPrimaries == 1 && header.transferCharacteristics == 13 && header.matrixCoefficients == 0) { + header.colorRange = true; + header.subsamplingX = false; + header.subsamplingY = false; + } + else { + header.colorRange = reader.Read(); + + if (header.profile == 0) { + header.subsamplingX = true; + header.subsamplingY = true; + } + else if (header.profile == 1) { + header.subsamplingX = false; + header.subsamplingY = false; + } + else if (header.bitDepth == 12) { + header.subsamplingX = reader.Read(); + header.subsamplingY = header.subsamplingX ? reader.Read() : false; + } + else { + header.subsamplingX = true; + header.subsamplingY = false; + } + + if (header.subsamplingX && header.subsamplingY) { + // chroma_sample_position. + reader.ConsumeBits(2); + } + } + + header.separateUvDeltaQ = reader.Read(); + } + + header.filmGrainParamsPresent = reader.Read(); + + if (!reader.Ok()) { + return false; + } + + sequence = header; + + return true; + } + + int Av1Parser::RelativeDistance(int a, int b) const + { + if (!sequence->enableOrderHint) { + return 0; + } + + const int bits = sequence->orderHintBits; + int diff = a - b; + const int m = 1 << (bits - 1); + + diff = (diff & (m - 1)) - (diff & m); + + return diff; + } + + bool Av1Parser::SetFrameRefs(Av1FrameHeader & header, uint8_t lastFrameIdx, uint8_t goldFrameIdx) const + { + int refFrameIdx[kRefsPerFrame]; + bool usedFrame[kNumRefFrames] = {}; + int shiftedOrderHints[kNumRefFrames]; + + std::fill(std::begin(refFrameIdx), std::end(refFrameIdx), -1); + + refFrameIdx[kLastFrame - kLastFrame] = lastFrameIdx; + refFrameIdx[kGoldenFrame - kLastFrame] = goldFrameIdx; + usedFrame[lastFrameIdx] = true; + usedFrame[goldFrameIdx] = true; + + const int curFrameHint = 1 << (sequence->orderHintBits - 1); + + for (int i = 0; i < kNumRefFrames; i++) { + shiftedOrderHints[i] = curFrameHint + RelativeDistance(references[i].orderHint, header.orderHint); + } + + if (shiftedOrderHints[lastFrameIdx] >= curFrameHint || shiftedOrderHints[goldFrameIdx] >= curFrameHint) { + return false; + } + + auto findLatestBackward = [&]() { + int ref = -1; + int latestOrderHint = 0; + + for (int i = 0; i < kNumRefFrames; i++) { + const int hint = shiftedOrderHints[i]; + + if (!usedFrame[i] && hint >= curFrameHint && (ref < 0 || hint >= latestOrderHint)) { + ref = i; + latestOrderHint = hint; + } + } + + return ref; + }; + auto findEarliestBackward = [&]() { + int ref = -1; + int earliestOrderHint = 0; + + for (int i = 0; i < kNumRefFrames; i++) { + const int hint = shiftedOrderHints[i]; + + if (!usedFrame[i] && hint >= curFrameHint && (ref < 0 || hint < earliestOrderHint)) { + ref = i; + earliestOrderHint = hint; + } + } + + return ref; + }; + auto findLatestForward = [&]() { + int ref = -1; + int latestOrderHint = 0; + + for (int i = 0; i < kNumRefFrames; i++) { + const int hint = shiftedOrderHints[i]; + + if (!usedFrame[i] && hint < curFrameHint && (ref < 0 || hint >= latestOrderHint)) { + ref = i; + latestOrderHint = hint; + } + } + + return ref; + }; + + int ref = findLatestBackward(); + + if (ref >= 0) { + refFrameIdx[kAltrefFrame - kLastFrame] = ref; + usedFrame[ref] = true; + } + + ref = findEarliestBackward(); + + if (ref >= 0) { + refFrameIdx[kBwdrefFrame - kLastFrame] = ref; + usedFrame[ref] = true; + } + + ref = findEarliestBackward(); + + if (ref >= 0) { + refFrameIdx[kAltref2Frame - kLastFrame] = ref; + usedFrame[ref] = true; + } + + constexpr int kRefFrameList[kRefsPerFrame - 2] = { + kLast2Frame, kLast3Frame, kBwdrefFrame, kAltref2Frame, kAltrefFrame + }; + + for (int refFrame : kRefFrameList) { + if (refFrameIdx[refFrame - kLastFrame] < 0) { + ref = findLatestForward(); + + if (ref >= 0) { + refFrameIdx[refFrame - kLastFrame] = ref; + usedFrame[ref] = true; + } + } + } + + ref = -1; + int earliestOrderHint = 0; + + for (int i = 0; i < kNumRefFrames; i++) { + const int hint = shiftedOrderHints[i]; + + if (ref < 0 || hint < earliestOrderHint) { + ref = i; + earliestOrderHint = hint; + } + } + + for (int i = 0; i < kRefsPerFrame; i++) { + header.refFrameIdx[i] = static_cast(refFrameIdx[i] < 0 ? ref : refFrameIdx[i]); + } + + return true; + } + + std::optional Av1Parser::ParseFrameHeader(std::span payload, const Av1Obu & obu) + { + if (!sequence) { + return std::nullopt; + } + + const Av1SequenceHeader & seq = *sequence; + webrtc::BitstreamReader reader(payload); + Av1FrameHeader header; + + auto temporalPointInfo = [&]() { + // frame_presentation_time. + reader.ConsumeBits(seq.framePresentationTimeLength); + }; + + if (seq.reducedStillPictureHeader) { + header.showExistingFrame = false; + header.frameType = kAv1KeyFrame; + header.frameIsIntra = true; + header.showFrame = true; + header.showableFrame = false; + } + else { + header.showExistingFrame = reader.Read(); + + if (header.showExistingFrame) { + header.frameToShow = static_cast(ReadBits(reader, 3)); + + if (seq.decoderModelInfoPresent && !seq.equalPictureInterval) { + temporalPointInfo(); + } + if (seq.frameIdNumbersPresent) { + // display_frame_id. + reader.ConsumeBits(seq.frameIdLength); + } + + const Reference & shown = references[header.frameToShow]; + + if (!reader.Ok() || !shown.valid) { + return std::nullopt; + } + + header.frameType = shown.frameType; + header.refreshFrameFlags = header.frameType == kAv1KeyFrame ? kAllFrames : 0; + header.upscaledWidth = shown.upscaledWidth; + header.frameWidth = shown.frameWidth; + header.frameHeight = shown.frameHeight; + header.renderWidth = shown.renderWidth; + header.renderHeight = shown.renderHeight; + header.showFrame = true; + + return header; + } + + header.frameType = static_cast(ReadBits(reader, 2)); + header.frameIsIntra = header.frameType == kAv1IntraOnlyFrame || header.frameType == kAv1KeyFrame; + header.showFrame = reader.Read(); + + if (header.showFrame && seq.decoderModelInfoPresent && !seq.equalPictureInterval) { + temporalPointInfo(); + } + + header.showableFrame = header.showFrame + ? header.frameType != kAv1KeyFrame + : reader.Read(); + + if (header.frameType == kAv1SwitchFrame || (header.frameType == kAv1KeyFrame && header.showFrame)) { + header.errorResilientMode = true; + } + else { + header.errorResilientMode = reader.Read(); + } + } + + if (header.frameType == kAv1KeyFrame && header.showFrame) { + for (Reference & reference : references) { + reference.valid = false; + reference.orderHint = 0; + } + } + + header.disableCdfUpdate = reader.Read(); + + header.allowScreenContentTools = seq.seqForceScreenContentTools == kSelectScreenContentTools + ? reader.Read() + : seq.seqForceScreenContentTools != 0; + + if (header.allowScreenContentTools) { + header.forceIntegerMv = seq.seqForceIntegerMv == kSelectIntegerMv + ? reader.Read() + : seq.seqForceIntegerMv != 0; + } + else { + header.forceIntegerMv = false; + } + + if (header.frameIsIntra) { + header.forceIntegerMv = true; + } + + if (seq.frameIdNumbersPresent) { + header.currentFrameId = ReadBits(reader, seq.frameIdLength); + } + + bool frameSizeOverride; + + if (header.frameType == kAv1SwitchFrame) { + frameSizeOverride = true; + } + else if (seq.reducedStillPictureHeader) { + frameSizeOverride = false; + } + else { + frameSizeOverride = reader.Read(); + } + + header.orderHint = static_cast(ReadBits(reader, seq.orderHintBits)); + + if (header.frameIsIntra || header.errorResilientMode) { + header.primaryRefFrame = kPrimaryRefNone; + } + else { + header.primaryRefFrame = static_cast(ReadBits(reader, 3)); + } + + if (seq.decoderModelInfoPresent) { + const bool bufferRemovalTimePresent = reader.Read(); + + if (bufferRemovalTimePresent) { + for (int op = 0; op < seq.operatingPointCount; op++) { + if (!seq.decoderModelPresentForOperatingPoint[op]) { + continue; + } + + const uint16_t idc = seq.operatingPointIdc[op]; + const bool inTemporalLayer = (idc >> obu.temporalId) & 1; + const bool inSpatialLayer = (idc >> (obu.spatialId + 8)) & 1; + + if (idc == 0 || (inTemporalLayer && inSpatialLayer)) { + // buffer_removal_time. + reader.ConsumeBits(seq.bufferRemovalTimeLength); + } + } + } + } + + if (header.frameType == kAv1SwitchFrame || (header.frameType == kAv1KeyFrame && header.showFrame)) { + header.refreshFrameFlags = kAllFrames; + } + else { + header.refreshFrameFlags = static_cast(ReadBits(reader, 8)); + } + + if ((!header.frameIsIntra || header.refreshFrameFlags != kAllFrames) + && header.errorResilientMode && seq.enableOrderHint) + { + for (Reference & reference : references) { + const uint8_t refOrderHint = static_cast(ReadBits(reader, seq.orderHintBits)); + + if (refOrderHint != reference.orderHint) { + reference.valid = false; + reference.orderHint = refOrderHint; + } + } + } + + // superres_params and compute_image_size. + auto superresParams = [&]() { + header.useSuperres = seq.enableSuperres ? reader.Read() : false; + header.superresDenom = header.useSuperres + ? static_cast(ReadBits(reader, kSuperresDenomBits) + kSuperresDenomMin) + : kSuperresNum; + header.upscaledWidth = header.frameWidth; + header.frameWidth = (header.upscaledWidth * kSuperresNum + header.superresDenom / 2) / header.superresDenom; + header.miCols = 2 * ((header.frameWidth + 7) >> 3); + header.miRows = 2 * ((header.frameHeight + 7) >> 3); + }; + auto frameSize = [&]() { + if (frameSizeOverride) { + header.frameWidth = ReadBits(reader, seq.frameWidthBits) + 1; + header.frameHeight = ReadBits(reader, seq.frameHeightBits) + 1; + } + else { + header.frameWidth = seq.maxFrameWidth; + header.frameHeight = seq.maxFrameHeight; + } + + superresParams(); + }; + auto renderSize = [&]() { + if (reader.Read()) { + header.renderWidth = ReadBits(reader, 16) + 1; + header.renderHeight = ReadBits(reader, 16) + 1; + } + else { + header.renderWidth = header.upscaledWidth; + header.renderHeight = header.frameHeight; + } + }; + + if (header.frameIsIntra) { + frameSize(); + renderSize(); + + if (header.allowScreenContentTools && header.upscaledWidth == header.frameWidth) { + header.allowIntrabc = reader.Read(); + } + } + else { + bool frameRefsShortSignaling = false; + + if (seq.enableOrderHint) { + frameRefsShortSignaling = reader.Read(); + + if (frameRefsShortSignaling) { + const uint8_t lastFrameIdx = static_cast(ReadBits(reader, 3)); + const uint8_t goldFrameIdx = static_cast(ReadBits(reader, 3)); + + if (!reader.Ok() || !SetFrameRefs(header, lastFrameIdx, goldFrameIdx)) { + return std::nullopt; + } + } + } + + for (int i = 0; i < kRefsPerFrame; i++) { + if (!frameRefsShortSignaling) { + header.refFrameIdx[i] = static_cast(ReadBits(reader, 3)); + } + if (seq.frameIdNumbersPresent) { + // delta_frame_id_minus_1. + reader.ConsumeBits(seq.deltaFrameIdLength); + } + } + + for (int i = 0; i < kRefsPerFrame; i++) { + if (!references[header.refFrameIdx[i]].valid) { + return std::nullopt; + } + } + + if (frameSizeOverride && !header.errorResilientMode) { + // frame_size_with_refs. + bool foundRef = false; + + for (int i = 0; i < kRefsPerFrame; i++) { + foundRef = reader.Read(); + + if (foundRef) { + const Reference & reference = references[header.refFrameIdx[i]]; + + header.upscaledWidth = reference.upscaledWidth; + header.frameWidth = header.upscaledWidth; + header.frameHeight = reference.frameHeight; + header.renderWidth = reference.renderWidth; + header.renderHeight = reference.renderHeight; + break; + } + } + + if (foundRef) { + superresParams(); + } + else { + frameSize(); + renderSize(); + } + } + else { + frameSize(); + renderSize(); + } + + header.allowHighPrecisionMv = header.forceIntegerMv ? false : reader.Read(); + + // read_interpolation_filter. + const bool isFilterSwitchable = reader.Read(); + header.interpolationFilter = isFilterSwitchable ? 4 : static_cast(ReadBits(reader, 2)); + + header.isMotionModeSwitchable = reader.Read(); + + if (header.errorResilientMode || !seq.enableRefFrameMvs) { + header.useRefFrameMvs = false; + } + else { + header.useRefFrameMvs = reader.Read(); + } + } + + if (seq.reducedStillPictureHeader || header.disableCdfUpdate) { + header.disableFrameEndUpdateCdf = true; + } + else { + header.disableFrameEndUpdateCdf = reader.Read(); + } + + // setup_past_independence or load_previous: what carries over. + std::array, 8> prevGmParams; + + if (header.primaryRefFrame == kPrimaryRefNone) { + prevGmParams = DefaultGmParams(); + header.loopFilterRefDeltas = kDefaultLoopFilterRefDeltas; + header.loopFilterModeDeltas = { 0, 0 }; + } + else { + const Reference & previous = references[header.refFrameIdx[header.primaryRefFrame]]; + + prevGmParams = previous.gmParams; + header.loopFilterRefDeltas = previous.loopFilterRefDeltas; + header.loopFilterModeDeltas = previous.loopFilterModeDeltas; + + std::copy(&previous.featureEnabled[0][0], &previous.featureEnabled[0][0] + kMaxSegments * kSegLvlMax, + &header.featureEnabled[0][0]); + std::copy(&previous.featureData[0][0], &previous.featureData[0][0] + kMaxSegments * kSegLvlMax, + &header.featureData[0][0]); + } + + // tile_info, section 5.9.15. + { + const int sbCols = seq.use128x128Superblock ? ((header.miCols + 31) >> 5) : ((header.miCols + 15) >> 4); + const int sbRows = seq.use128x128Superblock ? ((header.miRows + 31) >> 5) : ((header.miRows + 15) >> 4); + const int sbShift = seq.use128x128Superblock ? 5 : 4; + const int sbSize = sbShift + 2; + const int maxTileWidthSb = kMaxTileWidth >> sbSize; + int maxTileAreaSb = kMaxTileArea >> (2 * sbSize); + const int minLog2TileCols = TileLog2(maxTileWidthSb, sbCols); + const int maxLog2TileCols = TileLog2(1, std::min(sbCols, kMaxTileCols)); + const int maxLog2TileRows = TileLog2(1, std::min(sbRows, kMaxTileRows)); + const int minLog2Tiles = std::max(minLog2TileCols, TileLog2(maxTileAreaSb, sbRows * sbCols)); + + header.uniformTileSpacing = reader.Read(); + + if (header.uniformTileSpacing) { + int tileColsLog2 = minLog2TileCols; + + while (tileColsLog2 < maxLog2TileCols && reader.Ok()) { + if (!reader.Read()) { + break; + } + + tileColsLog2++; + } + + const int tileWidthSb = (sbCols + (1 << tileColsLog2) - 1) >> tileColsLog2; + int i = 0; + + for (int startSb = 0; startSb < sbCols; startSb += tileWidthSb) { + header.tileWidthSbs[i++] = static_cast(std::min(tileWidthSb, sbCols - startSb)); + } + + header.tileCols = static_cast(i); + header.tileColsLog2 = static_cast(tileColsLog2); + + const int minLog2TileRows = std::max(minLog2Tiles - tileColsLog2, 0); + int tileRowsLog2 = minLog2TileRows; + + while (tileRowsLog2 < maxLog2TileRows && reader.Ok()) { + if (!reader.Read()) { + break; + } + + tileRowsLog2++; + } + + const int tileHeightSb = (sbRows + (1 << tileRowsLog2) - 1) >> tileRowsLog2; + i = 0; + + for (int startSb = 0; startSb < sbRows; startSb += tileHeightSb) { + header.tileHeightSbs[i++] = static_cast(std::min(tileHeightSb, sbRows - startSb)); + } + + header.tileRows = static_cast(i); + header.tileRowsLog2 = static_cast(tileRowsLog2); + } + else { + int widestTileSb = 0; + int startSb = 0; + int i = 0; + + for (; startSb < sbCols && reader.Ok(); i++) { + if (i >= kMaxTileCols) { + return std::nullopt; + } + + const int maxWidth = std::min(sbCols - startSb, maxTileWidthSb); + const int sizeSb = static_cast(reader.ReadNonSymmetric(static_cast(maxWidth))) + 1; + + header.tileWidthSbs[i] = static_cast(sizeSb); + widestTileSb = std::max(sizeSb, widestTileSb); + startSb += sizeSb; + } + + header.tileCols = static_cast(i); + header.tileColsLog2 = static_cast(TileLog2(1, i)); + + if (minLog2Tiles > 0) { + maxTileAreaSb = (sbRows * sbCols) >> (minLog2Tiles + 1); + } + else { + maxTileAreaSb = sbRows * sbCols; + } + + const int maxTileHeightSb = std::max(maxTileAreaSb / std::max(widestTileSb, 1), 1); + + startSb = 0; + i = 0; + + for (; startSb < sbRows && reader.Ok(); i++) { + if (i >= kMaxTileRows) { + return std::nullopt; + } + + const int maxHeight = std::min(sbRows - startSb, maxTileHeightSb); + const int sizeSb = static_cast(reader.ReadNonSymmetric(static_cast(maxHeight))) + 1; + + header.tileHeightSbs[i] = static_cast(sizeSb); + startSb += sizeSb; + } + + header.tileRows = static_cast(i); + header.tileRowsLog2 = static_cast(TileLog2(1, i)); + } + + if (header.tileColsLog2 > 0 || header.tileRowsLog2 > 0) { + header.contextUpdateTileId = static_cast( + ReadBits(reader, header.tileRowsLog2 + header.tileColsLog2)); + header.tileSizeBytes = static_cast(ReadBits(reader, 2) + 1); + } + else { + header.contextUpdateTileId = 0; + } + } + + // quantization_params, section 5.9.12. + header.baseQIndex = static_cast(ReadBits(reader, 8)); + header.deltaQYDc = ReadDeltaQ(reader); + + if (!seq.monochrome) { + const bool diffUvDelta = seq.separateUvDeltaQ ? reader.Read() : false; + + header.deltaQUDc = ReadDeltaQ(reader); + header.deltaQUAc = ReadDeltaQ(reader); + + if (diffUvDelta) { + header.deltaQVDc = ReadDeltaQ(reader); + header.deltaQVAc = ReadDeltaQ(reader); + } + else { + header.deltaQVDc = header.deltaQUDc; + header.deltaQVAc = header.deltaQUAc; + } + } + + header.usingQmatrix = reader.Read(); + + if (header.usingQmatrix) { + header.qmY = static_cast(ReadBits(reader, 4)); + header.qmU = static_cast(ReadBits(reader, 4)); + header.qmV = seq.separateUvDeltaQ ? static_cast(ReadBits(reader, 4)) : header.qmU; + } + + // segmentation_params, section 5.9.14. + header.segmentationEnabled = reader.Read(); + + if (header.segmentationEnabled) { + if (header.primaryRefFrame == kPrimaryRefNone) { + header.segmentationUpdateMap = true; + header.segmentationTemporalUpdate = false; + header.segmentationUpdateData = true; + } + else { + header.segmentationUpdateMap = reader.Read(); + header.segmentationTemporalUpdate = header.segmentationUpdateMap ? reader.Read() : false; + header.segmentationUpdateData = reader.Read(); + } + + if (header.segmentationUpdateData) { + for (int i = 0; i < kMaxSegments; i++) { + for (int j = 0; j < kSegLvlMax; j++) { + const bool enabled = reader.Read(); + int clippedValue = 0; + + if (enabled) { + const int bits = kSegmentationFeatureBits[j]; + const int limit = kSegmentationFeatureMax[j]; + + if (kSegmentationFeatureSigned[j]) { + clippedValue = std::clamp(static_cast(ReadSu(reader, 1 + bits)), -limit, limit); + } + else { + clippedValue = std::clamp(static_cast(ReadBits(reader, bits)), 0, limit); + } + } + + header.featureEnabled[i][j] = enabled; + header.featureData[i][j] = static_cast(clippedValue); + } + } + } + } + else { + for (int i = 0; i < kMaxSegments; i++) { + for (int j = 0; j < kSegLvlMax; j++) { + header.featureEnabled[i][j] = false; + header.featureData[i][j] = 0; + } + } + } + + // delta_q_params and delta_lf_params. + if (header.baseQIndex > 0) { + header.deltaQPresent = reader.Read(); + } + if (header.deltaQPresent) { + header.deltaQRes = static_cast(ReadBits(reader, 2)); + + if (!header.allowIntrabc) { + header.deltaLfPresent = reader.Read(); + } + if (header.deltaLfPresent) { + header.deltaLfRes = static_cast(ReadBits(reader, 2)); + header.deltaLfMulti = reader.Read(); + } + } + + header.codedLossless = true; + + for (int segment = 0; segment < kMaxSegments; segment++) { + int qindex = header.baseQIndex; + + if (header.segmentationEnabled && header.featureEnabled[segment][kSegLvlAltQ]) { + qindex = std::clamp(qindex + header.featureData[segment][kSegLvlAltQ], 0, 255); + } + + const bool lossless = qindex == 0 && header.deltaQYDc == 0 && header.deltaQUAc == 0 + && header.deltaQUDc == 0 && header.deltaQVAc == 0 && header.deltaQVDc == 0; + + if (!lossless) { + header.codedLossless = false; + } + } + + header.allLossless = header.codedLossless && header.frameWidth == header.upscaledWidth; + + // loop_filter_params, section 5.9.11. + if (header.codedLossless || header.allowIntrabc) { + header.loopFilterLevel = {}; + header.loopFilterRefDeltas = kDefaultLoopFilterRefDeltas; + header.loopFilterModeDeltas = { 0, 0 }; + } + else { + header.loopFilterLevel[0] = static_cast(ReadBits(reader, 6)); + header.loopFilterLevel[1] = static_cast(ReadBits(reader, 6)); + + if (!seq.monochrome && (header.loopFilterLevel[0] || header.loopFilterLevel[1])) { + header.loopFilterLevel[2] = static_cast(ReadBits(reader, 6)); + header.loopFilterLevel[3] = static_cast(ReadBits(reader, 6)); + } + + header.loopFilterSharpness = static_cast(ReadBits(reader, 3)); + header.loopFilterDeltaEnabled = reader.Read(); + + if (header.loopFilterDeltaEnabled) { + header.loopFilterDeltaUpdate = reader.Read(); + + if (header.loopFilterDeltaUpdate) { + for (int i = 0; i < kTotalRefsPerFrame; i++) { + if (reader.Read()) { + header.loopFilterRefDeltas[i] = static_cast(ReadSu(reader, 7)); + } + } + for (int i = 0; i < 2; i++) { + if (reader.Read()) { + header.loopFilterModeDeltas[i] = static_cast(ReadSu(reader, 7)); + } + } + } + } + } + + // cdef_params, section 5.9.19. + if (header.codedLossless || header.allowIntrabc || !seq.enableCdef) { + header.cdefBits = 0; + header.cdefDampingMinus3 = 0; + } + else { + header.cdefDampingMinus3 = static_cast(ReadBits(reader, 2)); + header.cdefBits = static_cast(ReadBits(reader, 2)); + + for (int i = 0; i < (1 << header.cdefBits); i++) { + header.cdefYPriStrength[i] = static_cast(ReadBits(reader, 4)); + header.cdefYSecStrength[i] = static_cast(ReadBits(reader, 2)); + + if (!seq.monochrome) { + header.cdefUvPriStrength[i] = static_cast(ReadBits(reader, 4)); + header.cdefUvSecStrength[i] = static_cast(ReadBits(reader, 2)); + } + } + } + + // lr_params, section 5.9.20. + if (!header.allLossless && !header.allowIntrabc && seq.enableRestoration) { + bool usesLr = false; + bool usesChromaLr = false; + const int planes = seq.monochrome ? 1 : 3; + + for (int i = 0; i < planes; i++) { + header.frameRestorationType[i] = kRemapLrType[ReadBits(reader, 2)]; + + if (header.frameRestorationType[i] != 0) { + usesLr = true; + + if (i > 0) { + usesChromaLr = true; + } + } + } + + if (usesLr) { + if (seq.use128x128Superblock) { + header.lrUnitShift = static_cast(reader.ReadBit() + 1); + } + else { + header.lrUnitShift = static_cast(reader.ReadBit()); + + if (header.lrUnitShift) { + header.lrUnitShift += static_cast(reader.ReadBit()); + } + } + + if (seq.subsamplingX && seq.subsamplingY && usesChromaLr) { + header.lrUvShift = static_cast(reader.ReadBit()); + } + } + } + + // read_tx_mode. + if (header.codedLossless) { + header.txMode = 0; + } + else { + header.txMode = reader.Read() ? 2 : 1; + } + + // frame_reference_mode. + header.referenceSelect = header.frameIsIntra ? false : reader.Read(); + + // skip_mode_params, section 5.9.22. + bool skipModeAllowed = false; + + if (!header.frameIsIntra && header.referenceSelect && seq.enableOrderHint) { + int forwardIdx = -1; + int backwardIdx = -1; + int forwardHint = 0; + int backwardHint = 0; + + for (int i = 0; i < kRefsPerFrame; i++) { + const int refHint = references[header.refFrameIdx[i]].orderHint; + + if (RelativeDistance(refHint, header.orderHint) < 0) { + if (forwardIdx < 0 || RelativeDistance(refHint, forwardHint) > 0) { + forwardIdx = i; + forwardHint = refHint; + } + } + else if (RelativeDistance(refHint, header.orderHint) > 0) { + if (backwardIdx < 0 || RelativeDistance(refHint, backwardHint) < 0) { + backwardIdx = i; + backwardHint = refHint; + } + } + } + + if (forwardIdx < 0) { + skipModeAllowed = false; + } + else if (backwardIdx >= 0) { + skipModeAllowed = true; + } + else { + int secondForwardIdx = -1; + int secondForwardHint = 0; + + for (int i = 0; i < kRefsPerFrame; i++) { + const int refHint = references[header.refFrameIdx[i]].orderHint; + + if (RelativeDistance(refHint, forwardHint) < 0) { + if (secondForwardIdx < 0 || RelativeDistance(refHint, secondForwardHint) > 0) { + secondForwardIdx = i; + secondForwardHint = refHint; + } + } + } + + skipModeAllowed = secondForwardIdx >= 0; + } + } + + header.skipModePresent = skipModeAllowed ? reader.Read() : false; + + if (header.frameIsIntra || header.errorResilientMode || !seq.enableWarpedMotion) { + header.allowWarpedMotion = false; + } + else { + header.allowWarpedMotion = reader.Read(); + } + + header.reducedTxSet = reader.Read(); + + // global_motion_params, section 5.9.24. + header.gmParams = DefaultGmParams(); + header.gmType.fill(kIdentity); + header.gmValid.fill(true); + + if (!header.frameIsIntra) { + for (int ref = kLastFrame; ref <= kAltrefFrame; ref++) { + uint8_t type = kIdentity; + + if (reader.Read()) { + if (reader.Read()) { + type = kRotzoom; + } + else { + type = reader.Read() ? kTranslation : kAffine; + } + } + + header.gmType[ref] = type; + + auto readGlobalParam = [&](int idx) { + int absBits = kGmAbsAlphaBits; + int precBits = kGmAlphaPrecBits; + + if (idx < 2) { + if (type == kTranslation) { + absBits = kGmAbsTransOnlyBits - !header.allowHighPrecisionMv; + precBits = kGmTransOnlyPrecBits - !header.allowHighPrecisionMv; + } + else { + absBits = kGmAbsTransBits; + precBits = kGmTransPrecBits; + } + } + + const int precDiff = kWarpedModelPrecBits - precBits; + const int32_t round = (idx % 3) == 2 ? (1 << kWarpedModelPrecBits) : 0; + const int32_t sub = (idx % 3) == 2 ? (1 << precBits) : 0; + const int32_t mx = 1 << absBits; + const int32_t r = (prevGmParams[ref][idx] >> precDiff) - sub; + + header.gmParams[ref][idx] = (DecodeSignedSubexpWithRef(reader, -mx, mx + 1, r) * (1 << precDiff)) + + round; + }; + + if (type >= kRotzoom) { + readGlobalParam(2); + readGlobalParam(3); + + if (type == kAffine) { + readGlobalParam(4); + readGlobalParam(5); + } + else { + header.gmParams[ref][4] = -header.gmParams[ref][3]; + header.gmParams[ref][5] = header.gmParams[ref][2]; + } + } + if (type >= kTranslation) { + readGlobalParam(0); + readGlobalParam(1); + } + + header.gmValid[ref] = type <= kTranslation || ShearValid(header.gmParams[ref]); + } + } + + // film_grain_params, section 5.9.30. + if (seq.filmGrainParamsPresent && (header.showFrame || header.showableFrame)) { + header.applyGrain = reader.Read(); + + if (header.applyGrain) { + // Not parsed further: a frame with film grain is left to + // the software decoder. + return reader.Ok() ? std::optional(header) : std::nullopt; + } + } + + if (!reader.Ok()) { + return std::nullopt; + } + + const int bitsRead = static_cast(payload.size() * 8) - reader.RemainingBitCount(); + header.headerBytes = static_cast((bitsRead + 7) / 8); + + return header; + } + + std::optional Av1Parser::ParseTileGroup(std::span payload, + const Av1FrameHeader & header) const + { + webrtc::BitstreamReader reader(payload); + Av1TileGroup group; + + const int numTiles = header.tileCols * header.tileRows; + bool tileStartAndEndPresent = false; + + if (numTiles > 1) { + tileStartAndEndPresent = reader.Read(); + } + + if (numTiles == 1 || !tileStartAndEndPresent) { + group.start = 0; + group.end = static_cast(numTiles - 1); + } + else { + const int tileBits = header.tileColsLog2 + header.tileRowsLog2; + + group.start = static_cast(ReadBits(reader, tileBits)); + group.end = static_cast(ReadBits(reader, tileBits)); + } + + if (!reader.Ok() || group.end < group.start || group.end >= numTiles) { + return std::nullopt; + } + + // byte_alignment. + const int bitsRead = static_cast(payload.size() * 8) - reader.RemainingBitCount(); + size_t position = static_cast((bitsRead + 7) / 8); + + for (int tileNum = group.start; tileNum <= group.end; tileNum++) { + Av1Tile tile; + tile.row = static_cast(tileNum / header.tileCols); + tile.column = static_cast(tileNum % header.tileCols); + + size_t tileSize; + + if (tileNum == group.end) { + if (position >= payload.size()) { + return std::nullopt; + } + + tileSize = payload.size() - position; + } + else { + if (payload.size() - position < header.tileSizeBytes) { + return std::nullopt; + } + + // tile_size_minus_1, le(TileSizeBytes). + tileSize = 0; + + for (int i = 0; i < header.tileSizeBytes; i++) { + tileSize |= size_t(payload[position + i]) << (8 * i); + } + + tileSize += 1; + position += header.tileSizeBytes; + + if (tileSize > payload.size() - position) { + return std::nullopt; + } + } + + tile.offset = static_cast(position); + tile.size = static_cast(tileSize); + group.tiles.push_back(tile); + + position += tileSize; + } + + return group; + } + + void Av1Parser::UpdateReferences(const Av1FrameHeader & header) + { + if (header.showExistingFrame) { + // A key frame shown again is loaded, and then refreshes all slots. + if (header.frameType == kAv1KeyFrame) { + const Reference shown = references[header.frameToShow]; + + references.fill(shown); + } + + return; + } + + Reference reference; + reference.valid = true; + reference.frameId = header.currentFrameId; + reference.upscaledWidth = header.upscaledWidth; + reference.frameWidth = header.frameWidth; + reference.frameHeight = header.frameHeight; + reference.renderWidth = header.renderWidth; + reference.renderHeight = header.renderHeight; + reference.frameType = header.frameType; + reference.orderHint = header.orderHint; + reference.gmParams = header.gmParams; + reference.loopFilterRefDeltas = header.loopFilterRefDeltas; + reference.loopFilterModeDeltas = header.loopFilterModeDeltas; + + std::copy(&header.featureEnabled[0][0], &header.featureEnabled[0][0] + kMaxSegments * kSegLvlMax, + &reference.featureEnabled[0][0]); + std::copy(&header.featureData[0][0], &header.featureData[0][0] + kMaxSegments * kSegLvlMax, + &reference.featureData[0][0]); + + for (int slot = 0; slot < kNumRefFrames; slot++) { + if (header.refreshFrameFlags & (1 << slot)) { + references[slot] = reference; + } + } + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp index 1591e2ca..e96ec3ad 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp @@ -44,7 +44,7 @@ namespace jni std::vector> factories; // NVDEC on NVIDIA GPUs, for H.264, H.265, VP9 and AV1; VA-API for the - // GPUs of Intel and AMD, for VP9 and H.265. + // GPUs of Intel and AMD, for H.264, VP9, H.265 and AV1. if (auto nvdec = NvdecVideoDecoderFactory::Create()) { factories.push_back(std::move(nvdec)); } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Decoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Decoder.cpp new file mode 100644 index 00000000..ac48fd1a --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Decoder.cpp @@ -0,0 +1,716 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/linux/VaapiAv1Decoder.h" + +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" +#include "third_party/libyuv/include/libyuv/convert.h" + +#include + +namespace jni +{ + namespace + { + // The eight reference frames, the frame being decoded, and one more, + // so that a free surface is always there. + constexpr int kSurfaceCount = 10; + } + + VaapiAv1Decoder::VaapiAv1Decoder(VaapiLibrary & library) : + library(library), + va(library.Api()), + implementationName("VA-API (" + library.Vendor() + ")"), + callback(nullptr), + config(VA_INVALID_ID), + context(VA_INVALID_ID), + width(0), + height(0), + downloadImage(), + started(false), + failed(false) + { + downloadImage.image_id = VA_INVALID_ID; + slots.fill(-1); + } + + VaapiAv1Decoder::~VaapiAv1Decoder() + { + Release(); + } + + bool VaapiAv1Decoder::Check(VAStatus status, const char * operation) const + { + if (status == VA_STATUS_SUCCESS) { + return true; + } + + RTC_LOG(LS_WARNING) << implementationName << " AV1: " << operation << " failed: " + << library.ErrorString(status); + + return false; + } + + bool VaapiAv1Decoder::Configure(const Settings & settings) + { + if (settings.codec_type() != webrtc::kVideoCodecAV1) { + return false; + } + + Release(); + + RTC_LOG(LS_INFO) << implementationName << " AV1 decoder configured"; + + return true; + } + + int32_t VaapiAv1Decoder::Decode(const webrtc::EncodedImage & image, int64_t renderTimeMs) + { + if (callback == nullptr) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (image.size() == 0) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (failed) { + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + // Decoding starts with a key frame. Until one has come, WebRTC is + // asked for it. + if (!started && image.FrameType() != webrtc::VideoFrameType::kVideoFrameKey) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + std::vector frames; + + if (!ParseTemporalUnit(std::span(image.data(), image.size()), frames) + || !DecodeFrames(frames, image, renderTimeMs)) + { + failed = true; + + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VaapiAv1Decoder::RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback * decodeCallback) + { + callback = decodeCallback; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VaapiAv1Decoder::Release() + { + DestroySession(); + + parser.Reset(); + started = false; + failed = false; + + return WEBRTC_VIDEO_CODEC_OK; + } + + webrtc::VideoDecoder::DecoderInfo VaapiAv1Decoder::GetDecoderInfo() const + { + DecoderInfo info; + info.implementation_name = implementationName; + info.is_hardware_accelerated = true; + + return info; + } + + const char * VaapiAv1Decoder::ImplementationName() const + { + return implementationName.c_str(); + } + + bool VaapiAv1Decoder::ParseTemporalUnit(std::span data, std::vector & frames) + { + const auto obus = parser.SplitObus(data); + + if (!obus) { + RTC_LOG(LS_WARNING) << implementationName << " AV1: OBUs that do not fit into the frame"; + return false; + } + + for (const Av1Obu & obu : *obus) { + if (obu.spatialId > 0) { + // Spatial layers predict from frames of another size. + RTC_LOG(LS_INFO) << implementationName << " AV1: a stream with spatial layers"; + return false; + } + } + + // The size the frames are decoded at, which a key frame may change. + uint32_t frameWidth = context != VA_INVALID_ID ? width : 0; + uint32_t frameHeight = context != VA_INVALID_ID ? height : 0; + + // The frame whose tiles are coming. + std::optional pending; + + // Takes a tile group of the pending frame in, and the frame into the + // reference slots once it has all of its tiles. + auto addTileGroup = [&](std::span payload) { + if (!pending || !AddTileGroup(payload, *pending)) { + return false; + } + + const Av1FrameHeader & header = pending->header; + + if (pending->tiles.size() == size_t(header.tileCols) * header.tileRows) { + parser.UpdateReferences(header); + frames.push_back(std::move(*pending)); + pending.reset(); + } + + return true; + }; + + for (const Av1Obu & obu : *obus) { + switch (obu.type) { + case Av1ObuType::SequenceHeader: { + if (!parser.ParseSequenceHeader(obu.payload)) { + RTC_LOG(LS_WARNING) << implementationName << " AV1: a broken sequence header"; + return false; + } + + const Av1SequenceHeader & sequence = *parser.SequenceHeader(); + + if (sequence.profile != 0 || sequence.bitDepth != 8 || sequence.monochrome) { + RTC_LOG(LS_INFO) << implementationName << " AV1: a stream that is not 8 bit 4:2:0 profile 0"; + return false; + } + break; + } + + case Av1ObuType::FrameHeader: + case Av1ObuType::RedundantFrameHeader: + case Av1ObuType::Frame: { + if (pending) { + // Copies of the header of the frame whose tiles are + // coming; a frame OBU has to start a frame. + if (obu.type == Av1ObuType::Frame) { + return false; + } + break; + } + + const std::optional header = parser.ParseFrameHeader(obu.payload, obu); + + if (!header) { + RTC_LOG(LS_INFO) << implementationName << " AV1: a broken frame header, or one that " + "refers to a frame that is not there"; + return false; + } + + if (header->showExistingFrame) { + if (obu.type == Av1ObuType::Frame) { + return false; + } + + parser.UpdateReferences(*header); + frames.push_back(Frame { *header }); + break; + } + + if (header->applyGrain) { + RTC_LOG(LS_INFO) << implementationName << " AV1: a frame with film grain"; + return false; + } + if (header->useSuperres) { + RTC_LOG(LS_INFO) << implementationName << " AV1: a frame with superres"; + return false; + } + + if (header->upscaledWidth != frameWidth || header->frameHeight != frameHeight) { + // A key frame that is shown starts a new session, as it + // replaces all references; frames that scale their + // references are left to dav1d. + if (header->frameType != kAv1KeyFrame || !header->showFrame) { + RTC_LOG(LS_INFO) << implementationName << " AV1: a frame changes the size to " + << header->upscaledWidth << "x" << header->frameHeight; + return false; + } + + frameWidth = header->upscaledWidth; + frameHeight = header->frameHeight; + } + + pending = Frame { *header }; + + if (obu.type == Av1ObuType::Frame) { + if (header->headerBytes > obu.payload.size() + || !addTileGroup(obu.payload.subspan(header->headerBytes))) + { + return false; + } + } + break; + } + + case Av1ObuType::TileGroup: + if (!addTileGroup(obu.payload)) { + return false; + } + break; + + default: + // Temporal delimiters, metadata and padding. + break; + } + } + + if (pending) { + RTC_LOG(LS_WARNING) << implementationName << " AV1: a frame without all of its tiles"; + return false; + } + + return true; + } + + bool VaapiAv1Decoder::AddTileGroup(std::span payload, Frame & frame) + { + const std::optional group = parser.ParseTileGroup(payload, frame.header); + + if (!group || group->start != frame.tiles.size()) { + RTC_LOG(LS_WARNING) << implementationName << " AV1: a broken tile group, or one out of order"; + return false; + } + + const uint32_t base = static_cast(frame.tileData.size()); + + frame.tileData.insert(frame.tileData.end(), payload.begin(), payload.end()); + + for (const Av1Tile & tile : group->tiles) { + VASliceParameterBufferAV1 slice = {}; + slice.slice_data_size = tile.size; + slice.slice_data_offset = base + tile.offset; + slice.slice_data_flag = VA_SLICE_DATA_FLAG_ALL; + slice.tile_row = tile.row; + slice.tile_column = tile.column; + + frame.tiles.push_back(slice); + } + + return true; + } + + bool VaapiAv1Decoder::DecodeFrames(const std::vector & frames, const webrtc::EncodedImage & image, + int64_t renderTimeMs) + { + for (const Frame & frame : frames) { + const Av1FrameHeader & header = frame.header; + + if (header.showExistingFrame) { + const int surface = slots[header.frameToShow]; + + if (surface < 0 || !Output(surface, image, renderTimeMs)) { + return false; + } + + // A key frame shown again takes all reference slots. + if (header.frameType == kAv1KeyFrame) { + slots.fill(surface); + } + + continue; + } + + if (context == VA_INVALID_ID || header.upscaledWidth != width || header.frameHeight != height) { + if (!CreateSession(header.upscaledWidth, header.frameHeight)) { + return false; + } + } + + const int surface = FreeSurface(); + + if (surface < 0 || !DecodeFrame(frame, surface)) { + return false; + } + + for (int slot = 0; slot < 8; slot++) { + if (header.refreshFrameFlags & (1 << slot)) { + slots[slot] = surface; + } + } + + started = true; + + if (header.showFrame && !Output(surface, image, renderTimeMs)) { + return false; + } + } + + return true; + } + + bool VaapiAv1Decoder::CreateSession(uint32_t frameWidth, uint32_t frameHeight) + { + DestroySession(); + + VADisplay display = library.Display(); + + VAConfigAttrib attribute = {}; + attribute.type = VAConfigAttribRTFormat; + attribute.value = VA_RT_FORMAT_YUV420; + + if (!Check(va.CreateConfig(display, VAProfileAV1Profile0, VAEntrypointVLD, &attribute, 1, &config), + "creating the configuration")) + { + config = VA_INVALID_ID; + return false; + } + + surfaces.assign(kSurfaceCount, VA_INVALID_SURFACE); + + if (!Check(va.CreateSurfaces(display, VA_RT_FORMAT_YUV420, frameWidth, frameHeight, surfaces.data(), + kSurfaceCount, nullptr, 0), "creating surfaces")) + { + surfaces.clear(); + DestroySession(); + return false; + } + + if (!Check(va.CreateContext(display, config, static_cast(frameWidth), static_cast(frameHeight), + VA_PROGRESSIVE, surfaces.data(), kSurfaceCount, &context), "creating the context")) + { + context = VA_INVALID_ID; + DestroySession(); + return false; + } + + // Copying a surface into an image reads it from cached memory, which + // mapping it directly does not. Where the driver does not copy, the + // surface is mapped. + VAImageFormat format = {}; + format.fourcc = VA_FOURCC_NV12; + format.byte_order = VA_LSB_FIRST; + format.bits_per_pixel = 12; + + if (va.CreateImage(display, &format, static_cast(frameWidth), static_cast(frameHeight), + &downloadImage) != VA_STATUS_SUCCESS) + { + downloadImage.image_id = VA_INVALID_ID; + } + + width = frameWidth; + height = frameHeight; + + RTC_LOG(LS_INFO) << implementationName << " AV1 decoding " << width << "x" << height; + + return true; + } + + void VaapiAv1Decoder::DestroySession() + { + VADisplay display = library.Display(); + + if (downloadImage.image_id != VA_INVALID_ID) { + va.DestroyImage(display, downloadImage.image_id); + downloadImage.image_id = VA_INVALID_ID; + } + if (context != VA_INVALID_ID) { + va.DestroyContext(display, context); + context = VA_INVALID_ID; + } + if (!surfaces.empty()) { + va.DestroySurfaces(display, surfaces.data(), static_cast(surfaces.size())); + surfaces.clear(); + } + if (config != VA_INVALID_ID) { + va.DestroyConfig(display, config); + config = VA_INVALID_ID; + } + + width = 0; + height = 0; + slots.fill(-1); + } + + int VaapiAv1Decoder::FreeSurface() const + { + for (int surface = 0; surface < static_cast(surfaces.size()); surface++) { + if (std::find(slots.begin(), slots.end(), surface) == slots.end()) { + return surface; + } + } + + return -1; + } + + bool VaapiAv1Decoder::DecodeFrame(const Frame & frame, int surface) + { + const Av1FrameHeader & header = frame.header; + const Av1SequenceHeader & sequence = *parser.SequenceHeader(); + + VADecPictureParameterBufferAV1 picture = {}; + picture.profile = sequence.profile; + picture.order_hint_bits_minus_1 = static_cast(std::max(sequence.orderHintBits - 1, 0)); + picture.bit_depth_idx = 0; + picture.matrix_coefficients = sequence.matrixCoefficients; + + auto & seqFields = picture.seq_info_fields.fields; + seqFields.still_picture = sequence.stillPicture ? 1 : 0; + seqFields.use_128x128_superblock = sequence.use128x128Superblock ? 1 : 0; + seqFields.enable_filter_intra = sequence.enableFilterIntra ? 1 : 0; + seqFields.enable_intra_edge_filter = sequence.enableIntraEdgeFilter ? 1 : 0; + seqFields.enable_interintra_compound = sequence.enableInterintraCompound ? 1 : 0; + seqFields.enable_masked_compound = sequence.enableMaskedCompound ? 1 : 0; + seqFields.enable_dual_filter = sequence.enableDualFilter ? 1 : 0; + seqFields.enable_order_hint = sequence.enableOrderHint ? 1 : 0; + seqFields.enable_jnt_comp = sequence.enableJntComp ? 1 : 0; + seqFields.enable_cdef = sequence.enableCdef ? 1 : 0; + seqFields.mono_chrome = sequence.monochrome ? 1 : 0; + seqFields.color_range = sequence.colorRange ? 1 : 0; + seqFields.subsampling_x = sequence.subsamplingX ? 1 : 0; + seqFields.subsampling_y = sequence.subsamplingY ? 1 : 0; + seqFields.film_grain_params_present = sequence.filmGrainParamsPresent ? 1 : 0; + + picture.current_frame = surfaces[surface]; + picture.current_display_picture = surfaces[surface]; + picture.frame_width_minus1 = static_cast(header.upscaledWidth - 1); + picture.frame_height_minus1 = static_cast(header.frameHeight - 1); + + // A key frame that is shown refers to nothing. + const bool resets = header.frameType == kAv1KeyFrame && header.showFrame; + + for (int slot = 0; slot < 8; slot++) { + picture.ref_frame_map[slot] = !resets && slots[slot] >= 0 ? surfaces[slots[slot]] : VA_INVALID_SURFACE; + } + for (int i = 0; i < 7; i++) { + picture.ref_frame_idx[i] = header.refFrameIdx[i]; + } + + picture.primary_ref_frame = header.primaryRefFrame; + picture.order_hint = header.orderHint; + + auto & segmentation = picture.seg_info; + segmentation.segment_info_fields.bits.enabled = header.segmentationEnabled ? 1 : 0; + segmentation.segment_info_fields.bits.update_map = header.segmentationUpdateMap ? 1 : 0; + segmentation.segment_info_fields.bits.temporal_update = header.segmentationTemporalUpdate ? 1 : 0; + segmentation.segment_info_fields.bits.update_data = header.segmentationUpdateData ? 1 : 0; + + for (int segment = 0; segment < 8; segment++) { + for (int feature = 0; feature < 8; feature++) { + segmentation.feature_mask[segment] |= (header.featureEnabled[segment][feature] ? 1 : 0) << feature; + segmentation.feature_data[segment][feature] = header.featureData[segment][feature]; + } + } + + picture.tile_cols = header.tileCols; + picture.tile_rows = header.tileRows; + + for (int i = 0; i < header.tileCols && i < 63; i++) { + picture.width_in_sbs_minus_1[i] = static_cast(header.tileWidthSbs[i] - 1); + } + for (int i = 0; i < header.tileRows && i < 63; i++) { + picture.height_in_sbs_minus_1[i] = static_cast(header.tileHeightSbs[i] - 1); + } + + picture.context_update_tile_id = header.contextUpdateTileId; + + auto & picFields = picture.pic_info_fields.bits; + picFields.frame_type = header.frameType; + picFields.show_frame = header.showFrame ? 1 : 0; + picFields.showable_frame = header.showableFrame ? 1 : 0; + picFields.error_resilient_mode = header.errorResilientMode ? 1 : 0; + picFields.disable_cdf_update = header.disableCdfUpdate ? 1 : 0; + picFields.allow_screen_content_tools = header.allowScreenContentTools ? 1 : 0; + picFields.force_integer_mv = header.forceIntegerMv ? 1 : 0; + picFields.allow_intrabc = header.allowIntrabc ? 1 : 0; + picFields.use_superres = header.useSuperres ? 1 : 0; + picFields.allow_high_precision_mv = header.allowHighPrecisionMv ? 1 : 0; + picFields.is_motion_mode_switchable = header.isMotionModeSwitchable ? 1 : 0; + picFields.use_ref_frame_mvs = header.useRefFrameMvs ? 1 : 0; + picFields.disable_frame_end_update_cdf = header.disableFrameEndUpdateCdf ? 1 : 0; + picFields.uniform_tile_spacing_flag = header.uniformTileSpacing ? 1 : 0; + picFields.allow_warped_motion = header.allowWarpedMotion ? 1 : 0; + picFields.large_scale_tile = 0; + + picture.superres_scale_denominator = header.superresDenom; + picture.interp_filter = header.interpolationFilter; + picture.filter_level[0] = header.loopFilterLevel[0]; + picture.filter_level[1] = header.loopFilterLevel[1]; + picture.filter_level_u = header.loopFilterLevel[2]; + picture.filter_level_v = header.loopFilterLevel[3]; + + auto & loopFilterFields = picture.loop_filter_info_fields.bits; + loopFilterFields.sharpness_level = header.loopFilterSharpness; + loopFilterFields.mode_ref_delta_enabled = header.loopFilterDeltaEnabled ? 1 : 0; + loopFilterFields.mode_ref_delta_update = header.loopFilterDeltaUpdate ? 1 : 0; + + std::copy(header.loopFilterRefDeltas.begin(), header.loopFilterRefDeltas.end(), picture.ref_deltas); + std::copy(header.loopFilterModeDeltas.begin(), header.loopFilterModeDeltas.end(), picture.mode_deltas); + + picture.base_qindex = header.baseQIndex; + picture.y_dc_delta_q = header.deltaQYDc; + picture.u_dc_delta_q = header.deltaQUDc; + picture.u_ac_delta_q = header.deltaQUAc; + picture.v_dc_delta_q = header.deltaQVDc; + picture.v_ac_delta_q = header.deltaQVAc; + + auto & qmatrixFields = picture.qmatrix_fields.bits; + qmatrixFields.using_qmatrix = header.usingQmatrix ? 1 : 0; + qmatrixFields.qm_y = header.qmY; + qmatrixFields.qm_u = header.qmU; + qmatrixFields.qm_v = header.qmV; + + auto & modeFields = picture.mode_control_fields.bits; + modeFields.delta_q_present_flag = header.deltaQPresent ? 1 : 0; + modeFields.log2_delta_q_res = header.deltaQRes; + modeFields.delta_lf_present_flag = header.deltaLfPresent ? 1 : 0; + modeFields.log2_delta_lf_res = header.deltaLfRes; + modeFields.delta_lf_multi = header.deltaLfMulti ? 1 : 0; + modeFields.tx_mode = header.txMode; + modeFields.reference_select = header.referenceSelect ? 1 : 0; + modeFields.reduced_tx_set_used = header.reducedTxSet ? 1 : 0; + modeFields.skip_mode_present = header.skipModePresent ? 1 : 0; + + picture.cdef_damping_minus_3 = header.cdefDampingMinus3; + picture.cdef_bits = header.cdefBits; + + for (int i = 0; i < 8; i++) { + picture.cdef_y_strengths[i] = static_cast((header.cdefYPriStrength[i] << 2) + | header.cdefYSecStrength[i]); + picture.cdef_uv_strengths[i] = static_cast((header.cdefUvPriStrength[i] << 2) + | header.cdefUvSecStrength[i]); + } + + auto & restorationFields = picture.loop_restoration_fields.bits; + restorationFields.yframe_restoration_type = header.frameRestorationType[0]; + restorationFields.cbframe_restoration_type = header.frameRestorationType[1]; + restorationFields.crframe_restoration_type = header.frameRestorationType[2]; + restorationFields.lr_unit_shift = header.lrUnitShift; + restorationFields.lr_uv_shift = header.lrUvShift; + + for (int i = 0; i < 7; i++) { + const int ref = i + 1; + VAWarpedMotionParamsAV1 & warp = picture.wm[i]; + + warp.wmtype = static_cast(header.gmType[ref]); + + for (int j = 0; j < 6; j++) { + warp.wmmat[j] = header.gmParams[ref][j]; + } + + warp.invalid = header.gmValid[ref] ? 0 : 1; + } + + VADisplay display = library.Display(); + VABufferID buffers[3] = { VA_INVALID_ID, VA_INVALID_ID, VA_INVALID_ID }; + + // vaCreateBuffer copies the data, though it takes it as not const. + auto * tiles = const_cast(frame.tiles.data()); + auto * tileData = const_cast(frame.tileData.data()); + + bool created = Check(va.CreateBuffer(display, context, VAPictureParameterBufferType, sizeof(picture), 1, + &picture, &buffers[0]), "creating the picture parameters") + && Check(va.CreateBuffer(display, context, VASliceParameterBufferType, sizeof(VASliceParameterBufferAV1), + static_cast(frame.tiles.size()), tiles, &buffers[1]), "creating the tile parameters") + && Check(va.CreateBuffer(display, context, VASliceDataBufferType, + static_cast(frame.tileData.size()), 1, tileData, &buffers[2]), "creating the tile data"); + + bool decoded = false; + + if (created && Check(va.BeginPicture(display, context, surfaces[surface]), "beginning a picture")) { + const bool rendered = Check(va.RenderPicture(display, context, buffers, 3), "rendering a picture"); + // A picture that was begun has to be ended. + const bool ended = Check(va.EndPicture(display, context), "ending a picture"); + + decoded = rendered && ended && Check(va.SyncSurface(display, surfaces[surface]), "waiting for a picture"); + } + + for (VABufferID buffer : buffers) { + if (buffer != VA_INVALID_ID) { + va.DestroyBuffer(display, buffer); + } + } + + return decoded; + } + + bool VaapiAv1Decoder::Output(int surface, const webrtc::EncodedImage & image, int64_t renderTimeMs) + { + VADisplay display = library.Display(); + const int frameWidth = static_cast(width); + const int frameHeight = static_cast(height); + + VAImage mapped = {}; + mapped.image_id = VA_INVALID_ID; + bool derived = false; + + if (downloadImage.image_id != VA_INVALID_ID + && va.GetImage(display, surfaces[surface], 0, 0, width, height, downloadImage.image_id) + == VA_STATUS_SUCCESS) + { + mapped = downloadImage; + } + else if (va.DeriveImage(display, surfaces[surface], &mapped) == VA_STATUS_SUCCESS) { + derived = true; + + if (mapped.format.fourcc != VA_FOURCC_NV12) { + RTC_LOG(LS_WARNING) << implementationName << " AV1: the decoded surface is not NV12"; + va.DestroyImage(display, mapped.image_id); + return false; + } + } + else { + RTC_LOG(LS_WARNING) << implementationName << " AV1: the decoded surface cannot be read"; + return false; + } + + void * data = nullptr; + bool converted = false; + webrtc::scoped_refptr i420 = webrtc::I420Buffer::Create(frameWidth, frameHeight); + + if (Check(va.MapBuffer(display, mapped.buf, &data), "mapping a decoded picture")) { + const uint8_t * base = static_cast(data); + + converted = libyuv::NV12ToI420(base + mapped.offsets[0], static_cast(mapped.pitches[0]), + base + mapped.offsets[1], static_cast(mapped.pitches[1]), + i420->MutableDataY(), i420->StrideY(), i420->MutableDataU(), i420->StrideU(), + i420->MutableDataV(), i420->StrideV(), frameWidth, frameHeight) == 0; + + va.UnmapBuffer(display, mapped.buf); + } + + if (derived) { + va.DestroyImage(display, mapped.image_id); + } + + if (!converted) { + return false; + } + + webrtc::VideoFrame frame = webrtc::VideoFrame::Builder() + .set_video_frame_buffer(i420) + .set_rtp_timestamp(image.RtpTimestamp()) + .set_timestamp_ms(renderTimeMs) + .set_ntp_time_ms(image.ntp_time_ms_) + .set_rotation(image.rotation_) + .build(); + + callback->Decoded(frame, std::nullopt, std::nullopt); + + return true; + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoDecoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoDecoderFactory.cpp index 2848cadf..1b0abb31 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoDecoderFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoDecoderFactory.cpp @@ -18,6 +18,7 @@ #include "media/video/codec/linux/VaapiVideoDecoderFactory.h" #include "media/video/codec/H264Format.h" #include "media/video/codec/H265Format.h" +#include "media/video/codec/linux/VaapiAv1Decoder.h" #include "media/video/codec/linux/VaapiH264Decoder.h" #include "media/video/codec/linux/VaapiH265Decoder.h" #include "media/video/codec/linux/VaapiVp9Decoder.h" @@ -38,22 +39,26 @@ namespace jni const bool h264 = library->Decodes(VAProfileH264High); const bool vp9 = library->Decodes(VAProfileVP9Profile0); const bool h265 = library->Decodes(VAProfileHEVCMain); + const bool av1 = library->Decodes(VAProfileAV1Profile0); RTC_LOG(LS_INFO) << "VA-API (" << library->Vendor() << ") H.264 decoding: " << h264 << ", VP9 decoding: " - << vp9 << ", H.265 decoding: " << h265; + << vp9 << ", H.265 decoding: " << h265 << ", AV1 decoding: " << av1; - if (!h264 && !vp9 && !h265) { + if (!h264 && !vp9 && !h265 && !av1) { return nullptr; } - return std::unique_ptr(new VaapiVideoDecoderFactory(*library, h264, vp9, h265)); + return std::unique_ptr( + new VaapiVideoDecoderFactory(*library, h264, vp9, h265, av1)); } - VaapiVideoDecoderFactory::VaapiVideoDecoderFactory(VaapiLibrary & library, bool h264, bool vp9, bool h265) : + VaapiVideoDecoderFactory::VaapiVideoDecoderFactory(VaapiLibrary & library, bool h264, bool vp9, bool h265, + bool av1) : library(library), h264(h264), vp9(vp9), - h265(h265) + h265(h265), + av1(av1) { } @@ -70,6 +75,9 @@ namespace jni if (h265) { formats.push_back(CreateH265MainFormat()); } + if (av1) { + formats.push_back(webrtc::SdpVideoFormat::AV1Profile0()); + } return formats; } @@ -86,6 +94,9 @@ namespace jni if (vp9 && format.IsSameCodec(webrtc::SdpVideoFormat::VP9Profile0())) { return std::make_unique(library); } + if (av1 && format.IsSameCodec(webrtc::SdpVideoFormat::AV1Profile0())) { + return std::make_unique(library); + } return nullptr; } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java index e431ad47..00f4b5a7 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java @@ -55,9 +55,11 @@ * are decoded on NVIDIA GPUs with NVDEC, each where the GPU decodes it (AV1 * since Ampere), loaded from the NVIDIA driver when the factory is made; a * machine without the driver is not affected. Streams with spatial layers go - * to libvpx and dav1d. On the GPUs of Intel and AMD, H.264, VP9 (profile 0) - * and H.265 (Main profile) are decoded with VA-API, where the driver decodes - * them; interlaced H.264 goes to the software decoder. + * to libvpx and dav1d. On the GPUs of Intel and AMD, H.264, VP9 (profile 0), + * H.265 (Main profile) and AV1 (profile 0) are decoded with VA-API, where the + * driver decodes them; interlaced H.264 goes to the software decoder, and so + * do AV1 streams with film grain, superres, more than 8 bits or a single + * plane. *

* The supported codecs are those of a {@link DefaultVideoDecoderFactory}, in * the same order. On Windows and Linux, H.264 High 4:4:4 Predictive, diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java index 61f9738b..bd1cd244 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoDecoderIntegrationTest.java @@ -424,6 +424,21 @@ void defaultDecodesVp9InSoftware() throws Exception { void hardwareFollowsResolutionChange() throws Exception { assumeVp9Platform(); + assertFollowsResolutionChange(VP9, HardwareVideoDecoderIntegrationTest::expectVp9Hardware); + } + + @Test + void hardwareFollowsAv1ResolutionChange() throws Exception { + assertFollowsResolutionChange(AV1, implementation -> expectHardware(implementation, HARDWARE_AV1_REQUIRED)); + } + + /** + * Receives video that halves its size during the call, and checks that the + * hardware decoder follows it, rather than handing over to the software + * decoder. + */ + private void assertFollowsResolutionChange(Predicate codec, + Consumer expectHardware) throws Exception { PeerConnectionFactory hardware = PeerConnectionFactory.builder() .setAudioDeviceModule(audioDevModule) .setVideoDecoderFactory(new HardwareVideoDecoderFactory()) @@ -432,7 +447,7 @@ void hardwareFollowsResolutionChange() throws Exception { CountDownLatch full = new CountDownLatch(10); CountDownLatch half = new CountDownLatch(10); - try (TestMediaCall call = new TestMediaCall(hardware, true, false, VP9)) { + try (TestMediaCall call = new TestMediaCall(hardware, true, false, codec)) { // Large enough that half the size is still one every hardware // decoder takes: NVDEC does not decode VP9 below 128 pixels on the // shorter side, and falls back to libvpx for such a stream. @@ -462,7 +477,7 @@ else if (width == 320 && frame.buffer.getHeight() == 240) { String implementation = decoderImplementationOf(call); - expectVp9Hardware(implementation); + expectHardware.accept(implementation); // The sender restarts at half the size, with a key frame; the // decoder has to start a session for it. @@ -531,6 +546,28 @@ void vp9SpatialLayers() throws Exception { assertFalse(isHardware(result.decoder), result.decoder); } + @Test + void av1NeedsNoKeyFrames() throws Exception { + CallResult result = receiveVideo(hardwareFactory(), AV1, 320, 240, null, 60); + + expectHardware(result.decoder, HARDWARE_AV1_REQUIRED); + + // As with VP9: a stream that is decoded has the one key frame it + // started with. + assertTrue(result.pliCount <= 1, "key frames requested: " + result.pliCount); + assertTrue(result.keyFrames <= 2, "key frames decoded: " + result.keyFrames); + } + + @Test + void av1TemporalLayers() throws Exception { + // Temporal layers are one spatial layer, which the hardware decodes. + CallResult result = receiveVideo(hardwareFactory(), AV1, 320, 240, "L1T3", 60); + + expectHardware(result.decoder, HARDWARE_AV1_REQUIRED); + + assertTrue(result.pliCount <= 1, "key frames requested: " + result.pliCount); + } + private PeerConnectionFactory hardwareFactory() { return PeerConnectionFactory.builder() .setAudioDeviceModule(audioDevModule)