From ba5cf9c022b09c66c3f25eb9a23e0623a15ac1d1 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Thu, 15 Oct 2026 02:24:48 +0200 Subject: [PATCH] feat: decode H.264 with VA-API on Linux Adds an H.264 decoder for Intel and AMD GPUs, with its own parser. The H.264 and H.265 decoders share a new VaapiDecodeSession. --- docs/guide/advanced/video-codecs.md | 2 +- .../media/video/codec/linux/H264Parser.h | 198 ++++ .../video/codec/linux/VaapiDecodeSession.h | 98 ++ .../video/codec/linux/VaapiH264Decoder.h | 179 +++ .../video/codec/linux/VaapiH265Decoder.h | 9 +- .../codec/linux/VaapiVideoDecoderFactory.h | 8 +- .../media/video/codec/linux/H264Parser.cpp | 729 ++++++++++++ .../video/codec/linux/VaapiDecodeSession.cpp | 265 +++++ .../video/codec/linux/VaapiH264Decoder.cpp | 1049 +++++++++++++++++ .../video/codec/linux/VaapiH265Decoder.cpp | 198 +--- .../codec/linux/VaapiVideoDecoderFactory.cpp | 21 +- .../codec/HardwareVideoDecoderFactory.java | 5 +- 12 files changed, 2563 insertions(+), 198 deletions(-) create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/linux/H264Parser.h create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiDecodeSession.h create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH264Decoder.h create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/linux/H264Parser.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiDecodeSession.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Decoder.cpp diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index 0338fcb8..a255ed1c 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -62,7 +62,7 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder() | Platform | Hardware decoding | |---|---| | Windows | H.264, H.265 (Main profile), AV1 and VP9 (profile 0), on GPUs that decode them, through the Media Foundation decoders of Windows on Direct3D 11 (DXVA) | -| Linux | H.264, H.265 (Main profile), VP9 and AV1 (profile 0) on NVIDIA GPUs that decode them, through NVDEC; VP9 (profile 0) and H.265 (Main profile) on Intel and AMD GPUs through VA-API | +| Linux | H.264, H.265 (Main profile), VP9 and AV1 (profile 0) on NVIDIA GPUs that decode them, through NVDEC; H.264, VP9 (profile 0) and H.265 (Main profile) on Intel and AMD GPUs through VA-API | | macOS | H.264 through VideoToolbox, as with `DefaultVideoDecoderFactory`; VP9 (profile 0) and H.265 (Main profile) through the decoders of VideoToolbox, on Macs that have them, and AV1 (profile 0) on Apple M3 and newer | On Linux, NVDEC needs an NVIDIA driver, which provides `libcuda.so.1` and `libnvcuvid.so.1`; both are loaded when the factory is made, so a machine without the driver is not affected. Streams with spatial layers are decoded in software. diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/H264Parser.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/H264Parser.h new file mode 100644 index 00000000..a0a8c4ad --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/H264Parser.h @@ -0,0 +1,198 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_H264_PARSER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_H264_PARSER_H_ + +#include +#include +#include +#include +#include +#include + +namespace jni +{ + // The scaling lists of a picture (7.4.2.1.1.1), in the order they are + // coded, which is zigzag order. + struct H264ScalingLists + { + std::array, 6> list4x4 = {}; + std::array, 2> list8x8 = {}; + }; + + // What the decoder needs of a sequence parameter set (7.3.2.1.1). + struct H264Sps + { + uint32_t profileIdc = 0; + uint32_t levelIdc = 0; + uint32_t chromaFormatIdc = 1; + uint32_t bitDepthLumaMinus8 = 0; + uint32_t bitDepthChromaMinus8 = 0; + bool scalingMatrixPresent = false; + // The lists given or inferred, where scalingMatrixPresent is set. + H264ScalingLists scalingLists; + uint32_t log2MaxFrameNumMinus4 = 0; + uint32_t pocType = 0; + uint32_t log2MaxPocLsbMinus4 = 0; + bool deltaPicOrderAlwaysZero = false; + int32_t offsetForNonRefPic = 0; + int32_t offsetForTopToBottomField = 0; + std::vector offsetForRefFrame; + uint32_t maxNumRefFrames = 0; + bool gapsInFrameNumAllowed = false; + uint32_t widthInMbs = 0; + uint32_t heightInMbs = 0; + bool frameMbsOnly = true; + bool direct8x8Inference = false; + // The frame cropping, in luma samples. + uint32_t cropLeft = 0; + uint32_t cropRight = 0; + uint32_t cropTop = 0; + uint32_t cropBottom = 0; + // From the VUI, where it says. + std::optional maxNumReorderFrames; + + uint32_t MaxFrameNum() const; + }; + + // A picture parameter set (7.3.2.2). + struct H264Pps + { + uint32_t spsId = 0; + bool entropyCodingMode = false; + bool bottomFieldPicOrderInFramePresent = false; + uint32_t numSliceGroupsMinus1 = 0; + uint32_t numRefIdxL0DefaultMinus1 = 0; + uint32_t numRefIdxL1DefaultMinus1 = 0; + bool weightedPred = false; + uint32_t weightedBipredIdc = 0; + int32_t picInitQpMinus26 = 0; + int32_t picInitQsMinus26 = 0; + int32_t chromaQpIndexOffset = 0; + bool deblockingFilterControlPresent = false; + bool constrainedIntraPred = false; + bool redundantPicCntPresent = false; + bool transform8x8Mode = false; + int32_t secondChromaQpIndexOffset = 0; + bool scalingMatrixPresent = false; + // The lists as coded, and which were. + std::array scalingListPresent = {}; + std::array useDefaultScalingList = {}; + H264ScalingLists scalingLists; + }; + + // A reference picture list modification (7.3.3.1), or a memory + // management control operation (7.3.3.3). + struct H264Operation + { + uint32_t operation = 0; + uint32_t value = 0; + // For memory management: long_term_frame_idx of operations 3 and 6. + uint32_t longTermFrameIdx = 0; + }; + + // A slice header (7.3.3) of a frame. + struct H264SliceHeader + { + static constexpr int kMaxRefs = 32; + + uint8_t nalType = 0; + uint8_t nalRefIdc = 0; + uint32_t firstMbInSlice = 0; + // slice_type modulo 5. + uint32_t sliceType = 0; + uint32_t ppsId = 0; + uint32_t frameNum = 0; + uint32_t idrPicId = 0; + uint32_t pocLsb = 0; + int32_t deltaPocBottom = 0; + std::array deltaPoc = {}; + uint32_t redundantPicCnt = 0; + bool directSpatialMvPred = false; + uint32_t numRefIdxL0Minus1 = 0; + uint32_t numRefIdxL1Minus1 = 0; + std::array, 2> listModifications; + uint32_t lumaLog2WeightDenom = 0; + uint32_t chromaLog2WeightDenom = 0; + // Whether any weight of a list was given, and the weights and offsets, + // the default ones where none were given. + std::array lumaWeightsGiven = {}; + std::array chromaWeightsGiven = {}; + std::array, 2> lumaWeight = {}; + std::array, 2> lumaOffset = {}; + std::array, kMaxRefs>, 2> chromaWeight = {}; + std::array, kMaxRefs>, 2> chromaOffset = {}; + bool noOutputOfPriorPics = false; + bool longTermReference = false; + bool adaptiveRefPicMarking = false; + std::vector memoryManagement; + uint32_t cabacInitIdc = 0; + int32_t qpDelta = 0; + uint32_t disableDeblockingFilterIdc = 0; + int32_t alphaOffsetDiv2 = 0; + int32_t betaOffsetDiv2 = 0; + // Where the slice data begins, in bits from the NAL unit header, with + // emulation prevention bytes not counted, as VA-API takes it. + uint32_t dataBitOffset = 0; + + bool IsIdr() const; + bool IsIntra() const; + bool IsB() const; + }; + + // Parses the parameter sets and slice headers of an H.264 stream, as a + // VA-API decoder, which leaves that to the application, needs them. It + // takes progressive 8 bit 4:2:0 streams without slice groups; anything + // else is not parsed, so that the software decoder takes it. + class H264Parser + { + public: + // NAL unit types (table 7-1). + static constexpr uint8_t kNalSlice = 1; + static constexpr uint8_t kNalIdr = 5; + static constexpr uint8_t kNalSps = 7; + static constexpr uint8_t kNalPps = 8; + static constexpr uint8_t kNalEndOfSequence = 10; + + // Slice types modulo 5 (table 7-6). + static constexpr uint32_t kSliceP = 0; + static constexpr uint32_t kSliceB = 1; + static constexpr uint32_t kSliceI = 2; + + // Each takes a NAL unit from its header on, with emulation + // prevention bytes, and returns false if it is broken or not one + // the parser takes. + bool ParseSps(std::span nal); + bool ParsePps(std::span nal); + std::optional ParseSliceHeader(std::span nal) const; + + const H264Sps * Sps(uint32_t id) const; + const H264Pps * Pps(uint32_t id) const; + + // The scaling lists a picture is decoded with: of the PPS, falling + // back to those of the SPS, or flat ones (7.4.2.2). + static H264ScalingLists ScalingLists(const H264Sps & sps, const H264Pps & pps); + + void Reset(); + + private: + std::map spss; + std::map ppss; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiDecodeSession.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiDecodeSession.h new file mode 100644 index 00000000..245772d1 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiDecodeSession.h @@ -0,0 +1,98 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_DECODE_SESSION_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_DECODE_SESSION_H_ + +#include "media/video/codec/linux/VaapiLibrary.h" + +#include "api/scoped_refptr.h" +#include "api/video/i420_buffer.h" + +#include + +#include +#include +#include +#include + +namespace jni +{ + // What every VA-API decoder of a codec whose stream the decoder parses + // needs: a configuration and a context with the surfaces the decoded + // pictures go into, decoding a picture from the parameter and data + // buffers the codec's decoder fills in, and copying a decoded picture out + // as I420, which is what WebRTC's frames hold. + // + // Used by the H.264, H.265 and AV1 decoders; the VP9 one keeps its own, + // older copy of the same. + class VaapiDecodeSession + { + public: + // A parameter or data buffer of a picture. + struct Buffer + { + VABufferType type; + const void * data; + size_t size; + }; + + // The name says whose operations failed, in the log. + VaapiDecodeSession(VaapiLibrary & library, std::string name); + ~VaapiDecodeSession(); + + VaapiDecodeSession(const VaapiDecodeSession &) = delete; + VaapiDecodeSession & operator=(const VaapiDecodeSession &) = delete; + + // Creates the session for 4:2:0 pictures of the given size, which + // is the coded size, with the given number of surfaces. + bool Create(VAProfile profile, uint32_t width, uint32_t height, uint32_t surfaceCount); + void Destroy(); + + bool IsCreated() const; + uint32_t Width() const; + uint32_t Height() const; + int SurfaceCount() const; + VASurfaceID Surface(int index) const; + + // Decodes a picture into the surface from the buffers, in the + // order given, and waits for it. + bool Decode(int surface, const std::vector & buffers); + // The region of the decoded picture in the surface, as I420; null + // if it cannot be read. + webrtc::scoped_refptr Download(int surface, uint32_t x, uint32_t y, uint32_t width, + uint32_t height); + + bool Check(VAStatus status, const char * operation) const; + + private: + VaapiLibrary & library; + const VaapiFunctions & va; + const std::string name; + + VAConfigID config; + VAContextID context; + std::vector surfaces; + uint32_t width; + uint32_t height; + + // The image a decoded surface is copied into, where the driver + // copies surfaces at all. + VAImage downloadImage; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH264Decoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH264Decoder.h new file mode 100644 index 00000000..a686b28c --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH264Decoder.h @@ -0,0 +1,179 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_H264_DECODER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_H264_DECODER_H_ + +#include "media/video/codec/linux/H264Parser.h" +#include "media/video/codec/linux/VaapiDecodeSession.h" +#include "media/video/codec/linux/VaapiLibrary.h" + +#include "api/video/encoded_image.h" +#include "api/video/video_rotation.h" +#include "api/video_codecs/video_decoder.h" + +#include + +#include +#include +#include +#include + +namespace jni +{ + // Decodes H.264 with the GPU driver's VA-API decoder, as Intel and AMD + // drivers provide it on Linux. + // + // VA-API leaves the parsing of the stream to the application: the + // parameter sets and slice headers are parsed here, and the decoder keeps + // the picture order counts, the marking of reference pictures and the + // reference picture lists, the driver decoding the slice data. Decoding is + // synchronous; pictures come out in output order, as far as the VUI says + // they are reordered, and in decoding order otherwise, which is the order + // of real-time encoders. The decoded picture is cropped and converted to + // I420, which is what WebRTC's frames hold. + // + // Interlaced streams, 4:2:2 and 4:4:4, more than 8 bits, slice groups and + // SP and SI slices, and anything that fails, return + // WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE, so that the software decoder takes + // over; a picture that refers to one that is not there returns + // WEBRTC_VIDEO_CODEC_ERROR, and decoding starts again with the next key + // frame. + class VaapiH264Decoder : public webrtc::VideoDecoder + { + public: + explicit VaapiH264Decoder(VaapiLibrary & library); + ~VaapiH264Decoder() override; + + using webrtc::VideoDecoder::Decode; + + bool Configure(const Settings & settings) override; + int32_t Decode(const webrtc::EncodedImage & image, int64_t renderTimeMs) override; + int32_t RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback * callback) override; + int32_t Release() override; + DecoderInfo GetDecoderInfo() const override; + const char * ImplementationName() const override; + + private: + // A decoded frame, as long as it is referenced or waits for output. + struct Picture + { + int surface = -1; + uint32_t frameNum = 0; + int32_t topPoc = 0; + int32_t bottomPoc = 0; + bool shortTerm = false; + bool longTerm = false; + uint32_t longTermFrameIdx = 0; + bool waitingOutput = false; + + // Of the image it came in, for the frame that shows it. + uint32_t rtpTimestamp = 0; + int64_t ntpTimeMs = 0; + int64_t renderTimeMs = 0; + webrtc::VideoRotation rotation = webrtc::kVideoRotation_0; + + int32_t Poc() const; + bool IsReference() const; + }; + + // A slice of the picture being decoded. + struct Slice + { + H264SliceHeader header; + std::span nal; + }; + + enum class Result + { + kOk, + // A reference picture is missing; a key frame starts again. + kMissingReference, + kFailed + }; + + Result DecodePicture(const std::vector & slices, const webrtc::EncodedImage & image, + int64_t renderTimeMs); + + // The picture order counts of the picture (8.2.1), top and bottom. + void PictureOrderCount(const H264SliceHeader & header, const H264Sps & sps, int32_t & top, + int32_t & bottom); + // The marking of the reference pictures after the current one is + // decoded (8.2.5), which becomes one itself unless it is not used + // for reference. + Result MarkReferences(const H264SliceHeader & header, const H264Sps & sps, Picture & current); + + // The reference picture lists of a slice (8.2.4), as surfaces, -1 + // where there is no picture. + bool BuildReferenceLists(const H264SliceHeader & header, const H264Sps & sps, int32_t currentPoc, + std::vector & list0, std::vector & list1) const; + // PicNum (8-28) of a short-term reference picture. + int32_t PicNum(const Picture & picture, uint32_t currentFrameNum, uint32_t maxFrameNum) const; + + VAPictureH264 ToVaPicture(const Picture & picture) const; + const Picture * FindBySurface(int surface) const; + + // Hands pictures to WebRTC in output order until at most the given + // number wait. + bool Bump(uint32_t keep); + bool Output(const Picture & picture); + void RemoveUnused(); + + int FreeSurface() const; + + private: + VaapiLibrary & library; + const std::string implementationName; + + webrtc::DecodedImageCallback * callback; + + H264Parser parser; + VaapiDecodeSession session; + + uint32_t widthInMbs; + uint32_t heightInMbs; + uint32_t maxNumRefFrames; + uint32_t maxNumReorder; + uint32_t cropX; + uint32_t cropY; + uint32_t width; + uint32_t height; + + std::vector dpb; + // MaxLongTermFrameIdx plus one, 0 for "no long-term frame indices". + uint32_t maxLongTermFrameIdxPlus1; + + // The state the picture order count and frame_num of the next + // picture derive from (8.2.1). + int32_t prevPocMsb; + int32_t prevPocLsb; + uint32_t prevFrameNum; + uint32_t prevFrameNumOffset; + uint32_t prevRefFrameNum; + // FrameNumOffset and PicOrderCntMsb of the current picture, and + // whether it had a memory management control operation 5. + uint32_t frameNumOffset; + int32_t pocMsb; + bool mmco5; + + // Whether a key frame has been decoded, and whether anything since + // then failed for good. + bool started; + bool failed; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Decoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Decoder.h index 356a91fa..c180cb48 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Decoder.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Decoder.h @@ -18,6 +18,7 @@ #define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_H265_DECODER_H_ #include "media/video/codec/linux/H265Parser.h" +#include "media/video/codec/linux/VaapiDecodeSession.h" #include "media/video/codec/linux/VaapiLibrary.h" #include "api/video/encoded_image.h" @@ -132,20 +133,15 @@ namespace jni void DestroySession(); int FreeSurface() const; - bool Check(VAStatus status, const char * operation) const; - private: VaapiLibrary & library; - const VaapiFunctions & va; const std::string implementationName; webrtc::DecodedImageCallback * callback; H265Parser parser; - VAConfigID config; - VAContextID context; - std::vector surfaces; + VaapiDecodeSession session; uint32_t codedWidth; uint32_t codedHeight; uint32_t maxDecPicBuffering; @@ -155,7 +151,6 @@ namespace jni uint32_t cropY; uint32_t width; uint32_t height; - VAImage downloadImage; // The decoded picture buffer, and the surfaces of the pictures the // current one refers to, by the subset of its reference picture diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoDecoderFactory.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoDecoderFactory.h index 98fff8bf..3c83aedc 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoDecoderFactory.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoDecoderFactory.h @@ -29,8 +29,9 @@ namespace jni { - // Creates VA-API decoders on the GPUs of Intel and AMD, for VP9 profile 0 - // and H.265 Main where the driver decodes them. + // Creates VA-API decoders on the GPUs of Intel and AMD, for H.264 (but + // High 4:4:4 Predictive), VP9 profile 0 and H.265 Main where the driver + // decodes them. class VaapiVideoDecoderFactory : public webrtc::VideoDecoderFactory { public: @@ -45,10 +46,11 @@ namespace jni const webrtc::SdpVideoFormat & format) override; private: - VaapiVideoDecoderFactory(VaapiLibrary & library, bool vp9, bool h265); + VaapiVideoDecoderFactory(VaapiLibrary & library, bool h264, bool vp9, bool h265); private: VaapiLibrary & library; + const bool h264; const bool vp9; const bool h265; }; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/H264Parser.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/H264Parser.cpp new file mode 100644 index 00000000..d8f306ee --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/H264Parser.cpp @@ -0,0 +1,729 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/linux/H264Parser.h" + +#include "rtc_base/bitstream_reader.h" + +#include + +namespace jni +{ + namespace + { + constexpr uint32_t kMaxSpsCount = 32; + constexpr uint32_t kMaxPpsCount = 256; + + // The default scaling lists (tables 7-3 and 7-4), in coded order. + constexpr std::array kDefault4x4Intra = { + 6, 13, 13, 20, 20, 20, 28, 28, 28, 28, 32, 32, 32, 37, 37, 42 + }; + constexpr std::array kDefault4x4Inter = { + 10, 14, 14, 20, 20, 20, 24, 24, 24, 24, 27, 27, 27, 30, 30, 34 + }; + constexpr std::array kDefault8x8Intra = { + 6, 10, 10, 13, 11, 13, 16, 16, 16, 16, 18, 18, 18, 18, 18, 23, + 23, 23, 23, 23, 23, 25, 25, 25, 25, 25, 25, 25, 27, 27, 27, 27, + 27, 27, 27, 27, 29, 29, 29, 29, 29, 29, 29, 31, 31, 31, 31, 31, + 31, 33, 33, 33, 33, 33, 36, 36, 36, 36, 38, 38, 38, 40, 40, 42 + }; + constexpr std::array kDefault8x8Inter = { + 9, 13, 13, 15, 13, 15, 17, 17, 17, 17, 19, 19, 19, 19, 19, 21, + 21, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 24, 24, 24, 24, + 24, 24, 24, 24, 25, 25, 25, 25, 25, 25, 25, 27, 27, 27, 27, 27, + 27, 28, 28, 28, 28, 28, 30, 30, 30, 30, 32, 32, 32, 33, 33, 35 + }; + + // The profiles whose SPS has chroma format, bit depth and scaling + // matrices. + bool HasHighProfileFields(uint32_t profile) + { + switch (profile) { + case 100: case 110: case 122: case 244: case 44: case 83: case 86: case 118: case 128: case 138: + case 139: case 134: case 135: + return true; + default: + return false; + } + } + + // The RBSP of a NAL unit, without its one byte header. + std::vector Unescape(std::span nal) + { + std::vector rbsp; + rbsp.reserve(nal.size()); + + int zeros = 0; + + for (size_t i = 1; i < nal.size(); i++) { + if (zeros >= 2 && nal[i] == 3) { + zeros = 0; + continue; + } + + rbsp.push_back(nal[i]); + zeros = nal[i] == 0 ? zeros + 1 : 0; + } + + return rbsp; + } + + int ConsumedBits(const webrtc::BitstreamReader & reader, size_t size) + { + return static_cast(size * 8) - reader.RemainingBitCount(); + } + + // scaling_list() (7.3.2.1.1.1); returns whether the default list is + // to be used. + template + bool ParseScalingList(webrtc::BitstreamReader & reader, std::array & list) + { + int32_t lastScale = 8; + int32_t nextScale = 8; + + for (size_t j = 0; j < Size; j++) { + if (nextScale != 0) { + const int32_t delta = reader.ReadSignedExponentialGolomb(); + + nextScale = (lastScale + delta + 256) % 256; + + if (j == 0 && nextScale == 0) { + return true; + } + } + + list[j] = static_cast(nextScale == 0 ? lastScale : nextScale); + lastScale = list[j]; + } + + return false; + } + + // The lists of a set as coded, where present, or by fall-back rule A + // (table 7-2) or B, given the lists rule B falls back to. + void ResolveScalingLists(const std::array & present, const std::array & useDefault, + const H264ScalingLists & coded, const H264ScalingLists * fallback, H264ScalingLists & lists) + { + for (int i = 0; i < 6; i++) { + const auto & defaultList = i < 3 ? kDefault4x4Intra : kDefault4x4Inter; + + if (present[i]) { + lists.list4x4[i] = useDefault[i] ? defaultList : coded.list4x4[i]; + } + else if (i == 0 || i == 3) { + lists.list4x4[i] = fallback != nullptr ? fallback->list4x4[i] : defaultList; + } + else { + lists.list4x4[i] = lists.list4x4[i - 1]; + } + } + + for (int i = 0; i < 2; i++) { + const auto & defaultList = i == 0 ? kDefault8x8Intra : kDefault8x8Inter; + + if (present[6 + i]) { + lists.list8x8[i] = useDefault[6 + i] ? defaultList : coded.list8x8[i]; + } + else { + lists.list8x8[i] = fallback != nullptr ? fallback->list8x8[i] : defaultList; + } + } + } + + // The scaling lists of a set: whether each is present, whether its + // default is to be used, and the lists coded. + void ParseScalingLists(webrtc::BitstreamReader & reader, int count, std::array & present, + std::array & useDefault, H264ScalingLists & coded) + { + for (int i = 0; i < count; i++) { + present[i] = reader.Read(); + + if (!present[i]) { + continue; + } + + useDefault[i] = i < 6 ? ParseScalingList(reader, coded.list4x4[i]) + : ParseScalingList(reader, coded.list8x8[i - 6]); + } + } + + void SkipHrdParameters(webrtc::BitstreamReader & reader) + { + const uint32_t cpbCount = reader.ReadExponentialGolomb() + 1; + + if (cpbCount > 32) { + reader.Invalidate(); + return; + } + + // bit_rate_scale and cpb_size_scale. + reader.ConsumeBits(8); + + for (uint32_t i = 0; i < cpbCount; i++) { + reader.ReadExponentialGolomb(); + reader.ReadExponentialGolomb(); + reader.ConsumeBits(1); + } + + // The lengths of the delays and of the time offset. + reader.ConsumeBits(20); + } + + // vui_parameters() (E.1.1), for the reordering it allows. + void ParseVui(webrtc::BitstreamReader & reader, H264Sps & sps) + { + if (reader.Read()) { + if (reader.ReadBits(8) == 255) { + reader.ConsumeBits(32); + } + } + if (reader.Read()) { + reader.ConsumeBits(1); + } + if (reader.Read()) { + reader.ConsumeBits(4); + + if (reader.Read()) { + reader.ConsumeBits(24); + } + } + if (reader.Read()) { + reader.ReadExponentialGolomb(); + reader.ReadExponentialGolomb(); + } + if (reader.Read()) { + reader.ConsumeBits(65); + } + + const bool nalHrd = reader.Read(); + + if (nalHrd) { + SkipHrdParameters(reader); + } + + const bool vclHrd = reader.Read(); + + if (vclHrd) { + SkipHrdParameters(reader); + } + if (nalHrd || vclHrd) { + reader.ConsumeBits(1); + } + + // pic_struct_present_flag. + reader.ConsumeBits(1); + + if (reader.Read()) { + reader.ConsumeBits(1); + + for (int i = 0; i < 4; i++) { + reader.ReadExponentialGolomb(); + } + + const uint32_t maxNumReorderFrames = reader.ReadExponentialGolomb(); + // max_dec_frame_buffering. + reader.ReadExponentialGolomb(); + + if (reader.Ok()) { + sps.maxNumReorderFrames = maxNumReorderFrames; + } + } + } + + // pred_weight_table() (7.3.3.2) of one list. + void ParseWeights(webrtc::BitstreamReader & reader, H264SliceHeader & header, int list, uint32_t count) + { + for (uint32_t i = 0; i < count; i++) { + header.lumaWeight[list][i] = 1 << header.lumaLog2WeightDenom; + header.lumaOffset[list][i] = 0; + + if (reader.Read()) { + header.lumaWeightsGiven[list] = true; + header.lumaWeight[list][i] = reader.ReadSignedExponentialGolomb(); + header.lumaOffset[list][i] = reader.ReadSignedExponentialGolomb(); + } + + const bool chroma = reader.Read(); + + for (int j = 0; j < 2; j++) { + header.chromaWeight[list][i][j] = 1 << header.chromaLog2WeightDenom; + header.chromaOffset[list][i][j] = 0; + + if (chroma) { + header.chromaWeightsGiven[list] = true; + header.chromaWeight[list][i][j] = reader.ReadSignedExponentialGolomb(); + header.chromaOffset[list][i][j] = reader.ReadSignedExponentialGolomb(); + } + } + } + } + + // ref_pic_list_modification() (7.3.3.1) of one list. + bool ParseListModification(webrtc::BitstreamReader & reader, std::vector & operations) + { + if (!reader.Read()) { + return true; + } + + while (true) { + H264Operation operation; + operation.operation = reader.ReadExponentialGolomb(); + + if (operation.operation == 3) { + return true; + } + if (operation.operation > 3 || operations.size() > H264SliceHeader::kMaxRefs || !reader.Ok()) { + return false; + } + + operation.value = reader.ReadExponentialGolomb(); + operations.push_back(operation); + } + } + + // dec_ref_pic_marking() (7.3.3.3). + bool ParseRefPicMarking(webrtc::BitstreamReader & reader, H264SliceHeader & header) + { + if (header.IsIdr()) { + header.noOutputOfPriorPics = reader.Read(); + header.longTermReference = reader.Read(); + + return true; + } + + header.adaptiveRefPicMarking = reader.Read(); + + if (!header.adaptiveRefPicMarking) { + return true; + } + + while (true) { + H264Operation operation; + operation.operation = reader.ReadExponentialGolomb(); + + if (operation.operation == 0) { + return true; + } + if (operation.operation > 6 || header.memoryManagement.size() > 66 || !reader.Ok()) { + return false; + } + + if (operation.operation == 1 || operation.operation == 2 || operation.operation == 3 + || operation.operation == 4) + { + operation.value = reader.ReadExponentialGolomb(); + } + if (operation.operation == 3 || operation.operation == 6) { + operation.longTermFrameIdx = reader.ReadExponentialGolomb(); + } + + header.memoryManagement.push_back(operation); + } + } + } + + uint32_t H264Sps::MaxFrameNum() const + { + return 1u << (log2MaxFrameNumMinus4 + 4); + } + + bool H264SliceHeader::IsIdr() const + { + return nalType == H264Parser::kNalIdr; + } + + bool H264SliceHeader::IsIntra() const + { + return sliceType == H264Parser::kSliceI; + } + + bool H264SliceHeader::IsB() const + { + return sliceType == H264Parser::kSliceB; + } + + bool H264Parser::ParseSps(std::span nal) + { + const std::vector rbsp = Unescape(nal); + webrtc::BitstreamReader reader(rbsp); + H264Sps sps; + + sps.profileIdc = reader.ReadBits(8); + // The constraint flags. + reader.ConsumeBits(8); + sps.levelIdc = reader.ReadBits(8); + + const uint32_t id = reader.ReadExponentialGolomb(); + + if (id >= kMaxSpsCount) { + return false; + } + + if (HasHighProfileFields(sps.profileIdc)) { + sps.chromaFormatIdc = reader.ReadExponentialGolomb(); + + if (sps.chromaFormatIdc == 3) { + return false; + } + + sps.bitDepthLumaMinus8 = reader.ReadExponentialGolomb(); + sps.bitDepthChromaMinus8 = reader.ReadExponentialGolomb(); + // qpprime_y_zero_transform_bypass_flag. + reader.ConsumeBits(1); + sps.scalingMatrixPresent = reader.Read(); + + if (sps.scalingMatrixPresent) { + std::array present = {}; + std::array useDefault = {}; + H264ScalingLists coded; + + ParseScalingLists(reader, 8, present, useDefault, coded); + ResolveScalingLists(present, useDefault, coded, nullptr, sps.scalingLists); + } + } + + sps.log2MaxFrameNumMinus4 = reader.ReadExponentialGolomb(); + sps.pocType = reader.ReadExponentialGolomb(); + + if (sps.log2MaxFrameNumMinus4 > 12 || sps.pocType > 2) { + return false; + } + + if (sps.pocType == 0) { + sps.log2MaxPocLsbMinus4 = reader.ReadExponentialGolomb(); + + if (sps.log2MaxPocLsbMinus4 > 12) { + return false; + } + } + else if (sps.pocType == 1) { + sps.deltaPicOrderAlwaysZero = reader.Read(); + sps.offsetForNonRefPic = reader.ReadSignedExponentialGolomb(); + sps.offsetForTopToBottomField = reader.ReadSignedExponentialGolomb(); + + const uint32_t cycleLength = reader.ReadExponentialGolomb(); + + if (cycleLength > 255) { + return false; + } + + for (uint32_t i = 0; i < cycleLength; i++) { + sps.offsetForRefFrame.push_back(reader.ReadSignedExponentialGolomb()); + } + } + + sps.maxNumRefFrames = reader.ReadExponentialGolomb(); + sps.gapsInFrameNumAllowed = reader.Read(); + sps.widthInMbs = reader.ReadExponentialGolomb() + 1; + sps.heightInMbs = reader.ReadExponentialGolomb() + 1; + sps.frameMbsOnly = reader.Read(); + + // Interlaced streams are left to the software decoder. + if (!sps.frameMbsOnly || sps.maxNumRefFrames > 16 || sps.widthInMbs > 1024 || sps.heightInMbs > 1024) { + return false; + } + + sps.direct8x8Inference = reader.Read(); + + if (reader.Read()) { + // In units of two luma samples for 4:2:0 frames. + const uint32_t unitX = sps.chromaFormatIdc == 0 ? 1 : 2; + const uint32_t unitY = sps.chromaFormatIdc == 1 ? 2 : 1; + + sps.cropLeft = reader.ReadExponentialGolomb() * unitX; + sps.cropRight = reader.ReadExponentialGolomb() * unitX; + sps.cropTop = reader.ReadExponentialGolomb() * unitY; + sps.cropBottom = reader.ReadExponentialGolomb() * unitY; + } + + if (!reader.Ok() || sps.chromaFormatIdc != 1 || sps.bitDepthLumaMinus8 != 0 || sps.bitDepthChromaMinus8 != 0 + || sps.cropLeft + sps.cropRight >= sps.widthInMbs * 16 + || sps.cropTop + sps.cropBottom >= sps.heightInMbs * 16) + { + return false; + } + + if (reader.Read()) { + // A VUI that cannot be read leaves the reordering unknown. + webrtc::BitstreamReader vui = reader; + ParseVui(vui, sps); + + if (!vui.Ok()) { + sps.maxNumReorderFrames.reset(); + } + } + + spss[id] = std::move(sps); + + return true; + } + + bool H264Parser::ParsePps(std::span nal) + { + const std::vector rbsp = Unescape(nal); + webrtc::BitstreamReader reader(rbsp); + H264Pps pps; + + const uint32_t id = reader.ReadExponentialGolomb(); + pps.spsId = reader.ReadExponentialGolomb(); + + if (id >= kMaxPpsCount || pps.spsId >= kMaxSpsCount) { + return false; + } + + pps.entropyCodingMode = reader.Read(); + pps.bottomFieldPicOrderInFramePresent = reader.Read(); + pps.numSliceGroupsMinus1 = reader.ReadExponentialGolomb(); + + // Slice groups are left to the software decoder. + if (pps.numSliceGroupsMinus1 > 0) { + return false; + } + + pps.numRefIdxL0DefaultMinus1 = reader.ReadExponentialGolomb(); + pps.numRefIdxL1DefaultMinus1 = reader.ReadExponentialGolomb(); + pps.weightedPred = reader.Read(); + pps.weightedBipredIdc = reader.ReadBits(2); + pps.picInitQpMinus26 = reader.ReadSignedExponentialGolomb(); + pps.picInitQsMinus26 = reader.ReadSignedExponentialGolomb(); + pps.chromaQpIndexOffset = reader.ReadSignedExponentialGolomb(); + pps.deblockingFilterControlPresent = reader.Read(); + pps.constrainedIntraPred = reader.Read(); + pps.redundantPicCntPresent = reader.Read(); + pps.secondChromaQpIndexOffset = pps.chromaQpIndexOffset; + + if (pps.numRefIdxL0DefaultMinus1 > 31 || pps.numRefIdxL1DefaultMinus1 > 31 || pps.weightedBipredIdc > 2) { + return false; + } + + // more_rbsp_data(): anything before the stop bit, the last one bit. + size_t lastOne = 0; + + for (size_t i = rbsp.size(); i > 0; i--) { + if (rbsp[i - 1] != 0) { + int bit = 0; + + while (((rbsp[i - 1] >> bit) & 1) == 0) { + bit++; + } + + lastOne = (i - 1) * 8 + (7 - bit); + break; + } + } + + if (reader.Ok() && static_cast(ConsumedBits(reader, rbsp.size())) < lastOne) { + pps.transform8x8Mode = reader.Read(); + pps.scalingMatrixPresent = reader.Read(); + + if (pps.scalingMatrixPresent) { + ParseScalingLists(reader, pps.transform8x8Mode ? 8 : 6, pps.scalingListPresent, + pps.useDefaultScalingList, pps.scalingLists); + } + + pps.secondChromaQpIndexOffset = reader.ReadSignedExponentialGolomb(); + } + + if (!reader.Ok()) { + return false; + } + + ppss[id] = std::move(pps); + + return true; + } + + std::optional H264Parser::ParseSliceHeader(std::span nal) const + { + if (nal.size() < 2) { + return std::nullopt; + } + + const std::vector rbsp = Unescape(nal); + webrtc::BitstreamReader reader(rbsp); + H264SliceHeader header; + + header.nalType = nal[0] & 0x1F; + header.nalRefIdc = (nal[0] >> 5) & 0x03; + header.firstMbInSlice = reader.ReadExponentialGolomb(); + + const uint32_t sliceType = reader.ReadExponentialGolomb(); + + // SP and SI slices are left to the software decoder. + if (sliceType > 9 || sliceType % 5 > kSliceI) { + return std::nullopt; + } + + header.sliceType = sliceType % 5; + header.ppsId = reader.ReadExponentialGolomb(); + + const H264Pps * pps = Pps(header.ppsId); + const H264Sps * sps = pps != nullptr ? Sps(pps->spsId) : nullptr; + + if (sps == nullptr) { + return std::nullopt; + } + + header.frameNum = static_cast(reader.ReadBits(static_cast(sps->log2MaxFrameNumMinus4) + 4)); + + if (header.IsIdr()) { + header.idrPicId = reader.ReadExponentialGolomb(); + } + + if (sps->pocType == 0) { + header.pocLsb = static_cast(reader.ReadBits(static_cast(sps->log2MaxPocLsbMinus4) + 4)); + + if (pps->bottomFieldPicOrderInFramePresent) { + header.deltaPocBottom = reader.ReadSignedExponentialGolomb(); + } + } + else if (sps->pocType == 1 && !sps->deltaPicOrderAlwaysZero) { + header.deltaPoc[0] = reader.ReadSignedExponentialGolomb(); + + if (pps->bottomFieldPicOrderInFramePresent) { + header.deltaPoc[1] = reader.ReadSignedExponentialGolomb(); + } + } + + if (pps->redundantPicCntPresent) { + header.redundantPicCnt = reader.ReadExponentialGolomb(); + } + + if (header.IsB()) { + header.directSpatialMvPred = reader.Read(); + } + + if (!header.IsIntra()) { + header.numRefIdxL0Minus1 = pps->numRefIdxL0DefaultMinus1; + header.numRefIdxL1Minus1 = pps->numRefIdxL1DefaultMinus1; + + if (reader.Read()) { + header.numRefIdxL0Minus1 = reader.ReadExponentialGolomb(); + + if (header.IsB()) { + header.numRefIdxL1Minus1 = reader.ReadExponentialGolomb(); + } + } + + // 32 references are for fields, which are not taken. + if (header.numRefIdxL0Minus1 > 15 || header.numRefIdxL1Minus1 > 15) { + return std::nullopt; + } + + if (!ParseListModification(reader, header.listModifications[0]) + || (header.IsB() && !ParseListModification(reader, header.listModifications[1]))) + { + return std::nullopt; + } + + if ((pps->weightedPred && header.sliceType == kSliceP) || (pps->weightedBipredIdc == 1 && header.IsB())) { + header.lumaLog2WeightDenom = reader.ReadExponentialGolomb(); + header.chromaLog2WeightDenom = reader.ReadExponentialGolomb(); + + if (header.lumaLog2WeightDenom > 7 || header.chromaLog2WeightDenom > 7) { + return std::nullopt; + } + + ParseWeights(reader, header, 0, header.numRefIdxL0Minus1 + 1); + + if (header.IsB()) { + ParseWeights(reader, header, 1, header.numRefIdxL1Minus1 + 1); + } + } + } + + if (header.nalRefIdc != 0 && !ParseRefPicMarking(reader, header)) { + return std::nullopt; + } + + if (pps->entropyCodingMode && !header.IsIntra()) { + header.cabacInitIdc = reader.ReadExponentialGolomb(); + + if (header.cabacInitIdc > 2) { + return std::nullopt; + } + } + + header.qpDelta = reader.ReadSignedExponentialGolomb(); + + if (pps->deblockingFilterControlPresent) { + header.disableDeblockingFilterIdc = reader.ReadExponentialGolomb(); + + if (header.disableDeblockingFilterIdc > 2) { + return std::nullopt; + } + + if (header.disableDeblockingFilterIdc != 1) { + header.alphaOffsetDiv2 = reader.ReadSignedExponentialGolomb(); + header.betaOffsetDiv2 = reader.ReadSignedExponentialGolomb(); + } + } + + if (!reader.Ok()) { + return std::nullopt; + } + + // With the header byte. + header.dataBitOffset = static_cast(ConsumedBits(reader, rbsp.size()) + 8); + + return header; + } + + const H264Sps * H264Parser::Sps(uint32_t id) const + { + auto sps = spss.find(id); + + return sps != spss.end() ? &sps->second : nullptr; + } + + const H264Pps * H264Parser::Pps(uint32_t id) const + { + auto pps = ppss.find(id); + + return pps != ppss.end() ? &pps->second : nullptr; + } + + H264ScalingLists H264Parser::ScalingLists(const H264Sps & sps, const H264Pps & pps) + { + H264ScalingLists lists; + + if (pps.scalingMatrixPresent) { + // Fall-back rule B to the lists of the SPS, or rule A where it has + // none. + ResolveScalingLists(pps.scalingListPresent, pps.useDefaultScalingList, pps.scalingLists, + sps.scalingMatrixPresent ? &sps.scalingLists : nullptr, lists); + } + else if (sps.scalingMatrixPresent) { + lists = sps.scalingLists; + } + else { + for (auto & list : lists.list4x4) { + list.fill(16); + } + for (auto & list : lists.list8x8) { + list.fill(16); + } + } + + return lists; + } + + void H264Parser::Reset() + { + spss.clear(); + ppss.clear(); + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiDecodeSession.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiDecodeSession.cpp new file mode 100644 index 00000000..9f7761ed --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiDecodeSession.cpp @@ -0,0 +1,265 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/linux/VaapiDecodeSession.h" + +#include "rtc_base/logging.h" +#include "third_party/libyuv/include/libyuv/convert.h" + +#include + +namespace jni +{ + VaapiDecodeSession::VaapiDecodeSession(VaapiLibrary & library, std::string name) : + library(library), + va(library.Api()), + name(std::move(name)), + config(VA_INVALID_ID), + context(VA_INVALID_ID), + width(0), + height(0), + downloadImage() + { + downloadImage.image_id = VA_INVALID_ID; + } + + VaapiDecodeSession::~VaapiDecodeSession() + { + Destroy(); + } + + bool VaapiDecodeSession::Check(VAStatus status, const char * operation) const + { + if (status == VA_STATUS_SUCCESS) { + return true; + } + + RTC_LOG(LS_WARNING) << name << ": " << operation << " failed: " << library.ErrorString(status); + + return false; + } + + bool VaapiDecodeSession::Create(VAProfile profile, uint32_t pictureWidth, uint32_t pictureHeight, + uint32_t surfaceCount) + { + Destroy(); + + VADisplay display = library.Display(); + + VAConfigAttrib attribute = {}; + attribute.type = VAConfigAttribRTFormat; + attribute.value = VA_RT_FORMAT_YUV420; + + if (!Check(va.CreateConfig(display, profile, VAEntrypointVLD, &attribute, 1, &config), + "creating the configuration")) + { + config = VA_INVALID_ID; + return false; + } + + surfaces.assign(surfaceCount, VA_INVALID_SURFACE); + + if (!Check(va.CreateSurfaces(display, VA_RT_FORMAT_YUV420, pictureWidth, pictureHeight, surfaces.data(), + surfaceCount, nullptr, 0), "creating surfaces")) + { + surfaces.clear(); + Destroy(); + return false; + } + + if (!Check(va.CreateContext(display, config, static_cast(pictureWidth), static_cast(pictureHeight), + VA_PROGRESSIVE, surfaces.data(), static_cast(surfaceCount), &context), "creating the context")) + { + context = VA_INVALID_ID; + Destroy(); + return false; + } + + width = pictureWidth; + height = pictureHeight; + + return true; + } + + void VaapiDecodeSession::Destroy() + { + VADisplay display = library.Display(); + + if (downloadImage.image_id != VA_INVALID_ID) { + va.DestroyImage(display, downloadImage.image_id); + downloadImage.image_id = VA_INVALID_ID; + } + if (context != VA_INVALID_ID) { + va.DestroyContext(display, context); + context = VA_INVALID_ID; + } + if (!surfaces.empty()) { + va.DestroySurfaces(display, surfaces.data(), static_cast(surfaces.size())); + surfaces.clear(); + } + if (config != VA_INVALID_ID) { + va.DestroyConfig(display, config); + config = VA_INVALID_ID; + } + + width = 0; + height = 0; + } + + bool VaapiDecodeSession::IsCreated() const + { + return context != VA_INVALID_ID; + } + + uint32_t VaapiDecodeSession::Width() const + { + return width; + } + + uint32_t VaapiDecodeSession::Height() const + { + return height; + } + + int VaapiDecodeSession::SurfaceCount() const + { + return static_cast(surfaces.size()); + } + + VASurfaceID VaapiDecodeSession::Surface(int index) const + { + return surfaces[index]; + } + + bool VaapiDecodeSession::Decode(int surface, const std::vector & buffers) + { + VADisplay display = library.Display(); + std::vector created; + bool ok = true; + + for (const Buffer & buffer : buffers) { + VABufferID id = VA_INVALID_ID; + + if (!Check(va.CreateBuffer(display, context, buffer.type, static_cast(buffer.size), 1, + const_cast(buffer.data), &id), "creating a buffer")) + { + ok = false; + break; + } + + created.push_back(id); + } + + bool decoded = false; + + if (ok && Check(va.BeginPicture(display, context, surfaces[surface]), "beginning a picture")) { + const bool rendered = Check(va.RenderPicture(display, context, created.data(), + static_cast(created.size())), "rendering a picture"); + // A picture that was begun has to be ended. + const bool ended = Check(va.EndPicture(display, context), "ending a picture"); + + decoded = rendered && ended && Check(va.SyncSurface(display, surfaces[surface]), "waiting for a picture"); + } + + for (VABufferID id : created) { + va.DestroyBuffer(display, id); + } + + return decoded; + } + + webrtc::scoped_refptr VaapiDecodeSession::Download(int surface, uint32_t x, uint32_t y, + uint32_t regionWidth, uint32_t regionHeight) + { + VADisplay display = library.Display(); + + // Copying a surface into an image reads it from cached memory, which + // mapping it directly does not. Where the driver does not copy, the + // surface is mapped. + if (downloadImage.image_id != VA_INVALID_ID + && (downloadImage.width != regionWidth || downloadImage.height != regionHeight)) + { + va.DestroyImage(display, downloadImage.image_id); + downloadImage.image_id = VA_INVALID_ID; + } + + if (downloadImage.image_id == VA_INVALID_ID) { + VAImageFormat format = {}; + format.fourcc = VA_FOURCC_NV12; + format.byte_order = VA_LSB_FIRST; + format.bits_per_pixel = 12; + + if (va.CreateImage(display, &format, static_cast(regionWidth), static_cast(regionHeight), + &downloadImage) != VA_STATUS_SUCCESS) + { + downloadImage.image_id = VA_INVALID_ID; + } + } + + VAImage mapped = {}; + mapped.image_id = VA_INVALID_ID; + bool derived = false; + size_t offsetY = 0; + size_t offsetUv = 0; + + if (downloadImage.image_id != VA_INVALID_ID + && va.GetImage(display, surfaces[surface], static_cast(x), static_cast(y), regionWidth, + regionHeight, downloadImage.image_id) == VA_STATUS_SUCCESS) + { + mapped = downloadImage; + } + else if (va.DeriveImage(display, surfaces[surface], &mapped) == VA_STATUS_SUCCESS) { + derived = true; + + if (mapped.format.fourcc != VA_FOURCC_NV12) { + RTC_LOG(LS_WARNING) << name << ": the decoded surface is not NV12"; + va.DestroyImage(display, mapped.image_id); + return nullptr; + } + + // The region within the whole surface. + offsetY = y * mapped.pitches[0] + x; + offsetUv = (y / 2) * mapped.pitches[1] + x; + } + else { + RTC_LOG(LS_WARNING) << name << ": the decoded surface cannot be read"; + return nullptr; + } + + void * data = nullptr; + bool converted = false; + webrtc::scoped_refptr i420 = webrtc::I420Buffer::Create(static_cast(regionWidth), + static_cast(regionHeight)); + + if (Check(va.MapBuffer(display, mapped.buf, &data), "mapping a decoded picture")) { + const uint8_t * base = static_cast(data); + + converted = libyuv::NV12ToI420(base + mapped.offsets[0] + offsetY, static_cast(mapped.pitches[0]), + base + mapped.offsets[1] + offsetUv, static_cast(mapped.pitches[1]), + i420->MutableDataY(), i420->StrideY(), i420->MutableDataU(), i420->StrideU(), + i420->MutableDataV(), i420->StrideV(), static_cast(regionWidth), + static_cast(regionHeight)) == 0; + + va.UnmapBuffer(display, mapped.buf); + } + + if (derived) { + va.DestroyImage(display, mapped.image_id); + } + + return converted ? i420 : nullptr; + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Decoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Decoder.cpp new file mode 100644 index 00000000..6bdd4af1 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Decoder.cpp @@ -0,0 +1,1049 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/linux/VaapiH264Decoder.h" + +#include "api/video/video_frame.h" +#include "common_video/h264/h264_common.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" + +#include + +namespace jni +{ + namespace + { + constexpr size_t kMaxReferenceFrames = 16; + constexpr size_t kMaxListEntries = 32; + + // Data partitions (table 7-1), which the decoder does not take. + constexpr uint8_t kNalPartitionA = 2; + constexpr uint8_t kNalPartitionC = 4; + + // The raster positions of the coefficients in zigzag order (8.5.6). + constexpr uint8_t kZigzag4x4[16] = { 0, 1, 4, 8, 5, 2, 3, 6, 9, 12, 13, 10, 7, 11, 14, 15 }; + constexpr uint8_t kZigzag8x8[64] = { + 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, + 12, 19, 26, 33, 40, 48, 41, 34, 27, 20, 13, 6, 7, 14, 21, 28, + 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, 30, 37, 44, 51, + 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63 + }; + + void InvalidatePicture(VAPictureH264 & picture) + { + picture = {}; + picture.picture_id = VA_INVALID_SURFACE; + picture.flags = VA_PICTURE_H264_INVALID; + } + } + + int32_t VaapiH264Decoder::Picture::Poc() const + { + return std::min(topPoc, bottomPoc); + } + + bool VaapiH264Decoder::Picture::IsReference() const + { + return shortTerm || longTerm; + } + + VaapiH264Decoder::VaapiH264Decoder(VaapiLibrary & library) : + library(library), + implementationName("VA-API (" + library.Vendor() + ")"), + callback(nullptr), + session(library, implementationName + " H.264"), + widthInMbs(0), + heightInMbs(0), + maxNumRefFrames(0), + maxNumReorder(0), + cropX(0), + cropY(0), + width(0), + height(0), + maxLongTermFrameIdxPlus1(0), + prevPocMsb(0), + prevPocLsb(0), + prevFrameNum(0), + prevFrameNumOffset(0), + prevRefFrameNum(0), + frameNumOffset(0), + pocMsb(0), + mmco5(false), + started(false), + failed(false) + { + } + + VaapiH264Decoder::~VaapiH264Decoder() + { + Release(); + } + + bool VaapiH264Decoder::Configure(const Settings & settings) + { + if (settings.codec_type() != webrtc::kVideoCodecH264) { + return false; + } + + Release(); + + RTC_LOG(LS_INFO) << implementationName << " H.264 decoder configured"; + + return true; + } + + int32_t VaapiH264Decoder::Decode(const webrtc::EncodedImage & image, int64_t renderTimeMs) + { + if (callback == nullptr) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (image.size() == 0) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (failed) { + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + // Decoding starts with a key frame. Until one has come, WebRTC is + // asked for it. + if (!started && image.FrameType() != webrtc::VideoFrameType::kVideoFrameKey) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + const std::span data(image.data(), image.size()); + std::vector slices; + + const auto decode = [&]() { + const Result result = DecodePicture(slices, image, renderTimeMs); + + slices.clear(); + + if (result == Result::kMissingReference) { + RTC_LOG(LS_INFO) << implementationName << " H.264: a picture refers to one that is not there; " + "waiting for a key frame"; + + dpb.clear(); + started = false; + + return WEBRTC_VIDEO_CODEC_ERROR; + } + if (result == Result::kFailed) { + failed = true; + + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + return WEBRTC_VIDEO_CODEC_OK; + }; + + for (const webrtc::H264::NaluIndex & index : webrtc::H264::FindNaluIndices(data)) { + const std::span nal = data.subspan(index.payload_start_offset, index.payload_size); + + if (nal.empty()) { + continue; + } + + const uint8_t type = nal[0] & 0x1F; + + if (type == H264Parser::kNalSps || type == H264Parser::kNalPps) { + // The parameter sets of the next picture. + if (!slices.empty()) { + if (const int32_t result = decode(); result != WEBRTC_VIDEO_CODEC_OK) { + return result; + } + } + + const bool parsed = type == H264Parser::kNalSps ? parser.ParseSps(nal) : parser.ParsePps(nal); + + if (!parsed) { + RTC_LOG(LS_INFO) << implementationName << " H.264: a parameter set of a stream that is not " + "progressive 8 bit 4:2:0 without slice groups; decoding it in software"; + failed = true; + + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + continue; + } + + if (type == H264Parser::kNalEndOfSequence) { + if (!slices.empty()) { + if (const int32_t result = decode(); result != WEBRTC_VIDEO_CODEC_OK) { + return result; + } + } + + started = false; + continue; + } + + if (type >= kNalPartitionA && type <= kNalPartitionC) { + RTC_LOG(LS_INFO) << implementationName << " H.264: data partitions; decoding them in software"; + failed = true; + + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + if (type != H264Parser::kNalSlice && type != H264Parser::kNalIdr) { + continue; + } + + std::optional header = parser.ParseSliceHeader(nal); + + if (!header) { + RTC_LOG(LS_INFO) << implementationName << " H.264: a slice header that is broken, of an SP or SI " + "slice, or refers to a parameter set that is not there"; + failed = true; + + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + // Redundant pictures are for decoders that lost the primary one. + if (header->redundantPicCnt > 0) { + continue; + } + + if (header->firstMbInSlice == 0 && !slices.empty()) { + if (const int32_t result = decode(); result != WEBRTC_VIDEO_CODEC_OK) { + return result; + } + } + + slices.push_back(Slice { *header, nal }); + } + + if (slices.empty()) { + return WEBRTC_VIDEO_CODEC_OK; + } + + return decode(); + } + + int32_t VaapiH264Decoder::RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback * decodeCallback) + { + callback = decodeCallback; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VaapiH264Decoder::Release() + { + session.Destroy(); + dpb.clear(); + + parser.Reset(); + widthInMbs = 0; + heightInMbs = 0; + maxLongTermFrameIdxPlus1 = 0; + started = false; + failed = false; + + return WEBRTC_VIDEO_CODEC_OK; + } + + webrtc::VideoDecoder::DecoderInfo VaapiH264Decoder::GetDecoderInfo() const + { + DecoderInfo info; + info.implementation_name = implementationName; + info.is_hardware_accelerated = true; + + return info; + } + + const char * VaapiH264Decoder::ImplementationName() const + { + return implementationName.c_str(); + } + + VaapiH264Decoder::Result VaapiH264Decoder::DecodePicture(const std::vector & slices, + const webrtc::EncodedImage & image, int64_t renderTimeMs) + { + const H264SliceHeader & first = slices.front().header; + + if (first.firstMbInSlice != 0) { + return Result::kMissingReference; + } + + const H264Pps * pps = parser.Pps(first.ppsId); + const H264Sps * sps = pps != nullptr ? parser.Sps(pps->spsId) : nullptr; + + if (sps == nullptr) { + return Result::kFailed; + } + + const bool idr = first.IsIdr(); + const uint32_t maxFrameNum = sps->MaxFrameNum(); + + if (!idr && !started) { + return Result::kMissingReference; + } + + if (idr) { + // The frames of the sequence before are output, unless the picture + // says not to, and all references are let go. + if (first.noOutputOfPriorPics) { + dpb.clear(); + } + else if (!Bump(0)) { + return Result::kFailed; + } + + dpb.clear(); + + const uint32_t spsMaxNumReorder = sps->maxNumReorderFrames.value_or(0); + + if (!session.IsCreated() || sps->widthInMbs != widthInMbs || sps->heightInMbs != heightInMbs + || sps->maxNumRefFrames != maxNumRefFrames || spsMaxNumReorder != maxNumReorder) + { + // The reference frames, the frames waiting for output, the + // one being decoded and one more. + const uint32_t count = std::max(sps->maxNumRefFrames, 1u) + spsMaxNumReorder + 2; + + if (!session.Create(VAProfileH264High, sps->widthInMbs * 16, sps->heightInMbs * 16, count)) { + return Result::kFailed; + } + + widthInMbs = sps->widthInMbs; + heightInMbs = sps->heightInMbs; + maxNumRefFrames = sps->maxNumRefFrames; + maxNumReorder = spsMaxNumReorder; + + RTC_LOG(LS_INFO) << implementationName << " H.264 decoding " << widthInMbs * 16 - sps->cropLeft + - sps->cropRight << "x" << heightInMbs * 16 - sps->cropTop - sps->cropBottom; + } + } + else { + if (sps->widthInMbs != widthInMbs || sps->heightInMbs != heightInMbs) { + RTC_LOG(LS_INFO) << implementationName << " H.264: the size changes without a key frame"; + return Result::kFailed; + } + + // A frame that is lost leaves a gap in frame_num. + if (first.frameNum != prevRefFrameNum && first.frameNum != (prevRefFrameNum + 1) % maxFrameNum) { + return Result::kMissingReference; + } + } + + cropX = sps->cropLeft; + cropY = sps->cropTop; + width = widthInMbs * 16 - sps->cropLeft - sps->cropRight; + height = heightInMbs * 16 - sps->cropTop - sps->cropBottom; + + Picture current; + current.frameNum = first.frameNum; + PictureOrderCount(first, *sps, current.topPoc, current.bottomPoc); + + // A full buffer makes room, in output order. + RemoveUnused(); + + while (static_cast(dpb.size()) + 1 >= session.SurfaceCount()) { + const uint32_t waiting = static_cast(std::count_if(dpb.begin(), dpb.end(), + [](const Picture & picture) { + return picture.waitingOutput; + })); + + if (waiting == 0) { + break; + } + if (!Bump(waiting - 1)) { + return Result::kFailed; + } + } + + current.surface = FreeSurface(); + + if (current.surface < 0) { + RTC_LOG(LS_WARNING) << implementationName << " H.264: no surface is free"; + return Result::kFailed; + } + + VAPictureParameterBufferH264 picture = {}; + picture.CurrPic.picture_id = session.Surface(current.surface); + picture.CurrPic.frame_idx = first.frameNum; + picture.CurrPic.flags = 0; + picture.CurrPic.TopFieldOrderCnt = current.topPoc; + picture.CurrPic.BottomFieldOrderCnt = current.bottomPoc; + + for (VAPictureH264 & reference : picture.ReferenceFrames) { + InvalidatePicture(reference); + } + + size_t count = 0; + + for (const Picture & reference : dpb) { + if (reference.IsReference() && count < kMaxReferenceFrames) { + picture.ReferenceFrames[count++] = ToVaPicture(reference); + } + } + + picture.picture_width_in_mbs_minus1 = static_cast(sps->widthInMbs - 1); + picture.picture_height_in_mbs_minus1 = static_cast(sps->heightInMbs - 1); + picture.bit_depth_luma_minus8 = static_cast(sps->bitDepthLumaMinus8); + picture.bit_depth_chroma_minus8 = static_cast(sps->bitDepthChromaMinus8); + picture.num_ref_frames = static_cast(sps->maxNumRefFrames); + + auto & seq = picture.seq_fields.bits; + seq.chroma_format_idc = sps->chromaFormatIdc; + seq.gaps_in_frame_num_value_allowed_flag = sps->gapsInFrameNumAllowed ? 1 : 0; + seq.frame_mbs_only_flag = 1; + seq.mb_adaptive_frame_field_flag = 0; + seq.direct_8x8_inference_flag = sps->direct8x8Inference ? 1 : 0; + // Bi-prediction of blocks smaller than 8x8 is left out from level 3.1 + // on (A.3.3.2). + seq.MinLumaBiPredSize8x8 = sps->levelIdc >= 31 ? 1 : 0; + seq.log2_max_frame_num_minus4 = sps->log2MaxFrameNumMinus4; + seq.pic_order_cnt_type = sps->pocType; + seq.log2_max_pic_order_cnt_lsb_minus4 = sps->log2MaxPocLsbMinus4; + seq.delta_pic_order_always_zero_flag = sps->deltaPicOrderAlwaysZero ? 1 : 0; + + picture.pic_init_qp_minus26 = static_cast(pps->picInitQpMinus26); + picture.pic_init_qs_minus26 = static_cast(pps->picInitQsMinus26); + picture.chroma_qp_index_offset = static_cast(pps->chromaQpIndexOffset); + picture.second_chroma_qp_index_offset = static_cast(pps->secondChromaQpIndexOffset); + + auto & fields = picture.pic_fields.bits; + fields.entropy_coding_mode_flag = pps->entropyCodingMode ? 1 : 0; + fields.weighted_pred_flag = pps->weightedPred ? 1 : 0; + fields.weighted_bipred_idc = pps->weightedBipredIdc; + fields.transform_8x8_mode_flag = pps->transform8x8Mode ? 1 : 0; + fields.field_pic_flag = 0; + fields.constrained_intra_pred_flag = pps->constrainedIntraPred ? 1 : 0; + fields.pic_order_present_flag = pps->bottomFieldPicOrderInFramePresent ? 1 : 0; + fields.deblocking_filter_control_present_flag = pps->deblockingFilterControlPresent ? 1 : 0; + fields.redundant_pic_cnt_present_flag = pps->redundantPicCntPresent ? 1 : 0; + fields.reference_pic_flag = first.nalRefIdc != 0 ? 1 : 0; + + picture.frame_num = static_cast(first.frameNum); + + const H264ScalingLists lists = H264Parser::ScalingLists(*sps, *pps); + VAIQMatrixBufferH264 matrix = {}; + + // VA-API takes the lists in raster order, as the coefficients of a + // block are laid out; they are coded in zigzag order. + for (int i = 0; i < 6; i++) { + for (int j = 0; j < 16; j++) { + matrix.ScalingList4x4[i][kZigzag4x4[j]] = lists.list4x4[i][j]; + } + } + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 64; j++) { + matrix.ScalingList8x8[i][kZigzag8x8[j]] = lists.list8x8[i][j]; + } + } + + std::vector sliceParameters(slices.size()); + + for (size_t i = 0; i < slices.size(); i++) { + const H264SliceHeader & header = slices[i].header; + VASliceParameterBufferH264 & parameters = sliceParameters[i]; + std::vector referenceLists[2]; + + if (!BuildReferenceLists(header, *sps, current.Poc(), referenceLists[0], referenceLists[1])) { + return Result::kMissingReference; + } + + parameters.slice_data_size = static_cast(slices[i].nal.size()); + parameters.slice_data_offset = 0; + parameters.slice_data_flag = VA_SLICE_DATA_FLAG_ALL; + parameters.slice_data_bit_offset = static_cast(header.dataBitOffset); + parameters.first_mb_in_slice = static_cast(header.firstMbInSlice); + parameters.slice_type = static_cast(header.sliceType); + parameters.direct_spatial_mv_pred_flag = header.directSpatialMvPred ? 1 : 0; + parameters.num_ref_idx_l0_active_minus1 = header.IsIntra() ? 0 : static_cast(header.numRefIdxL0Minus1); + parameters.num_ref_idx_l1_active_minus1 = header.IsB() ? static_cast(header.numRefIdxL1Minus1) : 0; + parameters.cabac_init_idc = static_cast(header.cabacInitIdc); + parameters.slice_qp_delta = static_cast(header.qpDelta); + parameters.disable_deblocking_filter_idc = static_cast(header.disableDeblockingFilterIdc); + parameters.slice_alpha_c0_offset_div2 = static_cast(header.alphaOffsetDiv2); + parameters.slice_beta_offset_div2 = static_cast(header.betaOffsetDiv2); + + VAPictureH264 * vaLists[2] = { parameters.RefPicList0, parameters.RefPicList1 }; + + for (int list = 0; list < 2; list++) { + for (size_t j = 0; j < kMaxListEntries; j++) { + InvalidatePicture(vaLists[list][j]); + } + + for (size_t j = 0; j < referenceLists[list].size() && j < kMaxListEntries; j++) { + const Picture * reference = FindBySurface(referenceLists[list][j]); + + if (reference != nullptr) { + vaLists[list][j] = ToVaPicture(*reference); + } + } + } + + parameters.luma_log2_weight_denom = static_cast(header.lumaLog2WeightDenom); + parameters.chroma_log2_weight_denom = static_cast(header.chromaLog2WeightDenom); + parameters.luma_weight_l0_flag = header.lumaWeightsGiven[0] ? 1 : 0; + parameters.chroma_weight_l0_flag = header.chromaWeightsGiven[0] ? 1 : 0; + parameters.luma_weight_l1_flag = header.lumaWeightsGiven[1] ? 1 : 0; + parameters.chroma_weight_l1_flag = header.chromaWeightsGiven[1] ? 1 : 0; + + for (size_t j = 0; j < kMaxListEntries; j++) { + parameters.luma_weight_l0[j] = static_cast(header.lumaWeight[0][j]); + parameters.luma_offset_l0[j] = static_cast(header.lumaOffset[0][j]); + parameters.luma_weight_l1[j] = static_cast(header.lumaWeight[1][j]); + parameters.luma_offset_l1[j] = static_cast(header.lumaOffset[1][j]); + + for (int k = 0; k < 2; k++) { + parameters.chroma_weight_l0[j][k] = static_cast(header.chromaWeight[0][j][k]); + parameters.chroma_offset_l0[j][k] = static_cast(header.chromaOffset[0][j][k]); + parameters.chroma_weight_l1[j][k] = static_cast(header.chromaWeight[1][j][k]); + parameters.chroma_offset_l1[j][k] = static_cast(header.chromaOffset[1][j][k]); + } + } + } + + std::vector buffers = { + { VAPictureParameterBufferType, &picture, sizeof(picture) }, + { VAIQMatrixBufferType, &matrix, sizeof(matrix) } + }; + + for (size_t i = 0; i < slices.size(); i++) { + buffers.push_back({ VASliceParameterBufferType, &sliceParameters[i], sizeof(sliceParameters[i]) }); + buffers.push_back({ VASliceDataBufferType, slices[i].nal.data(), slices[i].nal.size() }); + } + + if (!session.Decode(current.surface, buffers)) { + return Result::kFailed; + } + + current.waitingOutput = true; + current.rtpTimestamp = image.RtpTimestamp(); + current.ntpTimeMs = image.ntp_time_ms_; + current.renderTimeMs = renderTimeMs; + current.rotation = image.rotation_; + + mmco5 = false; + + if (first.nalRefIdc != 0) { + const Result marked = MarkReferences(first, *sps, current); + + if (marked != Result::kOk) { + return marked; + } + } + + // The state the next picture's order count and frame_num derive from; + // operation 5 starts them over, as an IDR picture does. + prevFrameNum = mmco5 ? 0 : first.frameNum; + prevFrameNumOffset = mmco5 ? 0 : frameNumOffset; + + if (first.nalRefIdc != 0) { + prevRefFrameNum = mmco5 ? 0 : first.frameNum; + prevPocMsb = mmco5 ? 0 : pocMsb; + prevPocLsb = mmco5 ? current.topPoc : static_cast(first.pocLsb); + } + + dpb.push_back(current); + started = true; + + return Bump(maxNumReorder) ? Result::kOk : Result::kFailed; + } + + void VaapiH264Decoder::PictureOrderCount(const H264SliceHeader & header, const H264Sps & sps, int32_t & top, + int32_t & bottom) + { + const bool idr = header.IsIdr(); + const uint32_t maxFrameNum = sps.MaxFrameNum(); + + if (sps.pocType == 0) { + const int32_t maxPocLsb = 1 << (sps.log2MaxPocLsbMinus4 + 4); + const int32_t pocLsb = static_cast(header.pocLsb); + const int32_t prevMsb = idr ? 0 : prevPocMsb; + const int32_t prevLsb = idr ? 0 : prevPocLsb; + + pocMsb = prevMsb; + + if (pocLsb < prevLsb && prevLsb - pocLsb >= maxPocLsb / 2) { + pocMsb = prevMsb + maxPocLsb; + } + else if (pocLsb > prevLsb && pocLsb - prevLsb > maxPocLsb / 2) { + pocMsb = prevMsb - maxPocLsb; + } + + top = pocMsb + pocLsb; + bottom = top + header.deltaPocBottom; + + return; + } + + // FrameNumOffset (8-6, 8-11) for the other two types. + if (idr) { + frameNumOffset = 0; + } + else if (prevFrameNum > header.frameNum) { + frameNumOffset = prevFrameNumOffset + maxFrameNum; + } + else { + frameNumOffset = prevFrameNumOffset; + } + + if (sps.pocType == 1) { + const uint32_t cycleLength = static_cast(sps.offsetForRefFrame.size()); + uint32_t absFrameNum = cycleLength != 0 ? frameNumOffset + header.frameNum : 0; + + if (header.nalRefIdc == 0 && absFrameNum > 0) { + absFrameNum--; + } + + int32_t expected = 0; + + if (absFrameNum > 0) { + int32_t deltaPerCycle = 0; + + for (int32_t offset : sps.offsetForRefFrame) { + deltaPerCycle += offset; + } + + const uint32_t cycleCount = (absFrameNum - 1) / cycleLength; + const uint32_t inCycle = (absFrameNum - 1) % cycleLength; + + expected = static_cast(cycleCount) * deltaPerCycle; + + for (uint32_t i = 0; i <= inCycle; i++) { + expected += sps.offsetForRefFrame[i]; + } + } + + if (header.nalRefIdc == 0) { + expected += sps.offsetForNonRefPic; + } + + top = expected + header.deltaPoc[0]; + bottom = top + sps.offsetForTopToBottomField + header.deltaPoc[1]; + + return; + } + + // Type 2: the order of decoding. + int32_t temp = 0; + + if (!idr) { + temp = 2 * static_cast(frameNumOffset + header.frameNum) - (header.nalRefIdc == 0 ? 1 : 0); + } + + top = temp; + bottom = temp; + } + + VaapiH264Decoder::Result VaapiH264Decoder::MarkReferences(const H264SliceHeader & header, const H264Sps & sps, + Picture & current) + { + const uint32_t maxFrameNum = sps.MaxFrameNum(); + + if (header.IsIdr()) { + for (Picture & picture : dpb) { + picture.shortTerm = false; + picture.longTerm = false; + } + + if (header.longTermReference) { + current.longTerm = true; + current.longTermFrameIdx = 0; + maxLongTermFrameIdxPlus1 = 1; + } + else { + current.shortTerm = true; + maxLongTermFrameIdxPlus1 = 0; + } + + return Result::kOk; + } + + const auto shortTermByPicNum = [&](int32_t picNum) -> Picture * { + for (Picture & picture : dpb) { + if (picture.shortTerm && PicNum(picture, header.frameNum, maxFrameNum) == picNum) { + return &picture; + } + } + + return nullptr; + }; + + const auto releaseLongTermIdx = [&](uint32_t idx, const Picture * except) { + for (Picture & picture : dpb) { + if (picture.longTerm && picture.longTermFrameIdx == idx && &picture != except) { + picture.longTerm = false; + } + } + }; + + if (header.adaptiveRefPicMarking) { + for (const H264Operation & operation : header.memoryManagement) { + const int32_t picNumX = static_cast(header.frameNum) - static_cast(operation.value + 1); + + switch (operation.operation) { + case 1: + if (Picture * picture = shortTermByPicNum(picNumX)) { + picture->shortTerm = false; + } + break; + + case 2: + for (Picture & picture : dpb) { + if (picture.longTerm && picture.longTermFrameIdx == operation.value) { + picture.longTerm = false; + } + } + break; + + case 3: + if (Picture * picture = shortTermByPicNum(picNumX)) { + releaseLongTermIdx(operation.longTermFrameIdx, picture); + + picture->shortTerm = false; + picture->longTerm = true; + picture->longTermFrameIdx = operation.longTermFrameIdx; + } + break; + + case 4: + maxLongTermFrameIdxPlus1 = operation.value; + + for (Picture & picture : dpb) { + if (picture.longTerm && picture.longTermFrameIdx >= maxLongTermFrameIdxPlus1) { + picture.longTerm = false; + } + } + break; + + case 5: + for (Picture & picture : dpb) { + picture.shortTerm = false; + picture.longTerm = false; + } + + maxLongTermFrameIdxPlus1 = 0; + mmco5 = true; + break; + + case 6: + releaseLongTermIdx(operation.longTermFrameIdx, nullptr); + + current.longTerm = true; + current.longTermFrameIdx = operation.longTermFrameIdx; + break; + } + } + } + else { + // The sliding window (8.2.5.3): the oldest short-term frame goes + // when the buffer of references is full. + const auto references = std::count_if(dpb.begin(), dpb.end(), [](const Picture & picture) { + return picture.IsReference(); + }); + + if (references >= static_cast(std::max(sps.maxNumRefFrames, 1u))) { + Picture * oldest = nullptr; + + for (Picture & picture : dpb) { + if (picture.shortTerm && (oldest == nullptr + || PicNum(picture, header.frameNum, maxFrameNum) < PicNum(*oldest, header.frameNum, maxFrameNum))) + { + oldest = &picture; + } + } + + if (oldest != nullptr) { + oldest->shortTerm = false; + } + } + } + + if (!current.longTerm) { + current.shortTerm = true; + } + + if (mmco5) { + // The picture counts as the first of a new sequence (8.2.1). + const int32_t temp = current.Poc(); + + current.topPoc -= temp; + current.bottomPoc -= temp; + current.frameNum = 0; + } + + return Result::kOk; + } + + int32_t VaapiH264Decoder::PicNum(const Picture & picture, uint32_t currentFrameNum, uint32_t maxFrameNum) const + { + // FrameNumWrap (8-27), which is PicNum for frames. + return picture.frameNum > currentFrameNum ? static_cast(picture.frameNum) - static_cast(maxFrameNum) + : static_cast(picture.frameNum); + } + + bool VaapiH264Decoder::BuildReferenceLists(const H264SliceHeader & header, const H264Sps & sps, + int32_t currentPoc, std::vector & list0, std::vector & list1) const + { + list0.clear(); + list1.clear(); + + if (header.IsIntra()) { + return true; + } + + const uint32_t maxFrameNum = sps.MaxFrameNum(); + std::vector shortTerm; + std::vector longTerm; + + for (const Picture & picture : dpb) { + if (picture.shortTerm) { + shortTerm.push_back(&picture); + } + else if (picture.longTerm) { + longTerm.push_back(&picture); + } + } + + std::sort(longTerm.begin(), longTerm.end(), [](const Picture * a, const Picture * b) { + return a->longTermFrameIdx < b->longTermFrameIdx; + }); + + std::vector initial[2]; + + if (!header.IsB()) { + // 8.2.4.2.1: by PicNum, highest first. + std::sort(shortTerm.begin(), shortTerm.end(), [&](const Picture * a, const Picture * b) { + return PicNum(*a, header.frameNum, maxFrameNum) > PicNum(*b, header.frameNum, maxFrameNum); + }); + + for (const Picture * picture : shortTerm) { + initial[0].push_back(picture->surface); + } + } + else { + // 8.2.4.2.3: the frames before the current one, nearest first, + // then those after it, and the other way round for list 1. + std::vector before; + std::vector after; + + for (const Picture * picture : shortTerm) { + (picture->Poc() < currentPoc ? before : after).push_back(picture); + } + + std::sort(before.begin(), before.end(), [](const Picture * a, const Picture * b) { + return a->Poc() > b->Poc(); + }); + std::sort(after.begin(), after.end(), [](const Picture * a, const Picture * b) { + return a->Poc() < b->Poc(); + }); + + for (const Picture * picture : before) { + initial[0].push_back(picture->surface); + } + for (const Picture * picture : after) { + initial[0].push_back(picture->surface); + initial[1].push_back(picture->surface); + } + for (const Picture * picture : before) { + initial[1].push_back(picture->surface); + } + } + + for (int list = 0; list < (header.IsB() ? 2 : 1); list++) { + for (const Picture * picture : longTerm) { + initial[list].push_back(picture->surface); + } + } + + if (header.IsB() && initial[1].size() > 1 && initial[1] == initial[0]) { + std::swap(initial[1][0], initial[1][1]); + } + + // The modifications (8.2.4.3). + for (int list = 0; list < (header.IsB() ? 2 : 1); list++) { + const uint32_t numActive = (list == 0 ? header.numRefIdxL0Minus1 : header.numRefIdxL1Minus1) + 1; + std::vector & entries = initial[list]; + + entries.resize(numActive + 1, -1); + + const int32_t maxPicNum = static_cast(maxFrameNum); + int32_t picNumPred = static_cast(header.frameNum); + uint32_t refIdx = 0; + + for (const H264Operation & modification : header.listModifications[list]) { + if (refIdx >= numActive) { + return false; + } + + const Picture * target = nullptr; + int32_t picNum = 0; + + if (modification.operation == 2) { + for (const Picture * picture : longTerm) { + if (picture->longTermFrameIdx == modification.value) { + target = picture; + } + } + } + else { + const int32_t absDiff = static_cast(modification.value) + 1; + + if (absDiff > maxPicNum) { + return false; + } + + int32_t picNumNoWrap = modification.operation == 0 ? picNumPred - absDiff : picNumPred + absDiff; + + if (picNumNoWrap < 0) { + picNumNoWrap += maxPicNum; + } + else if (picNumNoWrap >= maxPicNum) { + picNumNoWrap -= maxPicNum; + } + + picNumPred = picNumNoWrap; + picNum = picNumNoWrap > static_cast(header.frameNum) ? picNumNoWrap - maxPicNum : picNumNoWrap; + + for (const Picture * picture : shortTerm) { + if (PicNum(*picture, header.frameNum, maxFrameNum) == picNum) { + target = picture; + } + } + } + + if (target == nullptr) { + return false; + } + + // The picture goes to refIdx, and its later entry out. + for (uint32_t c = numActive; c > refIdx; c--) { + entries[c] = entries[c - 1]; + } + + entries[refIdx++] = target->surface; + + uint32_t n = refIdx; + + for (uint32_t c = refIdx; c <= numActive; c++) { + if (entries[c] != target->surface) { + entries[n++] = entries[c]; + } + } + } + + entries.resize(numActive); + } + + list0 = std::move(initial[0]); + + if (header.IsB()) { + list1 = std::move(initial[1]); + } + + return true; + } + + VAPictureH264 VaapiH264Decoder::ToVaPicture(const Picture & picture) const + { + VAPictureH264 vaPicture = {}; + vaPicture.picture_id = session.Surface(picture.surface); + vaPicture.frame_idx = picture.longTerm ? picture.longTermFrameIdx : picture.frameNum; + vaPicture.flags = picture.longTerm ? VA_PICTURE_H264_LONG_TERM_REFERENCE + : (picture.shortTerm ? VA_PICTURE_H264_SHORT_TERM_REFERENCE : 0); + vaPicture.TopFieldOrderCnt = picture.topPoc; + vaPicture.BottomFieldOrderCnt = picture.bottomPoc; + + return vaPicture; + } + + const VaapiH264Decoder::Picture * VaapiH264Decoder::FindBySurface(int surface) const + { + for (const Picture & picture : dpb) { + if (picture.surface == surface) { + return &picture; + } + } + + return nullptr; + } + + bool VaapiH264Decoder::Bump(uint32_t keep) + { + while (true) { + auto next = dpb.end(); + uint32_t waiting = 0; + + for (auto picture = dpb.begin(); picture != dpb.end(); ++picture) { + if (!picture->waitingOutput) { + continue; + } + + waiting++; + + if (next == dpb.end() || picture->Poc() < next->Poc()) { + next = picture; + } + } + + if (waiting <= keep || next == dpb.end()) { + return true; + } + + next->waitingOutput = false; + + if (!Output(*next)) { + return false; + } + + RemoveUnused(); + } + } + + void VaapiH264Decoder::RemoveUnused() + { + dpb.erase(std::remove_if(dpb.begin(), dpb.end(), [](const Picture & picture) { + return !picture.IsReference() && !picture.waitingOutput; + }), dpb.end()); + } + + bool VaapiH264Decoder::Output(const Picture & picture) + { + webrtc::scoped_refptr i420 = session.Download(picture.surface, cropX, cropY, width, + height); + + if (!i420) { + return false; + } + + webrtc::VideoFrame frame = webrtc::VideoFrame::Builder() + .set_video_frame_buffer(i420) + .set_rtp_timestamp(picture.rtpTimestamp) + .set_timestamp_ms(picture.renderTimeMs) + .set_ntp_time_ms(picture.ntpTimeMs) + .set_rotation(picture.rotation) + .build(); + + callback->Decoded(frame, std::nullopt, std::nullopt); + + return true; + } + + int VaapiH264Decoder::FreeSurface() const + { + for (int surface = 0; surface < session.SurfaceCount(); surface++) { + if (FindBySurface(surface) == nullptr) { + return surface; + } + } + + return -1; + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Decoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Decoder.cpp index c4289bce..fae25dd0 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Decoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Decoder.cpp @@ -16,12 +16,10 @@ #include "media/video/codec/linux/VaapiH265Decoder.h" -#include "api/video/i420_buffer.h" #include "api/video/video_frame.h" #include "common_video/h265/h265_common.h" #include "modules/video_coding/include/video_error_codes.h" #include "rtc_base/logging.h" -#include "third_party/libyuv/include/libyuv/convert.h" #include @@ -79,11 +77,9 @@ namespace jni VaapiH265Decoder::VaapiH265Decoder(VaapiLibrary & library) : library(library), - va(library.Api()), implementationName("VA-API (" + library.Vendor() + ")"), callback(nullptr), - config(VA_INVALID_ID), - context(VA_INVALID_ID), + session(library, implementationName + " H.265"), codedWidth(0), codedHeight(0), maxDecPicBuffering(0), @@ -92,13 +88,11 @@ namespace jni cropY(0), width(0), height(0), - downloadImage(), prevTid0Poc(0), skipRasl(false), started(false), failed(false) { - downloadImage.image_id = VA_INVALID_ID; } VaapiH265Decoder::~VaapiH265Decoder() @@ -106,18 +100,6 @@ namespace jni Release(); } - bool VaapiH265Decoder::Check(VAStatus status, const char * operation) const - { - if (status == VA_STATUS_SUCCESS) { - return true; - } - - RTC_LOG(LS_WARNING) << implementationName << " H.265: " << operation << " failed: " - << library.ErrorString(status); - - return false; - } - bool VaapiH265Decoder::Configure(const Settings & settings) { if (settings.codec_type() != webrtc::kVideoCodecH265) { @@ -346,7 +328,7 @@ namespace jni const uint32_t spsMaxDecPicBuffering = sps->maxDecPicBufferingMinus1 + 1; - if (context == VA_INVALID_ID || sps->width != codedWidth || sps->height != codedHeight + if (!session.IsCreated() || sps->width != codedWidth || sps->height != codedHeight || spsMaxDecPicBuffering != maxDecPicBuffering || sps->maxNumReorderPics != maxNumReorder) { if (!CreateSession(*sps)) { @@ -604,7 +586,7 @@ namespace jni void VaapiH265Decoder::FillPicture(VAPictureParameterBufferHEVC & parameters, const H265Sps & sps, const H265Pps & pps, const H265SliceHeader & header, int surface) const { - parameters.CurrPic.picture_id = surfaces[surface]; + parameters.CurrPic.picture_id = session.Surface(surface); parameters.CurrPic.flags = 0; for (VAPictureHEVC & reference : parameters.ReferenceFrames) { @@ -620,7 +602,7 @@ namespace jni const int index = dpb[j].surface; VAPictureHEVC & reference = parameters.ReferenceFrames[count++]; - reference.picture_id = surfaces[dpb[j].surface]; + reference.picture_id = session.Surface(dpb[j].surface); reference.pic_order_cnt = dpb[j].poc; reference.flags = dpb[j].longTerm ? VA_PICTURE_HEVC_LONG_TERM_REFERENCE : 0; @@ -838,47 +820,16 @@ namespace jni const std::vector & sliceParameters, const std::vector & slices, int surface) { - VADisplay display = library.Display(); - std::vector buffers; - - const auto create = [&](VABufferType type, size_t size, const void * data, const char * operation) { - VABufferID buffer = VA_INVALID_ID; - - if (!Check(va.CreateBuffer(display, context, type, static_cast(size), 1, - const_cast(data), &buffer), operation)) - { - return false; - } - - buffers.push_back(buffer); - - return true; + std::vector buffers = { + { VAPictureParameterBufferType, &picture, sizeof(picture) } }; - bool created = create(VAPictureParameterBufferType, sizeof(picture), &picture, "creating the picture parameters"); - - for (size_t i = 0; created && i < slices.size(); i++) { - created = create(VASliceParameterBufferType, sizeof(sliceParameters[i]), &sliceParameters[i], - "creating the slice parameters") - && create(VASliceDataBufferType, slices[i].nal.size(), slices[i].nal.data(), "creating the slice data"); - } - - bool decoded = false; - - if (created && Check(va.BeginPicture(display, context, surfaces[surface]), "beginning a picture")) { - const bool rendered = Check(va.RenderPicture(display, context, buffers.data(), - static_cast(buffers.size())), "rendering a picture"); - // A picture that was begun has to be ended. - const bool ended = Check(va.EndPicture(display, context), "ending a picture"); - - decoded = rendered && ended && Check(va.SyncSurface(display, surfaces[surface]), "waiting for a picture"); - } - - for (VABufferID buffer : buffers) { - va.DestroyBuffer(display, buffer); + for (size_t i = 0; i < slices.size(); i++) { + buffers.push_back({ VASliceParameterBufferType, &sliceParameters[i], sizeof(sliceParameters[i]) }); + buffers.push_back({ VASliceDataBufferType, slices[i].nal.data(), slices[i].nal.size() }); } - return decoded; + return session.Decode(surface, buffers); } bool VaapiH265Decoder::Bump(uint32_t keep) @@ -922,61 +873,10 @@ namespace jni bool VaapiH265Decoder::Output(const Picture & picture) { - VADisplay display = library.Display(); - const int frameWidth = static_cast(width); - const int frameHeight = static_cast(height); - - VAImage mapped = {}; - mapped.image_id = VA_INVALID_ID; - bool derived = false; - size_t offsetY = 0; - size_t offsetUv = 0; - - if (downloadImage.image_id != VA_INVALID_ID - && downloadImage.width == width && downloadImage.height == height - && va.GetImage(display, surfaces[picture.surface], static_cast(cropX), static_cast(cropY), - width, height, downloadImage.image_id) == VA_STATUS_SUCCESS) - { - mapped = downloadImage; - } - else if (va.DeriveImage(display, surfaces[picture.surface], &mapped) == VA_STATUS_SUCCESS) { - derived = true; - - if (mapped.format.fourcc != VA_FOURCC_NV12) { - RTC_LOG(LS_WARNING) << implementationName << " H.265: the decoded surface is not NV12"; - va.DestroyImage(display, mapped.image_id); - return false; - } - - // The conformance window within the whole surface. - offsetY = cropY * mapped.pitches[0] + cropX; - offsetUv = (cropY / 2) * mapped.pitches[1] + cropX; - } - else { - RTC_LOG(LS_WARNING) << implementationName << " H.265: the decoded surface cannot be read"; - return false; - } - - void * data = nullptr; - bool converted = false; - webrtc::scoped_refptr i420 = webrtc::I420Buffer::Create(frameWidth, frameHeight); - - if (Check(va.MapBuffer(display, mapped.buf, &data), "mapping a decoded picture")) { - const uint8_t * base = static_cast(data); - - converted = libyuv::NV12ToI420(base + mapped.offsets[0] + offsetY, static_cast(mapped.pitches[0]), - base + mapped.offsets[1] + offsetUv, static_cast(mapped.pitches[1]), - i420->MutableDataY(), i420->StrideY(), i420->MutableDataU(), i420->StrideU(), - i420->MutableDataV(), i420->StrideV(), frameWidth, frameHeight) == 0; - - va.UnmapBuffer(display, mapped.buf); - } - - if (derived) { - va.DestroyImage(display, mapped.image_id); - } + webrtc::scoped_refptr i420 = session.Download(picture.surface, cropX, cropY, width, + height); - if (!converted) { + if (!i420) { return false; } @@ -997,87 +897,25 @@ namespace jni { DestroySession(); - VADisplay display = library.Display(); - - VAConfigAttrib attribute = {}; - attribute.type = VAConfigAttribRTFormat; - attribute.value = VA_RT_FORMAT_YUV420; - - if (!Check(va.CreateConfig(display, VAProfileHEVCMain, VAEntrypointVLD, &attribute, 1, &config), - "creating the configuration")) - { - config = VA_INVALID_ID; - return false; - } - // The pictures the stream keeps, and the one being decoded. - const uint32_t count = sps.maxDecPicBufferingMinus1 + 2; - - surfaces.assign(count, VA_INVALID_SURFACE); - - if (!Check(va.CreateSurfaces(display, VA_RT_FORMAT_YUV420, sps.width, sps.height, surfaces.data(), - count, nullptr, 0), "creating surfaces")) - { - surfaces.clear(); - DestroySession(); - return false; - } - - if (!Check(va.CreateContext(display, config, static_cast(sps.width), static_cast(sps.height), - VA_PROGRESSIVE, surfaces.data(), static_cast(count), &context), "creating the context")) - { - context = VA_INVALID_ID; - DestroySession(); + if (!session.Create(VAProfileHEVCMain, sps.width, sps.height, sps.maxDecPicBufferingMinus1 + 2)) { return false; } - const uint32_t frameWidth = sps.width - sps.cropLeft - sps.cropRight; - const uint32_t frameHeight = sps.height - sps.cropTop - sps.cropBottom; - - // Copying a surface into an image reads it from cached memory, which - // mapping it directly does not. Where the driver does not copy, the - // surface is mapped. - VAImageFormat format = {}; - format.fourcc = VA_FOURCC_NV12; - format.byte_order = VA_LSB_FIRST; - format.bits_per_pixel = 12; - - if (va.CreateImage(display, &format, static_cast(frameWidth), static_cast(frameHeight), - &downloadImage) != VA_STATUS_SUCCESS) - { - downloadImage.image_id = VA_INVALID_ID; - } - codedWidth = sps.width; codedHeight = sps.height; maxDecPicBuffering = sps.maxDecPicBufferingMinus1 + 1; maxNumReorder = sps.maxNumReorderPics; - RTC_LOG(LS_INFO) << implementationName << " H.265 decoding " << frameWidth << "x" << frameHeight; + RTC_LOG(LS_INFO) << implementationName << " H.265 decoding " << sps.width - sps.cropLeft - sps.cropRight + << "x" << sps.height - sps.cropTop - sps.cropBottom; return true; } void VaapiH265Decoder::DestroySession() { - VADisplay display = library.Display(); - - if (downloadImage.image_id != VA_INVALID_ID) { - va.DestroyImage(display, downloadImage.image_id); - downloadImage.image_id = VA_INVALID_ID; - } - if (context != VA_INVALID_ID) { - va.DestroyContext(display, context); - context = VA_INVALID_ID; - } - if (!surfaces.empty()) { - va.DestroySurfaces(display, surfaces.data(), static_cast(surfaces.size())); - surfaces.clear(); - } - if (config != VA_INVALID_ID) { - va.DestroyConfig(display, config); - config = VA_INVALID_ID; - } + session.Destroy(); codedWidth = 0; codedHeight = 0; @@ -1091,7 +929,7 @@ namespace jni int VaapiH265Decoder::FreeSurface() const { - for (int surface = 0; surface < static_cast(surfaces.size()); surface++) { + for (int surface = 0; surface < session.SurfaceCount(); surface++) { const bool used = std::any_of(dpb.begin(), dpb.end(), [surface](const Picture & picture) { return picture.surface == surface; }); diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoDecoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoDecoderFactory.cpp index 1870d950..2848cadf 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoDecoderFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoDecoderFactory.cpp @@ -16,10 +16,13 @@ #include "media/video/codec/linux/VaapiVideoDecoderFactory.h" +#include "media/video/codec/H264Format.h" #include "media/video/codec/H265Format.h" +#include "media/video/codec/linux/VaapiH264Decoder.h" #include "media/video/codec/linux/VaapiH265Decoder.h" #include "media/video/codec/linux/VaapiVp9Decoder.h" +#include "api/video_codecs/video_codec.h" #include "rtc_base/logging.h" namespace jni @@ -32,21 +35,23 @@ namespace jni return nullptr; } + const bool h264 = library->Decodes(VAProfileH264High); const bool vp9 = library->Decodes(VAProfileVP9Profile0); const bool h265 = library->Decodes(VAProfileHEVCMain); - RTC_LOG(LS_INFO) << "VA-API (" << library->Vendor() << ") VP9 decoding: " << vp9 << ", H.265 decoding: " - << h265; + RTC_LOG(LS_INFO) << "VA-API (" << library->Vendor() << ") H.264 decoding: " << h264 << ", VP9 decoding: " + << vp9 << ", H.265 decoding: " << h265; - if (!vp9 && !h265) { + if (!h264 && !vp9 && !h265) { return nullptr; } - return std::unique_ptr(new VaapiVideoDecoderFactory(*library, vp9, h265)); + return std::unique_ptr(new VaapiVideoDecoderFactory(*library, h264, vp9, h265)); } - VaapiVideoDecoderFactory::VaapiVideoDecoderFactory(VaapiLibrary & library, bool vp9, bool h265) : + VaapiVideoDecoderFactory::VaapiVideoDecoderFactory(VaapiLibrary & library, bool h264, bool vp9, bool h265) : library(library), + h264(h264), vp9(vp9), h265(h265) { @@ -56,6 +61,9 @@ namespace jni { std::vector formats; + if (h264) { + formats = CreateH264HardwareDecoderFormats(); + } if (vp9) { formats.push_back(webrtc::SdpVideoFormat::VP9Profile0()); } @@ -69,6 +77,9 @@ namespace jni std::unique_ptr VaapiVideoDecoderFactory::Create(const webrtc::Environment & env, const webrtc::SdpVideoFormat & format) { + if (h264 && webrtc::PayloadStringToCodecType(format.name) == webrtc::kVideoCodecH264) { + return std::make_unique(library); + } if (h265 && format.IsSameCodec(CreateH265MainFormat())) { return std::make_unique(library); } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java index 814e0dd6..e431ad47 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java @@ -55,8 +55,9 @@ * are decoded on NVIDIA GPUs with NVDEC, each where the GPU decodes it (AV1 * since Ampere), loaded from the NVIDIA driver when the factory is made; a * machine without the driver is not affected. Streams with spatial layers go - * to libvpx and dav1d. On the GPUs of Intel and AMD, VP9 (profile 0) and - * H.265 (Main profile) are decoded with VA-API, where the driver decodes them. + * to libvpx and dav1d. On the GPUs of Intel and AMD, H.264, VP9 (profile 0) + * and H.265 (Main profile) are decoded with VA-API, where the driver decodes + * them; interlaced H.264 goes to the software decoder. *

* The supported codecs are those of a {@link DefaultVideoDecoderFactory}, in * the same order. On Windows and Linux, H.264 High 4:4:4 Predictive,