From 2394b9d60ed12e9c89b0bb037b450b52316e48d8 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Thu, 15 Oct 2026 01:38:05 +0200 Subject: [PATCH 1/2] fix: code small H.265 pictures at AMD's minimum size AMD's encoder takes no picture below 384x128, which WebRTC scales down to. Smaller frames are coded in a picture of that size and cropped by the SPS. --- .../media/video/codec/linux/VaapiH265Encoder.h | 6 +++--- .../src/media/video/codec/linux/VaapiH265Encoder.cpp | 12 ++++++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Encoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Encoder.h index c6a82026..553dd443 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Encoder.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Encoder.h @@ -48,9 +48,9 @@ namespace jni // the same previous picture ("low delay B"), as FFmpeg does. The block // sizes and coding tools are those the driver says it takes, as FFmpeg's // VA-API encoder chooses them. The frame size is kept to a multiple of the - // smallest coding block; the coded picture is that aligned to 16, which - // AMD's encoder needs, and cropped to the frame by the conformance window - // of the SPS. There is no software H.265 to fall back to: anything that + // smallest coding block; the coded picture is that aligned to 16 and at + // least 384x128, which AMD's encoder needs, and cropped to the frame by + // the conformance window of the SPS. There is no software H.265 to fall back to: anything that // fails, including a driver that writes no parameter sets into key // frames, is an error, upon which WebRTC switches to the next codec it // negotiated. diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Encoder.cpp index e2a3171f..37670174 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Encoder.cpp @@ -45,6 +45,12 @@ namespace jni // blocks and leaves the conformance window to the SPS it is given. constexpr uint32_t kPictureAlignment = 16; + // AMD's encoder takes no picture narrower than 384 or lower than 128 + // luma samples; a smaller frame is coded in a picture of that size and + // cropped, as WebRTC scales frames down that far. + constexpr uint32_t kMinPictureWidth = 384; + constexpr uint32_t kMinPictureHeight = 128; + // slice_pic_order_cnt_lsb has 8 bits. constexpr uint32_t kLog2MaxPicOrderCntLsb = 8; @@ -222,8 +228,10 @@ namespace jni codecSettings.width -= codecSettings.width % alignment; codecSettings.height -= codecSettings.height % alignment; const uint32_t pictureAlignment = std::max(alignment, kPictureAlignment); - codedWidth = (codecSettings.width + pictureAlignment - 1) / pictureAlignment * pictureAlignment; - codedHeight = (codecSettings.height + pictureAlignment - 1) / pictureAlignment * pictureAlignment; + codedWidth = std::max((codecSettings.width + pictureAlignment - 1) / pictureAlignment * pictureAlignment, + kMinPictureWidth); + codedHeight = std::max((codecSettings.height + pictureAlignment - 1) / pictureAlignment * pictureAlignment, + kMinPictureHeight); widthInCtbs = (codedWidth + ctbSize - 1) / ctbSize; heightInCtbs = (codedHeight + ctbSize - 1) / ctbSize; bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; From bdcf83aebc9ccf4bc0a384582f5590928620e8b8 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Thu, 15 Oct 2026 01:38:05 +0200 Subject: [PATCH 2/2] feat: decode H.265 with VA-API on Linux Adds an H.265 Main decoder for Intel and AMD GPUs, with its own parser for the parameter sets and slice headers. VA-API decoding no longer needs webrtc.hardware.vaapiExperimental, which is removed. --- docs/guide/advanced/video-codecs.md | 4 +- .../media/video/codec/HardwareCodecOptions.h | 7 - .../media/video/codec/linux/H265Parser.h | 245 ++++ .../video/codec/linux/VaapiH265Decoder.h | 180 +++ .../codec/linux/VaapiVideoDecoderFactory.h | 7 +- .../media/video/codec/linux/VaapiVp9Decoder.h | 3 +- .../codec/DefaultVideoCodecFactories.cpp | 10 +- .../video/codec/HardwareCodecOptions.cpp | 5 - .../media/video/codec/linux/H265Parser.cpp | 975 +++++++++++++++ .../LinuxHardwareVideoCodecFactories.cpp | 9 +- .../video/codec/linux/VaapiH265Decoder.cpp | 1106 +++++++++++++++++ .../codec/linux/VaapiVideoDecoderFactory.cpp | 37 +- .../codec/HardwareVideoDecoderFactory.java | 6 +- 13 files changed, 2552 insertions(+), 42 deletions(-) create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/linux/H265Parser.h create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Decoder.h create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/linux/H265Parser.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Decoder.cpp diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index b7a235c5..0338fcb8 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -40,7 +40,7 @@ System.setProperty("webrtc.hardware.vp9Encoder", "true"); Where the GPU has no VP9 encoder, or the stream asks for SVC, VP9 is still encoded with libvpx. -On Linux, H.265 is encoded with the VA-API encoder of Intel and AMD drivers where the driver encodes H.265 Main. It is offered only for sending: Intel and AMD GPUs do not decode H.265 here yet, so the other end has to. +On Linux, H.265 is encoded with the VA-API encoder of Intel and AMD drivers where the driver encodes H.265 Main. Receiving it needs a hardware H.265 decoder too, since there is no software one: on Intel and AMD GPUs that is the VA-API decoder (see below). NVENC needs an NVIDIA driver of version 522 or newer on Windows, 520 or newer on Linux. VA-API needs libva 2 and a driver that encodes H.264, such as Intel's `intel-media-va-driver` (iHD) or Mesa's `mesa-va-drivers` for AMD, and access to a render node in `/dev/dri`. All of them are loaded at run time, so nothing needs to be installed on machines without them. @@ -62,7 +62,7 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder() | Platform | Hardware decoding | |---|---| | Windows | H.264, H.265 (Main profile), AV1 and VP9 (profile 0), on GPUs that decode them, through the Media Foundation decoders of Windows on Direct3D 11 (DXVA) | -| Linux | H.264, H.265 (Main profile), VP9 and AV1 (profile 0) on NVIDIA GPUs that decode them, through NVDEC; VP9 (profile 0) on Intel and AMD GPUs through VA-API, when turned on with `webrtc.hardware.vaapiExperimental` | +| Linux | H.264, H.265 (Main profile), VP9 and AV1 (profile 0) on NVIDIA GPUs that decode them, through NVDEC; VP9 (profile 0) and H.265 (Main profile) on Intel and AMD GPUs through VA-API | | macOS | H.264 through VideoToolbox, as with `DefaultVideoDecoderFactory`; VP9 (profile 0) and H.265 (Main profile) through the decoders of VideoToolbox, on Macs that have them, and AV1 (profile 0) on Apple M3 and newer | On Linux, NVDEC needs an NVIDIA driver, which provides `libcuda.so.1` and `libnvcuvid.so.1`; both are loaded when the factory is made, so a machine without the driver is not affected. Streams with spatial layers are decoded in software. diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareCodecOptions.h b/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareCodecOptions.h index 33726a64..2b60dcd3 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareCodecOptions.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/HardwareCodecOptions.h @@ -24,17 +24,10 @@ namespace jni // tested on one. constexpr const char * kHardwareVp9EncoderProperty = "webrtc.hardware.vp9Encoder"; - // The Java system property that turns on VA-API decoding, which is off by - // default: it has not been tested on an Intel or AMD GPU under Linux yet. - constexpr const char * kExperimentalVaapiProperty = "webrtc.hardware.vaapiExperimental"; - // Whether hardware VP9 encoding is turned on: the Java system property // above is "true", as Boolean.getBoolean has it. Callable from any thread; // false where Java cannot be asked, for one with an exception pending. bool IsHardwareVp9EncoderEnabled(); - - // Whether VA-API decoding is turned on, the same way. - bool IsExperimentalVaapiEnabled(); } #endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/H265Parser.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/H265Parser.h new file mode 100644 index 00000000..380d707f --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/H265Parser.h @@ -0,0 +1,245 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_H265_PARSER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_H265_PARSER_H_ + +#include +#include +#include +#include +#include +#include + +namespace jni +{ + // A short-term reference picture set (7.3.7), as the POC differences of + // its pictures to the current one. + struct H265ShortTermRps + { + static constexpr int kMaxPictures = 16; + + int numNegative = 0; + int numPositive = 0; + std::array deltaPocS0 = {}; + std::array usedS0 = {}; + std::array deltaPocS1 = {}; + std::array usedS1 = {}; + }; + + // What the decoder needs of a sequence parameter set (7.3.2.2), up to the + // VUI, which decoding does not depend on. + struct H265Sps + { + uint32_t generalProfileIdc = 0; + uint32_t generalProfileCompatibility = 0; + uint32_t chromaFormatIdc = 0; + bool separateColourPlane = false; + uint32_t width = 0; + uint32_t height = 0; + // The conformance window, in luma samples. + uint32_t cropLeft = 0; + uint32_t cropRight = 0; + uint32_t cropTop = 0; + uint32_t cropBottom = 0; + uint32_t bitDepthLumaMinus8 = 0; + uint32_t bitDepthChromaMinus8 = 0; + uint32_t log2MaxPocLsbMinus4 = 0; + // Of the highest sub-layer. + uint32_t maxDecPicBufferingMinus1 = 0; + uint32_t maxNumReorderPics = 0; + uint32_t log2MinCbMinus3 = 0; + uint32_t log2DiffMaxMinCb = 0; + uint32_t log2MinTbMinus2 = 0; + uint32_t log2DiffMaxMinTb = 0; + uint32_t maxTransformDepthInter = 0; + uint32_t maxTransformDepthIntra = 0; + bool scalingListEnabled = false; + bool amp = false; + bool sao = false; + bool pcm = false; + uint32_t pcmBitDepthLumaMinus1 = 0; + uint32_t pcmBitDepthChromaMinus1 = 0; + uint32_t log2MinPcmCbMinus3 = 0; + uint32_t log2DiffMaxMinPcmCb = 0; + bool pcmLoopFilterDisabled = false; + std::vector shortTermRps; + bool longTermRefsPresent = false; + std::vector ltRefPicPocLsb; + std::vector ltUsedByCurrPic; + bool temporalMvp = false; + bool strongIntraSmoothing = false; + + uint32_t Log2CtbSize() const; + uint32_t WidthInCtbs() const; + uint32_t HeightInCtbs() const; + }; + + // A picture parameter set (7.3.2.3). + struct H265Pps + { + uint32_t spsId = 0; + bool dependentSliceSegments = false; + bool outputFlagPresent = false; + uint32_t numExtraSliceHeaderBits = 0; + bool signDataHiding = false; + bool cabacInitPresent = false; + uint32_t numRefIdxL0DefaultMinus1 = 0; + uint32_t numRefIdxL1DefaultMinus1 = 0; + int32_t initQpMinus26 = 0; + bool constrainedIntraPred = false; + bool transformSkip = false; + bool cuQpDelta = false; + uint32_t diffCuQpDeltaDepth = 0; + int32_t cbQpOffset = 0; + int32_t crQpOffset = 0; + bool sliceChromaQpOffsetsPresent = false; + bool weightedPred = false; + bool weightedBipred = false; + bool transquantBypass = false; + bool tiles = false; + bool entropyCodingSync = false; + uint32_t numTileColumnsMinus1 = 0; + uint32_t numTileRowsMinus1 = 0; + bool uniformSpacing = true; + // Of all columns and rows but the last, where the spacing is not + // uniform. + std::vector columnWidthMinus1; + std::vector rowHeightMinus1; + bool loopFilterAcrossTiles = false; + bool loopFilterAcrossSlices = false; + bool deblockingFilterOverrideEnabled = false; + bool deblockingFilterDisabled = false; + int32_t betaOffsetDiv2 = 0; + int32_t tcOffsetDiv2 = 0; + bool scalingListDataPresent = false; + bool listsModificationPresent = false; + uint32_t log2ParallelMergeLevelMinus2 = 0; + bool sliceHeaderExtensionPresent = false; + bool extensionPresent = false; + }; + + // A slice segment header (7.3.6.1). A dependent slice segment carries the + // fields of the independent one before it, which the parser fills in. + struct H265SliceHeader + { + static constexpr int kMaxRefs = 16; + + uint8_t nalType = 0; + uint8_t temporalId = 0; + bool firstSliceInPicture = false; + bool noOutputOfPriorPics = false; + uint32_t ppsId = 0; + bool dependent = false; + uint32_t segmentAddress = 0; + uint32_t sliceType = 0; + bool picOutput = true; + uint32_t pocLsb = 0; + // The short-term set of the picture, and how many bits it took in the + // header, 0 when it is one of the SPS. + H265ShortTermRps shortTermRps; + uint32_t shortTermRpsBits = 0; + // The long-term pictures: the POC (LSBs, or all of it where the MSBs + // are given), whether the MSBs are given and whether the current + // picture uses them. + std::vector longTermPoc; + std::vector longTermMsbPresent; + std::vector longTermUsed; + bool temporalMvp = false; + bool saoLuma = false; + bool saoChroma = false; + uint32_t numRefIdxL0Minus1 = 0; + uint32_t numRefIdxL1Minus1 = 0; + bool listModificationL0 = false; + bool listModificationL1 = false; + std::array listEntryL0 = {}; + std::array listEntryL1 = {}; + bool mvdL1Zero = false; + bool cabacInit = false; + bool collocatedFromL0 = true; + uint32_t collocatedRefIdx = 0; + uint32_t lumaLog2WeightDenom = 0; + int32_t deltaChromaLog2WeightDenom = 0; + std::array, 2> deltaLumaWeight = {}; + std::array, 2> lumaOffset = {}; + std::array, kMaxRefs>, 2> deltaChromaWeight = {}; + std::array, kMaxRefs>, 2> chromaOffset = {}; + uint32_t fiveMinusMaxNumMergeCand = 0; + int32_t qpDelta = 0; + int32_t cbQpOffset = 0; + int32_t crQpOffset = 0; + bool deblockingFilterDisabled = false; + int32_t betaOffsetDiv2 = 0; + int32_t tcOffsetDiv2 = 0; + bool loopFilterAcrossSlices = false; + uint32_t numEntryPoints = 0; + // Where the slice data begins, in bytes from the NAL unit header, with + // emulation prevention bytes counted. + uint32_t dataByteOffset = 0; + + bool IsIntra() const; + bool IsB() const; + // NumPicTotalCurr (7-55). + uint32_t NumPicTotalCurr() const; + }; + + // Parses the parameter sets and slice segment headers of an H.265 stream, + // as a VA-API decoder, which leaves that to the application, needs them. + // It takes 8 bit 4:2:0 Main and Main Still Picture streams; anything else, + // including scaling lists and extensions, is not parsed. + class H265Parser + { + public: + // NAL unit types (table 7-1). + static constexpr uint8_t kNalRaslN = 8; + static constexpr uint8_t kNalRaslR = 9; + static constexpr uint8_t kNalBlaWLp = 16; + static constexpr uint8_t kNalIdrWRadl = 19; + static constexpr uint8_t kNalIdrNLp = 20; + static constexpr uint8_t kNalCraNut = 21; + static constexpr uint8_t kNalReservedIrap23 = 23; + static constexpr uint8_t kNalVps = 32; + static constexpr uint8_t kNalSps = 33; + static constexpr uint8_t kNalPps = 34; + static constexpr uint8_t kNalEos = 36; + + // Slice types (7.4.7.1). + static constexpr uint32_t kSliceB = 0; + static constexpr uint32_t kSliceP = 1; + static constexpr uint32_t kSliceI = 2; + + // Each takes a NAL unit from its header on, with emulation + // prevention bytes, and returns false if it is broken or not one + // the parser takes. + bool ParseSps(std::span nal); + bool ParsePps(std::span nal); + std::optional ParseSliceHeader(std::span nal); + + const H265Sps * Sps(uint32_t id) const; + const H265Pps * Pps(uint32_t id) const; + + void Reset(); + + private: + std::map spss; + std::map ppss; + // The last independent slice segment, whose fields a dependent one + // takes. + std::optional lastIndependent; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Decoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Decoder.h new file mode 100644 index 00000000..356a91fa --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Decoder.h @@ -0,0 +1,180 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_H265_DECODER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_H265_DECODER_H_ + +#include "media/video/codec/linux/H265Parser.h" +#include "media/video/codec/linux/VaapiLibrary.h" + +#include "api/video/encoded_image.h" +#include "api/video/video_rotation.h" +#include "api/video_codecs/video_decoder.h" + +#include +#include + +#include +#include +#include +#include + +namespace jni +{ + // Decodes H.265 Main with the GPU driver's VA-API decoder, as Intel and + // AMD drivers provide it on Linux. + // + // VA-API leaves the parsing of the stream to the application: the + // parameter sets and slice headers are parsed here, and the decoder keeps + // the picture order counts, the reference picture sets and the decoded + // picture buffer, the driver decoding the slice data. Decoding is + // synchronous; pictures come out in output order, which is decoding order + // for a stream without reordering, as real-time encoders send. The decoded + // picture is copied from its surface in NV12, cropped to the conformance + // window, and converted to I420, which is what WebRTC's frames hold. + // + // A stream it does not take, such as one with scaling lists, and anything + // that fails return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; a picture that + // refers to one that is not there returns WEBRTC_VIDEO_CODEC_ERROR, and + // decoding starts again with the next key frame. + class VaapiH265Decoder : public webrtc::VideoDecoder + { + public: + explicit VaapiH265Decoder(VaapiLibrary & library); + ~VaapiH265Decoder() override; + + using webrtc::VideoDecoder::Decode; + + bool Configure(const Settings & settings) override; + int32_t Decode(const webrtc::EncodedImage & image, int64_t renderTimeMs) override; + int32_t RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback * callback) override; + int32_t Release() override; + DecoderInfo GetDecoderInfo() const override; + const char * ImplementationName() const override; + + private: + // What a decoded picture is kept for. + struct Picture + { + int surface = -1; + int32_t poc = 0; + bool shortTerm = false; + bool longTerm = false; + bool waitingOutput = false; + + // Of the image it came in, for the frame that shows it. + uint32_t rtpTimestamp = 0; + int64_t ntpTimeMs = 0; + int64_t renderTimeMs = 0; + webrtc::VideoRotation rotation = webrtc::kVideoRotation_0; + + bool IsReference() const; + }; + + // A slice of the picture being decoded. + struct Slice + { + H265SliceHeader header; + std::span nal; + }; + + enum class Result + { + kOk, + // A reference picture is missing; a key frame starts again. + kMissingReference, + kFailed + }; + + Result DecodePicture(const std::vector & slices, const webrtc::EncodedImage & image, + int64_t renderTimeMs); + + // The picture order count of the picture (8.3.1), and the reference + // picture set it marks the decoded picture buffer with (8.3.2). + int32_t PictureOrderCount(const H265SliceHeader & header, bool noRaslOutput); + Result ApplyReferencePictureSet(const H265SliceHeader & header, const H265Sps & sps, int32_t poc); + + // The reference picture lists of a slice (8.3.4), as surfaces. + bool BuildReferenceLists(const H265SliceHeader & header, std::vector & list0, + std::vector & list1) const; + + void FillPicture(VAPictureParameterBufferHEVC & parameters, const H265Sps & sps, const H265Pps & pps, + const H265SliceHeader & header, int surface) const; + bool FillSlice(VASliceParameterBufferHEVC & parameters, const Slice & slice, bool last) const; + // The index in ReferenceFrames of the reference picture in the + // surface: they are in the order of the decoded picture buffer. + int ReferenceFrameIndex(int surface) const; + + bool Render(const VAPictureParameterBufferHEVC & picture, + const std::vector & sliceParameters, const std::vector & slices, + int surface); + + // Hands pictures to WebRTC in output order until at most the given + // number wait. + bool Bump(uint32_t keep); + bool Output(const Picture & picture); + void RemoveUnused(); + + bool CreateSession(const H265Sps & sps); + void DestroySession(); + int FreeSurface() const; + + bool Check(VAStatus status, const char * operation) const; + + private: + VaapiLibrary & library; + const VaapiFunctions & va; + const std::string implementationName; + + webrtc::DecodedImageCallback * callback; + + H265Parser parser; + + VAConfigID config; + VAContextID context; + std::vector surfaces; + uint32_t codedWidth; + uint32_t codedHeight; + uint32_t maxDecPicBuffering; + uint32_t maxNumReorder; + // The conformance window of the stream. + uint32_t cropX; + uint32_t cropY; + uint32_t width; + uint32_t height; + VAImage downloadImage; + + // The decoded picture buffer, and the surfaces of the pictures the + // current one refers to, by the subset of its reference picture + // set they are in. + std::vector dpb; + std::vector stCurrBefore; + std::vector stCurrAfter; + std::vector ltCurr; + + // prevTid0Pic of 8.3.1. + int32_t prevTid0Poc; + // Whether the RASL pictures of the IRAP picture before are skipped. + bool skipRasl; + + // Whether a key frame has been decoded, and whether anything since + // then failed for good. + bool started; + bool failed; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoDecoderFactory.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoDecoderFactory.h index e666a219..98fff8bf 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoDecoderFactory.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVideoDecoderFactory.h @@ -30,8 +30,7 @@ namespace jni { // Creates VA-API decoders on the GPUs of Intel and AMD, for VP9 profile 0 - // where the driver decodes it, the format WebRTC's software decoder - // offers too. + // and H.265 Main where the driver decodes them. class VaapiVideoDecoderFactory : public webrtc::VideoDecoderFactory { public: @@ -46,10 +45,12 @@ namespace jni const webrtc::SdpVideoFormat & format) override; private: - explicit VaapiVideoDecoderFactory(VaapiLibrary & library); + VaapiVideoDecoderFactory(VaapiLibrary & library, bool vp9, bool h265); private: VaapiLibrary & library; + const bool vp9; + const bool h265; }; } diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Decoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Decoder.h index 263c23f8..e106aca9 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Decoder.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Decoder.h @@ -35,8 +35,7 @@ namespace jni { // Decodes VP9 profile 0 with the GPU driver's VA-API decoder, as Intel and - // AMD drivers provide it on Linux. Experimental: it has not run on such a - // GPU yet, and is used only where it is turned on. + // AMD drivers provide it on Linux. // // VA-API leaves the parsing of the stream to the application: the frame // headers are parsed here, the driver decodes the rest, the compressed diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp index fee88d76..6c465ca8 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp @@ -62,25 +62,23 @@ namespace jni } // Probes for the hardware factories of the platform once for each - // setting of the options that change what is found, and shares what + // setting of the option that changes what is found, and shares what // it found: a probe takes tens of milliseconds, and so every factory // works with the same codecs. A GPU, driver or codec installed or // removed later is noticed only by a new process. template std::shared_ptr GetPlatformHardwareFactories(Factories (*probe)()) { - using Options = std::pair; - static std::mutex mutex; // Never destroyed: the factories use libraries that may be gone // by the time static objects are destroyed as the process exits. - static auto * cache = new std::map>(); + static auto * cache = new std::map>(); - const Options options(IsHardwareVp9EncoderEnabled(), IsExperimentalVaapiEnabled()); + const bool vp9Encoder = IsHardwareVp9EncoderEnabled(); std::lock_guard lock(mutex); - std::shared_ptr & factories = (*cache)[options]; + std::shared_ptr & factories = (*cache)[vp9Encoder]; if (!factories) { factories = std::make_shared(probe()); diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareCodecOptions.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareCodecOptions.cpp index bb74c835..77f5dad8 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareCodecOptions.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/HardwareCodecOptions.cpp @@ -70,9 +70,4 @@ namespace jni { return JavaBooleanProperty(kHardwareVp9EncoderProperty); } - - bool IsExperimentalVaapiEnabled() - { - return JavaBooleanProperty(kExperimentalVaapiProperty); - } } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/H265Parser.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/H265Parser.cpp new file mode 100644 index 00000000..64268fdc --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/H265Parser.cpp @@ -0,0 +1,975 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/linux/H265Parser.h" + +#include "common_video/h265/h265_sps_parser.h" +#include "rtc_base/bitstream_reader.h" + +#include + +namespace jni +{ + namespace + { + constexpr uint32_t kMaxSpsCount = 16; + constexpr uint32_t kMaxPpsCount = 64; + constexpr uint32_t kMaxShortTermRpsCount = 64; + constexpr uint32_t kMaxLongTermRefsSps = 32; + constexpr uint32_t kMaxTiles = 20; + + // Main and Main Still Picture. + constexpr uint32_t kProfileMain = 1; + constexpr uint32_t kProfileMainStill = 3; + + // A NAL unit without its two byte header and without emulation + // prevention bytes, with where those were. + struct Rbsp + { + std::vector bytes; + // The positions in the NAL unit of the bytes removed. + std::vector removed; + }; + + Rbsp Unescape(std::span nal) + { + Rbsp rbsp; + rbsp.bytes.reserve(nal.size()); + + int zeros = 0; + + for (size_t i = 2; i < nal.size(); i++) { + if (zeros >= 2 && nal[i] == 3) { + rbsp.removed.push_back(i); + zeros = 0; + continue; + } + + rbsp.bytes.push_back(nal[i]); + zeros = nal[i] == 0 ? zeros + 1 : 0; + } + + return rbsp; + } + + // The position in the NAL unit of a byte of its RBSP. + size_t NalOffset(const Rbsp & rbsp, size_t rbspOffset) + { + size_t offset = rbspOffset + 2; + + for (size_t position : rbsp.removed) { + if (position <= offset) { + offset++; + } + } + + return offset; + } + + // Ceil(Log2(value)). + int CeilLog2(uint32_t value) + { + int bits = 0; + + while ((1u << bits) < value) { + bits++; + } + + return bits; + } + + int ConsumedBits(const webrtc::BitstreamReader & reader, size_t size) + { + return static_cast(size * 8) - reader.RemainingBitCount(); + } + + void SkipProfileTierLevel(webrtc::BitstreamReader & reader, uint32_t maxSubLayersMinus1, H265Sps & sps) + { + // general_profile_space, general_tier_flag. + reader.ConsumeBits(3); + sps.generalProfileIdc = reader.ReadBits(5); + sps.generalProfileCompatibility = static_cast(reader.ReadBits(32)); + // The source and constraint flags, and general_level_idc. + reader.ConsumeBits(48); + reader.ConsumeBits(8); + + bool profilePresent[8] = {}; + bool levelPresent[8] = {}; + + for (uint32_t i = 0; i < maxSubLayersMinus1; i++) { + profilePresent[i] = reader.Read(); + levelPresent[i] = reader.Read(); + } + + if (maxSubLayersMinus1 > 0) { + reader.ConsumeBits(2 * (8 - static_cast(maxSubLayersMinus1))); + } + + for (uint32_t i = 0; i < maxSubLayersMinus1; i++) { + if (profilePresent[i]) { + reader.ConsumeBits(88); + } + if (levelPresent[i]) { + reader.ConsumeBits(8); + } + } + } + + // st_ref_pic_set(index) (7.3.7), with the derivation of an inter + // predicted set (7.4.8). + bool ParseShortTermRps(webrtc::BitstreamReader & reader, uint32_t index, + const std::vector & sets, H265ShortTermRps & rps) + { + const uint32_t count = static_cast(sets.size()); + bool interPrediction = false; + + if (index != 0) { + interPrediction = reader.Read(); + } + + if (interPrediction) { + uint32_t deltaIdxMinus1 = 0; + + if (index == count) { + deltaIdxMinus1 = reader.ReadExponentialGolomb(); + } + if (deltaIdxMinus1 + 1 > index) { + return false; + } + + const H265ShortTermRps & reference = sets[index - (deltaIdxMinus1 + 1)]; + const bool sign = reader.Read(); + const uint32_t absDeltaMinus1 = reader.ReadExponentialGolomb(); + + if (absDeltaMinus1 > 0x7FFF) { + return false; + } + + const int32_t deltaRps = (sign ? -1 : 1) * static_cast(absDeltaMinus1 + 1); + const int numDeltaPocs = reference.numNegative + reference.numPositive; + + bool usedByCurr[H265ShortTermRps::kMaxPictures * 2 + 1] = {}; + bool useDelta[H265ShortTermRps::kMaxPictures * 2 + 1] = {}; + + for (int j = 0; j <= numDeltaPocs; j++) { + usedByCurr[j] = reader.Read(); + useDelta[j] = usedByCurr[j] ? true : reader.Read(); + } + + const auto add = [](int & count, std::array & deltas, + std::array & used, int32_t delta, bool isUsed) + { + if (count >= H265ShortTermRps::kMaxPictures) { + return false; + } + + deltas[count] = delta; + used[count] = isUsed; + count++; + + return true; + }; + + int i = 0; + bool ok = true; + + for (int j = reference.numPositive - 1; j >= 0; j--) { + const int32_t dPoc = reference.deltaPocS1[j] + deltaRps; + + if (dPoc < 0 && useDelta[reference.numNegative + j]) { + ok &= add(i, rps.deltaPocS0, rps.usedS0, dPoc, usedByCurr[reference.numNegative + j]); + } + } + if (deltaRps < 0 && useDelta[numDeltaPocs]) { + ok &= add(i, rps.deltaPocS0, rps.usedS0, deltaRps, usedByCurr[numDeltaPocs]); + } + for (int j = 0; j < reference.numNegative; j++) { + const int32_t dPoc = reference.deltaPocS0[j] + deltaRps; + + if (dPoc < 0 && useDelta[j]) { + ok &= add(i, rps.deltaPocS0, rps.usedS0, dPoc, usedByCurr[j]); + } + } + + rps.numNegative = i; + i = 0; + + for (int j = reference.numNegative - 1; j >= 0; j--) { + const int32_t dPoc = reference.deltaPocS0[j] + deltaRps; + + if (dPoc > 0 && useDelta[j]) { + ok &= add(i, rps.deltaPocS1, rps.usedS1, dPoc, usedByCurr[j]); + } + } + if (deltaRps > 0 && useDelta[numDeltaPocs]) { + ok &= add(i, rps.deltaPocS1, rps.usedS1, deltaRps, usedByCurr[numDeltaPocs]); + } + for (int j = 0; j < reference.numPositive; j++) { + const int32_t dPoc = reference.deltaPocS1[j] + deltaRps; + + if (dPoc > 0 && useDelta[reference.numNegative + j]) { + ok &= add(i, rps.deltaPocS1, rps.usedS1, dPoc, usedByCurr[reference.numNegative + j]); + } + } + + rps.numPositive = i; + + return ok && reader.Ok(); + } + + const uint32_t numNegative = reader.ReadExponentialGolomb(); + const uint32_t numPositive = reader.ReadExponentialGolomb(); + + if (numNegative > H265ShortTermRps::kMaxPictures || numPositive > H265ShortTermRps::kMaxPictures + || numNegative + numPositive > H265ShortTermRps::kMaxPictures) + { + return false; + } + + rps.numNegative = static_cast(numNegative); + rps.numPositive = static_cast(numPositive); + + int32_t poc = 0; + + for (uint32_t i = 0; i < numNegative; i++) { + const uint32_t deltaMinus1 = reader.ReadExponentialGolomb(); + + if (deltaMinus1 > 0x7FFF) { + return false; + } + + poc -= static_cast(deltaMinus1) + 1; + rps.deltaPocS0[i] = poc; + rps.usedS0[i] = reader.Read(); + } + + poc = 0; + + for (uint32_t i = 0; i < numPositive; i++) { + const uint32_t deltaMinus1 = reader.ReadExponentialGolomb(); + + if (deltaMinus1 > 0x7FFF) { + return false; + } + + poc += static_cast(deltaMinus1) + 1; + rps.deltaPocS1[i] = poc; + rps.usedS1[i] = reader.Read(); + } + + return reader.Ok(); + } + + // pred_weight_table() (7.3.6.3) of one list, with the chroma offsets + // derived (7-56) for 8 bit samples. + void ParseWeights(webrtc::BitstreamReader & reader, H265SliceHeader & header, int list, uint32_t count, + bool chroma) + { + const uint32_t chromaLog2Denom = static_cast( + static_cast(header.lumaLog2WeightDenom) + header.deltaChromaLog2WeightDenom); + constexpr int32_t kOffsetHalfRange = 1 << 7; + + bool lumaFlags[H265SliceHeader::kMaxRefs] = {}; + bool chromaFlags[H265SliceHeader::kMaxRefs] = {}; + + for (uint32_t i = 0; i < count; i++) { + lumaFlags[i] = reader.Read(); + } + if (chroma) { + for (uint32_t i = 0; i < count; i++) { + chromaFlags[i] = reader.Read(); + } + } + + for (uint32_t i = 0; i < count; i++) { + if (lumaFlags[i]) { + header.deltaLumaWeight[list][i] = reader.ReadSignedExponentialGolomb(); + header.lumaOffset[list][i] = reader.ReadSignedExponentialGolomb(); + } + if (chromaFlags[i]) { + for (int j = 0; j < 2; j++) { + const int32_t deltaWeight = reader.ReadSignedExponentialGolomb(); + const int32_t deltaOffset = reader.ReadSignedExponentialGolomb(); + const int32_t weight = (1 << chromaLog2Denom) + deltaWeight; + const int32_t offset = kOffsetHalfRange + - ((kOffsetHalfRange * weight) >> chromaLog2Denom) + deltaOffset; + + header.deltaChromaWeight[list][i][j] = deltaWeight; + header.chromaOffset[list][i][j] = std::clamp(offset, -kOffsetHalfRange, kOffsetHalfRange - 1); + } + } + } + } + } + + uint32_t H265Sps::Log2CtbSize() const + { + return log2MinCbMinus3 + 3 + log2DiffMaxMinCb; + } + + uint32_t H265Sps::WidthInCtbs() const + { + const uint32_t ctbSize = 1u << Log2CtbSize(); + + return (width + ctbSize - 1) / ctbSize; + } + + uint32_t H265Sps::HeightInCtbs() const + { + const uint32_t ctbSize = 1u << Log2CtbSize(); + + return (height + ctbSize - 1) / ctbSize; + } + + bool H265SliceHeader::IsIntra() const + { + return sliceType == H265Parser::kSliceI; + } + + bool H265SliceHeader::IsB() const + { + return sliceType == H265Parser::kSliceB; + } + + uint32_t H265SliceHeader::NumPicTotalCurr() const + { + uint32_t count = 0; + + for (int i = 0; i < shortTermRps.numNegative; i++) { + count += shortTermRps.usedS0[i] ? 1 : 0; + } + for (int i = 0; i < shortTermRps.numPositive; i++) { + count += shortTermRps.usedS1[i] ? 1 : 0; + } + for (bool used : longTermUsed) { + count += used ? 1 : 0; + } + + return count; + } + + bool H265Parser::ParseSps(std::span nal) + { + const Rbsp rbsp = Unescape(nal); + webrtc::BitstreamReader reader(rbsp.bytes); + H265Sps sps; + + // sps_video_parameter_set_id. + reader.ConsumeBits(4); + const uint32_t maxSubLayersMinus1 = reader.ReadBits(3); + // sps_temporal_id_nesting_flag. + reader.ConsumeBits(1); + + if (maxSubLayersMinus1 > 6) { + return false; + } + + SkipProfileTierLevel(reader, maxSubLayersMinus1, sps); + + const uint32_t id = reader.ReadExponentialGolomb(); + + if (id >= kMaxSpsCount) { + return false; + } + + sps.chromaFormatIdc = reader.ReadExponentialGolomb(); + + if (sps.chromaFormatIdc == 3) { + sps.separateColourPlane = reader.Read(); + } + + sps.width = reader.ReadExponentialGolomb(); + sps.height = reader.ReadExponentialGolomb(); + + if (reader.Read()) { + // In chroma samples, two luma samples each for 4:2:0. + const uint32_t subWidth = sps.chromaFormatIdc == 1 || sps.chromaFormatIdc == 2 ? 2 : 1; + const uint32_t subHeight = sps.chromaFormatIdc == 1 ? 2 : 1; + + sps.cropLeft = reader.ReadExponentialGolomb() * subWidth; + sps.cropRight = reader.ReadExponentialGolomb() * subWidth; + sps.cropTop = reader.ReadExponentialGolomb() * subHeight; + sps.cropBottom = reader.ReadExponentialGolomb() * subHeight; + } + + sps.bitDepthLumaMinus8 = reader.ReadExponentialGolomb(); + sps.bitDepthChromaMinus8 = reader.ReadExponentialGolomb(); + sps.log2MaxPocLsbMinus4 = reader.ReadExponentialGolomb(); + + if (sps.log2MaxPocLsbMinus4 > 12) { + return false; + } + + const bool orderingInfoPresent = reader.Read(); + + for (uint32_t i = orderingInfoPresent ? 0 : maxSubLayersMinus1; i <= maxSubLayersMinus1; i++) { + sps.maxDecPicBufferingMinus1 = reader.ReadExponentialGolomb(); + sps.maxNumReorderPics = reader.ReadExponentialGolomb(); + // sps_max_latency_increase_plus1. + reader.ReadExponentialGolomb(); + } + + if (sps.maxDecPicBufferingMinus1 > 15 || sps.maxNumReorderPics > sps.maxDecPicBufferingMinus1) { + return false; + } + + sps.log2MinCbMinus3 = reader.ReadExponentialGolomb(); + sps.log2DiffMaxMinCb = reader.ReadExponentialGolomb(); + sps.log2MinTbMinus2 = reader.ReadExponentialGolomb(); + sps.log2DiffMaxMinTb = reader.ReadExponentialGolomb(); + sps.maxTransformDepthInter = reader.ReadExponentialGolomb(); + sps.maxTransformDepthIntra = reader.ReadExponentialGolomb(); + + if (sps.log2MinCbMinus3 + 3 + sps.log2DiffMaxMinCb > 6 || sps.log2MinTbMinus2 > 3 + || sps.log2DiffMaxMinTb > 3) + { + return false; + } + + sps.scalingListEnabled = reader.Read(); + + if (sps.scalingListEnabled && reader.Read() + && !webrtc::H265SpsParser::ParseScalingListData(reader)) + { + return false; + } + + sps.amp = reader.Read(); + sps.sao = reader.Read(); + sps.pcm = reader.Read(); + + if (sps.pcm) { + sps.pcmBitDepthLumaMinus1 = reader.ReadBits(4); + sps.pcmBitDepthChromaMinus1 = reader.ReadBits(4); + sps.log2MinPcmCbMinus3 = reader.ReadExponentialGolomb(); + sps.log2DiffMaxMinPcmCb = reader.ReadExponentialGolomb(); + sps.pcmLoopFilterDisabled = reader.Read(); + } + + const uint32_t numShortTermRps = reader.ReadExponentialGolomb(); + + if (numShortTermRps > kMaxShortTermRpsCount) { + return false; + } + + for (uint32_t i = 0; i < numShortTermRps; i++) { + H265ShortTermRps rps; + + if (!ParseShortTermRps(reader, i, sps.shortTermRps, rps)) { + return false; + } + + sps.shortTermRps.push_back(rps); + } + + sps.longTermRefsPresent = reader.Read(); + + if (sps.longTermRefsPresent) { + const uint32_t count = reader.ReadExponentialGolomb(); + + if (count > kMaxLongTermRefsSps) { + return false; + } + + for (uint32_t i = 0; i < count; i++) { + sps.ltRefPicPocLsb.push_back(static_cast(reader.ReadBits(sps.log2MaxPocLsbMinus4 + 4))); + sps.ltUsedByCurrPic.push_back(reader.Read()); + } + } + + sps.temporalMvp = reader.Read(); + sps.strongIntraSmoothing = reader.Read(); + + const bool main = sps.generalProfileIdc == kProfileMain || sps.generalProfileIdc == kProfileMainStill + || (sps.generalProfileCompatibility & (1u << (31 - kProfileMain))) != 0; + + if (!reader.Ok() || !main || sps.chromaFormatIdc != 1 || sps.bitDepthLumaMinus8 != 0 + || sps.bitDepthChromaMinus8 != 0 || sps.width == 0 || sps.height == 0 + || sps.cropLeft + sps.cropRight >= sps.width || sps.cropTop + sps.cropBottom >= sps.height) + { + return false; + } + + spss[id] = std::move(sps); + + return true; + } + + bool H265Parser::ParsePps(std::span nal) + { + const Rbsp rbsp = Unescape(nal); + webrtc::BitstreamReader reader(rbsp.bytes); + H265Pps pps; + + const uint32_t id = reader.ReadExponentialGolomb(); + pps.spsId = reader.ReadExponentialGolomb(); + + if (id >= kMaxPpsCount || pps.spsId >= kMaxSpsCount) { + return false; + } + + pps.dependentSliceSegments = reader.Read(); + pps.outputFlagPresent = reader.Read(); + pps.numExtraSliceHeaderBits = reader.ReadBits(3); + pps.signDataHiding = reader.Read(); + pps.cabacInitPresent = reader.Read(); + pps.numRefIdxL0DefaultMinus1 = reader.ReadExponentialGolomb(); + pps.numRefIdxL1DefaultMinus1 = reader.ReadExponentialGolomb(); + pps.initQpMinus26 = reader.ReadSignedExponentialGolomb(); + pps.constrainedIntraPred = reader.Read(); + pps.transformSkip = reader.Read(); + pps.cuQpDelta = reader.Read(); + + if (pps.cuQpDelta) { + pps.diffCuQpDeltaDepth = reader.ReadExponentialGolomb(); + } + + pps.cbQpOffset = reader.ReadSignedExponentialGolomb(); + pps.crQpOffset = reader.ReadSignedExponentialGolomb(); + pps.sliceChromaQpOffsetsPresent = reader.Read(); + pps.weightedPred = reader.Read(); + pps.weightedBipred = reader.Read(); + pps.transquantBypass = reader.Read(); + pps.tiles = reader.Read(); + pps.entropyCodingSync = reader.Read(); + + if (pps.numRefIdxL0DefaultMinus1 > 14 || pps.numRefIdxL1DefaultMinus1 > 14) { + return false; + } + + if (pps.tiles) { + pps.numTileColumnsMinus1 = reader.ReadExponentialGolomb(); + pps.numTileRowsMinus1 = reader.ReadExponentialGolomb(); + + if (pps.numTileColumnsMinus1 >= kMaxTiles - 1 || pps.numTileRowsMinus1 >= kMaxTiles + 1) { + return false; + } + + pps.uniformSpacing = reader.Read(); + + if (!pps.uniformSpacing) { + for (uint32_t i = 0; i < pps.numTileColumnsMinus1; i++) { + pps.columnWidthMinus1.push_back(reader.ReadExponentialGolomb()); + } + for (uint32_t i = 0; i < pps.numTileRowsMinus1; i++) { + pps.rowHeightMinus1.push_back(reader.ReadExponentialGolomb()); + } + } + + pps.loopFilterAcrossTiles = reader.Read(); + } + + pps.loopFilterAcrossSlices = reader.Read(); + + if (reader.Read()) { + pps.deblockingFilterOverrideEnabled = reader.Read(); + pps.deblockingFilterDisabled = reader.Read(); + + if (!pps.deblockingFilterDisabled) { + pps.betaOffsetDiv2 = reader.ReadSignedExponentialGolomb(); + pps.tcOffsetDiv2 = reader.ReadSignedExponentialGolomb(); + } + } + + pps.scalingListDataPresent = reader.Read(); + + if (pps.scalingListDataPresent && !webrtc::H265SpsParser::ParseScalingListData(reader)) { + return false; + } + + pps.listsModificationPresent = reader.Read(); + pps.log2ParallelMergeLevelMinus2 = reader.ReadExponentialGolomb(); + pps.sliceHeaderExtensionPresent = reader.Read(); + pps.extensionPresent = reader.Read(); + + if (!reader.Ok()) { + return false; + } + + ppss[id] = std::move(pps); + + return true; + } + + std::optional H265Parser::ParseSliceHeader(std::span nal) + { + if (nal.size() < 3) { + return std::nullopt; + } + + const Rbsp rbsp = Unescape(nal); + webrtc::BitstreamReader reader(rbsp.bytes); + H265SliceHeader header; + + header.nalType = (nal[0] >> 1) & 0x3F; + header.temporalId = static_cast((nal[1] & 0x07) - 1); + header.firstSliceInPicture = reader.Read(); + + const bool irap = header.nalType >= kNalBlaWLp && header.nalType <= kNalReservedIrap23; + const bool idr = header.nalType == kNalIdrWRadl || header.nalType == kNalIdrNLp; + + if (irap) { + header.noOutputOfPriorPics = reader.Read(); + } + + header.ppsId = reader.ReadExponentialGolomb(); + + const H265Pps * pps = Pps(header.ppsId); + const H265Sps * sps = pps != nullptr ? Sps(pps->spsId) : nullptr; + + if (sps == nullptr) { + return std::nullopt; + } + + if (!header.firstSliceInPicture) { + if (pps->dependentSliceSegments) { + header.dependent = reader.Read(); + } + + header.segmentAddress = static_cast( + reader.ReadBits(CeilLog2(sps->WidthInCtbs() * sps->HeightInCtbs()))); + } + + if (header.dependent) { + // The fields of the independent slice segment, but those of this + // one's own parsed below. + if (!lastIndependent || lastIndependent->ppsId != header.ppsId) { + return std::nullopt; + } + + const H265SliceHeader own = header; + + header = *lastIndependent; + header.firstSliceInPicture = own.firstSliceInPicture; + header.dependent = true; + header.segmentAddress = own.segmentAddress; + header.numEntryPoints = 0; + } + else { + reader.ConsumeBits(static_cast(pps->numExtraSliceHeaderBits)); + header.sliceType = reader.ReadExponentialGolomb(); + + if (header.sliceType > kSliceI) { + return std::nullopt; + } + + if (pps->outputFlagPresent) { + header.picOutput = reader.Read(); + } + + if (!idr) { + const int log2MaxPocLsb = static_cast(sps->log2MaxPocLsbMinus4) + 4; + + header.pocLsb = static_cast(reader.ReadBits(log2MaxPocLsb)); + + const bool fromSps = reader.Read(); + + if (!fromSps) { + const int before = ConsumedBits(reader, rbsp.bytes.size()); + + if (!ParseShortTermRps(reader, static_cast(sps->shortTermRps.size()), sps->shortTermRps, + header.shortTermRps)) + { + return std::nullopt; + } + + header.shortTermRpsBits = static_cast(ConsumedBits(reader, rbsp.bytes.size()) - before); + } + else { + if (sps->shortTermRps.empty()) { + return std::nullopt; + } + + uint32_t index = 0; + + if (sps->shortTermRps.size() > 1) { + index = static_cast( + reader.ReadBits(CeilLog2(static_cast(sps->shortTermRps.size())))); + } + if (index >= sps->shortTermRps.size()) { + return std::nullopt; + } + + header.shortTermRps = sps->shortTermRps[index]; + } + + if (sps->longTermRefsPresent) { + uint32_t numFromSps = 0; + + if (!sps->ltRefPicPocLsb.empty()) { + numFromSps = reader.ReadExponentialGolomb(); + } + + const uint32_t numInHeader = reader.ReadExponentialGolomb(); + + if (numFromSps > sps->ltRefPicPocLsb.size() || numFromSps + numInHeader > H265SliceHeader::kMaxRefs) { + return std::nullopt; + } + + const int32_t maxPocLsb = 1 << log2MaxPocLsb; + int32_t msbCycle = 0; + + for (uint32_t i = 0; i < numFromSps + numInHeader; i++) { + int32_t poc = 0; + bool used = false; + + if (i < numFromSps) { + uint32_t index = 0; + + if (sps->ltRefPicPocLsb.size() > 1) { + index = static_cast( + reader.ReadBits(CeilLog2(static_cast(sps->ltRefPicPocLsb.size())))); + } + if (index >= sps->ltRefPicPocLsb.size()) { + return std::nullopt; + } + + poc = static_cast(sps->ltRefPicPocLsb[index]); + used = sps->ltUsedByCurrPic[index]; + } + else { + poc = static_cast(reader.ReadBits(log2MaxPocLsb)); + used = reader.Read(); + } + + const bool msbPresent = reader.Read(); + + if (msbPresent) { + const int32_t cycle = static_cast(reader.ReadExponentialGolomb()); + + // DeltaPocMsbCycleLt (7-52) accumulates within each + // of the two groups. + msbCycle = (i == 0 || i == numFromSps) ? cycle : cycle + msbCycle; + } + else if (i == 0 || i == numFromSps) { + msbCycle = 0; + } + + // With the MSBs, the POC relative to the current + // picture's MSBs, which the decoder adds. + if (msbPresent) { + poc -= msbCycle * maxPocLsb; + } + + header.longTermPoc.push_back(poc); + header.longTermMsbPresent.push_back(msbPresent); + header.longTermUsed.push_back(used); + } + } + + if (sps->temporalMvp) { + header.temporalMvp = reader.Read(); + } + } + + if (sps->sao) { + header.saoLuma = reader.Read(); + header.saoChroma = reader.Read(); + } + + if (!header.IsIntra()) { + header.numRefIdxL0Minus1 = pps->numRefIdxL0DefaultMinus1; + header.numRefIdxL1Minus1 = pps->numRefIdxL1DefaultMinus1; + + if (reader.Read()) { + header.numRefIdxL0Minus1 = reader.ReadExponentialGolomb(); + + if (header.IsB()) { + header.numRefIdxL1Minus1 = reader.ReadExponentialGolomb(); + } + } + + if (header.numRefIdxL0Minus1 > 14 || header.numRefIdxL1Minus1 > 14) { + return std::nullopt; + } + + const uint32_t numPicTotalCurr = header.NumPicTotalCurr(); + + if (numPicTotalCurr == 0) { + return std::nullopt; + } + + if (pps->listsModificationPresent && numPicTotalCurr > 1) { + const int bits = CeilLog2(numPicTotalCurr); + + header.listModificationL0 = reader.Read(); + + if (header.listModificationL0) { + for (uint32_t i = 0; i <= header.numRefIdxL0Minus1; i++) { + header.listEntryL0[i] = static_cast(reader.ReadBits(bits)); + } + } + + if (header.IsB()) { + header.listModificationL1 = reader.Read(); + + if (header.listModificationL1) { + for (uint32_t i = 0; i <= header.numRefIdxL1Minus1; i++) { + header.listEntryL1[i] = static_cast(reader.ReadBits(bits)); + } + } + } + } + + if (header.IsB()) { + header.mvdL1Zero = reader.Read(); + } + if (pps->cabacInitPresent) { + header.cabacInit = reader.Read(); + } + + if (header.temporalMvp) { + if (header.IsB()) { + header.collocatedFromL0 = reader.Read(); + } + + if ((header.collocatedFromL0 && header.numRefIdxL0Minus1 > 0) + || (!header.collocatedFromL0 && header.numRefIdxL1Minus1 > 0)) + { + header.collocatedRefIdx = reader.ReadExponentialGolomb(); + } + } + + if ((pps->weightedPred && header.sliceType == kSliceP) || (pps->weightedBipred && header.IsB())) { + header.lumaLog2WeightDenom = reader.ReadExponentialGolomb(); + header.deltaChromaLog2WeightDenom = reader.ReadSignedExponentialGolomb(); + + const int32_t chromaDenom = static_cast(header.lumaLog2WeightDenom) + + header.deltaChromaLog2WeightDenom; + + if (header.lumaLog2WeightDenom > 7 || chromaDenom < 0 || chromaDenom > 7) { + return std::nullopt; + } + + ParseWeights(reader, header, 0, header.numRefIdxL0Minus1 + 1, true); + + if (header.IsB()) { + ParseWeights(reader, header, 1, header.numRefIdxL1Minus1 + 1, true); + } + } + + header.fiveMinusMaxNumMergeCand = reader.ReadExponentialGolomb(); + } + + header.qpDelta = reader.ReadSignedExponentialGolomb(); + + if (pps->sliceChromaQpOffsetsPresent) { + header.cbQpOffset = reader.ReadSignedExponentialGolomb(); + header.crQpOffset = reader.ReadSignedExponentialGolomb(); + } + + bool deblockingOverride = false; + + if (pps->deblockingFilterOverrideEnabled) { + deblockingOverride = reader.Read(); + } + + if (deblockingOverride) { + header.deblockingFilterDisabled = reader.Read(); + + if (!header.deblockingFilterDisabled) { + header.betaOffsetDiv2 = reader.ReadSignedExponentialGolomb(); + header.tcOffsetDiv2 = reader.ReadSignedExponentialGolomb(); + } + } + else { + header.deblockingFilterDisabled = pps->deblockingFilterDisabled; + header.betaOffsetDiv2 = pps->betaOffsetDiv2; + header.tcOffsetDiv2 = pps->tcOffsetDiv2; + } + + header.loopFilterAcrossSlices = pps->loopFilterAcrossSlices; + + if (pps->loopFilterAcrossSlices + && (header.saoLuma || header.saoChroma || !header.deblockingFilterDisabled)) + { + header.loopFilterAcrossSlices = reader.Read(); + } + } + + if (pps->tiles || pps->entropyCodingSync) { + header.numEntryPoints = reader.ReadExponentialGolomb(); + + if (header.numEntryPoints > sps->WidthInCtbs() * sps->HeightInCtbs()) { + return std::nullopt; + } + + if (header.numEntryPoints > 0) { + const uint32_t offsetLenMinus1 = reader.ReadExponentialGolomb(); + + if (offsetLenMinus1 > 31) { + return std::nullopt; + } + + for (uint32_t i = 0; i < header.numEntryPoints; i++) { + reader.ConsumeBits(static_cast(offsetLenMinus1) + 1); + } + } + } + + if (pps->sliceHeaderExtensionPresent) { + const uint32_t length = reader.ReadExponentialGolomb(); + + if (length > 256) { + return std::nullopt; + } + + reader.ConsumeBits(static_cast(length) * 8); + } + + // byte_alignment(): a one bit and zero bits up to the next byte. + if (reader.ReadBit() != 1) { + return std::nullopt; + } + + const int consumed = ConsumedBits(reader, rbsp.bytes.size()); + reader.ConsumeBits((8 - consumed % 8) % 8); + + if (!reader.Ok()) { + return std::nullopt; + } + + header.dataByteOffset = static_cast( + NalOffset(rbsp, static_cast(ConsumedBits(reader, rbsp.bytes.size()) / 8))); + + if (!header.dependent) { + lastIndependent = header; + } + + return header; + } + + const H265Sps * H265Parser::Sps(uint32_t id) const + { + auto sps = spss.find(id); + + return sps != spss.end() ? &sps->second : nullptr; + } + + const H265Pps * H265Parser::Pps(uint32_t id) const + { + auto pps = ppss.find(id); + + return pps != ppss.end() ? &pps->second : nullptr; + } + + void H265Parser::Reset() + { + spss.clear(); + ppss.clear(); + lastIndependent.reset(); + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp index 48c0cf62..3b05a400 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/LinuxHardwareVideoCodecFactories.cpp @@ -48,15 +48,12 @@ namespace jni std::vector> factories; // NVDEC on NVIDIA GPUs, for H.264, H.265, VP9 and AV1; VA-API for the - // GPUs of Intel and AMD, only if asked for, since it has not been - // tested on such a GPU yet. + // GPUs of Intel and AMD, for VP9 and H.265. if (auto nvdec = NvdecVideoDecoderFactory::Create()) { factories.push_back(std::move(nvdec)); } - if (IsExperimentalVaapiEnabled()) { - if (auto vaapi = VaapiVideoDecoderFactory::Create()) { - factories.push_back(std::move(vaapi)); - } + if (auto vaapi = VaapiVideoDecoderFactory::Create()) { + factories.push_back(std::move(vaapi)); } return factories; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Decoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Decoder.cpp new file mode 100644 index 00000000..c4289bce --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Decoder.cpp @@ -0,0 +1,1106 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/linux/VaapiH265Decoder.h" + +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "common_video/h265/h265_common.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" +#include "third_party/libyuv/include/libyuv/convert.h" + +#include + +namespace jni +{ + namespace + { + // The reference frames VA-API takes. + constexpr size_t kMaxReferenceFrames = 15; + + constexpr uint8_t kInvalidReference = 0xFF; + + // NAL unit types (table 7-1) of the pictures that matter to the + // picture order count. + constexpr uint8_t kNalRadlN = 6; + constexpr uint8_t kNalRaslR = 9; + constexpr uint8_t kNalVclMax = 31; + constexpr uint8_t kNalReservedNonIrapMin = 10; + constexpr uint8_t kNalReservedNonIrapMax = 15; + constexpr uint8_t kNalReservedIrap22 = 22; + + bool IsIrap(uint8_t type) + { + return type >= H265Parser::kNalBlaWLp && type <= H265Parser::kNalReservedIrap23; + } + + bool IsIdr(uint8_t type) + { + return type == H265Parser::kNalIdrWRadl || type == H265Parser::kNalIdrNLp; + } + + bool IsRasl(uint8_t type) + { + return type == H265Parser::kNalRaslN || type == H265Parser::kNalRaslR; + } + + // RADL, RASL and sub-layer non-reference pictures. + bool SkippedForPrevTid0(uint8_t type) + { + return (type >= kNalRadlN && type <= kNalRaslR) || (type <= 14 && type % 2 == 0); + } + + void InvalidatePicture(VAPictureHEVC & picture) + { + picture = {}; + picture.picture_id = VA_INVALID_SURFACE; + picture.flags = VA_PICTURE_HEVC_INVALID; + } + } + + bool VaapiH265Decoder::Picture::IsReference() const + { + return shortTerm || longTerm; + } + + VaapiH265Decoder::VaapiH265Decoder(VaapiLibrary & library) : + library(library), + va(library.Api()), + implementationName("VA-API (" + library.Vendor() + ")"), + callback(nullptr), + config(VA_INVALID_ID), + context(VA_INVALID_ID), + codedWidth(0), + codedHeight(0), + maxDecPicBuffering(0), + maxNumReorder(0), + cropX(0), + cropY(0), + width(0), + height(0), + downloadImage(), + prevTid0Poc(0), + skipRasl(false), + started(false), + failed(false) + { + downloadImage.image_id = VA_INVALID_ID; + } + + VaapiH265Decoder::~VaapiH265Decoder() + { + Release(); + } + + bool VaapiH265Decoder::Check(VAStatus status, const char * operation) const + { + if (status == VA_STATUS_SUCCESS) { + return true; + } + + RTC_LOG(LS_WARNING) << implementationName << " H.265: " << operation << " failed: " + << library.ErrorString(status); + + return false; + } + + bool VaapiH265Decoder::Configure(const Settings & settings) + { + if (settings.codec_type() != webrtc::kVideoCodecH265) { + return false; + } + + Release(); + + RTC_LOG(LS_INFO) << implementationName << " H.265 decoder configured"; + + return true; + } + + int32_t VaapiH265Decoder::Decode(const webrtc::EncodedImage & image, int64_t renderTimeMs) + { + if (callback == nullptr) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (image.size() == 0) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (failed) { + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + // Decoding starts with a key frame. Until one has come, WebRTC is + // asked for it. + if (!started && image.FrameType() != webrtc::VideoFrameType::kVideoFrameKey) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + const std::span data(image.data(), image.size()); + std::vector slices; + + const auto decode = [&]() { + const Result result = DecodePicture(slices, image, renderTimeMs); + + slices.clear(); + + if (result == Result::kMissingReference) { + RTC_LOG(LS_INFO) << implementationName << " H.265: a picture refers to one that is not there; " + "waiting for a key frame"; + + dpb.clear(); + started = false; + + return WEBRTC_VIDEO_CODEC_ERROR; + } + if (result == Result::kFailed) { + failed = true; + + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + return WEBRTC_VIDEO_CODEC_OK; + }; + + for (const webrtc::H265::NaluIndex & index : webrtc::H265::FindNaluIndices(data)) { + const std::span nal = data.subspan(index.payload_start_offset, index.payload_size); + + if (nal.size() < 2) { + continue; + } + + const uint8_t type = (nal[0] >> 1) & 0x3F; + // Only the base layer. + const uint8_t layerId = static_cast(((nal[0] & 1) << 5) | (nal[1] >> 3)); + + if (layerId != 0) { + continue; + } + + if (type == H265Parser::kNalSps || type == H265Parser::kNalPps) { + // The parameter sets of the next picture. + if (!slices.empty()) { + if (const int32_t result = decode(); result != WEBRTC_VIDEO_CODEC_OK) { + return result; + } + } + + const bool parsed = type == H265Parser::kNalSps ? parser.ParseSps(nal) : parser.ParsePps(nal); + + if (!parsed) { + RTC_LOG(LS_INFO) << implementationName << " H.265: a parameter set that is not 8 bit 4:2:0 " + "Main, or one the decoder does not take"; + failed = true; + + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + continue; + } + + if (type == H265Parser::kNalEos) { + // The next picture starts anew, as the first one does. + if (!slices.empty()) { + if (const int32_t result = decode(); result != WEBRTC_VIDEO_CODEC_OK) { + return result; + } + } + + started = false; + continue; + } + + // Reserved picture types are ignored, as are all other NAL units. + if (type > kNalVclMax || type >= kNalReservedIrap22 + || (type >= kNalReservedNonIrapMin && type <= kNalReservedNonIrapMax)) + { + continue; + } + + std::optional header = parser.ParseSliceHeader(nal); + + if (!header) { + RTC_LOG(LS_INFO) << implementationName << " H.265: a slice header that is broken, or refers to " + "a parameter set that is not there"; + failed = true; + + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + if (header->firstSliceInPicture && !slices.empty()) { + if (const int32_t result = decode(); result != WEBRTC_VIDEO_CODEC_OK) { + return result; + } + } + + slices.push_back(Slice { *header, nal }); + } + + if (slices.empty()) { + return WEBRTC_VIDEO_CODEC_OK; + } + + return decode(); + } + + int32_t VaapiH265Decoder::RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback * decodeCallback) + { + callback = decodeCallback; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VaapiH265Decoder::Release() + { + DestroySession(); + + parser.Reset(); + prevTid0Poc = 0; + skipRasl = false; + started = false; + failed = false; + + return WEBRTC_VIDEO_CODEC_OK; + } + + webrtc::VideoDecoder::DecoderInfo VaapiH265Decoder::GetDecoderInfo() const + { + DecoderInfo info; + info.implementation_name = implementationName; + info.is_hardware_accelerated = true; + + return info; + } + + const char * VaapiH265Decoder::ImplementationName() const + { + return implementationName.c_str(); + } + + VaapiH265Decoder::Result VaapiH265Decoder::DecodePicture(const std::vector & slices, + const webrtc::EncodedImage & image, int64_t renderTimeMs) + { + const H265SliceHeader & first = slices.front().header; + + if (!first.firstSliceInPicture) { + return Result::kMissingReference; + } + + const H265Pps * pps = parser.Pps(first.ppsId); + const H265Sps * sps = pps != nullptr ? parser.Sps(pps->spsId) : nullptr; + + if (sps == nullptr) { + return Result::kFailed; + } + + if (sps->scalingListEnabled || pps->scalingListDataPresent || pps->extensionPresent) { + RTC_LOG(LS_INFO) << implementationName << " H.265: a stream with scaling lists or extensions"; + return Result::kFailed; + } + + const uint8_t type = first.nalType; + const bool irap = IsIrap(type); + + // The leading pictures of a CRA picture that decoding starts with + // refer to pictures before it, which are not there. + if (IsRasl(type) && skipRasl) { + return Result::kOk; + } + + if (!irap && !started) { + return Result::kMissingReference; + } + + // An IDR or BLA picture, or a CRA picture that decoding starts with, + // begins a new coded video sequence (NoRaslOutputFlag). + const bool noRaslOutput = irap && (IsIdr(type) || type < H265Parser::kNalCraNut || !started); + + if (irap) { + skipRasl = noRaslOutput; + } + + if (noRaslOutput) { + // The pictures of the sequence before are output, unless the + // picture says not to, and all references are let go. + if (first.noOutputOfPriorPics) { + dpb.clear(); + } + else if (!Bump(0)) { + return Result::kFailed; + } + + dpb.clear(); + + const uint32_t spsMaxDecPicBuffering = sps->maxDecPicBufferingMinus1 + 1; + + if (context == VA_INVALID_ID || sps->width != codedWidth || sps->height != codedHeight + || spsMaxDecPicBuffering != maxDecPicBuffering || sps->maxNumReorderPics != maxNumReorder) + { + if (!CreateSession(*sps)) { + return Result::kFailed; + } + } + } + else if (sps->width != codedWidth || sps->height != codedHeight) { + RTC_LOG(LS_INFO) << implementationName << " H.265: the size changes without a key frame"; + return Result::kFailed; + } + + cropX = sps->cropLeft; + cropY = sps->cropTop; + width = sps->width - sps->cropLeft - sps->cropRight; + height = sps->height - sps->cropTop - sps->cropBottom; + + const int32_t poc = PictureOrderCount(first, noRaslOutput); + const Result rps = ApplyReferencePictureSet(first, *sps, poc); + + if (rps != Result::kOk) { + return rps; + } + + // The pictures neither referenced nor waiting for output are gone; and + // a full buffer makes room, in output order (C.5.2.2). + RemoveUnused(); + + while (dpb.size() >= maxDecPicBuffering) { + const uint32_t waiting = static_cast(std::count_if(dpb.begin(), dpb.end(), + [](const Picture & picture) { + return picture.waitingOutput; + })); + + // Only references are left, which the stream should not have. + if (waiting == 0) { + break; + } + if (!Bump(waiting - 1)) { + return Result::kFailed; + } + } + + const int surface = FreeSurface(); + + if (surface < 0) { + RTC_LOG(LS_WARNING) << implementationName << " H.265: no surface is free"; + return Result::kFailed; + } + + VAPictureParameterBufferHEVC pictureParameters = {}; + FillPicture(pictureParameters, *sps, *pps, first, surface); + pictureParameters.CurrPic.pic_order_cnt = poc; + + std::vector sliceParameters(slices.size()); + + for (size_t i = 0; i < slices.size(); i++) { + if (!FillSlice(sliceParameters[i], slices[i], i + 1 == slices.size())) { + return Result::kMissingReference; + } + } + + if (!Render(pictureParameters, sliceParameters, slices, surface)) { + return Result::kFailed; + } + + Picture picture; + picture.surface = surface; + picture.poc = poc; + picture.shortTerm = true; + picture.waitingOutput = first.picOutput; + picture.rtpTimestamp = image.RtpTimestamp(); + picture.ntpTimeMs = image.ntp_time_ms_; + picture.renderTimeMs = renderTimeMs; + picture.rotation = image.rotation_; + + dpb.push_back(picture); + + if (first.temporalId == 0 && !SkippedForPrevTid0(type)) { + prevTid0Poc = poc; + } + + started = true; + + return Bump(maxNumReorder) ? Result::kOk : Result::kFailed; + } + + int32_t VaapiH265Decoder::PictureOrderCount(const H265SliceHeader & header, bool noRaslOutput) + { + const H265Sps * sps = parser.Sps(parser.Pps(header.ppsId)->spsId); + const int32_t maxPocLsb = 1 << (sps->log2MaxPocLsbMinus4 + 4); + const int32_t pocLsb = static_cast(header.pocLsb); + + if (IsIrap(header.nalType) && noRaslOutput) { + return pocLsb; + } + + // 8.3.1, from the picture order count of prevTid0Pic. + const int32_t prevLsb = prevTid0Poc & (maxPocLsb - 1); + const int32_t prevMsb = prevTid0Poc - prevLsb; + int32_t msb = prevMsb; + + if (pocLsb < prevLsb && prevLsb - pocLsb >= maxPocLsb / 2) { + msb = prevMsb + maxPocLsb; + } + else if (pocLsb > prevLsb && pocLsb - prevLsb > maxPocLsb / 2) { + msb = prevMsb - maxPocLsb; + } + + return msb + pocLsb; + } + + VaapiH265Decoder::Result VaapiH265Decoder::ApplyReferencePictureSet(const H265SliceHeader & header, + const H265Sps & sps, int32_t poc) + { + stCurrBefore.clear(); + stCurrAfter.clear(); + ltCurr.clear(); + + const int32_t maxPocLsb = 1 << (sps.log2MaxPocLsbMinus4 + 4); + const H265ShortTermRps & rps = header.shortTermRps; + // Whether each picture stays a short-term or a long-term reference. + std::vector shortTerm(dpb.size(), false); + std::vector longTerm(dpb.size(), false); + + if (!IsIdr(header.nalType)) { + // The long-term pictures first, by their POC or its LSBs. + for (size_t i = 0; i < header.longTermPoc.size(); i++) { + int32_t ltPoc = header.longTermPoc[i]; + + if (header.longTermMsbPresent[i]) { + ltPoc += poc - (poc & (maxPocLsb - 1)); + } + + int found = -1; + + for (size_t j = 0; j < dpb.size(); j++) { + const int32_t candidate = header.longTermMsbPresent[i] ? dpb[j].poc + : (dpb[j].poc & (maxPocLsb - 1)); + + if (dpb[j].IsReference() && !longTerm[j] && candidate == ltPoc) { + found = static_cast(j); + break; + } + } + + if (found < 0) { + if (header.longTermUsed[i]) { + return Result::kMissingReference; + } + + continue; + } + + longTerm[found] = true; + + if (header.longTermUsed[i]) { + ltCurr.push_back(dpb[found].surface); + } + } + + const auto findShortTerm = [&](int32_t target, bool used, std::vector & list) { + for (size_t j = 0; j < dpb.size(); j++) { + if (dpb[j].shortTerm && !longTerm[j] && !shortTerm[j] && dpb[j].poc == target) { + shortTerm[j] = true; + + if (used) { + list.push_back(dpb[j].surface); + } + + return true; + } + } + + return !used; + }; + + for (int i = 0; i < rps.numNegative; i++) { + if (!findShortTerm(poc + rps.deltaPocS0[i], rps.usedS0[i], stCurrBefore)) { + return Result::kMissingReference; + } + } + for (int i = 0; i < rps.numPositive; i++) { + if (!findShortTerm(poc + rps.deltaPocS1[i], rps.usedS1[i], stCurrAfter)) { + return Result::kMissingReference; + } + } + } + + // The pictures in no subset are no longer references. + for (size_t j = 0; j < dpb.size(); j++) { + dpb[j].shortTerm = shortTerm[j]; + dpb[j].longTerm = longTerm[j]; + } + + return Result::kOk; + } + + bool VaapiH265Decoder::BuildReferenceLists(const H265SliceHeader & header, std::vector & list0, + std::vector & list1) const + { + list0.clear(); + list1.clear(); + + if (header.IsIntra()) { + return true; + } + + const size_t total = stCurrBefore.size() + stCurrAfter.size() + ltCurr.size(); + + if (total == 0) { + return false; + } + + const auto build = [&](uint32_t numActiveMinus1, bool modification, + const std::array & entries, const std::vector & firstSet, + const std::vector & secondSet, std::vector & list) + { + const size_t numTemp = std::max(numActiveMinus1 + 1, total); + std::vector temp; + + while (temp.size() < numTemp) { + for (const std::vector * set : { &firstSet, &secondSet, <Curr }) { + for (int index : *set) { + if (temp.size() < numTemp) { + temp.push_back(index); + } + } + } + } + + for (uint32_t i = 0; i <= numActiveMinus1; i++) { + const uint32_t entry = modification ? entries[i] : i; + + if (entry >= temp.size()) { + return false; + } + + list.push_back(temp[entry]); + } + + return true; + }; + + if (!build(header.numRefIdxL0Minus1, header.listModificationL0, header.listEntryL0, stCurrBefore, stCurrAfter, + list0)) + { + return false; + } + + return !header.IsB() || build(header.numRefIdxL1Minus1, header.listModificationL1, header.listEntryL1, + stCurrAfter, stCurrBefore, list1); + } + + void VaapiH265Decoder::FillPicture(VAPictureParameterBufferHEVC & parameters, const H265Sps & sps, + const H265Pps & pps, const H265SliceHeader & header, int surface) const + { + parameters.CurrPic.picture_id = surfaces[surface]; + parameters.CurrPic.flags = 0; + + for (VAPictureHEVC & reference : parameters.ReferenceFrames) { + InvalidatePicture(reference); + } + + size_t count = 0; + + for (size_t j = 0; j < dpb.size() && count < kMaxReferenceFrames; j++) { + if (!dpb[j].IsReference()) { + continue; + } + + const int index = dpb[j].surface; + VAPictureHEVC & reference = parameters.ReferenceFrames[count++]; + reference.picture_id = surfaces[dpb[j].surface]; + reference.pic_order_cnt = dpb[j].poc; + reference.flags = dpb[j].longTerm ? VA_PICTURE_HEVC_LONG_TERM_REFERENCE : 0; + + if (std::find(stCurrBefore.begin(), stCurrBefore.end(), index) != stCurrBefore.end()) { + reference.flags |= VA_PICTURE_HEVC_RPS_ST_CURR_BEFORE; + } + else if (std::find(stCurrAfter.begin(), stCurrAfter.end(), index) != stCurrAfter.end()) { + reference.flags |= VA_PICTURE_HEVC_RPS_ST_CURR_AFTER; + } + else if (std::find(ltCurr.begin(), ltCurr.end(), index) != ltCurr.end()) { + reference.flags |= VA_PICTURE_HEVC_RPS_LT_CURR; + } + } + + parameters.pic_width_in_luma_samples = static_cast(sps.width); + parameters.pic_height_in_luma_samples = static_cast(sps.height); + + auto & fields = parameters.pic_fields.bits; + fields.chroma_format_idc = sps.chromaFormatIdc; + fields.separate_colour_plane_flag = sps.separateColourPlane ? 1 : 0; + fields.pcm_enabled_flag = sps.pcm ? 1 : 0; + fields.scaling_list_enabled_flag = sps.scalingListEnabled ? 1 : 0; + fields.transform_skip_enabled_flag = pps.transformSkip ? 1 : 0; + fields.amp_enabled_flag = sps.amp ? 1 : 0; + fields.strong_intra_smoothing_enabled_flag = sps.strongIntraSmoothing ? 1 : 0; + fields.sign_data_hiding_enabled_flag = pps.signDataHiding ? 1 : 0; + fields.constrained_intra_pred_flag = pps.constrainedIntraPred ? 1 : 0; + fields.cu_qp_delta_enabled_flag = pps.cuQpDelta ? 1 : 0; + fields.weighted_pred_flag = pps.weightedPred ? 1 : 0; + fields.weighted_bipred_flag = pps.weightedBipred ? 1 : 0; + fields.transquant_bypass_enabled_flag = pps.transquantBypass ? 1 : 0; + fields.tiles_enabled_flag = pps.tiles ? 1 : 0; + fields.entropy_coding_sync_enabled_flag = pps.entropyCodingSync ? 1 : 0; + fields.pps_loop_filter_across_slices_enabled_flag = pps.loopFilterAcrossSlices ? 1 : 0; + fields.loop_filter_across_tiles_enabled_flag = pps.loopFilterAcrossTiles ? 1 : 0; + fields.pcm_loop_filter_disabled_flag = sps.pcmLoopFilterDisabled ? 1 : 0; + + parameters.sps_max_dec_pic_buffering_minus1 = static_cast(sps.maxDecPicBufferingMinus1); + parameters.bit_depth_luma_minus8 = static_cast(sps.bitDepthLumaMinus8); + parameters.bit_depth_chroma_minus8 = static_cast(sps.bitDepthChromaMinus8); + parameters.pcm_sample_bit_depth_luma_minus1 = static_cast(sps.pcmBitDepthLumaMinus1); + parameters.pcm_sample_bit_depth_chroma_minus1 = static_cast(sps.pcmBitDepthChromaMinus1); + parameters.log2_min_luma_coding_block_size_minus3 = static_cast(sps.log2MinCbMinus3); + parameters.log2_diff_max_min_luma_coding_block_size = static_cast(sps.log2DiffMaxMinCb); + parameters.log2_min_transform_block_size_minus2 = static_cast(sps.log2MinTbMinus2); + parameters.log2_diff_max_min_transform_block_size = static_cast(sps.log2DiffMaxMinTb); + parameters.log2_min_pcm_luma_coding_block_size_minus3 = static_cast(sps.log2MinPcmCbMinus3); + parameters.log2_diff_max_min_pcm_luma_coding_block_size = static_cast(sps.log2DiffMaxMinPcmCb); + parameters.max_transform_hierarchy_depth_intra = static_cast(sps.maxTransformDepthIntra); + parameters.max_transform_hierarchy_depth_inter = static_cast(sps.maxTransformDepthInter); + parameters.init_qp_minus26 = static_cast(pps.initQpMinus26); + parameters.diff_cu_qp_delta_depth = static_cast(pps.diffCuQpDeltaDepth); + parameters.pps_cb_qp_offset = static_cast(pps.cbQpOffset); + parameters.pps_cr_qp_offset = static_cast(pps.crQpOffset); + parameters.log2_parallel_merge_level_minus2 = static_cast(pps.log2ParallelMergeLevelMinus2); + + if (pps.tiles) { + const uint32_t columns = pps.numTileColumnsMinus1 + 1; + const uint32_t rows = pps.numTileRowsMinus1 + 1; + const uint32_t widthInCtbs = sps.WidthInCtbs(); + const uint32_t heightInCtbs = sps.HeightInCtbs(); + + parameters.num_tile_columns_minus1 = static_cast(pps.numTileColumnsMinus1); + parameters.num_tile_rows_minus1 = static_cast(pps.numTileRowsMinus1); + + // The width of every column and the height of every row (6-3, 6-4), + // the last ones taking what is left. + uint32_t used = 0; + + for (uint32_t i = 0; i < columns; i++) { + uint32_t columnWidth = pps.uniformSpacing + ? ((i + 1) * widthInCtbs) / columns - (i * widthInCtbs) / columns + : (i + 1 < columns ? pps.columnWidthMinus1[i] + 1 : widthInCtbs - used); + + parameters.column_width_minus1[i] = static_cast(columnWidth - 1); + used += columnWidth; + } + + used = 0; + + for (uint32_t i = 0; i < rows; i++) { + uint32_t rowHeight = pps.uniformSpacing + ? ((i + 1) * heightInCtbs) / rows - (i * heightInCtbs) / rows + : (i + 1 < rows ? pps.rowHeightMinus1[i] + 1 : heightInCtbs - used); + + parameters.row_height_minus1[i] = static_cast(rowHeight - 1); + used += rowHeight; + } + } + + auto & parsing = parameters.slice_parsing_fields.bits; + parsing.lists_modification_present_flag = pps.listsModificationPresent ? 1 : 0; + parsing.long_term_ref_pics_present_flag = sps.longTermRefsPresent ? 1 : 0; + parsing.sps_temporal_mvp_enabled_flag = sps.temporalMvp ? 1 : 0; + parsing.cabac_init_present_flag = pps.cabacInitPresent ? 1 : 0; + parsing.output_flag_present_flag = pps.outputFlagPresent ? 1 : 0; + parsing.dependent_slice_segments_enabled_flag = pps.dependentSliceSegments ? 1 : 0; + parsing.pps_slice_chroma_qp_offsets_present_flag = pps.sliceChromaQpOffsetsPresent ? 1 : 0; + parsing.sample_adaptive_offset_enabled_flag = sps.sao ? 1 : 0; + parsing.deblocking_filter_override_enabled_flag = pps.deblockingFilterOverrideEnabled ? 1 : 0; + parsing.pps_disable_deblocking_filter_flag = pps.deblockingFilterDisabled ? 1 : 0; + parsing.slice_segment_header_extension_present_flag = pps.sliceHeaderExtensionPresent ? 1 : 0; + parsing.RapPicFlag = IsIrap(header.nalType) ? 1 : 0; + parsing.IdrPicFlag = IsIdr(header.nalType) ? 1 : 0; + parsing.IntraPicFlag = IsIrap(header.nalType) ? 1 : 0; + + parameters.log2_max_pic_order_cnt_lsb_minus4 = static_cast(sps.log2MaxPocLsbMinus4); + parameters.num_short_term_ref_pic_sets = static_cast(sps.shortTermRps.size()); + parameters.num_long_term_ref_pic_sps = static_cast(sps.ltRefPicPocLsb.size()); + parameters.num_ref_idx_l0_default_active_minus1 = static_cast(pps.numRefIdxL0DefaultMinus1); + parameters.num_ref_idx_l1_default_active_minus1 = static_cast(pps.numRefIdxL1DefaultMinus1); + parameters.pps_beta_offset_div2 = static_cast(pps.betaOffsetDiv2); + parameters.pps_tc_offset_div2 = static_cast(pps.tcOffsetDiv2); + parameters.num_extra_slice_header_bits = static_cast(pps.numExtraSliceHeaderBits); + parameters.st_rps_bits = header.shortTermRpsBits; + } + + bool VaapiH265Decoder::FillSlice(VASliceParameterBufferHEVC & parameters, const Slice & slice, bool last) const + { + const H265SliceHeader & header = slice.header; + std::vector lists[2]; + + if (!BuildReferenceLists(header, lists[0], lists[1])) { + return false; + } + + parameters.slice_data_size = static_cast(slice.nal.size()); + parameters.slice_data_offset = 0; + parameters.slice_data_flag = VA_SLICE_DATA_FLAG_ALL; + parameters.slice_data_byte_offset = header.dataByteOffset; + parameters.slice_segment_address = header.segmentAddress; + + // The references, as indices into ReferenceFrames, which holds the + // reference pictures in the order of the decoded picture buffer. + std::fill(¶meters.RefPicList[0][0], ¶meters.RefPicList[0][0] + 2 * 15, kInvalidReference); + + for (int list = 0; list < 2; list++) { + for (size_t i = 0; i < lists[list].size() && i < kMaxReferenceFrames; i++) { + const int frameIndex = ReferenceFrameIndex(lists[list][i]); + + if (frameIndex < 0) { + return false; + } + + parameters.RefPicList[list][i] = static_cast(frameIndex); + } + } + + auto & flags = parameters.LongSliceFlags.fields; + flags.LastSliceOfPic = last ? 1 : 0; + flags.dependent_slice_segment_flag = header.dependent ? 1 : 0; + flags.slice_type = header.sliceType; + flags.color_plane_id = 0; + flags.slice_sao_luma_flag = header.saoLuma ? 1 : 0; + flags.slice_sao_chroma_flag = header.saoChroma ? 1 : 0; + flags.mvd_l1_zero_flag = header.mvdL1Zero ? 1 : 0; + flags.cabac_init_flag = header.cabacInit ? 1 : 0; + flags.slice_temporal_mvp_enabled_flag = header.temporalMvp ? 1 : 0; + flags.slice_deblocking_filter_disabled_flag = header.deblockingFilterDisabled ? 1 : 0; + flags.collocated_from_l0_flag = header.collocatedFromL0 ? 1 : 0; + flags.slice_loop_filter_across_slices_enabled_flag = header.loopFilterAcrossSlices ? 1 : 0; + + parameters.collocated_ref_idx = header.temporalMvp ? static_cast(header.collocatedRefIdx) + : kInvalidReference; + parameters.num_ref_idx_l0_active_minus1 = header.IsIntra() ? 0 : static_cast(header.numRefIdxL0Minus1); + parameters.num_ref_idx_l1_active_minus1 = header.IsB() ? static_cast(header.numRefIdxL1Minus1) : 0; + parameters.slice_qp_delta = static_cast(header.qpDelta); + parameters.slice_cb_qp_offset = static_cast(header.cbQpOffset); + parameters.slice_cr_qp_offset = static_cast(header.crQpOffset); + parameters.slice_beta_offset_div2 = static_cast(header.betaOffsetDiv2); + parameters.slice_tc_offset_div2 = static_cast(header.tcOffsetDiv2); + parameters.luma_log2_weight_denom = static_cast(header.lumaLog2WeightDenom); + parameters.delta_chroma_log2_weight_denom = static_cast(header.deltaChromaLog2WeightDenom); + + for (size_t i = 0; i < kMaxReferenceFrames; i++) { + parameters.delta_luma_weight_l0[i] = static_cast(header.deltaLumaWeight[0][i]); + parameters.luma_offset_l0[i] = static_cast(header.lumaOffset[0][i]); + parameters.delta_luma_weight_l1[i] = static_cast(header.deltaLumaWeight[1][i]); + parameters.luma_offset_l1[i] = static_cast(header.lumaOffset[1][i]); + + for (int j = 0; j < 2; j++) { + parameters.delta_chroma_weight_l0[i][j] = static_cast(header.deltaChromaWeight[0][i][j]); + parameters.ChromaOffsetL0[i][j] = static_cast(header.chromaOffset[0][i][j]); + parameters.delta_chroma_weight_l1[i][j] = static_cast(header.deltaChromaWeight[1][i][j]); + parameters.ChromaOffsetL1[i][j] = static_cast(header.chromaOffset[1][i][j]); + } + } + + parameters.five_minus_max_num_merge_cand = static_cast(header.fiveMinusMaxNumMergeCand); + parameters.num_entry_point_offsets = static_cast(header.numEntryPoints); + parameters.entry_offset_to_subset_array = 0; + + return true; + } + + int VaapiH265Decoder::ReferenceFrameIndex(int surface) const + { + int index = 0; + + for (const Picture & picture : dpb) { + if (!picture.IsReference()) { + continue; + } + if (picture.surface == surface) { + return index < static_cast(kMaxReferenceFrames) ? index : -1; + } + + index++; + } + + return -1; + } + + bool VaapiH265Decoder::Render(const VAPictureParameterBufferHEVC & picture, + const std::vector & sliceParameters, const std::vector & slices, + int surface) + { + VADisplay display = library.Display(); + std::vector buffers; + + const auto create = [&](VABufferType type, size_t size, const void * data, const char * operation) { + VABufferID buffer = VA_INVALID_ID; + + if (!Check(va.CreateBuffer(display, context, type, static_cast(size), 1, + const_cast(data), &buffer), operation)) + { + return false; + } + + buffers.push_back(buffer); + + return true; + }; + + bool created = create(VAPictureParameterBufferType, sizeof(picture), &picture, "creating the picture parameters"); + + for (size_t i = 0; created && i < slices.size(); i++) { + created = create(VASliceParameterBufferType, sizeof(sliceParameters[i]), &sliceParameters[i], + "creating the slice parameters") + && create(VASliceDataBufferType, slices[i].nal.size(), slices[i].nal.data(), "creating the slice data"); + } + + bool decoded = false; + + if (created && Check(va.BeginPicture(display, context, surfaces[surface]), "beginning a picture")) { + const bool rendered = Check(va.RenderPicture(display, context, buffers.data(), + static_cast(buffers.size())), "rendering a picture"); + // A picture that was begun has to be ended. + const bool ended = Check(va.EndPicture(display, context), "ending a picture"); + + decoded = rendered && ended && Check(va.SyncSurface(display, surfaces[surface]), "waiting for a picture"); + } + + for (VABufferID buffer : buffers) { + va.DestroyBuffer(display, buffer); + } + + return decoded; + } + + bool VaapiH265Decoder::Bump(uint32_t keep) + { + while (true) { + auto next = dpb.end(); + uint32_t waiting = 0; + + for (auto picture = dpb.begin(); picture != dpb.end(); ++picture) { + if (!picture->waitingOutput) { + continue; + } + + waiting++; + + if (next == dpb.end() || picture->poc < next->poc) { + next = picture; + } + } + + if (waiting <= keep || next == dpb.end()) { + return true; + } + + next->waitingOutput = false; + + if (!Output(*next)) { + return false; + } + + RemoveUnused(); + } + } + + void VaapiH265Decoder::RemoveUnused() + { + dpb.erase(std::remove_if(dpb.begin(), dpb.end(), [](const Picture & picture) { + return !picture.IsReference() && !picture.waitingOutput; + }), dpb.end()); + } + + bool VaapiH265Decoder::Output(const Picture & picture) + { + VADisplay display = library.Display(); + const int frameWidth = static_cast(width); + const int frameHeight = static_cast(height); + + VAImage mapped = {}; + mapped.image_id = VA_INVALID_ID; + bool derived = false; + size_t offsetY = 0; + size_t offsetUv = 0; + + if (downloadImage.image_id != VA_INVALID_ID + && downloadImage.width == width && downloadImage.height == height + && va.GetImage(display, surfaces[picture.surface], static_cast(cropX), static_cast(cropY), + width, height, downloadImage.image_id) == VA_STATUS_SUCCESS) + { + mapped = downloadImage; + } + else if (va.DeriveImage(display, surfaces[picture.surface], &mapped) == VA_STATUS_SUCCESS) { + derived = true; + + if (mapped.format.fourcc != VA_FOURCC_NV12) { + RTC_LOG(LS_WARNING) << implementationName << " H.265: the decoded surface is not NV12"; + va.DestroyImage(display, mapped.image_id); + return false; + } + + // The conformance window within the whole surface. + offsetY = cropY * mapped.pitches[0] + cropX; + offsetUv = (cropY / 2) * mapped.pitches[1] + cropX; + } + else { + RTC_LOG(LS_WARNING) << implementationName << " H.265: the decoded surface cannot be read"; + return false; + } + + void * data = nullptr; + bool converted = false; + webrtc::scoped_refptr i420 = webrtc::I420Buffer::Create(frameWidth, frameHeight); + + if (Check(va.MapBuffer(display, mapped.buf, &data), "mapping a decoded picture")) { + const uint8_t * base = static_cast(data); + + converted = libyuv::NV12ToI420(base + mapped.offsets[0] + offsetY, static_cast(mapped.pitches[0]), + base + mapped.offsets[1] + offsetUv, static_cast(mapped.pitches[1]), + i420->MutableDataY(), i420->StrideY(), i420->MutableDataU(), i420->StrideU(), + i420->MutableDataV(), i420->StrideV(), frameWidth, frameHeight) == 0; + + va.UnmapBuffer(display, mapped.buf); + } + + if (derived) { + va.DestroyImage(display, mapped.image_id); + } + + if (!converted) { + return false; + } + + webrtc::VideoFrame frame = webrtc::VideoFrame::Builder() + .set_video_frame_buffer(i420) + .set_rtp_timestamp(picture.rtpTimestamp) + .set_timestamp_ms(picture.renderTimeMs) + .set_ntp_time_ms(picture.ntpTimeMs) + .set_rotation(picture.rotation) + .build(); + + callback->Decoded(frame, std::nullopt, std::nullopt); + + return true; + } + + bool VaapiH265Decoder::CreateSession(const H265Sps & sps) + { + DestroySession(); + + VADisplay display = library.Display(); + + VAConfigAttrib attribute = {}; + attribute.type = VAConfigAttribRTFormat; + attribute.value = VA_RT_FORMAT_YUV420; + + if (!Check(va.CreateConfig(display, VAProfileHEVCMain, VAEntrypointVLD, &attribute, 1, &config), + "creating the configuration")) + { + config = VA_INVALID_ID; + return false; + } + + // The pictures the stream keeps, and the one being decoded. + const uint32_t count = sps.maxDecPicBufferingMinus1 + 2; + + surfaces.assign(count, VA_INVALID_SURFACE); + + if (!Check(va.CreateSurfaces(display, VA_RT_FORMAT_YUV420, sps.width, sps.height, surfaces.data(), + count, nullptr, 0), "creating surfaces")) + { + surfaces.clear(); + DestroySession(); + return false; + } + + if (!Check(va.CreateContext(display, config, static_cast(sps.width), static_cast(sps.height), + VA_PROGRESSIVE, surfaces.data(), static_cast(count), &context), "creating the context")) + { + context = VA_INVALID_ID; + DestroySession(); + return false; + } + + const uint32_t frameWidth = sps.width - sps.cropLeft - sps.cropRight; + const uint32_t frameHeight = sps.height - sps.cropTop - sps.cropBottom; + + // Copying a surface into an image reads it from cached memory, which + // mapping it directly does not. Where the driver does not copy, the + // surface is mapped. + VAImageFormat format = {}; + format.fourcc = VA_FOURCC_NV12; + format.byte_order = VA_LSB_FIRST; + format.bits_per_pixel = 12; + + if (va.CreateImage(display, &format, static_cast(frameWidth), static_cast(frameHeight), + &downloadImage) != VA_STATUS_SUCCESS) + { + downloadImage.image_id = VA_INVALID_ID; + } + + codedWidth = sps.width; + codedHeight = sps.height; + maxDecPicBuffering = sps.maxDecPicBufferingMinus1 + 1; + maxNumReorder = sps.maxNumReorderPics; + + RTC_LOG(LS_INFO) << implementationName << " H.265 decoding " << frameWidth << "x" << frameHeight; + + return true; + } + + void VaapiH265Decoder::DestroySession() + { + VADisplay display = library.Display(); + + if (downloadImage.image_id != VA_INVALID_ID) { + va.DestroyImage(display, downloadImage.image_id); + downloadImage.image_id = VA_INVALID_ID; + } + if (context != VA_INVALID_ID) { + va.DestroyContext(display, context); + context = VA_INVALID_ID; + } + if (!surfaces.empty()) { + va.DestroySurfaces(display, surfaces.data(), static_cast(surfaces.size())); + surfaces.clear(); + } + if (config != VA_INVALID_ID) { + va.DestroyConfig(display, config); + config = VA_INVALID_ID; + } + + codedWidth = 0; + codedHeight = 0; + maxDecPicBuffering = 0; + maxNumReorder = 0; + dpb.clear(); + stCurrBefore.clear(); + stCurrAfter.clear(); + ltCurr.clear(); + } + + int VaapiH265Decoder::FreeSurface() const + { + for (int surface = 0; surface < static_cast(surfaces.size()); surface++) { + const bool used = std::any_of(dpb.begin(), dpb.end(), [surface](const Picture & picture) { + return picture.surface == surface; + }); + + if (!used) { + return surface; + } + } + + return -1; + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoDecoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoDecoderFactory.cpp index 7d30a0e8..1870d950 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoDecoderFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoDecoderFactory.cpp @@ -14,7 +14,10 @@ * limitations under the License. */ + #include "media/video/codec/linux/VaapiVideoDecoderFactory.h" +#include "media/video/codec/H265Format.h" +#include "media/video/codec/linux/VaapiH265Decoder.h" #include "media/video/codec/linux/VaapiVp9Decoder.h" #include "rtc_base/logging.h" @@ -30,29 +33,49 @@ namespace jni } const bool vp9 = library->Decodes(VAProfileVP9Profile0); + const bool h265 = library->Decodes(VAProfileHEVCMain); - RTC_LOG(LS_INFO) << "VA-API (" << library->Vendor() << ") VP9 decoding: " << vp9; + RTC_LOG(LS_INFO) << "VA-API (" << library->Vendor() << ") VP9 decoding: " << vp9 << ", H.265 decoding: " + << h265; - if (!vp9) { + if (!vp9 && !h265) { return nullptr; } - return std::unique_ptr(new VaapiVideoDecoderFactory(*library)); + return std::unique_ptr(new VaapiVideoDecoderFactory(*library, vp9, h265)); } - VaapiVideoDecoderFactory::VaapiVideoDecoderFactory(VaapiLibrary & library) : - library(library) + VaapiVideoDecoderFactory::VaapiVideoDecoderFactory(VaapiLibrary & library, bool vp9, bool h265) : + library(library), + vp9(vp9), + h265(h265) { } std::vector VaapiVideoDecoderFactory::GetSupportedFormats() const { - return { webrtc::SdpVideoFormat::VP9Profile0() }; + std::vector formats; + + if (vp9) { + formats.push_back(webrtc::SdpVideoFormat::VP9Profile0()); + } + if (h265) { + formats.push_back(CreateH265MainFormat()); + } + + return formats; } std::unique_ptr VaapiVideoDecoderFactory::Create(const webrtc::Environment & env, const webrtc::SdpVideoFormat & format) { - return std::make_unique(library); + if (h265 && format.IsSameCodec(CreateH265MainFormat())) { + return std::make_unique(library); + } + if (vp9 && format.IsSameCodec(webrtc::SdpVideoFormat::VP9Profile0())) { + return std::make_unique(library); + } + + return nullptr; } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java index 9023deaf..814e0dd6 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoDecoderFactory.java @@ -55,10 +55,8 @@ * are decoded on NVIDIA GPUs with NVDEC, each where the GPU decodes it (AV1 * since Ampere), loaded from the NVIDIA driver when the factory is made; a * machine without the driver is not affected. Streams with spatial layers go - * to libvpx and dav1d. VP9 (profile 0) is decoded with VA-API on the GPUs of - * Intel and AMD only with the system property - * {@code webrtc.hardware.vaapiExperimental} set to {@code true}: the decoder - * has not been tested on such a GPU yet. + * to libvpx and dav1d. On the GPUs of Intel and AMD, VP9 (profile 0) and + * H.265 (Main profile) are decoded with VA-API, where the driver decodes them. *

* The supported codecs are those of a {@link DefaultVideoDecoderFactory}, in * the same order. On Windows and Linux, H.264 High 4:4:4 Predictive,