diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index a8d739bf..84ab7eb2 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -30,7 +30,7 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder() | Linux | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) where it encodes H.265 | NVENC on NVIDIA RTX 40 and newer, otherwise the VA-API encoder of the GPU driver where it encodes AV1 (e.g. AMD RDNA3 and newer) | | macOS | VideoToolbox, as with `DefaultVideoEncoderFactory` | VideoToolbox, on Macs with a hardware HEVC encoder, such as all with Apple silicon | Software | -Hardware encoders produce a single layer, but for the VA-API encoders of AV1 and VP9, which encode temporal layers too (`L1T2` and `L1T3`), with a rate control of each layer, so that an SFU can forward a lower frame rate. A stream that asks for spatial layers, or for temporal layers from another hardware encoder, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom. +Hardware encoders produce a single layer, but for the VA-API encoders, which encode temporal layers too (`L1T2` and `L1T3`) in every codec, so that an SFU can forward a lower frame rate. Each layer gets a rate control of its own where the driver has one that works (AMD's for H.265 and AV1, Intel's for VP9); otherwise the layers share one, which treats them alike. A stream that asks for spatial layers, or for temporal layers from another hardware encoder, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom. VP9 can be encoded in hardware too, with the VP9 encoder of the GPU driver, which Intel GPUs have: through VA-API on Linux, and through Media Foundation on Windows. On Windows it is off by default, since it has not been tested on such a GPU yet; turn it on with a system property before the factory is created: diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h b/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h index fbc761fd..81328ac7 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h @@ -48,9 +48,9 @@ namespace jni // or the sequence header of AV1. Encoders put them in front of the first // key frame, not necessarily in front of every one, so they are kept and // put back where missing. The frame dependencies WebRTC needs for AV1 and - // H.265 are those of a stream without layers, unless an AV1 or VP9 encoder - // that makes temporal layers sets its scalability mode and asks for the - // layer of each frame before it encodes it. VP9 frames go out as they + // H.265 are those of a stream without layers, unless an encoder that + // makes temporal layers sets its scalability mode and asks for the layer + // of each frame before it encodes it. VP9 frames go out as they // come, with the VP9 information of a single layer, as the Java encoders // get it, or with temporal layers in flexible mode, as libvpx gives it. class EncoderOutputProcessor @@ -61,8 +61,8 @@ namespace jni // Forgets the parameter sets, for a stream that starts over. void Reset(); - // Sets the layers of the stream, temporal ones of AV1 and VP9 only, - // as in L1T3. Returns false for a mode WebRTC has no structure of. + // Sets the layers of the stream, temporal ones only, as in L1T3. + // Returns false for a mode WebRTC has no structure of. bool SetScalabilityMode(webrtc::ScalabilityMode mode); // Lets the layers follow the bitrate WebRTC gives each. void OnRatesUpdated(const webrtc::VideoBitrateAllocation & bitrates); diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h index 23a4ec43..a4029fa8 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h @@ -35,8 +35,7 @@ namespace jni // parameter buffers of a frame, uploading a frame as NV12, encoding it // and reading what came out. The codec's encoder fills in the parameters. // - // Used by the H.265 and VP9 encoders; the H.264 one keeps its own, older - // copy of the same. + // Used by every VA-API encoder. class VaapiEncodeSession { public: diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH264Encoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH264Encoder.h index a94be3a5..2f7ae31d 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH264Encoder.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH264Encoder.h @@ -19,7 +19,9 @@ #include "media/video/codec/EncoderOutputProcessor.h" #include "media/video/codec/linux/NalUnitWriter.h" +#include "media/video/codec/linux/VaapiEncodeSession.h" #include "media/video/codec/linux/VaapiLibrary.h" +#include "media/video/codec/linux/VaapiTemporalLayers.h" #include "api/video/video_frame.h" #include "api/video_codecs/sdp_video_format.h" @@ -29,6 +31,7 @@ #include #include +#include #include #include #include @@ -38,15 +41,20 @@ namespace jni // Encodes H.264 Constrained Baseline with the GPU driver's VA-API encoder, // as Intel and AMD drivers provide it on Linux. // - // Encoding is synchronous, with one reference frame and P-frames only, so - // each frame is encoded before Encode() returns. The encoder writes the - // parameter sets and slice headers itself and hands them to the driver as - // packed headers, as FFmpeg does, since Intel's iHD driver writes none of - // its own; a driver that takes no packed headers writes them from the - // parameters it is given. A driver that leaves the parameter sets out of a - // key frame, which WebRTC cannot send without them, makes the encoder give - // up, as does anything else that fails, so that the software encoder takes - // over. + // Encoding is synchronous, with P-frames only, so each frame is encoded + // before Encode() returns. The encoder writes the parameter sets and slice + // headers itself and hands them to the driver as packed headers, as FFmpeg + // does, since Intel's iHD driver writes none of its own; a driver that + // takes no packed headers writes them from the parameters it is given. + // Each frame refers to the frame just before it, or, with temporal layers + // (L1T2, L1T3), which need a driver that takes the slice headers, to the + // one of its layer or a lower one that WebRTC's structure of the mode + // names: the top layer is not used for reference, frame_num may skip the + // frames of the layers a receiver goes without, and each slice puts its + // reference first in the list. A driver that leaves the parameter sets out + // of a key frame, which WebRTC cannot send without them, makes the encoder + // give up, as does anything else that fails, so that the software encoder + // takes over. class VaapiH264Encoder : public webrtc::VideoEncoder { public: @@ -61,43 +69,40 @@ namespace jni EncoderInfo GetEncoderInfo() const override; private: - bool CreateSession(); - void DestroySession(); - bool Upload(const webrtc::VideoFrame & frame); - bool Submit(bool idr); - bool ReadOutput(std::vector & output); + // A frame being encoded: its layer, its frame_num and whether it is + // used for reference. + struct Picture + { + VaapiTemporalLayers::Frame frame; + uint32_t frameNum = 0; + bool reference = true; + }; - bool AddBuffer(VABufferType type, void * data, size_t size); - bool AddMiscParameter(VAEncMiscParameterType type, const void * data, size_t size); - bool AddPackedHeader(VAEncPackedHeaderType type, const std::vector & data, size_t bits); - void DestroyFrameBuffers(); + bool Submit(bool idr, const Picture & picture); void FillSequence(VAEncSequenceParameterBufferH264 & sequence) const; - void FillPicture(VAEncPictureParameterBufferH264 & picture, bool idr) const; - void FillSlice(VAEncSliceParameterBufferH264 & slice, bool idr) const; + void FillPicture(VAEncPictureParameterBufferH264 & parameters, bool idr, const Picture & picture) const; + void FillSlice(VAEncSliceParameterBufferH264 & slice, bool idr, const Picture & picture) const; + + // The picture a reference slot holds, as VA-API names it. + VAPictureH264 Reference(int slot) const; + // The order count of a picture, as pic_order_cnt_type 2 derives it. + static int32_t OrderCount(uint32_t frameNum, bool reference); // The SPS and PPS, and the header of the slice, matching the // parameters above. Each returns the bits it appended. size_t WriteParameterSets(std::vector & output) const; - size_t WriteSliceHeader(std::vector & output, bool idr) const; - - bool Check(VAStatus status, const char * operation) const; + size_t WriteSliceHeader(std::vector & output, bool idr, const Picture & picture) const; private: VaapiLibrary & library; - const VaapiFunctions & va; const std::string implementationName; + VaapiEncodeSession session; - VAConfigID config; // The VA_ENC_PACKED_HEADER_* the driver takes from the encoder. uint32_t packedHeaders; - VAContextID context; - VASurfaceID inputSurface; - // The frame being encoded, and the one it refers to. - VASurfaceID reconstructed[2]; - VABufferID codedBuffer; - std::vector frameBuffers; + bool initialized; webrtc::VideoCodec codecSettings; uint32_t widthInMbs; uint32_t heightInMbs; @@ -105,11 +110,13 @@ namespace jni uint32_t framerate; bool ratesChanged; - // The frame number within the current IDR period, and which of - // the reconstructed surfaces holds the reference. - uint32_t frameNum; + // The frame_num of the last reference picture, and the temporal + // layers with the frame_num of the picture each reference slot + // holds. + uint32_t previousReferenceFrameNum; uint32_t idrPicId; - int current; + VaapiTemporalLayers layers; + std::array slotFrameNums; bool referenceValid; webrtc::EncodedImageCallback * callback; diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Encoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Encoder.h index 553dd443..c4d15077 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Encoder.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Encoder.h @@ -21,6 +21,7 @@ #include "media/video/codec/linux/NalUnitWriter.h" #include "media/video/codec/linux/VaapiEncodeSession.h" #include "media/video/codec/linux/VaapiLibrary.h" +#include "media/video/codec/linux/VaapiTemporalLayers.h" #include "api/video/video_frame.h" #include "api/video_codecs/sdp_video_format.h" @@ -30,6 +31,7 @@ #include #include +#include #include #include #include @@ -45,7 +47,13 @@ namespace jni // headers for a driver that takes them, as Intel's and AMD's do. A driver // that predicts only with both reference lists filled, as Intel's // low-power encoder, gets each P-frame as a B-frame whose two lists hold - // the same previous picture ("low delay B"), as FFmpeg does. The block + // the same previous picture ("low delay B"), as FFmpeg does. With + // temporal layers (L1T2, L1T3), which need a driver that takes the slice + // headers, a picture refers to the one of its layer or a lower one that + // WebRTC's structure of the mode names, its slice NAL unit carries its + // temporal id, and its reference picture set keeps the pictures of the + // reference slots of its layer and below, so that a receiver without the + // upper layers misses none of them. The block // sizes and coding tools are those the driver says it takes, as FFmpeg's // VA-API encoder chooses them. The frame size is kept to a multiple of the // smallest coding block; the coded picture is that aligned to 16 and at @@ -70,21 +78,31 @@ namespace jni private: // What the driver says about the HEVC encoder it has. void QueryCapabilities(); - bool Submit(bool idr); + bool Submit(bool idr, const VaapiTemporalLayers::Frame & frame); void FillSequence(VAEncSequenceParameterBufferHEVC & sequence) const; - void FillPicture(VAEncPictureParameterBufferHEVC & picture, bool idr) const; - void FillSlice(VAEncSliceParameterBufferHEVC & slice, bool idr) const; + void FillPicture(VAEncPictureParameterBufferHEVC & picture, bool idr, + const VaapiTemporalLayers::Frame & frame) const; + void FillSlice(VAEncSliceParameterBufferHEVC & slice, bool idr, const VaapiTemporalLayers::Frame & frame) const; // The VPS, SPS and PPS, and the slice segment header, matching // the parameters above. Each returns the bits it appended. size_t WriteParameterSets(std::vector & output) const; - size_t WriteSliceHeader(std::vector & output, bool idr) const; + size_t WriteSliceHeader(std::vector & output, bool idr, const VaapiTemporalLayers::Frame & frame) const; void WriteProfileTierLevel(NalUnitWriter & writer) const; // diff_cu_qp_delta_depth. uint32_t CuQpDeltaDepth() const; - // The picture the current one refers to. - VAPictureHEVC Reference() const; + // The pictures the reference slots hold, newest first, each once, + // and whether the current frame refers to it: its reference + // picture set. + struct ReferencePicture + { + int slot; + int32_t pictureOrder; + bool used; + }; + std::vector ReferencePictureSet(const VaapiTemporalLayers::Frame & frame) const; + VAPictureHEVC Picture(const ReferencePicture & reference) const; private: VaapiLibrary & library; @@ -122,10 +140,13 @@ namespace jni uint32_t framerate; bool ratesChanged; - // The picture order count since the last IDR picture, and which of - // the reconstructed surfaces holds the current picture. + // The picture order count since the last IDR picture, the temporal + // layers, and the picture order count and temporal id of the + // picture each of their reference slots holds. int32_t pictureOrder; - int current; + VaapiTemporalLayers layers; + std::array slotOrders; + std::array slotTemporalIds; bool referenceValid; webrtc::EncodedImageCallback * callback; diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiTemporalLayers.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiTemporalLayers.h index 769fc7ea..67197f98 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiTemporalLayers.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiTemporalLayers.h @@ -18,10 +18,12 @@ #define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_TEMPORAL_LAYERS_H_ #include "media/video/codec/linux/VaapiEncodeSession.h" +#include "media/video/codec/linux/VaapiLibrary.h" #include "api/video/video_bitrate_allocation.h" #include "api/video_codecs/scalability_mode.h" #include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/svc/scalability_mode_util.h" #include "modules/video_coding/svc/scalable_video_controller.h" #include @@ -36,7 +38,10 @@ namespace jni // it replaces, as WebRTC's structure of the mode says, each buffer of the // structure being the slot of the same number; which reconstructed surface // each slot holds; and the rate control of each layer, without which - // AMD's driver gives the upper layers next to no bits. + // AMD's driver gives the upper layers next to no bits, where the driver + // says it has one and the encoder wants it. Intel's H.264 and H.265 + // encoders say not, and refuse the layer structure that goes with it; + // their rate control treats every layer alike. class VaapiTemporalLayers { public: @@ -52,17 +57,28 @@ namespace jni // The slots of its references, the first being its last frame, // the second, if any, its golden frame. std::vector references; + // The slots it replaces, all for a key frame, and of those the + // ones WebRTC's structure goes on to use: a key frame leaves + // the others empty, which nothing refers to then, and which a + // codec would otherwise keep as references for good. uint8_t refreshFlags = 0; + uint8_t keptFlags = 0; // The reconstructed surface it is encoded into. int surface = -1; }; VaapiTemporalLayers(); + // Whether the driver's encoder of the profile has a rate control + // of each temporal layer, for as many layers as given. + static bool LayerRateControl(VaapiLibrary & library, VAProfile profile, VAEntrypoint entrypoint, + int layers); + // Starts over in the mode, splitting the bitrate across the layers - // until WebRTC gives its own split. Returns false for a mode other - // than L1T1, L1T2 and L1T3. - bool Configure(webrtc::ScalabilityMode mode, uint32_t bitrateBps); + // until WebRTC gives its own split, for a rate control of each layer + // where there is one. Returns false for a mode other than L1T1, L1T2 + // and L1T3. + bool Configure(webrtc::ScalabilityMode mode, uint32_t bitrateBps, bool layerRateControl); // The number of temporal layers, one without layers. int Count() const; @@ -74,6 +90,8 @@ namespace jni const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame) const; // Takes the frame, once encoded, into the slots it replaces. void Commit(const Frame & frame); + // Empties a slot whose picture the decoder no longer keeps. + void Forget(int slot); // The slots kept, and the reconstructed surface each holds, -1 for // none. A key frame fills the slots beyond too, but no frame refers @@ -93,6 +111,7 @@ namespace jni private: int count; + bool layerRateControl; std::array slots; // The bitrate of each layer, counting the layers below it. std::vector bitratesBps; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp index c401dbdd..86e40ebe 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp @@ -394,12 +394,28 @@ namespace jni info.codecSpecific.H264.temporal_idx = webrtc::kNoTemporalIdx; info.codecSpecific.H264.base_layer_sync = false; info.codecSpecific.H264.idr_frame = keyFrame; - return; + + if (scalabilityMode == webrtc::ScalabilityMode::kL1T1) { + return; + } + + // With temporal layers, the layer, and whether an upper layer frame + // refers to the base layer only, as OpenH264 gives them; the frame + // dependencies follow below. The base layer is buffer 0 in WebRTC's + // structures. + bool baseOnly = true; + + for (const webrtc::CodecBufferUsage & buffer : layerFrame.Buffers()) { + baseOnly &= !buffer.referenced || buffer.id == 0; + } + + info.codecSpecific.H264.temporal_idx = static_cast(layerFrame.TemporalId()); + info.codecSpecific.H264.base_layer_sync = layerFrame.TemporalId() > 0 && baseOnly; } // AV1 and H.265 have no codec specific information; WebRTC sends - // their frame dependencies, and those of VP9, from these, in the - // dependency descriptor. + // their frame dependencies, and those of VP9 and of H.264 with layers, + // from these, in the dependency descriptor. info.generic_frame_info = svcController->OnEncodeDone(layerFrame); info.scalability_mode = scalabilityMode; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp index dc61ffc7..eecfb1b1 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp @@ -350,7 +350,8 @@ namespace jni referenceValid = false; frameNumber = 0; outputProcessor.Reset(); - layers.Configure(mode, bitrateBps); + layers.Configure(mode, bitrateBps, VaapiTemporalLayers::LayerRateControl(library, VAProfileAV1Profile0, + entrypoint, webrtc::ScalabilityModeToNumTemporalLayers(mode))); if (!outputProcessor.SetScalabilityMode(mode)) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; @@ -436,6 +437,7 @@ namespace jni session.DropBuffers(); + if (!encoded) { return kEncodeFailed; } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Encoder.cpp index 97f9840c..ad8f2437 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Encoder.cpp @@ -17,15 +17,11 @@ #include "media/video/codec/linux/VaapiH264Encoder.h" #include "api/video/encoded_image.h" -#include "api/video/i420_buffer.h" #include "modules/video_coding/include/video_codec_interface.h" #include "modules/video_coding/include/video_error_codes.h" #include "rtc_base/logging.h" -#include "third_party/libyuv/include/libyuv/convert_from.h" #include -#include -#include namespace jni { @@ -44,7 +40,7 @@ namespace jni constexpr uint8_t kSliceTypeI = 2; // H.264 NAL unit types (table 7-1), and the nal_ref_idc of everything - // the encoder writes, all of which is used for reference. + // the encoder writes that is used for reference. constexpr uint32_t kNalSlice = 1; constexpr uint32_t kNalIdrSlice = 5; constexpr uint32_t kNalSps = 7; @@ -60,10 +56,10 @@ namespace jni constexpr uint32_t kInitialQp = 26; - // How many seconds of video the rate control may buffer, as a - // multiple of the bitrate. - constexpr uint32_t kHrdBufferNumerator = 3; - constexpr uint32_t kHrdBufferDenominator = 2; + // modification_of_pic_nums_idc: subtract from the predicted picture + // number, or end the list of modifications. + constexpr uint32_t kSubtractPicNum = 0; + constexpr uint32_t kEndModifications = 3; // The lowest H.264 level whose limits a stream fits in, going by its // frame size and macroblock rate, per table A-1 of the standard. @@ -107,27 +103,32 @@ namespace jni VaapiH264Encoder::VaapiH264Encoder(VaapiLibrary & library, const webrtc::SdpVideoFormat & format) : library(library), - va(library.Api()), implementationName("VA-API (" + library.Vendor() + ")"), - config(VA_INVALID_ID), - packedHeaders(0), - context(VA_INVALID_ID), - inputSurface(VA_INVALID_SURFACE), - reconstructed{ VA_INVALID_SURFACE, VA_INVALID_SURFACE }, - codedBuffer(VA_INVALID_ID), + session(library, "VA-API H.264"), + packedHeaders(VA_ENC_PACKED_HEADER_NONE), + initialized(false), codecSettings(), widthInMbs(0), heightInMbs(0), bitrateBps(0), framerate(30), ratesChanged(false), - frameNum(0), + previousReferenceFrameNum(0), idrPicId(0), - current(0), + slotFrameNums{ 0, 0 }, referenceValid(false), callback(nullptr), outputProcessor(webrtc::kVideoCodecH264, format) { + VAConfigAttrib packed = {}; + packed.type = VAConfigAttribEncPackedHeaders; + + if (library.Api().GetConfigAttributes(library.Display(), VAProfileH264ConstrainedBaseline, library.Entrypoint(), + &packed, 1) == VA_STATUS_SUCCESS && packed.value != VA_ATTRIB_NOT_SUPPORTED + && (packed.value & kPackedHeaders) == kPackedHeaders) + { + packedHeaders = kPackedHeaders; + } } VaapiH264Encoder::~VaapiH264Encoder() @@ -135,17 +136,6 @@ namespace jni Release(); } - bool VaapiH264Encoder::Check(VAStatus status, const char * operation) const - { - if (status == VA_STATUS_SUCCESS) { - return true; - } - - RTC_LOG(LS_WARNING) << "VA-API: " << operation << " failed: " << library.ErrorString(status); - - return false; - } - int32_t VaapiH264Encoder::InitEncode(const webrtc::VideoCodec * settings, const Settings & encoderSettings) { if (settings == nullptr || settings->width == 0 || settings->height == 0) { @@ -165,6 +155,16 @@ namespace jni return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } + // Temporal layers, where the encoder writes the slice headers, which + // carry what makes the layers; the rest is left to OpenH264. + const webrtc::ScalabilityMode mode = settings->GetScalabilityMode().value_or(webrtc::ScalabilityMode::kL1T1); + + if (mode != webrtc::ScalabilityMode::kL1T1 && (packedHeaders != kPackedHeaders + || (mode != webrtc::ScalabilityMode::kL1T2 && mode != webrtc::ScalabilityMode::kL1T3))) + { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + Release(); codecSettings = *settings; @@ -175,89 +175,35 @@ namespace jni bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; framerate = std::max(1u, codecSettings.maxFramerate); ratesChanged = true; - frameNum = 0; - current = 0; + previousReferenceFrameNum = 0; referenceValid = false; outputProcessor.Reset(); + // One rate control for all layers, which then treats them alike: + // Intel's driver has none of each layer for H.264, and AMD's, which + // says it has, overshoots the bitrate by half again with it. + layers.Configure(mode, bitrateBps, false); - if (!CreateSession()) { - Release(); - return WEBRTC_VIDEO_CODEC_ERROR; - } - - RTC_LOG(LS_INFO) << implementationName << " initialized: " << codecSettings.width << "x" - << codecSettings.height << " at " << bitrateBps << " bps"; - - return WEBRTC_VIDEO_CODEC_OK; - } - - bool VaapiH264Encoder::CreateSession() - { - VADisplay display = library.Display(); - - VAConfigAttrib packed = {}; - packed.type = VAConfigAttribEncPackedHeaders; - - packedHeaders = 0; - - if (va.GetConfigAttributes(display, VAProfileH264ConstrainedBaseline, library.Entrypoint(), &packed, 1) == - VA_STATUS_SUCCESS && packed.value != VA_ATTRIB_NOT_SUPPORTED - && (packed.value & kPackedHeaders) == kPackedHeaders) - { - packedHeaders = kPackedHeaders; - } - - VAConfigAttrib attributes[3] = {}; - attributes[0].type = VAConfigAttribRTFormat; - attributes[0].value = VA_RT_FORMAT_YUV420; - attributes[1].type = VAConfigAttribRateControl; - attributes[1].value = VA_RC_CBR; - attributes[2].type = VAConfigAttribEncPackedHeaders; - attributes[2].value = packedHeaders; - - const int attributeCount = packedHeaders != 0 ? 3 : 2; - - if (!Check(va.CreateConfig(display, VAProfileH264ConstrainedBaseline, library.Entrypoint(), attributes, - attributeCount, &config), "creating the configuration")) - { - config = VA_INVALID_ID; - return false; + if (!outputProcessor.SetScalabilityMode(mode)) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } // Whole macroblocks; the stream is cropped to the frame size. - const unsigned int width = widthInMbs * 16; - const unsigned int height = heightInMbs * 16; - - VASurfaceID surfaces[3] = { VA_INVALID_SURFACE, VA_INVALID_SURFACE, VA_INVALID_SURFACE }; + const uint32_t width = widthInMbs * 16; + const uint32_t height = heightInMbs * 16; - if (!Check(va.CreateSurfaces(display, VA_RT_FORMAT_YUV420, width, height, surfaces, 3, nullptr, 0), - "creating surfaces")) + if (!session.Create(VAProfileH264ConstrainedBaseline, library.Entrypoint(), width, height, width, height, + packedHeaders, VaapiTemporalLayers::kReconstructedSurfaces)) { - return false; - } - - inputSurface = surfaces[0]; - reconstructed[0] = surfaces[1]; - reconstructed[1] = surfaces[2]; - - if (!Check(va.CreateContext(display, config, width, height, VA_PROGRESSIVE, surfaces, 3, &context), - "creating the context")) - { - context = VA_INVALID_ID; - return false; + Release(); + return WEBRTC_VIDEO_CODEC_ERROR; } - // Room for a frame that does not compress at all. - const unsigned int codedSize = width * height * 3 / 2 + 64 * 1024; + initialized = true; - if (!Check(va.CreateBuffer(display, context, VAEncCodedBufferType, codedSize, 1, nullptr, &codedBuffer), - "creating the output buffer")) - { - codedBuffer = VA_INVALID_ID; - return false; - } + RTC_LOG(LS_INFO) << implementationName << " initialized: " << codecSettings.width << "x" + << codecSettings.height << " at " << bitrateBps << " bps, " << webrtc::ScalabilityModeToString(mode); - return true; + return WEBRTC_VIDEO_CODEC_OK; } int32_t VaapiH264Encoder::RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback * encodeCallback) @@ -269,48 +215,18 @@ namespace jni int32_t VaapiH264Encoder::Release() { - DestroySession(); - - return WEBRTC_VIDEO_CODEC_OK; - } - - void VaapiH264Encoder::DestroySession() - { - VADisplay display = library.Display(); - - DestroyFrameBuffers(); - - if (codedBuffer != VA_INVALID_ID) { - va.DestroyBuffer(display, codedBuffer); - codedBuffer = VA_INVALID_ID; - } - if (context != VA_INVALID_ID) { - va.DestroyContext(display, context); - context = VA_INVALID_ID; - } - - VASurfaceID surfaces[3] = { inputSurface, reconstructed[0], reconstructed[1] }; - - if (inputSurface != VA_INVALID_SURFACE) { - va.DestroySurfaces(display, surfaces, 3); - } - - inputSurface = VA_INVALID_SURFACE; - reconstructed[0] = VA_INVALID_SURFACE; - reconstructed[1] = VA_INVALID_SURFACE; - - if (config != VA_INVALID_ID) { - va.DestroyConfig(display, config); - config = VA_INVALID_ID; - } + session.Destroy(); + initialized = false; referenceValid = false; + + return WEBRTC_VIDEO_CODEC_OK; } int32_t VaapiH264Encoder::Encode(const webrtc::VideoFrame & frame, const std::vector * frameTypes) { - if (context == VA_INVALID_ID || callback == nullptr) { + if (!initialized || callback == nullptr) { return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } @@ -322,18 +238,42 @@ namespace jni }); } - if (idr) { - frameNum = 0; + const auto layerFrame = outputProcessor.NextFrameConfig(idr); + + // No layer is encoded now, as when WebRTC gives none a bitrate. + if (!layerFrame) { + return WEBRTC_VIDEO_CODEC_OK; } - std::vector output; + const auto layerPlan = layers.Plan(idr, *layerFrame); - if (!Upload(frame) || !Submit(idr) || !ReadOutput(output)) { - DestroyFrameBuffers(); + if (!layerPlan) { + RTC_LOG(LS_WARNING) << implementationName << " H.264: a frame refers to a slot that holds none"; return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; } - DestroyFrameBuffers(); + // frame_num goes up after each reference picture only, so that a + // receiver misses no frame_num for a picture of the top layer it goes + // without. + Picture picture; + picture.frame = *layerPlan; + picture.reference = idr || layerPlan->refreshFlags != 0; + picture.frameNum = idr ? 0 : (previousReferenceFrameNum + 1) % kMaxFrameNum; + + std::vector output; + + // WebRTC initializes the encoder again when the frame size changes, + // so a frame of another size is one that raced with that. An odd frame + // is cropped to the even size the encoder took, which needs no copy. + const bool encoded = session.Upload(frame, static_cast(codecSettings.width), + static_cast(codecSettings.height), 1) && Submit(idr, picture) && session.ReadOutput(output); + + session.DropBuffers(); + + + if (!encoded) { + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } webrtc::EncodedImage image; image._encodedWidth = codecSettings.width; @@ -347,14 +287,24 @@ namespace jni // A driver that writes no parameter sets into key frames makes a // stream nothing can decode; better in software then. - if (!outputProcessor.Process(output, idr, image, info)) { + if (!outputProcessor.Process(output, idr, *layerFrame, image, info)) { return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; } - // The frame just encoded is the reference of the next one. + // The slots the frame replaces hold it now. + layers.Commit(*layerPlan); + + for (int slot = 0; slot < layers.SlotCount(); slot++) { + if (layerPlan->keptFlags & (1 << slot)) { + slotFrameNums[slot] = picture.frameNum; + } + } + + if (picture.reference) { + previousReferenceFrameNum = picture.frameNum; + } + referenceValid = true; - current = 1 - current; - frameNum = (frameNum + 1) % kMaxFrameNum; ratesChanged = false; if (idr) { @@ -366,120 +316,36 @@ namespace jni return WEBRTC_VIDEO_CODEC_OK; } - bool VaapiH264Encoder::Upload(const webrtc::VideoFrame & frame) - { - webrtc::scoped_refptr i420 = frame.video_frame_buffer()->ToI420(); - - if (!i420) { - return false; - } - - const int width = static_cast(codecSettings.width); - const int height = static_cast(codecSettings.height); - - // WebRTC initializes the encoder again when the frame size changes, - // so a frame of another size is one that raced with that. An odd frame - // is cropped to the even size the encoder took, which needs no copy. - const bool cropped = i420->width() - width >= 0 && i420->width() - width <= 1 - && i420->height() - height >= 0 && i420->height() - height <= 1; - - if (!cropped) { - webrtc::scoped_refptr scaled = webrtc::I420Buffer::Create(width, height); - scaled->ScaleFrom(*i420); - i420 = scaled; - } - - VADisplay display = library.Display(); - VAImage image = {}; - image.image_id = VA_INVALID_ID; - - // Writing into the surface directly where the driver allows it, - // otherwise into an image that is copied into the surface. - bool derived = va.DeriveImage(display, inputSurface, &image) == VA_STATUS_SUCCESS - && image.format.fourcc == VA_FOURCC_NV12; - - if (!derived) { - if (image.image_id != VA_INVALID_ID) { - va.DestroyImage(display, image.image_id); - } - - VAImageFormat format = {}; - format.fourcc = VA_FOURCC_NV12; - format.byte_order = VA_LSB_FIRST; - format.bits_per_pixel = 12; - - if (!Check(va.CreateImage(display, &format, width, height, &image), "creating an image")) { - return false; - } - } - - void * data = nullptr; - bool uploaded = Check(va.MapBuffer(display, image.buf, &data), "mapping an image"); - - if (uploaded) { - uint8_t * base = static_cast(data); - - libyuv::I420ToNV12(i420->DataY(), i420->StrideY(), i420->DataU(), i420->StrideU(), - i420->DataV(), i420->StrideV(), - base + image.offsets[0], static_cast(image.pitches[0]), - base + image.offsets[1], static_cast(image.pitches[1]), - width, height); - - va.UnmapBuffer(display, image.buf); - - if (!derived) { - uploaded = Check(va.PutImage(display, inputSurface, image.image_id, 0, 0, width, height, - 0, 0, width, height), "copying an image"); - } - } - - va.DestroyImage(display, image.image_id); - - return uploaded; - } - - bool VaapiH264Encoder::Submit(bool idr) + bool VaapiH264Encoder::Submit(bool idr, const Picture & picture) { if (idr) { VAEncSequenceParameterBufferH264 sequence = {}; FillSequence(sequence); - if (!AddBuffer(VAEncSequenceParameterBufferType, &sequence, sizeof(sequence))) { + if (!session.AddBuffer(VAEncSequenceParameterBufferType, &sequence, sizeof(sequence))) { return false; } } if (idr || ratesChanged) { - VAEncMiscParameterRateControl rateControl = {}; - rateControl.bits_per_second = bitrateBps; - rateControl.target_percentage = 100; - rateControl.window_size = 1000; - rateControl.initial_qp = kInitialQp; - - VAEncMiscParameterFrameRate frameRate = {}; - // The numerator in the low 16 bits, the denominator in the high. - frameRate.framerate = framerate | (1u << 16); - - VAEncMiscParameterHRD hrd = {}; - hrd.buffer_size = static_cast( - static_cast(bitrateBps) * kHrdBufferNumerator / kHrdBufferDenominator); - hrd.initial_buffer_fullness = hrd.buffer_size / 2; - - if (!AddMiscParameter(VAEncMiscParameterTypeRateControl, &rateControl, sizeof(rateControl)) || - !AddMiscParameter(VAEncMiscParameterTypeFrameRate, &frameRate, sizeof(frameRate)) || - !AddMiscParameter(VAEncMiscParameterTypeHRD, &hrd, sizeof(hrd))) - { + // Without layers, from the initial QP the encoder has always + // started from. + const bool added = layers.Count() == 1 + ? session.AddRateControl(bitrateBps, framerate, kInitialQp) + : layers.AddRateControl(session, bitrateBps, framerate, idr); + + if (!added) { return false; } } - VAEncPictureParameterBufferH264 picture = {}; - FillPicture(picture, idr); + VAEncPictureParameterBufferH264 parameters = {}; + FillPicture(parameters, idr, picture); VAEncSliceParameterBufferH264 slice = {}; - FillSlice(slice, idr); + FillSlice(slice, idr, picture); - if (!AddBuffer(VAEncPictureParameterBufferType, &picture, sizeof(picture))) { + if (!session.AddBuffer(VAEncPictureParameterBufferType, ¶meters, sizeof(parameters))) { return false; } @@ -487,117 +353,21 @@ namespace jni std::vector parameterSets; const size_t bits = WriteParameterSets(parameterSets); - if (!AddPackedHeader(VAEncPackedHeaderSequence, parameterSets, bits)) { + if (!session.AddPackedHeader(VAEncPackedHeaderSequence, parameterSets, bits)) { return false; } } if (packedHeaders & VA_ENC_PACKED_HEADER_SLICE) { std::vector sliceHeader; - const size_t bits = WriteSliceHeader(sliceHeader, idr); + const size_t bits = WriteSliceHeader(sliceHeader, idr, picture); - if (!AddPackedHeader(VAEncPackedHeaderSlice, sliceHeader, bits)) { + if (!session.AddPackedHeader(VAEncPackedHeaderSlice, sliceHeader, bits)) { return false; } } - if (!AddBuffer(VAEncSliceParameterBufferType, &slice, sizeof(slice))) { - return false; - } - - VADisplay display = library.Display(); - - if (!Check(va.BeginPicture(display, context, inputSurface), "beginning a picture")) { - return false; - } - - bool rendered = Check(va.RenderPicture(display, context, frameBuffers.data(), - static_cast(frameBuffers.size())), "rendering a picture"); - - // A picture that was begun has to be ended. - bool ended = Check(va.EndPicture(display, context), "ending a picture"); - - return rendered && ended && Check(va.SyncSurface(display, inputSurface), "waiting for a picture"); - } - - bool VaapiH264Encoder::ReadOutput(std::vector & output) - { - VADisplay display = library.Display(); - void * data = nullptr; - - if (!Check(va.MapBuffer(display, codedBuffer, &data), "mapping the output")) { - return false; - } - - bool complete = true; - - for (auto * segment = static_cast(data); segment != nullptr; - segment = static_cast(segment->next)) - { - if (segment->status & VA_CODED_BUF_STATUS_SLICE_OVERFLOW_MASK) { - complete = false; - } - - const uint8_t * bytes = static_cast(segment->buf); - output.insert(output.end(), bytes, bytes + segment->size); - } - - va.UnmapBuffer(display, codedBuffer); - - if (!complete || output.empty()) { - RTC_LOG(LS_WARNING) << "VA-API: a frame did not fit the output buffer"; - return false; - } - - return true; - } - - bool VaapiH264Encoder::AddBuffer(VABufferType type, void * data, size_t size) - { - VABufferID buffer = VA_INVALID_ID; - - if (!Check(va.CreateBuffer(library.Display(), context, type, static_cast(size), 1, data, - &buffer), "creating a parameter buffer")) - { - return false; - } - - frameBuffers.push_back(buffer); - - return true; - } - - bool VaapiH264Encoder::AddMiscParameter(VAEncMiscParameterType type, const void * data, size_t size) - { - // A VAEncMiscParameterBuffer: the type, followed by the parameter. - std::vector buffer(sizeof(VAEncMiscParameterBuffer) + size); - - auto * header = reinterpret_cast(buffer.data()); - header->type = type; - std::memcpy(header->data, data, size); - - return AddBuffer(VAEncMiscParameterBufferType, buffer.data(), buffer.size()); - } - - bool VaapiH264Encoder::AddPackedHeader(VAEncPackedHeaderType type, const std::vector & data, - size_t bits) - { - VAEncPackedHeaderParameterBuffer parameters = {}; - parameters.type = type; - parameters.bit_length = static_cast(bits); - parameters.has_emulation_bytes = 1; - - return AddBuffer(VAEncPackedHeaderParameterBufferType, ¶meters, sizeof(parameters)) - && AddBuffer(VAEncPackedHeaderDataBufferType, const_cast(data.data()), data.size()); - } - - void VaapiH264Encoder::DestroyFrameBuffers() - { - for (VABufferID buffer : frameBuffers) { - va.DestroyBuffer(library.Display(), buffer); - } - - frameBuffers.clear(); + return session.AddBuffer(VAEncSliceParameterBufferType, &slice, sizeof(slice)) && session.Encode(); } void VaapiH264Encoder::FillSequence(VAEncSequenceParameterBufferH264 & sequence) const @@ -608,7 +378,8 @@ namespace jni sequence.intra_idr_period = kIntraPeriod; sequence.ip_period = 1; sequence.bits_per_second = bitrateBps; - sequence.max_num_ref_frames = 1; + // The pictures of both reference slots with layers. + sequence.max_num_ref_frames = layers.Count() > 1 ? 2 : 1; sequence.picture_width_in_mbs = static_cast(widthInMbs); sequence.picture_height_in_mbs = static_cast(heightInMbs); @@ -636,47 +407,75 @@ namespace jni sequence.time_scale = framerate * 2; } - void VaapiH264Encoder::FillPicture(VAEncPictureParameterBufferH264 & picture, bool idr) const + int32_t VaapiH264Encoder::OrderCount(uint32_t frameNum, bool reference) { - picture.CurrPic.picture_id = reconstructed[current]; - picture.CurrPic.frame_idx = frameNum; - picture.CurrPic.flags = 0; - picture.CurrPic.TopFieldOrderCnt = static_cast(frameNum * 2); - picture.CurrPic.BottomFieldOrderCnt = picture.CurrPic.TopFieldOrderCnt; + // Twice frame_num for a reference picture, one less for another. + return static_cast(frameNum * 2) - (reference ? 0 : 1); + } - for (VAPictureH264 & reference : picture.ReferenceFrames) { + VAPictureH264 VaapiH264Encoder::Reference(int slot) const + { + VAPictureH264 reference = {}; + reference.picture_id = session.Reconstructed(layers.Surface(slot)); + reference.frame_idx = slotFrameNums[slot]; + reference.flags = VA_PICTURE_H264_SHORT_TERM_REFERENCE; + reference.TopFieldOrderCnt = OrderCount(slotFrameNums[slot], true); + reference.BottomFieldOrderCnt = reference.TopFieldOrderCnt; + + return reference; + } + + void VaapiH264Encoder::FillPicture(VAEncPictureParameterBufferH264 & parameters, bool idr, + const Picture & picture) const + { + parameters.CurrPic.picture_id = session.Reconstructed(picture.frame.surface); + parameters.CurrPic.frame_idx = picture.frameNum; + parameters.CurrPic.flags = 0; + parameters.CurrPic.TopFieldOrderCnt = OrderCount(picture.frameNum, picture.reference); + parameters.CurrPic.BottomFieldOrderCnt = parameters.CurrPic.TopFieldOrderCnt; + + for (VAPictureH264 & reference : parameters.ReferenceFrames) { InvalidatePicture(reference); } if (!idr) { - const uint32_t previous = (frameNum + kMaxFrameNum - 1) % kMaxFrameNum; - - VAPictureH264 & reference = picture.ReferenceFrames[0]; - reference.picture_id = reconstructed[1 - current]; - reference.frame_idx = previous; - reference.flags = VA_PICTURE_H264_SHORT_TERM_REFERENCE; - reference.TopFieldOrderCnt = static_cast(previous * 2); - reference.BottomFieldOrderCnt = reference.TopFieldOrderCnt; - } - - picture.coded_buf = codedBuffer; - picture.pic_parameter_set_id = 0; - picture.seq_parameter_set_id = 0; - picture.last_picture = 0; - picture.frame_num = static_cast(frameNum); - picture.pic_init_qp = kInitialQp; - picture.num_ref_idx_l0_active_minus1 = 0; - picture.num_ref_idx_l1_active_minus1 = 0; - - picture.pic_fields.bits.idr_pic_flag = idr ? 1 : 0; - picture.pic_fields.bits.reference_pic_flag = 1; + // The pictures the decoder keeps: the one referred to, and with + // layers the other slot's too. + int count = 0; + + for (int slot = 0; slot < layers.SlotCount(); slot++) { + const bool referred = slot == picture.frame.references[0]; + const bool kept = layers.Count() > 1 && layers.Surface(slot) >= 0; + bool listed = false; + + for (int i = 0; i < count; i++) { + listed |= parameters.ReferenceFrames[i].picture_id == session.Reconstructed(layers.Surface(slot)); + } + + if ((referred || kept) && !listed) { + parameters.ReferenceFrames[count++] = Reference(slot); + } + } + } + + parameters.coded_buf = session.CodedBuffer(); + parameters.pic_parameter_set_id = 0; + parameters.seq_parameter_set_id = 0; + parameters.last_picture = 0; + parameters.frame_num = static_cast(picture.frameNum); + parameters.pic_init_qp = kInitialQp; + parameters.num_ref_idx_l0_active_minus1 = 0; + parameters.num_ref_idx_l1_active_minus1 = 0; + + parameters.pic_fields.bits.idr_pic_flag = idr ? 1 : 0; + parameters.pic_fields.bits.reference_pic_flag = picture.reference ? 1 : 0; // CAVLC: Constrained Baseline has no CABAC. - picture.pic_fields.bits.entropy_coding_mode_flag = 0; - picture.pic_fields.bits.transform_8x8_mode_flag = 0; - picture.pic_fields.bits.deblocking_filter_control_present_flag = 1; + parameters.pic_fields.bits.entropy_coding_mode_flag = 0; + parameters.pic_fields.bits.transform_8x8_mode_flag = 0; + parameters.pic_fields.bits.deblocking_filter_control_present_flag = 1; } - void VaapiH264Encoder::FillSlice(VAEncSliceParameterBufferH264 & slice, bool idr) const + void VaapiH264Encoder::FillSlice(VAEncSliceParameterBufferH264 & slice, bool idr, const Picture & picture) const { slice.macroblock_address = 0; slice.num_macroblocks = widthInMbs * heightInMbs; @@ -696,14 +495,7 @@ namespace jni } if (!idr) { - const uint32_t previous = (frameNum + kMaxFrameNum - 1) % kMaxFrameNum; - - VAPictureH264 & reference = slice.RefPicList0[0]; - reference.picture_id = reconstructed[1 - current]; - reference.frame_idx = previous; - reference.flags = VA_PICTURE_H264_SHORT_TERM_REFERENCE; - reference.TopFieldOrderCnt = static_cast(previous * 2); - reference.BottomFieldOrderCnt = reference.TopFieldOrderCnt; + slice.RefPicList0[0] = Reference(picture.frame.references[0]); } slice.slice_qp_delta = 0; @@ -712,6 +504,8 @@ namespace jni size_t VaapiH264Encoder::WriteParameterSets(std::vector & output) const { + const bool layered = layers.Count() > 1; + // The SPS (7.3.2.1.1) of FillSequence(). NalUnitWriter sps; sps.Bits(0, 1); @@ -726,8 +520,10 @@ namespace jni sps.Ue(0); sps.Ue(kLog2MaxFrameNum - 4); sps.Ue(2); - sps.Ue(1); - sps.Flag(false); + // max_num_ref_frames, and gaps_in_frame_num_value_allowed_flag, which + // a receiver without the upper layers needs. + sps.Ue(layered ? 2 : 1); + sps.Flag(layered); sps.Ue(widthInMbs - 1); sps.Ue(heightInMbs - 1); sps.Flag(true); @@ -790,38 +586,54 @@ namespace jni return sps.AppendTo(output) + pps.AppendTo(output); } - size_t VaapiH264Encoder::WriteSliceHeader(std::vector & output, bool idr) const + size_t VaapiH264Encoder::WriteSliceHeader(std::vector & output, bool idr, const Picture & picture) const { // The slice header (7.3.3) of FillSlice(), which the driver goes on // from with the slice data. NalUnitWriter header; header.Bits(0, 1); - header.Bits(kNalRefIdc, 2); + header.Bits(picture.reference ? kNalRefIdc : 0, 2); header.Bits(idr ? kNalIdrSlice : kNalSlice, 5); header.Ue(0); header.Ue(idr ? kSliceTypeI : kSliceTypeP); header.Ue(0); - header.Bits(frameNum, kLog2MaxFrameNum); + header.Bits(picture.frameNum, kLog2MaxFrameNum); if (idr) { header.Ue(idrPicId); } else { - // num_ref_idx_active_override_flag and - // ref_pic_list_modification_flag_l0. - header.Flag(false); + // num_ref_idx_active_override_flag. header.Flag(false); - } - // dec_ref_pic_marking(): no_output_of_prior_pics_flag and - // long_term_reference_flag, or adaptive_ref_pic_marking_mode_flag. - if (idr) { - header.Flag(false); - header.Flag(false); + // ref_pic_list_modification(): with layers, the picture referred + // to goes first in the list, which would otherwise start with the + // newest reference picture. + const bool layered = layers.Count() > 1; + + header.Flag(layered); + + if (layered) { + const uint32_t referenceFrameNum = slotFrameNums[picture.frame.references[0]]; + const uint32_t difference = (picture.frameNum + kMaxFrameNum - referenceFrameNum) % kMaxFrameNum; + + header.Ue(kSubtractPicNum); + header.Ue(difference - 1); + header.Ue(kEndModifications); + } } - else { - header.Flag(false); + + // dec_ref_pic_marking() of a reference picture: no_output_of_prior_pics_flag + // and long_term_reference_flag, or adaptive_ref_pic_marking_mode_flag. + if (picture.reference) { + if (idr) { + header.Flag(false); + header.Flag(false); + } + else { + header.Flag(false); + } } header.Se(0); @@ -844,6 +656,8 @@ namespace jni } bitrateBps = bitrate; + outputProcessor.OnRatesUpdated(parameters.bitrate); + layers.OnRatesUpdated(parameters.bitrate); if (parameters.framerate_fps >= 1.0) { framerate = static_cast(parameters.framerate_fps + 0.5); @@ -864,6 +678,8 @@ namespace jni // NV12 needs even dimensions. info.requested_resolution_alignment = 2; + layers.FillFpsAllocation(info); + return info; } } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Encoder.cpp index 37670174..43a2cf45 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Encoder.cpp @@ -35,6 +35,7 @@ namespace jni constexpr uint8_t kSliceTypeB = 0; constexpr uint8_t kSliceTypeP = 1; constexpr uint8_t kSliceTypeI = 2; + constexpr uint8_t kNalTrailN = 0; constexpr uint8_t kNalTrailR = 1; constexpr uint8_t kNalIdrWRadl = 19; constexpr uint8_t kNalVps = 32; @@ -142,7 +143,8 @@ namespace jni framerate(30), ratesChanged(false), pictureOrder(0), - current(0), + slotOrders{ 0, 0 }, + slotTemporalIds{ 0, 0 }, referenceValid(false), callback(nullptr), outputProcessor(webrtc::kVideoCodecH265, format) @@ -220,6 +222,16 @@ namespace jni return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } + // Temporal layers, where the encoder writes the slice headers, which + // carry the layers; spatial ones are left to the next codec. + const webrtc::ScalabilityMode mode = settings->GetScalabilityMode().value_or(webrtc::ScalabilityMode::kL1T1); + + if (mode != webrtc::ScalabilityMode::kL1T1 && (packedHeaders != kPackedHeaders + || (mode != webrtc::ScalabilityMode::kL1T2 && mode != webrtc::ScalabilityMode::kL1T3))) + { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + Release(); const uint32_t ctbSize = 1u << log2CtbSize; @@ -238,14 +250,19 @@ namespace jni framerate = std::max(1u, codecSettings.maxFramerate); ratesChanged = true; pictureOrder = 0; - current = 0; referenceValid = false; outputProcessor.Reset(); + layers.Configure(mode, bitrateBps, VaapiTemporalLayers::LayerRateControl(library, VAProfileHEVCMain, + entrypoint, webrtc::ScalabilityModeToNumTemporalLayers(mode))); + + if (!outputProcessor.SetScalabilityMode(mode)) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } // Surfaces of whole coding tree blocks, which the driver may write all // of. if (!session.Create(VAProfileHEVCMain, entrypoint, codedWidth, codedHeight, widthInCtbs << log2CtbSize, - heightInCtbs << log2CtbSize, packedHeaders)) + heightInCtbs << log2CtbSize, packedHeaders, VaapiTemporalLayers::kReconstructedSurfaces)) { Release(); return WEBRTC_VIDEO_CODEC_ERROR; @@ -254,7 +271,8 @@ namespace jni initialized = true; RTC_LOG(LS_INFO) << implementationName << " H.265 initialized: " << codecSettings.width << "x" - << codecSettings.height << " at " << bitrateBps << " bps, CTB " << ctbSize << ", min CB " << alignment; + << codecSettings.height << " at " << bitrateBps << " bps, CTB " << ctbSize << ", min CB " << alignment << ", " + << webrtc::ScalabilityModeToString(mode); return WEBRTC_VIDEO_CODEC_OK; } @@ -295,14 +313,30 @@ namespace jni pictureOrder = 0; } + const auto layerFrame = outputProcessor.NextFrameConfig(idr); + + // No layer is encoded now, as when WebRTC gives none a bitrate. + if (!layerFrame) { + return WEBRTC_VIDEO_CODEC_OK; + } + + const auto layerPlan = layers.Plan(idr, *layerFrame); + + if (!layerPlan) { + RTC_LOG(LS_WARNING) << implementationName << " H.265: a picture refers to a slot that holds none"; + return WEBRTC_VIDEO_CODEC_ERROR; + } + const int alignment = 1 << log2MinCbSize; std::vector output; const bool encoded = session.Upload(frame, static_cast(codecSettings.width), - static_cast(codecSettings.height), alignment - 1) && Submit(idr) && session.ReadOutput(output); + static_cast(codecSettings.height), alignment - 1) && Submit(idr, *layerPlan) + && session.ReadOutput(output); session.DropBuffers(); + if (!encoded) { return WEBRTC_VIDEO_CODEC_ERROR; } @@ -319,14 +353,27 @@ namespace jni // A driver that writes no parameter sets into key frames makes a // stream nothing can decode. - if (!outputProcessor.Process(output, idr, image, info)) { + if (!outputProcessor.Process(output, idr, *layerFrame, image, info)) { RTC_LOG(LS_WARNING) << implementationName << " H.265: the driver wrote no parameter sets"; return WEBRTC_VIDEO_CODEC_ERROR; } - // The frame just encoded is the reference of the next one. + // The slots the picture replaces hold it now. + layers.Commit(*layerPlan); + + for (int slot = 0; slot < layers.SlotCount(); slot++) { + if (layerPlan->keptFlags & (1 << slot)) { + slotOrders[slot] = pictureOrder; + slotTemporalIds[slot] = layerPlan->temporalId; + } + else if (slotTemporalIds[slot] > layerPlan->temporalId) { + // Left out of the picture's reference picture set, so gone from + // the decoder's. + layers.Forget(slot); + } + } + referenceValid = true; - current = 1 - current; pictureOrder++; ratesChanged = false; @@ -335,7 +382,7 @@ namespace jni return WEBRTC_VIDEO_CODEC_OK; } - bool VaapiH265Encoder::Submit(bool idr) + bool VaapiH265Encoder::Submit(bool idr, const VaapiTemporalLayers::Frame & frame) { if (idr) { VAEncSequenceParameterBufferHEVC sequence = {}; @@ -346,15 +393,23 @@ namespace jni } } - if ((idr || ratesChanged) && !session.AddRateControl(bitrateBps, framerate, kInitialQp)) { - return false; + if (idr || ratesChanged) { + // Without layers, from the initial QP the encoder has always + // started from. + const bool added = layers.Count() == 1 + ? session.AddRateControl(bitrateBps, framerate, kInitialQp) + : layers.AddRateControl(session, bitrateBps, framerate, idr); + + if (!added) { + return false; + } } VAEncPictureParameterBufferHEVC picture = {}; - FillPicture(picture, idr); + FillPicture(picture, idr, frame); VAEncSliceParameterBufferHEVC slice = {}; - FillSlice(slice, idr); + FillSlice(slice, idr, frame); if (!session.AddBuffer(VAEncPictureParameterBufferType, &picture, sizeof(picture))) { return false; @@ -371,7 +426,7 @@ namespace jni if (packedHeaders & VA_ENC_PACKED_HEADER_SLICE) { std::vector sliceHeader; - const size_t bits = WriteSliceHeader(sliceHeader, idr); + const size_t bits = WriteSliceHeader(sliceHeader, idr, frame); if (!session.AddPackedHeader(VAEncPackedHeaderSlice, sliceHeader, bits)) { return false; @@ -415,20 +470,55 @@ namespace jni sequence.vui_time_scale = framerate; } - VAPictureHEVC VaapiH265Encoder::Reference() const + std::vector VaapiH265Encoder::ReferencePictureSet( + const VaapiTemporalLayers::Frame & frame) const { - VAPictureHEVC reference = {}; - reference.picture_id = session.Reconstructed(1 - current); - reference.pic_order_cnt = pictureOrder - 1; - // A short-term reference before the current picture. - reference.flags = VA_PICTURE_HEVC_RPS_ST_CURR_BEFORE; + std::vector set; + + for (int slot = 0; slot < layers.SlotCount(); slot++) { + // A picture of a higher layer, which a receiver without that layer + // does not have. + if (layers.Surface(slot) < 0 || slotTemporalIds[slot] > frame.temporalId) { + continue; + } + + const bool used = std::find(frame.references.begin(), frame.references.end(), slot) + != frame.references.end(); + auto same = std::find_if(set.begin(), set.end(), [&](const ReferencePicture & picture) { + return layers.Surface(picture.slot) == layers.Surface(slot); + }); - return reference; + if (same != set.end()) { + same->used |= used; + } + else { + set.push_back({ slot, slotOrders[slot], used }); + } + } + + std::sort(set.begin(), set.end(), [](const ReferencePicture & a, const ReferencePicture & b) { + return a.pictureOrder > b.pictureOrder; + }); + + return set; + } + + VAPictureHEVC VaapiH265Encoder::Picture(const ReferencePicture & reference) const + { + VAPictureHEVC picture = {}; + picture.picture_id = session.Reconstructed(layers.Surface(reference.slot)); + picture.pic_order_cnt = reference.pictureOrder; + // A short-term reference before the current picture, which the + // current one refers to or keeps for later. + picture.flags = reference.used ? VA_PICTURE_HEVC_RPS_ST_CURR_BEFORE : 0; + + return picture; } - void VaapiH265Encoder::FillPicture(VAEncPictureParameterBufferHEVC & picture, bool idr) const + void VaapiH265Encoder::FillPicture(VAEncPictureParameterBufferHEVC & picture, bool idr, + const VaapiTemporalLayers::Frame & frame) const { - picture.decoded_curr_pic.picture_id = session.Reconstructed(current); + picture.decoded_curr_pic.picture_id = session.Reconstructed(frame.surface); picture.decoded_curr_pic.pic_order_cnt = pictureOrder; picture.decoded_curr_pic.flags = 0; @@ -437,7 +527,11 @@ namespace jni } if (!idr) { - picture.reference_frames[0] = Reference(); + const std::vector set = ReferencePictureSet(frame); + + for (size_t i = 0; i < set.size(); i++) { + picture.reference_frames[i] = Picture(set[i]); + } } picture.coded_buf = session.CodedBuffer(); @@ -453,18 +547,22 @@ namespace jni picture.num_ref_idx_l0_default_active_minus1 = 0; picture.num_ref_idx_l1_default_active_minus1 = 0; picture.slice_pic_parameter_set_id = 0; - picture.nal_unit_type = idr ? kNalIdrWRadl : kNalTrailR; + // A picture no other refers to, as of the top layer, is a sub-layer + // non-reference picture. + const bool reference = idr || frame.refreshFlags != 0; + picture.nal_unit_type = idr ? kNalIdrWRadl : (reference ? kNalTrailR : kNalTrailN); picture.pic_fields.bits.idr_pic_flag = idr ? 1 : 0; picture.pic_fields.bits.coding_type = idr ? kCodingTypeI : kCodingTypeP; - picture.pic_fields.bits.reference_pic_flag = 1; + picture.pic_fields.bits.reference_pic_flag = reference ? 1 : 0; picture.pic_fields.bits.sign_data_hiding_enabled_flag = signDataHiding ? 1 : 0; picture.pic_fields.bits.transform_skip_enabled_flag = transformSkip ? 1 : 0; // Needed for the rate control to change the QP within a picture. picture.pic_fields.bits.cu_qp_delta_enabled_flag = cuQpDelta ? 1 : 0; } - void VaapiH265Encoder::FillSlice(VAEncSliceParameterBufferHEVC & slice, bool idr) const + void VaapiH265Encoder::FillSlice(VAEncSliceParameterBufferHEVC & slice, bool idr, + const VaapiTemporalLayers::Frame & frame) const { slice.slice_segment_address = 0; slice.num_ctu_in_slice = widthInCtbs * heightInCtbs; @@ -481,10 +579,16 @@ namespace jni } if (!idr) { - slice.ref_pic_list0[0] = Reference(); - - if (lowDelayB) { - slice.ref_pic_list1[0] = Reference(); + // The one picture the current one refers to. + for (const ReferencePicture & reference : ReferencePictureSet(frame)) { + if (reference.used) { + slice.ref_pic_list0[0] = Picture(reference); + + if (lowDelayB) { + slice.ref_pic_list1[0] = Picture(reference); + } + break; + } } } @@ -519,6 +623,20 @@ namespace jni writer.Bits(0, 32); writer.Bits(0, 12); writer.Bits(LevelIdc(codedWidth * codedHeight, framerate), 8); + + // No profile or level of their own for the sub-layers. + const int maxSubLayersMinus1 = layers.Count() - 1; + + for (int i = 0; i < maxSubLayersMinus1; i++) { + writer.Flag(false); + writer.Flag(false); + } + + if (maxSubLayersMinus1 > 0) { + for (int i = maxSubLayersMinus1; i < 8; i++) { + writer.Bits(0, 2); + } + } } size_t VaapiH265Encoder::WriteParameterSets(std::vector & output) const @@ -530,6 +648,15 @@ namespace jni writer.Bits(1, 3); }; + // A sub-layer for each temporal layer. Without layers, the current + // picture and one reference picture are decoded at a time; with + // them, the current one and those of the two reference slots. There + // is no reordering. The temporal id nesting flag is set without + // layers, as the standard has it then, and left clear with them. + const uint32_t maxSubLayersMinus1 = static_cast(layers.Count() - 1); + const uint32_t maxDecPicBufferingMinus1 = layers.Count() > 1 ? 2 : 1; + const bool temporalIdNesting = layers.Count() == 1; + // The VPS (7.3.2.1). NalUnitWriter vps; nalHeader(vps, kNalVps); @@ -537,14 +664,13 @@ namespace jni vps.Flag(true); vps.Flag(true); vps.Bits(0, 6); - vps.Bits(0, 3); - vps.Flag(true); + vps.Bits(maxSubLayersMinus1, 3); + vps.Flag(temporalIdNesting); vps.Bits(0xFFFF, 16); WriteProfileTierLevel(vps); - // One sub-layer, decoding with the current and one reference - // picture, without reordering. + // The ordering information of the highest sub-layer stands for all. vps.Flag(false); - vps.Ue(1); + vps.Ue(maxDecPicBufferingMinus1); vps.Ue(0); vps.Ue(0); vps.Bits(0, 6); @@ -557,8 +683,8 @@ namespace jni NalUnitWriter sps; nalHeader(sps, kNalSps); sps.Bits(0, 4); - sps.Bits(0, 3); - sps.Flag(true); + sps.Bits(maxSubLayersMinus1, 3); + sps.Flag(temporalIdNesting); WriteProfileTierLevel(sps); sps.Ue(0); sps.Ue(1); @@ -583,7 +709,7 @@ namespace jni sps.Ue(0); sps.Ue(kLog2MaxPicOrderCntLsb - 4); sps.Flag(false); - sps.Ue(1); + sps.Ue(maxDecPicBufferingMinus1); sps.Ue(0); sps.Ue(0); sps.Ue(log2MinCbSize - 3); @@ -664,15 +790,19 @@ namespace jni return vps.AppendTo(output) + sps.AppendTo(output) + pps.AppendTo(output); } - size_t VaapiH265Encoder::WriteSliceHeader(std::vector & output, bool idr) const + size_t VaapiH265Encoder::WriteSliceHeader(std::vector & output, bool idr, + const VaapiTemporalLayers::Frame & frame) const { // The slice segment header (7.3.6.1) of FillSlice(), up to the slice - // data, which the driver writes. + // data, which the driver writes, in a NAL unit of the picture's + // temporal layer. + const bool reference = idr || frame.refreshFlags != 0; + NalUnitWriter header; header.Bits(0, 1); - header.Bits(idr ? kNalIdrWRadl : kNalTrailR, 6); + header.Bits(idr ? kNalIdrWRadl : (reference ? kNalTrailR : kNalTrailN), 6); header.Bits(0, 6); - header.Bits(1, 3); + header.Bits(static_cast(frame.temporalId + 1), 3); header.Flag(true); @@ -686,13 +816,22 @@ namespace jni if (!idr) { header.Bits(static_cast(pictureOrder) & ((1u << kLog2MaxPicOrderCntLsb) - 1), kLog2MaxPicOrderCntLsb); - // st_ref_pic_set(0) here, not in the SPS: the previous picture, - // used by this one. + // st_ref_pic_set(0) here, not in the SPS: the pictures of the + // reference slots, before this one, of which it uses one. Those it + // keeps for later may be missing at a receiver without the upper + // layers, which the standard allows. + const std::vector set = ReferencePictureSet(frame); + int32_t previous = pictureOrder; + header.Flag(false); - header.Ue(1); - header.Ue(0); + header.Ue(static_cast(set.size())); header.Ue(0); - header.Flag(true); + + for (const ReferencePicture & reference : set) { + header.Ue(static_cast(previous - reference.pictureOrder - 1)); + header.Flag(reference.used); + previous = reference.pictureOrder; + } } if (sao) { @@ -728,6 +867,8 @@ namespace jni } bitrateBps = bitrate; + outputProcessor.OnRatesUpdated(parameters.bitrate); + layers.OnRatesUpdated(parameters.bitrate); if (parameters.framerate_fps >= 1.0) { framerate = static_cast(parameters.framerate_fps + 0.5); @@ -748,6 +889,8 @@ namespace jni // Whole coding blocks, so that the coded picture is the frame. info.requested_resolution_alignment = 1 << log2MinCbSize; + layers.FillFpsAllocation(info); + return info; } } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiTemporalLayers.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiTemporalLayers.cpp index 1e8cfc70..ff0ec85b 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiTemporalLayers.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiTemporalLayers.cpp @@ -35,12 +35,32 @@ namespace jni } VaapiTemporalLayers::VaapiTemporalLayers() : - count(1) + count(1), + layerRateControl(false) { slots.fill(-1); } - bool VaapiTemporalLayers::Configure(webrtc::ScalabilityMode mode, uint32_t bitrateBps) + bool VaapiTemporalLayers::LayerRateControl(VaapiLibrary & library, VAProfile profile, VAEntrypoint entrypoint, + int layers) + { + VAConfigAttrib attribute = {}; + attribute.type = VAConfigAttribEncRateControlExt; + + if (library.Api().GetConfigAttributes(library.Display(), profile, entrypoint, &attribute, 1) + != VA_STATUS_SUCCESS || attribute.value == VA_ATTRIB_NOT_SUPPORTED) + { + return false; + } + + VAConfigAttribValEncRateControlExt extension; + extension.value = attribute.value; + + return extension.bits.temporal_layer_bitrate_control_flag + && static_cast(extension.bits.max_num_temporal_layers_minus1) + 1 >= layers; + } + + bool VaapiTemporalLayers::Configure(webrtc::ScalabilityMode mode, uint32_t bitrateBps, bool layerRates) { switch (mode) { case webrtc::ScalabilityMode::kL1T1: @@ -57,6 +77,7 @@ namespace jni } slots.fill(-1); + layerRateControl = layerRates; bitratesBps.clear(); for (int layer = 0; layer < count && count > 1; layer++) { @@ -88,6 +109,12 @@ namespace jni return std::nullopt; } + for (const webrtc::CodecBufferUsage & buffer : layerFrame.Buffers()) { + if (buffer.updated && buffer.id >= 0 && buffer.id < static_cast(slots.size())) { + frame.keptFlags |= static_cast(1 << buffer.id); + } + } + // A key frame replaces every slot. if (keyFrame) { frame.refreshFlags = 0xFF; @@ -120,12 +147,20 @@ namespace jni void VaapiTemporalLayers::Commit(const Frame & frame) { for (int slot = 0; slot < static_cast(slots.size()); slot++) { - if (frame.refreshFlags & (1 << slot)) { + if (frame.keptFlags & (1 << slot)) { slots[slot] = frame.surface; } + else if (frame.refreshFlags == 0xFF) { + slots[slot] = -1; + } } } + void VaapiTemporalLayers::Forget(int slot) + { + slots[slot] = -1; + } + int VaapiTemporalLayers::SlotCount() const { return static_cast(slots.size()); @@ -151,7 +186,7 @@ namespace jni bool VaapiTemporalLayers::AddRateControl(VaapiEncodeSession & session, uint32_t bitrateBps, uint32_t framerate, bool keyFrame) const { - if (count == 1) { + if (count == 1 || !layerRateControl) { return session.AddRateControl(bitrateBps, framerate, 0); } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp index 4bc6a9c1..175a10de 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp @@ -72,7 +72,14 @@ namespace jni }; if (h265) { - formats.push_back(CreateH265MainFormat()); + // H.265 has no software encoder whose format would say which + // scalability modes WebRTC may ask for: the temporal layers the + // encoder makes. + webrtc::SdpVideoFormat format = CreateH265MainFormat(); + format.scalability_modes = { + webrtc::ScalabilityMode::kL1T1, webrtc::ScalabilityMode::kL1T2, webrtc::ScalabilityMode::kL1T3 + }; + formats.push_back(format); } if (vp9) { formats.push_back(webrtc::SdpVideoFormat::VP9Profile0()); diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp index ecc507b9..af009f2e 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp @@ -114,7 +114,8 @@ namespace jni ratesChanged = true; referenceValid = false; outputProcessor.Reset(); - layers.Configure(mode, bitrateBps); + layers.Configure(mode, bitrateBps, VaapiTemporalLayers::LayerRateControl(library, VAProfileVP9Profile0, + entrypoint, webrtc::ScalabilityModeToNumTemporalLayers(mode))); if (!outputProcessor.SetScalabilityMode(mode)) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; @@ -193,6 +194,7 @@ namespace jni session.DropBuffers(); + if (!encoded) { return kEncodeFailed; } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java index 7b9ea334..50cc4e8e 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java @@ -37,17 +37,18 @@ * H.264 is encoded on an NVIDIA GPU with NVENC, on Windows and Linux. On * other GPUs it is encoded with the Media Foundation encoder of the driver on * Windows, and with its VA-API encoder on Linux. AV1 is encoded on GPUs that - * have an AV1 encoder, with NVENC, Media Foundation or VA-API, as a single - * layer, or with VA-API in temporal layers too (L1T2, L1T3); a stream that - * asks for other SVC modes is encoded in software. H.265 (Main profile) is - * encoded with NVENC or Media Foundation on GPUs that have an H.265 encoder. + * have an AV1 encoder, with NVENC, Media Foundation or VA-API. H.265 (Main + * profile) is encoded with NVENC or Media Foundation on GPUs that have an + * H.265 encoder. * VP9 (profile 0) is encoded on GPUs that have a VP9 encoder, which are * Intel's: with VA-API on Linux, and with Media Foundation on Windows only * with the system property {@code webrtc.hardware.vp9Encoder} set to * {@code true} when the factory is created, since that encoder has not been - * tested on such a GPU yet. With VA-API, VP9 is encoded in temporal layers - * too (L1T2, L1T3). Otherwise VP9 is encoded with libvpx. On Linux, H.265 is encoded with NVENC on NVIDIA + * tested on such a GPU yet. Otherwise VP9 is encoded with libvpx. On Linux, H.265 is encoded with NVENC on NVIDIA * GPUs and with VA-API on the GPUs of Intel and AMD whose driver encodes it. + * Hardware encoders produce a single layer, but for the VA-API encoders, which + * encode temporal layers (L1T2, L1T3) too; a stream that asks for other SVC + * modes is encoded in software. * A hardware encoder that fails to start, for example because the GPU has no * encoder sessions left, or fails while encoding, is replaced by the next one * in line, and finally by the software encoder of the same codec, so a stream diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java index 26e8aa61..8de5465e 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java @@ -54,6 +54,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; +import java.util.function.Supplier; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.parallel.Execution; @@ -524,7 +525,7 @@ void av1RestartsCleanly() throws Exception { @Test void av1EncodesTemporalLayers() throws Exception { - assertEncodesTemporalLayers(AV1, HARDWARE_AV1_REQUIRED); + assertEncodesTemporalLayers(this::hardwareFactory, AV1, HARDWARE_AV1_REQUIRED); } @Test @@ -532,20 +533,40 @@ void vp9EncodesTemporalLayers() throws Exception { // VA-API encodes VP9 in temporal layers, Media Foundation does not. assumeTrue(OS.contains("linux"), "hardware VP9 encodes temporal layers on Linux only"); - assertEncodesTemporalLayers(VP9, HARDWARE_VP9_REQUIRED); + assertEncodesTemporalLayers(this::hardwareFactory, VP9, HARDWARE_VP9_REQUIRED); + } + + @Test + void h264EncodesTemporalLayers() throws Exception { + // VA-API encodes H.264 in temporal layers; NVENC and Media Foundation + // leave them to OpenH264. Received with the hardware decoders, which + // decode the layers too. + assumeTrue(OS.contains("linux"), "hardware H.264 encodes temporal layers on Linux only"); + + assertEncodesTemporalLayers(this::hardwareCallFactory, H264, HARDWARE_REQUIRED); + } + + @Test + void h265EncodesTemporalLayers() throws Exception { + // There is no software H.265 to receive it, so the hardware decoders do. + assumeTrue(OS.contains("linux"), "hardware H.265 encodes temporal layers on Linux only"); + assumeH265(); + + assertEncodesTemporalLayers(this::hardwareCallFactory, H265, true); } /** - * Sends the codec in L1T2 and then L1T3, and checks that the hardware - * encodes the layers, rather than handing them to the software encoder. + * Sends the codec in L1T2 and then L1T3, each on a call of a factory the + * supplier makes, and checks that the hardware encodes the layers, rather + * than handing them to the software encoder. * Temporal layers are references between frames of one size, which a * hardware encoder can make; an SFU can then forward a lower frame rate * without asking for key frames. */ - private void assertEncodesTemporalLayers(Predicate codec, boolean required) - throws Exception { + private void assertEncodesTemporalLayers(Supplier factory, + Predicate codec, boolean required) throws Exception { for (String mode : new String[] { "L1T2", "L1T3" }) { - PeerConnectionFactory hardware = hardwareFactory(); + PeerConnectionFactory hardware = factory.get(); CountDownLatch received = new CountDownLatch(30); try (TestMediaCall call = new TestMediaCall(hardware, true, false, codec)) {