From 183228bbc0b8ae09666627f855cd6c6fd3c9f7a8 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Thu, 15 Oct 2026 18:57:00 +0200 Subject: [PATCH 1/3] feat: encode AV1 temporal layers with VA-API The VA-API AV1 encoder now takes L1T2 and L1T3, with WebRTC's reference structure and a rate control for each layer, rather than leaving them to libaom. A frame it cannot encode now falls back to libaom. --- docs/guide/advanced/video-codecs.md | 2 +- .../video/codec/EncoderOutputProcessor.h | 31 +- .../media/video/codec/linux/VaapiAv1Encoder.h | 53 +++- .../video/codec/linux/VaapiEncodeSession.h | 25 +- .../video/codec/EncoderOutputProcessor.cpp | 62 +++- .../video/codec/linux/VaapiAv1Encoder.cpp | 284 ++++++++++++++---- .../video/codec/linux/VaapiEncodeSession.cpp | 75 ++++- .../codec/HardwareVideoEncoderFactory.java | 3 +- .../HardwareVideoEncoderIntegrationTest.java | 74 +++++ 9 files changed, 507 insertions(+), 102 deletions(-) diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index c1063f01..e87bc22e 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -30,7 +30,7 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder() | Linux | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) where it encodes H.265 | NVENC on NVIDIA RTX 40 and newer, otherwise the VA-API encoder of the GPU driver where it encodes AV1 (e.g. AMD RDNA3 and newer) | | macOS | VideoToolbox, as with `DefaultVideoEncoderFactory` | VideoToolbox, on Macs with a hardware HEVC encoder, such as all with Apple silicon | Software | -Hardware encoders produce a single layer. A stream that asks for SVC, such as an AV1 stream with a scalability mode like `L1T3`, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom. +Hardware encoders produce a single layer, but for the VA-API AV1 encoder, which encodes temporal layers too (`L1T2` and `L1T3`), with a rate control of each layer, so that an SFU can forward a lower frame rate. A stream that asks for spatial layers, or for temporal layers from another hardware encoder, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom. VP9 can be encoded in hardware too, with the VP9 encoder of the GPU driver, which Intel GPUs have: through VA-API on Linux, and through Media Foundation on Windows. On Windows it is off by default, since it has not been tested on such a GPU yet; turn it on with a system property before the factory is created: diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h b/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h index 6b0f8ddf..0e158255 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h @@ -26,9 +26,13 @@ #include "modules/video_coding/codecs/h264/include/h264_globals.h" #include "modules/video_coding/codecs/vp9/include/vp9_globals.h" #include "modules/video_coding/include/video_codec_interface.h" -#include "modules/video_coding/svc/scalable_video_controller_no_layering.h" +#include "api/video/video_bitrate_allocation.h" +#include "api/video_codecs/scalability_mode.h" +#include "modules/video_coding/svc/scalable_video_controller.h" #include +#include +#include #include #include @@ -43,9 +47,10 @@ namespace jni // or the sequence header of AV1. Encoders put them in front of the first // key frame, not necessarily in front of every one, so they are kept and // put back where missing. The frame dependencies WebRTC needs for AV1 and - // H.265 are those of a stream without layers, which is all hardware - // encoders produce here. VP9 frames go out as they come, with the VP9 - // information of a single layer, as the Java encoders get it. + // H.265 are those of a stream without layers, unless an AV1 encoder that + // makes temporal layers sets its scalability mode and asks for the layer + // of each frame before it encodes it. VP9 frames go out as they come, with + // the VP9 information of a single layer, as the Java encoders get it. class EncoderOutputProcessor { public: @@ -54,6 +59,15 @@ namespace jni // Forgets the parameter sets, for a stream that starts over. void Reset(); + // Sets the layers of the stream, temporal ones of AV1 only, as in + // L1T3. Returns false for a mode WebRTC has no structure of. + bool SetScalabilityMode(webrtc::ScalabilityMode mode); + // Lets the layers follow the bitrate WebRTC gives each. + void OnRatesUpdated(const webrtc::VideoBitrateAllocation & bitrates); + // The layer of the next frame, which buffers it refers to and which + // it replaces, or nothing if no layer is to be encoded now. + std::optional NextFrameConfig(bool keyFrame); + // Fills the image with the bitstream of one frame, and the codec // specific information WebRTC needs to send it. The caller sets // the size of the image before, which the AV1 frame dependencies @@ -64,6 +78,10 @@ namespace jni // frame lacks parameter sets and there are none to put back. bool Process(std::span bitstream, bool keyFrame, webrtc::EncodedImage & image, webrtc::CodecSpecificInfo & info); + // Does what the above does, for a frame encoded as the given layer. + bool Process(std::span bitstream, bool keyFrame, + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::EncodedImage & image, + webrtc::CodecSpecificInfo & info); // The quality scaling settings that suit the codec: the thresholds // of WebRTC's H.264 encoder, for H.265 as well, those for VP9 of @@ -77,7 +95,7 @@ namespace jni bool ProcessAv1(std::span bitstream, bool keyFrame, std::vector & output); void FillCodecSpecificInfo(const webrtc::EncodedImage & image, bool keyFrame, - webrtc::CodecSpecificInfo & info); + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::CodecSpecificInfo & info); private: const webrtc::VideoCodecType codec; @@ -89,7 +107,8 @@ namespace jni webrtc::H264BitstreamParser h264Parser; webrtc::H265BitstreamParser h265Parser; - webrtc::ScalableVideoControllerNoLayering svcController; + webrtc::ScalabilityMode scalabilityMode; + std::unique_ptr svcController; // The group of frames VP9 describes: one layer, every frame // referring to the one before. diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h index 9e5dbbad..e4885707 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h @@ -25,10 +25,12 @@ #include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/svc/scalable_video_controller.h" #include #include +#include #include #include #include @@ -39,12 +41,16 @@ namespace jni // drivers provide it on Linux. // // It works as FFmpeg's VA-API encoder does without B-frames: synchronous, - // one tile, inter frames that refer to the frame before them only, which - // alternate between two reference slots. The sequence and frame headers + // one tile, inter frames that refer to one frame before them. Without + // layers that is the frame just before; with temporal layers (L1T2, L1T3) + // it is the one of the same layer or a lower one that WebRTC's structure + // of the mode names, so that a receiver can do without the upper layers. + // Each reference buffer of the structure is the reference slot of the + // same number. The sequence and frame headers // are written here and handed to the driver as packed headers, with where // the quantizer, loop filter and CDEF parameters are in the frame header, - // which the driver's rate control fills in. Anything that fails is an - // error, upon which libaom takes over. + // which the driver's rate control fills in. Spatial layers, and anything + // that fails, are left to libaom. class VaapiAv1Encoder : public webrtc::VideoEncoder { public: @@ -74,10 +80,32 @@ namespace jni // Reads which headers the driver takes and how it codes them. void QueryCapabilities(); - bool Submit(bool keyFrame); + // What a frame refers to: the slot each of the seven reference + // frames names, the second reference, if any, being golden; the + // slots the frame replaces; and its temporal layer. + struct FrameReferences + { + std::array refFrameIdx = {}; + bool golden = false; + uint8_t refreshFrameFlags = 0; + int temporalId = 0; + }; + + // Turns the buffers of the layer frame into reference slots. + // Returns false if it refers to a slot that holds no frame, or to + // one beyond those kept. + bool PlanReferences(bool keyFrame, const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, + FrameReferences & references) const; + // A reconstructed surface that no reference slot holds. + int FreeSurface() const; + // Adds the rate control of the stream, or of each temporal layer. + bool AddRateControl(bool keyFrame); + + bool Submit(bool keyFrame, const FrameReferences & references, int surface); void WriteSequenceHeader(std::vector & obu) const; - void WriteFrameHeader(std::vector & obu, FrameHeaderLayout & layout, bool keyFrame) const; + void WriteFrameHeader(std::vector & obu, FrameHeaderLayout & layout, bool keyFrame, + const FrameReferences & references) const; // The seq_level_idx the stream's size, frame rate and bitrate fit. uint8_t Level() const; @@ -100,9 +128,16 @@ namespace jni // The sequence header, kept for the key frames. std::vector sequenceHeader; - // Which of the reconstructed surfaces, and so which reference slot, - // holds the current frame. - int current; + // The temporal layers, one without layers, and the bitrate of each, + // counting the layers below it. + int temporalLayers; + std::vector layerBitratesBps; + + // The reconstructed surface each reference slot holds, -1 for + // none: those of the two buffers WebRTC's structures use, up to + // L1T3. A key frame fills the other six slots too, but no frame + // refers to them. + std::array slots; bool referenceValid; // The frames since the last key frame, whose low bits are the // order hint. diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h index 42dd6621..23a4ec43 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h @@ -30,8 +30,8 @@ namespace jni { // What every VA-API encoder of a single codec needs: a configuration and - // a context with an input surface and two reconstructed ones (the frame - // being encoded and the one it refers to), the output buffer, the + // a context with an input surface and reconstructed ones (by default two: + // the frame being encoded and the one it refers to), the output buffer, the // parameter buffers of a frame, uploading a frame as NV12, encoding it // and reading what came out. The codec's encoder fills in the parameters. // @@ -49,12 +49,14 @@ namespace jni // Creates the session for 4:2:0 at constant bitrate, for pictures // of the given size, in surfaces of the given size, which is that - // of the picture aligned as the codec needs, and with the - // VA_ENC_PACKED_HEADER_* the encoder hands the driver. Mesa takes - // the size of the coded picture from the context, so the context - // has the picture's. + // of the picture aligned as the codec needs, with the + // VA_ENC_PACKED_HEADER_* the encoder hands the driver, and with + // the given number of reconstructed surfaces. Mesa takes the size + // of the coded picture from the context, so the context has the + // picture's. bool Create(VAProfile profile, VAEntrypoint entrypoint, uint32_t pictureWidth, uint32_t pictureHeight, - uint32_t surfaceWidth, uint32_t surfaceHeight, uint32_t packedHeaders = VA_ENC_PACKED_HEADER_NONE); + uint32_t surfaceWidth, uint32_t surfaceHeight, uint32_t packedHeaders = VA_ENC_PACKED_HEADER_NONE, + int reconstructedCount = 2); void Destroy(); VASurfaceID InputSurface() const; @@ -75,6 +77,12 @@ namespace jni // The rate control, frame rate and HRD parameters, for constant // bitrate with the given initial quantizer. bool AddRateControl(uint32_t bitrateBps, uint32_t framerate, uint32_t initialQp); + // The same, with a rate control of each temporal layer: their + // bitrates, each counting the layers below it too, and the layer + // of each frame after a key frame, a pattern that repeats, which + // the driver is told along with the rate control of a key frame. + bool AddLayerRateControl(const std::vector & bitratesBps, uint32_t framerate, + const std::vector & pattern, bool keyFrame); // Encodes the input surface with the parameter buffers added, and // waits for it. @@ -92,7 +100,8 @@ namespace jni VAConfigID config; VAContextID context; - VASurfaceID surfaces[3]; + // The input surface, then the reconstructed ones. + std::vector surfaces; VABufferID codedBuffer; std::vector frameBuffers; }; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp index 26c7198f..1609dd16 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp @@ -21,6 +21,8 @@ #include "common_video/h264/h264_common.h" #include "common_video/h265/h265_common.h" #include "modules/video_coding/codecs/interface/common_constants.h" +#include "modules/video_coding/svc/create_scalability_structure.h" +#include "modules/video_coding/svc/scalable_video_controller_no_layering.h" #include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" #include "rtc_base/logging.h" @@ -119,7 +121,9 @@ namespace jni EncoderOutputProcessor::EncoderOutputProcessor(webrtc::VideoCodecType codec, const webrtc::SdpVideoFormat & format) : codec(codec), - packetizationMode(webrtc::H264PacketizationMode::NonInterleaved) + packetizationMode(webrtc::H264PacketizationMode::NonInterleaved), + scalabilityMode(webrtc::ScalabilityMode::kL1T1), + svcController(std::make_unique()) { auto mode = format.parameters.find("packetization-mode"); @@ -136,8 +140,49 @@ namespace jni gofIndex = 0; } + bool EncoderOutputProcessor::SetScalabilityMode(webrtc::ScalabilityMode mode) + { + std::unique_ptr controller = webrtc::CreateScalabilityStructure(mode); + + if (!controller) { + return false; + } + + scalabilityMode = mode; + svcController = std::move(controller); + + return true; + } + + void EncoderOutputProcessor::OnRatesUpdated(const webrtc::VideoBitrateAllocation & bitrates) + { + svcController->OnRatesUpdated(bitrates); + } + + std::optional EncoderOutputProcessor::NextFrameConfig( + bool keyFrame) + { + auto layerFrames = svcController->NextFrameConfig(/*restart=*/keyFrame); + + if (layerFrames.empty()) { + return std::nullopt; + } + + return layerFrames[0]; + } + bool EncoderOutputProcessor::Process(std::span bitstream, bool keyFrame, webrtc::EncodedImage & image, webrtc::CodecSpecificInfo & info) + { + // A stream without layers has a frame for every one encoded. + auto layerFrame = NextFrameConfig(keyFrame); + + return layerFrame && Process(bitstream, keyFrame, *layerFrame, image, info); + } + + bool EncoderOutputProcessor::Process(std::span bitstream, bool keyFrame, + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::EncodedImage & image, + webrtc::CodecSpecificInfo & info) { std::vector output; bool processed; @@ -189,7 +234,11 @@ namespace jni image.qp_ = -1; } - FillCodecSpecificInfo(image, keyFrame, info); + if (scalabilityMode != webrtc::ScalabilityMode::kL1T1) { + image.SetTemporalIndex(layerFrame.TemporalId()); + } + + FillCodecSpecificInfo(image, keyFrame, layerFrame, info); return true; } @@ -335,7 +384,7 @@ namespace jni } void EncoderOutputProcessor::FillCodecSpecificInfo(const webrtc::EncodedImage & image, bool keyFrame, - webrtc::CodecSpecificInfo & info) + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::CodecSpecificInfo & info) { info.codecType = codec; @@ -350,12 +399,11 @@ namespace jni // AV1 and H.265 have no codec specific information; WebRTC sends // their frame dependencies, and those of VP9, from these, in the // dependency descriptor. - auto layerFrames = svcController.NextFrameConfig(/*restart=*/keyFrame); - info.generic_frame_info = svcController.OnEncodeDone(layerFrames[0]); - info.scalability_mode = webrtc::ScalabilityMode::kL1T1; + info.generic_frame_info = svcController->OnEncodeDone(layerFrame); + info.scalability_mode = scalabilityMode; if (keyFrame) { - info.template_structure = svcController.DependencyStructure(); + info.template_structure = svcController->DependencyStructure(); info.template_structure->resolutions = { webrtc::RenderResolution(image._encodedWidth, image._encodedHeight) }; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp index 277bc3b2..ede55634 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp @@ -23,6 +23,7 @@ #include "rtc_base/logging.h" #include +#include namespace jni { @@ -55,7 +56,33 @@ namespace jni constexpr int kRefsPerFrame = 7; constexpr uint8_t kPrimaryRefNone = 7; + // The reference frames LAST and GOLDEN, and their index among the + // seven. constexpr uint32_t kLastFrame = 1; + constexpr uint32_t kGoldenFrame = 4; + constexpr int kGoldenIndex = 3; + // LAST is the first reference of a frame, and the one its probabilities + // and other state carry over from. + constexpr uint8_t kPrimaryRefLast = 0; + + // The slots of the two reference buffers L1T3 has, and the frame being + // encoded. + constexpr int kReconstructedSurfaces = 3; + + // The layers of the frames after a key frame, as + // VAEncMiscParameterTemporalLayerStructure takes them, of L1T2 and + // L1T3 as WebRTC's structures make them: L1T3 is 0, 2, 1, 2. + const std::vector kL1T2Pattern = { 1, 0 }; + const std::vector kL1T3Pattern = { 2, 1, 2, 0 }; + + // The share of the bitrate up to each layer until WebRTC gives its + // own, as WebRTC splits it for temporal layers. + constexpr double kL1T2Shares[] = { 0.6, 1.0 }; + constexpr double kL1T3Shares[] = { 0.4, 0.6, 1.0 }; + + // What a frame that cannot be encoded returns: libaom takes over the + // stream, which stays in AV1, rather than WebRTC changing the codec. + constexpr int32_t kEncodeFailed = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; constexpr uint32_t kOrderHintBits = 8; // TxMode: largest, or chosen per block. @@ -180,13 +207,20 @@ namespace jni uint32_t count = 0; }; - // An OBU of the payload: the header, without an extension, and - // obu_size as a leb128 of the given fixed length, which is what the - // driver rewrites the size of the frame header OBU as. - std::vector MakeObu(uint8_t type, const std::vector & payload, uint32_t sizeLength) + // An OBU of the payload: the header, with the extension that names + // the temporal layer if there is one, and obu_size as a leb128 of the + // given fixed length, which is what the driver rewrites the size of + // the frame header OBU as. + std::vector MakeObu(uint8_t type, const std::vector & payload, uint32_t sizeLength, + std::optional temporalId = std::nullopt) { std::vector obu; - obu.push_back(static_cast((type << 3) | 0x02)); + obu.push_back(static_cast((type << 3) | (temporalId ? 0x04 : 0) | 0x02)); + + if (temporalId) { + // temporal_id, and spatial_id 0. + obu.push_back(static_cast(*temporalId << 5)); + } const uint32_t size = static_cast(payload.size()); @@ -219,7 +253,7 @@ namespace jni ratesChanged(false), obuSizeLength(0), txMode(0), - current(0), + temporalLayers(1), referenceValid(false), frameNumber(0), callback(nullptr), @@ -279,8 +313,12 @@ namespace jni if (settings->numberOfSimulcastStreams > 1) { return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED; } - // A single layer; spatial or temporal layers are left to libaom. - if (settings->GetScalabilityMode().value_or(webrtc::ScalabilityMode::kL1T1) != webrtc::ScalabilityMode::kL1T1) { + // Temporal layers; spatial ones are left to libaom. + const webrtc::ScalabilityMode mode = settings->GetScalabilityMode().value_or(webrtc::ScalabilityMode::kL1T1); + + if (mode != webrtc::ScalabilityMode::kL1T1 && mode != webrtc::ScalabilityMode::kL1T2 + && mode != webrtc::ScalabilityMode::kL1T3) + { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } if (obuSizeLength == 0) { @@ -309,16 +347,28 @@ namespace jni bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; framerate = std::max(1u, codecSettings.maxFramerate); ratesChanged = true; - current = 0; + temporalLayers = mode == webrtc::ScalabilityMode::kL1T3 ? 3 : mode == webrtc::ScalabilityMode::kL1T2 ? 2 : 1; + slots.fill(-1); referenceValid = false; frameNumber = 0; outputProcessor.Reset(); + layerBitratesBps.clear(); + + for (int layer = 0; layer < temporalLayers && temporalLayers > 1; layer++) { + const double share = temporalLayers == 3 ? kL1T3Shares[layer] : kL1T2Shares[layer]; + layerBitratesBps.push_back(static_cast(bitrateBps * share)); + } + + if (!outputProcessor.SetScalabilityMode(mode)) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + const uint32_t surfaceWidth = Align(codecSettings.width, kSuperblockSize); const uint32_t surfaceHeight = Align(codecSettings.height, kSuperblockSize); if (!session.Create(VAProfileAV1Profile0, entrypoint, codecSettings.width, codecSettings.height, surfaceWidth, - surfaceHeight, kPackedHeaders)) + surfaceHeight, kPackedHeaders, kReconstructedSurfaces)) { Release(); return WEBRTC_VIDEO_CODEC_ERROR; @@ -329,7 +379,7 @@ namespace jni initialized = true; RTC_LOG(LS_INFO) << implementationName << " AV1 initialized: " << codecSettings.width << "x" - << codecSettings.height << " at " << bitrateBps << " bps"; + << codecSettings.height << " at " << bitrateBps << " bps, " << webrtc::ScalabilityModeToString(mode); return WEBRTC_VIDEO_CODEC_OK; } @@ -370,18 +420,33 @@ namespace jni frameNumber = 0; } + const auto layerFrame = outputProcessor.NextFrameConfig(keyFrame); + + // No layer is encoded now, as when WebRTC gives none a bitrate. + if (!layerFrame) { + return WEBRTC_VIDEO_CODEC_OK; + } + + FrameReferences references; + const int surface = FreeSurface(); + + if (surface < 0 || !PlanReferences(keyFrame, *layerFrame, references)) { + RTC_LOG(LS_WARNING) << implementationName << " AV1: a frame refers to a slot that holds no frame"; + return kEncodeFailed; + } + std::vector output; // A frame up to the alignment larger is cropped; one of another size, // which raced with a change of size, is scaled. const bool encoded = session.Upload(frame, static_cast(codecSettings.width), - static_cast(codecSettings.height), kSizeAlignment - 1) && Submit(keyFrame) + static_cast(codecSettings.height), kSizeAlignment - 1) && Submit(keyFrame, references, surface) && session.ReadOutput(output); session.DropBuffers(); if (!encoded) { - return WEBRTC_VIDEO_CODEC_ERROR; + return kEncodeFailed; } webrtc::EncodedImage image; @@ -394,13 +459,18 @@ namespace jni webrtc::CodecSpecificInfo info; - if (!outputProcessor.Process(output, keyFrame, image, info)) { - return WEBRTC_VIDEO_CODEC_ERROR; + if (!outputProcessor.Process(output, keyFrame, *layerFrame, image, info)) { + return kEncodeFailed; + } + + // The slots the frame replaces hold it now. + for (int slot = 0; slot < static_cast(slots.size()); slot++) { + if (references.refreshFrameFlags & (1 << slot)) { + slots[slot] = surface; + } } - // The frame just encoded is the reference of the next one. referenceValid = true; - current = 1 - current; frameNumber++; ratesChanged = false; @@ -409,13 +479,81 @@ namespace jni return WEBRTC_VIDEO_CODEC_OK; } - bool VaapiAv1Encoder::Submit(bool keyFrame) + bool VaapiAv1Encoder::PlanReferences(bool keyFrame, + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, FrameReferences & references) const + { + references.temporalId = layerFrame.TemporalId(); + + // A key frame replaces every slot. + if (keyFrame) { + references.refreshFrameFlags = 0xFF; + return true; + } + + std::vector referenced; + + for (const webrtc::CodecBufferUsage & buffer : layerFrame.Buffers()) { + if (buffer.id < 0 || buffer.id >= static_cast(slots.size())) { + return false; + } + if (buffer.referenced) { + if (slots[buffer.id] < 0) { + return false; + } + + referenced.push_back(buffer.id); + } + if (buffer.updated) { + references.refreshFrameFlags |= static_cast(1 << buffer.id); + } + } + + if (referenced.empty()) { + return false; + } + + // Every reference frame names the first reference, but golden, which + // names the second where there is one. + references.refFrameIdx.fill(static_cast(referenced[0])); + + if (referenced.size() > 1) { + references.refFrameIdx[kGoldenIndex] = static_cast(referenced[1]); + references.golden = true; + } + + return true; + } + + bool VaapiAv1Encoder::AddRateControl(bool keyFrame) + { + // No initial quantizer: the rate control picks its own, as FFmpeg + // leaves it. + if (temporalLayers == 1) { + return session.AddRateControl(bitrateBps, framerate, 0); + } + + // Without a rate control of each layer, AMD's driver gives the upper + // layers next to no bits. + return session.AddLayerRateControl(layerBitratesBps, framerate, + temporalLayers == 3 ? kL1T3Pattern : kL1T2Pattern, keyFrame); + } + + int VaapiAv1Encoder::FreeSurface() const + { + for (int surface = 0; surface < kReconstructedSurfaces; surface++) { + if (std::find(slots.begin(), slots.end(), surface) == slots.end()) { + return surface; + } + } + + return -1; + } + + bool VaapiAv1Encoder::Submit(bool keyFrame, const FrameReferences & references, int surface) { const uint32_t sbCols = Align(codecSettings.width, kSuperblockSize) / kSuperblockSize; const uint32_t sbRows = Align(codecSettings.height, kSuperblockSize) / kSuperblockSize; - // The slot of the frame this one refers to, which the reconstructed - // surfaces stand for. - const int referenceSlot = 1 - current; + const bool layered = temporalLayers > 1; if (keyFrame) { VAEncSequenceParameterBufferAV1 sequence = {}; @@ -425,6 +563,7 @@ namespace jni // Key frames only where asked for. sequence.intra_period = kKeyFrameInterval; sequence.ip_period = 1; + sequence.hierarchical_flag = layered ? 1 : 0; sequence.bits_per_second = bitrateBps; sequence.seq_fields.bits.enable_order_hint = 1; sequence.seq_fields.bits.enable_cdef = 1; @@ -435,21 +574,20 @@ namespace jni } } - // No initial quantizer: the rate control picks its own, as FFmpeg - // leaves it. - if ((keyFrame || ratesChanged) && !session.AddRateControl(bitrateBps, framerate, 0)) { + if ((keyFrame || ratesChanged) && !AddRateControl(keyFrame)) { return false; } std::vector frameHeader; FrameHeaderLayout layout; - WriteFrameHeader(frameHeader, layout, keyFrame); + WriteFrameHeader(frameHeader, layout, keyFrame, references); VAEncPictureParameterBufferAV1 picture = {}; picture.frame_width_minus_1 = static_cast(codecSettings.width - 1); picture.frame_height_minus_1 = static_cast(codecSettings.height - 1); - picture.reconstructed_frame = session.Reconstructed(current); + picture.reconstructed_frame = session.Reconstructed(surface); picture.coded_buf = session.CodedBuffer(); + picture.refresh_frame_flags = references.refreshFrameFlags; for (VASurfaceID & reference : picture.reference_frames) { reference = VA_INVALID_SURFACE; @@ -457,25 +595,35 @@ namespace jni if (keyFrame) { picture.primary_ref_frame = kPrimaryRefNone; - picture.refresh_frame_flags = 0xFF; picture.picture_flags.bits.frame_type = kKeyFrameType; picture.picture_flags.bits.error_resilient_mode = 1; } else { - picture.reference_frames[referenceSlot] = session.Reconstructed(referenceSlot); - - for (int i = 0; i < kRefsPerFrame; i++) { - picture.ref_frame_idx[i] = static_cast(referenceSlot); + for (int slot = 0; slot < static_cast(slots.size()); slot++) { + if (slots[slot] >= 0) { + picture.reference_frames[slot] = session.Reconstructed(slots[slot]); + } } - // Every reference frame names the same slot, so any of them is - // the primary one; FFmpeg takes the index of the slot. - picture.primary_ref_frame = static_cast(referenceSlot); - picture.refresh_frame_flags = static_cast(1 << current); + std::copy(references.refFrameIdx.begin(), references.refFrameIdx.end(), picture.ref_frame_idx); + + picture.primary_ref_frame = kPrimaryRefLast; picture.ref_frame_ctrl_l0.fields.search_idx0 = kLastFrame; + + if (references.golden) { + picture.ref_frame_ctrl_l0.fields.search_idx1 = kGoldenFrame; + } + picture.picture_flags.bits.frame_type = kInterFrameType; } + if (layered) { + picture.temporal_id = static_cast(references.temporalId); + picture.hierarchical_level_plus1 = static_cast(references.temporalId + 1); + picture.tile_group_obu_hdr_info.bits.obu_extension_flag = 1; + picture.tile_group_obu_hdr_info.bits.temporal_id = static_cast(references.temporalId); + } + picture.order_hint = static_cast(frameNumber); picture.base_qindex = kQIndex; picture.min_base_qindex = kMinQIndex; @@ -496,7 +644,8 @@ namespace jni picture.bit_offset_cdef_params = layout.cdefOffset; picture.size_in_bits_cdef_params = layout.cdefSize; picture.size_in_bits_frame_hdr_obu = layout.size; - picture.byte_offset_frame_hdr_obu_size = (keyFrame ? static_cast(sequenceHeader.size()) : 0) + 1; + picture.byte_offset_frame_hdr_obu_size = (keyFrame ? static_cast(sequenceHeader.size()) : 0) + + (layered ? 2 : 1); VAEncTileGroupBufferAV1 tileGroup = {}; tileGroup.tg_start = 0; @@ -522,17 +671,24 @@ namespace jni writer.Put(0, 3); writer.Put(0, 1); writer.Put(0, 1); - // timing_info_present_flag, initial_display_delay_present_flag, - // operating_points_cnt_minus_1, operating_point_idc[0]. + // timing_info_present_flag, initial_display_delay_present_flag. writer.Put(0, 1); writer.Put(0, 1); - writer.Put(0, 5); - writer.Put(0, 12); - // seq_level_idx[0], and seq_tier[0] above level 3.3. - writer.Put(level, 5); + // operating_points_cnt_minus_1: with temporal layers, an operating + // point for each, from all layers down to the lowest. Then + // operating_point_idc, of spatial layer 0 and those temporal layers, + // seq_level_idx, and seq_tier above level 3.3. + writer.Put(temporalLayers - 1, 5); - if (level > 7) { - writer.Put(0, 1); + for (int point = 0; point < temporalLayers; point++) { + const uint32_t idc = temporalLayers > 1 ? 0x100 | ((1u << (temporalLayers - point)) - 1) : 0; + + writer.Put(idc, 12); + writer.Put(level, 5); + + if (level > 7) { + writer.Put(0, 1); + } } writer.Put(widthBits - 1, 4); @@ -574,17 +730,18 @@ namespace jni obu = MakeObu(kObuSequenceHeader, writer.Bytes(), obuSizeLength); } - void VaapiAv1Encoder::WriteFrameHeader(std::vector & obu, FrameHeaderLayout & layout, bool keyFrame) const + void VaapiAv1Encoder::WriteFrameHeader(std::vector & obu, FrameHeaderLayout & layout, bool keyFrame, + const FrameReferences & references) const { // uncompressed_header, section 5.9, as FFmpeg's VA-API encoder writes // it, for the sequence header above. - const int referenceSlot = 1 - current; + const bool layered = temporalLayers > 1; const uint32_t miCols = 2 * ((codecSettings.width + 7) >> 3); const uint32_t miRows = 2 * ((codecSettings.height + 7) >> 3); const uint32_t sbCols = (miCols + 15) >> 4; const uint32_t sbRows = (miRows + 15) >> 4; - // The OBU header and obu_size come before the payload. - const uint32_t payloadOffset = (1 + obuSizeLength) * 8; + // The OBU header, its extension, and obu_size come before the payload. + const uint32_t payloadOffset = ((layered ? 2 : 1) + obuSizeLength) * 8; BitWriter writer; // show_existing_frame, frame_type, show_frame. @@ -604,14 +761,13 @@ namespace jni if (!keyFrame) { // primary_ref_frame, refresh_frame_flags. - writer.Put(referenceSlot, 3); - writer.Put(1u << current, 8); - // frame_refs_short_signaling, then ref_frame_idx: every reference - // is the frame before. + writer.Put(kPrimaryRefLast, 3); + writer.Put(references.refreshFrameFlags, 8); + // frame_refs_short_signaling, then ref_frame_idx. writer.Put(0, 1); for (int i = 0; i < kRefsPerFrame; i++) { - writer.Put(referenceSlot, 3); + writer.Put(references.refFrameIdx[i], 3); } } @@ -692,7 +848,8 @@ namespace jni writer.TrailingBits(); - obu = MakeObu(kObuFrameHeader, writer.Bytes(), obuSizeLength); + obu = MakeObu(kObuFrameHeader, writer.Bytes(), obuSizeLength, + layered ? std::optional(references.temporalId) : std::nullopt); layout.size = static_cast(obu.size() * 8); } @@ -724,6 +881,17 @@ namespace jni } bitrateBps = bitrate; + outputProcessor.OnRatesUpdated(parameters.bitrate); + + // The bitrate of each temporal layer, counting those below it. + uint32_t sum = 0; + + for (int layer = 0; layer < static_cast(layerBitratesBps.size()); layer++) { + sum += parameters.bitrate.GetBitrate(0, layer); + // A layer WebRTC gives no bitrate is not encoded; it keeps the + // rate of the one below, which the driver takes. + layerBitratesBps[layer] = std::max(sum, 1u); + } if (parameters.framerate_fps >= 1.0) { framerate = static_cast(parameters.framerate_fps + 0.5); @@ -743,6 +911,14 @@ namespace jni info.scaling_settings = outputProcessor.GetScalingSettings(); info.requested_resolution_alignment = kSizeAlignment; + // With temporal layers, the share of the frame rate up to each. + if (temporalLayers > 1) { + for (int layer = 0; layer < temporalLayers; layer++) { + info.fps_allocation[0].push_back(static_cast( + EncoderInfo::kMaxFramerateFraction >> (temporalLayers - 1 - layer))); + } + } + return info; } } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiEncodeSession.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiEncodeSession.cpp index 4f0bd54e..dab2bdb4 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiEncodeSession.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiEncodeSession.cpp @@ -38,7 +38,6 @@ namespace jni name(std::move(name)), config(VA_INVALID_ID), context(VA_INVALID_ID), - surfaces{ VA_INVALID_SURFACE, VA_INVALID_SURFACE, VA_INVALID_SURFACE }, codedBuffer(VA_INVALID_ID) { } @@ -60,7 +59,8 @@ namespace jni } bool VaapiEncodeSession::Create(VAProfile profile, VAEntrypoint entrypoint, uint32_t pictureWidth, - uint32_t pictureHeight, uint32_t surfaceWidth, uint32_t surfaceHeight, uint32_t packedHeaders) + uint32_t pictureHeight, uint32_t surfaceWidth, uint32_t surfaceHeight, uint32_t packedHeaders, + int reconstructedCount) { Destroy(); @@ -83,18 +83,18 @@ namespace jni return false; } - if (!Check(va.CreateSurfaces(display, VA_RT_FORMAT_YUV420, surfaceWidth, surfaceHeight, surfaces, 3, nullptr, 0), - "creating surfaces")) - { - for (VASurfaceID & surface : surfaces) { - surface = VA_INVALID_SURFACE; - } + const int surfaceCount = 1 + reconstructedCount; + surfaces.assign(surfaceCount, VA_INVALID_SURFACE); + if (!Check(va.CreateSurfaces(display, VA_RT_FORMAT_YUV420, surfaceWidth, surfaceHeight, surfaces.data(), + surfaceCount, nullptr, 0), "creating surfaces")) + { + surfaces.clear(); return false; } - if (!Check(va.CreateContext(display, config, pictureWidth, pictureHeight, VA_PROGRESSIVE, surfaces, 3, &context), - "creating the context")) + if (!Check(va.CreateContext(display, config, pictureWidth, pictureHeight, VA_PROGRESSIVE, surfaces.data(), + surfaceCount, &context), "creating the context")) { context = VA_INVALID_ID; return false; @@ -127,12 +127,9 @@ namespace jni va.DestroyContext(display, context); context = VA_INVALID_ID; } - if (surfaces[0] != VA_INVALID_SURFACE) { - va.DestroySurfaces(display, surfaces, 3); - } - - for (VASurfaceID & surface : surfaces) { - surface = VA_INVALID_SURFACE; + if (!surfaces.empty()) { + va.DestroySurfaces(display, surfaces.data(), static_cast(surfaces.size())); + surfaces.clear(); } if (config != VA_INVALID_ID) { @@ -283,6 +280,52 @@ namespace jni && AddMiscParameter(VAEncMiscParameterTypeHRD, &hrd, sizeof(hrd)); } + bool VaapiEncodeSession::AddLayerRateControl(const std::vector & bitratesBps, uint32_t framerate, + const std::vector & pattern, bool keyFrame) + { + const uint32_t layers = static_cast(bitratesBps.size()); + + if (keyFrame) { + VAEncMiscParameterTemporalLayerStructure structure = {}; + structure.number_of_layers = layers; + structure.periodicity = static_cast(pattern.size()); + + for (size_t i = 0; i < pattern.size() && i < 32; i++) { + structure.layer_id[i] = pattern[i]; + } + + if (!AddMiscParameter(VAEncMiscParameterTypeTemporalLayerStructure, &structure, sizeof(structure))) { + return false; + } + } + + for (uint32_t layer = 0; layer < layers; layer++) { + VAEncMiscParameterRateControl rateControl = {}; + rateControl.bits_per_second = bitratesBps[layer]; + rateControl.target_percentage = 100; + rateControl.window_size = 1000; + rateControl.rc_flags.bits.temporal_id = layer; + + // Each layer down has half the frame rate of the one above it. + VAEncMiscParameterFrameRate frameRate = {}; + frameRate.framerate = framerate | ((1u << (layers - 1 - layer)) << 16); + frameRate.framerate_flags.bits.temporal_id = layer; + + if (!AddMiscParameter(VAEncMiscParameterTypeRateControl, &rateControl, sizeof(rateControl)) + || !AddMiscParameter(VAEncMiscParameterTypeFrameRate, &frameRate, sizeof(frameRate))) + { + return false; + } + } + + VAEncMiscParameterHRD hrd = {}; + hrd.buffer_size = static_cast( + static_cast(bitratesBps.back()) * kHrdBufferNumerator / kHrdBufferDenominator); + hrd.initial_buffer_fullness = hrd.buffer_size / 2; + + return AddMiscParameter(VAEncMiscParameterTypeHRD, &hrd, sizeof(hrd)); + } + bool VaapiEncodeSession::Encode() { VADisplay display = library.Display(); diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java index 88d922aa..84c517f0 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java @@ -38,7 +38,8 @@ * other GPUs it is encoded with the Media Foundation encoder of the driver on * Windows, and with its VA-API encoder on Linux. AV1 is encoded on GPUs that * have an AV1 encoder, with NVENC, Media Foundation or VA-API, as a single - * layer; a stream that asks for SVC is encoded in software. H.265 (Main profile) is + * layer, or with VA-API in temporal layers too (L1T2, L1T3); a stream that + * asks for other SVC modes is encoded in software. H.265 (Main profile) is * encoded with NVENC or Media Foundation on GPUs that have an H.265 encoder. * VP9 (profile 0) is encoded on GPUs that have a VP9 encoder, which are * Intel's: with VA-API on Linux, and with Media Foundation on Windows only diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java index c2a7f2f4..5486c161 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java @@ -18,6 +18,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeFalse; @@ -41,6 +42,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -520,6 +522,78 @@ void av1RestartsCleanly() throws Exception { } } + @Test + void av1EncodesTemporalLayers() throws Exception { + // Temporal layers are references between frames of one size, which a + // hardware encoder can make; an SFU can then forward a lower frame + // rate without asking for key frames. + for (String mode : new String[] { "L1T2", "L1T3" }) { + PeerConnectionFactory hardware = hardwareFactory(); + CountDownLatch received = new CountDownLatch(30); + + try (TestMediaCall call = new TestMediaCall(hardware, true, false, AV1)) { + call.setVideoSize(640, 360); + call.negotiate(); + + RTCRtpReceiver receiver = call.getReceiver("video"); + VideoTrack track = (VideoTrack) receiver.getTrack(); + VideoTrackSink sink = frame -> { + frame.release(); + received.countDown(); + }; + track.addSink(sink); + + call.awaitConnected(); + + RTCRtpSendParameters parameters = call.getVideoSender().getParameters(); + parameters.encodings.get(0).scalabilityMode = mode; + call.getVideoSender().setParameters(parameters); + + call.startMedia(); + + assertTrue(received.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), "too few frames received in " + mode); + + Map outbound = awaitOutboundStats(call, "encoderImplementation", "scalabilityMode"); + Map inbound = call.getInboundVideoStats(); + Object pliCount = inbound == null ? null : inbound.get("pliCount"); + + track.removeSink(sink); + receiver.dispose(); + + assertHardware(String.valueOf(outbound.get("encoderImplementation")), HARDWARE_AV1_REQUIRED); + assertEquals(mode, outbound.get("scalabilityMode")); + assertTrue(!(pliCount instanceof Number) || ((Number) pliCount).longValue() <= 1, + "key frames requested: " + pliCount); + } + finally { + hardware.dispose(); + } + } + } + + /** + * Returns the outbound video statistics once they have all the given + * members, which come shortly after the encoder. + */ + private static Map awaitOutboundStats(TestMediaCall call, String... members) + throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + Map outbound = null; + + while (System.nanoTime() < deadline) { + outbound = call.getOutboundVideoStats(); + + if (outbound != null && Arrays.stream(members).allMatch(outbound::containsKey)) { + return outbound; + } + + Thread.sleep(100); + } + + assertNotNull(outbound, "no outbound statistics"); + return outbound; + } + @Test void av1BelowHardwareMinimumStaysAv1() throws Exception { PeerConnectionFactory hardware = hardwareFactory(); From f41d5247f110c54b19c9db1e3fa1c51aee2c7e4d Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Thu, 15 Oct 2026 19:24:24 +0200 Subject: [PATCH 2/3] feat: encode VP9 temporal layers with VA-API The VA-API VP9 encoder takes L1T2 and L1T3 too, which Intel GPUs encode, in error resilient mode and with the VP9 information of flexible mode, as libvpx does. The AV1 and VP9 encoders share the layer handling. --- docs/guide/advanced/video-codecs.md | 2 +- .../video/codec/EncoderOutputProcessor.h | 24 ++- .../media/video/codec/linux/VaapiAv1Encoder.h | 37 +--- .../video/codec/linux/VaapiTemporalLayers.h | 102 ++++++++++ .../media/video/codec/linux/VaapiVp9Encoder.h | 26 +-- .../video/codec/EncoderOutputProcessor.cpp | 77 ++++++- .../video/codec/linux/VaapiAv1Encoder.cpp | 189 +++++------------- .../video/codec/linux/VaapiTemporalLayers.cpp | 173 ++++++++++++++++ .../video/codec/linux/VaapiVp9Encoder.cpp | 110 +++++++--- .../codec/HardwareVideoEncoderFactory.java | 3 +- .../HardwareVideoEncoderIntegrationTest.java | 27 ++- 11 files changed, 542 insertions(+), 228 deletions(-) create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiTemporalLayers.h create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiTemporalLayers.cpp diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index e87bc22e..a8d739bf 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -30,7 +30,7 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder() | Linux | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) where it encodes H.265 | NVENC on NVIDIA RTX 40 and newer, otherwise the VA-API encoder of the GPU driver where it encodes AV1 (e.g. AMD RDNA3 and newer) | | macOS | VideoToolbox, as with `DefaultVideoEncoderFactory` | VideoToolbox, on Macs with a hardware HEVC encoder, such as all with Apple silicon | Software | -Hardware encoders produce a single layer, but for the VA-API AV1 encoder, which encodes temporal layers too (`L1T2` and `L1T3`), with a rate control of each layer, so that an SFU can forward a lower frame rate. A stream that asks for spatial layers, or for temporal layers from another hardware encoder, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom. +Hardware encoders produce a single layer, but for the VA-API encoders of AV1 and VP9, which encode temporal layers too (`L1T2` and `L1T3`), with a rate control of each layer, so that an SFU can forward a lower frame rate. A stream that asks for spatial layers, or for temporal layers from another hardware encoder, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom. VP9 can be encoded in hardware too, with the VP9 encoder of the GPU driver, which Intel GPUs have: through VA-API on Linux, and through Media Foundation on Windows. On Windows it is off by default, since it has not been tested on such a GPU yet; turn it on with a system property before the factory is created: diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h b/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h index 0e158255..fbc761fd 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h @@ -30,6 +30,7 @@ #include "api/video_codecs/scalability_mode.h" #include "modules/video_coding/svc/scalable_video_controller.h" +#include #include #include #include @@ -47,10 +48,11 @@ namespace jni // or the sequence header of AV1. Encoders put them in front of the first // key frame, not necessarily in front of every one, so they are kept and // put back where missing. The frame dependencies WebRTC needs for AV1 and - // H.265 are those of a stream without layers, unless an AV1 encoder that - // makes temporal layers sets its scalability mode and asks for the layer - // of each frame before it encodes it. VP9 frames go out as they come, with - // the VP9 information of a single layer, as the Java encoders get it. + // H.265 are those of a stream without layers, unless an AV1 or VP9 encoder + // that makes temporal layers sets its scalability mode and asks for the + // layer of each frame before it encodes it. VP9 frames go out as they + // come, with the VP9 information of a single layer, as the Java encoders + // get it, or with temporal layers in flexible mode, as libvpx gives it. class EncoderOutputProcessor { public: @@ -59,8 +61,8 @@ namespace jni // Forgets the parameter sets, for a stream that starts over. void Reset(); - // Sets the layers of the stream, temporal ones of AV1 only, as in - // L1T3. Returns false for a mode WebRTC has no structure of. + // Sets the layers of the stream, temporal ones of AV1 and VP9 only, + // as in L1T3. Returns false for a mode WebRTC has no structure of. bool SetScalabilityMode(webrtc::ScalabilityMode mode); // Lets the layers follow the bitrate WebRTC gives each. void OnRatesUpdated(const webrtc::VideoBitrateAllocation & bitrates); @@ -96,6 +98,10 @@ namespace jni void FillCodecSpecificInfo(const webrtc::EncodedImage & image, bool keyFrame, const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::CodecSpecificInfo & info); + // The VP9 information of a picture of a stream with temporal + // layers. + void FillVp9TemporalLayers(const webrtc::EncodedImage & image, bool keyFrame, + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::CodecSpecificInfo & info); private: const webrtc::VideoCodecType codec; @@ -114,6 +120,12 @@ namespace jni // referring to the one before. webrtc::GofInfoVP9 gof; size_t gofIndex = 0; + + // With temporal layers: the pictures since the key frame, and the + // picture each reference buffer holds, which the references of a + // picture are told by. + size_t vp9Picture = 0; + std::array vp9BufferPictures = {}; }; } diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h index e4885707..9197bccc 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiAv1Encoder.h @@ -20,6 +20,7 @@ #include "media/video/codec/EncoderOutputProcessor.h" #include "media/video/codec/linux/VaapiEncodeSession.h" #include "media/video/codec/linux/VaapiLibrary.h" +#include "media/video/codec/linux/VaapiTemporalLayers.h" #include "api/video/video_frame.h" #include "api/video_codecs/sdp_video_format.h" @@ -80,32 +81,11 @@ namespace jni // Reads which headers the driver takes and how it codes them. void QueryCapabilities(); - // What a frame refers to: the slot each of the seven reference - // frames names, the second reference, if any, being golden; the - // slots the frame replaces; and its temporal layer. - struct FrameReferences - { - std::array refFrameIdx = {}; - bool golden = false; - uint8_t refreshFrameFlags = 0; - int temporalId = 0; - }; - - // Turns the buffers of the layer frame into reference slots. - // Returns false if it refers to a slot that holds no frame, or to - // one beyond those kept. - bool PlanReferences(bool keyFrame, const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, - FrameReferences & references) const; - // A reconstructed surface that no reference slot holds. - int FreeSurface() const; - // Adds the rate control of the stream, or of each temporal layer. - bool AddRateControl(bool keyFrame); - - bool Submit(bool keyFrame, const FrameReferences & references, int surface); + bool Submit(bool keyFrame, const VaapiTemporalLayers::Frame & frame); void WriteSequenceHeader(std::vector & obu) const; void WriteFrameHeader(std::vector & obu, FrameHeaderLayout & layout, bool keyFrame, - const FrameReferences & references) const; + const VaapiTemporalLayers::Frame & frame) const; // The seq_level_idx the stream's size, frame rate and bitrate fit. uint8_t Level() const; @@ -128,16 +108,7 @@ namespace jni // The sequence header, kept for the key frames. std::vector sequenceHeader; - // The temporal layers, one without layers, and the bitrate of each, - // counting the layers below it. - int temporalLayers; - std::vector layerBitratesBps; - - // The reconstructed surface each reference slot holds, -1 for - // none: those of the two buffers WebRTC's structures use, up to - // L1T3. A key frame fills the other six slots too, but no frame - // refers to them. - std::array slots; + VaapiTemporalLayers layers; bool referenceValid; // The frames since the last key frame, whose low bits are the // order hint. diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiTemporalLayers.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiTemporalLayers.h new file mode 100644 index 00000000..769fc7ea --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiTemporalLayers.h @@ -0,0 +1,102 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_TEMPORAL_LAYERS_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_TEMPORAL_LAYERS_H_ + +#include "media/video/codec/linux/VaapiEncodeSession.h" + +#include "api/video/video_bitrate_allocation.h" +#include "api/video_codecs/scalability_mode.h" +#include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/svc/scalable_video_controller.h" + +#include +#include +#include +#include + +namespace jni +{ + // The temporal layers of a VA-API encoder of VP9 or AV1, none or those of + // L1T2 and L1T3: which reference slot each frame refers to and which ones + // it replaces, as WebRTC's structure of the mode says, each buffer of the + // structure being the slot of the same number; which reconstructed surface + // each slot holds; and the rate control of each layer, without which + // AMD's driver gives the upper layers next to no bits. + class VaapiTemporalLayers + { + public: + // The reconstructed surfaces a session needs: those of the two + // reference buffers WebRTC's structures use up to L1T3, and that + // of the frame being encoded. + static constexpr int kReconstructedSurfaces = 3; + + // What a frame refers to and replaces. + struct Frame + { + int temporalId = 0; + // The slots of its references, the first being its last frame, + // the second, if any, its golden frame. + std::vector references; + uint8_t refreshFlags = 0; + // The reconstructed surface it is encoded into. + int surface = -1; + }; + + VaapiTemporalLayers(); + + // Starts over in the mode, splitting the bitrate across the layers + // until WebRTC gives its own split. Returns false for a mode other + // than L1T1, L1T2 and L1T3. + bool Configure(webrtc::ScalabilityMode mode, uint32_t bitrateBps); + + // The number of temporal layers, one without layers. + int Count() const; + + // Plans a frame of the layer WebRTC's structure gives. Returns + // nothing if it refers to a slot that holds no frame, or to one + // beyond those kept, or no surface is free. + std::optional Plan(bool keyFrame, + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame) const; + // Takes the frame, once encoded, into the slots it replaces. + void Commit(const Frame & frame); + + // The slots kept, and the reconstructed surface each holds, -1 for + // none. A key frame fills the slots beyond too, but no frame refers + // to them. + int SlotCount() const; + int Surface(int slot) const; + + // Takes the bitrate WebRTC gives each layer. + void OnRatesUpdated(const webrtc::VideoBitrateAllocation & bitrates); + // Adds the rate control of the stream, or of each layer, with the + // layer structure along with that of a key frame. No initial + // quantizer: the rate control picks its own, as FFmpeg leaves it. + bool AddRateControl(VaapiEncodeSession & session, uint32_t bitrateBps, uint32_t framerate, + bool keyFrame) const; + // Fills in the share of the frame rate up to each layer. + void FillFpsAllocation(webrtc::VideoEncoder::EncoderInfo & info) const; + + private: + int count; + std::array slots; + // The bitrate of each layer, counting the layers below it. + std::vector bitratesBps; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Encoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Encoder.h index 9bcfb3aa..54d41d28 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Encoder.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiVp9Encoder.h @@ -20,6 +20,7 @@ #include "media/video/codec/EncoderOutputProcessor.h" #include "media/video/codec/linux/VaapiEncodeSession.h" #include "media/video/codec/linux/VaapiLibrary.h" +#include "media/video/codec/linux/VaapiTemporalLayers.h" #include "api/video/video_frame.h" #include "api/video_codecs/sdp_video_format.h" @@ -36,15 +37,16 @@ namespace jni { // Encodes VP9 profile 0 with the GPU driver's VA-API encoder, as Intel - // drivers provide it on Linux. Like hardware VP9 encoding on Windows, it - // has not run on such a GPU yet, and is used only where it is turned on. + // drivers provide it on Linux. // - // It works as the H.265 one does: synchronous, one reference frame, inter - // frames only, the driver writing the uncompressed frame header from the - // parameters it is given, as FFmpeg's VA-API encoder sets them without - // B-frames. Every frame refreshes all eight reference slots, so that the - // next one can refer to it as the last frame whichever slot it names. - // Anything that fails is an error, upon which libvpx takes over. + // It works as the H.265 one does: synchronous, inter frames only, the + // driver writing the uncompressed frame header from the parameters it is + // given, as FFmpeg's VA-API encoder sets them without B-frames. Each frame + // refers to one frame before it: without layers the frame just before; + // with temporal layers (L1T2, L1T3) the one of the same layer or a lower + // one that WebRTC's structure of the mode names, so that a receiver can do + // without the upper layers. Spatial layers, and anything that fails, are + // left to libvpx. class VaapiVp9Encoder : public webrtc::VideoEncoder { public: @@ -59,10 +61,11 @@ namespace jni EncoderInfo GetEncoderInfo() const override; private: - bool Submit(bool keyFrame); + bool Submit(bool keyFrame, const VaapiTemporalLayers::Frame & frame); void FillSequence(VAEncSequenceParameterBufferVP9 & sequence) const; - void FillPicture(VAEncPictureParameterBufferVP9 & picture, bool keyFrame) const; + void FillPicture(VAEncPictureParameterBufferVP9 & picture, bool keyFrame, + const VaapiTemporalLayers::Frame & frame) const; private: VaapiLibrary & library; @@ -76,8 +79,7 @@ namespace jni uint32_t framerate; bool ratesChanged; - // Which of the reconstructed surfaces holds the current frame. - int current; + VaapiTemporalLayers layers; bool referenceValid; webrtc::EncodedImageCallback * callback; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp index 1609dd16..c401dbdd 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp @@ -22,6 +22,7 @@ #include "common_video/h265/h265_common.h" #include "modules/video_coding/codecs/interface/common_constants.h" #include "modules/video_coding/svc/create_scalability_structure.h" +#include "modules/video_coding/svc/scalability_mode_util.h" #include "modules/video_coding/svc/scalable_video_controller_no_layering.h" #include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" #include "rtc_base/logging.h" @@ -409,7 +410,10 @@ namespace jni }; } - if (codec == webrtc::kVideoCodecVP9) { + if (codec == webrtc::kVideoCodecVP9 && scalabilityMode != webrtc::ScalabilityMode::kL1T1) { + FillVp9TemporalLayers(image, keyFrame, layerFrame, info); + } + else if (codec == webrtc::kVideoCodecVP9) { // VP9 has its own RTP header, which WebRTC fills from these: one // spatial and one temporal layer, the group of frames sent with // every key frame. @@ -436,6 +440,77 @@ namespace jni } } + void EncoderOutputProcessor::FillVp9TemporalLayers(const webrtc::EncodedImage & image, bool keyFrame, + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, webrtc::CodecSpecificInfo & info) + { + // One spatial layer and temporal ones, in flexible mode, as libvpx + // describes them: each picture names the pictures it refers to, by + // how many pictures back they are. + vp9Picture = keyFrame ? 0 : vp9Picture + 1; + + webrtc::CodecSpecificInfoVP9 & vp9 = info.codecSpecific.VP9; + vp9.first_frame_in_picture = true; + vp9.flexible_mode = true; + vp9.ss_data_available = keyFrame; + vp9.non_ref_for_inter_layer_pred = true; + vp9.temporal_idx = static_cast(layerFrame.TemporalId()); + vp9.inter_layer_predicted = false; + vp9.gof_idx = webrtc::kNoGofIdx; + vp9.num_spatial_layers = 1; + vp9.first_active_layer = 0; + vp9.spatial_layer_resolution_present = keyFrame; + vp9.num_ref_pics = 0; + + if (!keyFrame) { + for (const webrtc::CodecBufferUsage & buffer : layerFrame.Buffers()) { + if (!buffer.referenced || buffer.id < 0 || buffer.id >= webrtc::kMaxEncoderBuffers) { + continue; + } + + const size_t pDiff = vp9Picture - vp9BufferPictures[buffer.id]; + bool listed = false; + + for (uint8_t i = 0; i < vp9.num_ref_pics; i++) { + listed |= vp9.p_diff[i] == pDiff; + } + + if (!listed && pDiff > 0 && vp9.num_ref_pics < webrtc::kMaxVp9RefPics) { + vp9.p_diff[vp9.num_ref_pics++] = static_cast(pDiff); + } + } + } + + vp9.inter_pic_predicted = !keyFrame && vp9.num_ref_pics > 0; + + // A switch up to a higher layer is possible where all layers above + // this one are switch points, as libvpx works it out. + const int temporalLayers = webrtc::ScalabilityModeToNumTemporalLayers(scalabilityMode); + vp9.temporal_up_switch = true; + + for (int layer = layerFrame.TemporalId() + 1; layer < temporalLayers && info.generic_frame_info; layer++) { + const auto & indications = info.generic_frame_info->decode_target_indications; + + vp9.temporal_up_switch &= layer < static_cast(indications.size()) + && indications[layer] == webrtc::DecodeTargetIndication::kSwitch; + } + + if (keyFrame) { + vp9.width[0] = image._encodedWidth; + vp9.height[0] = image._encodedHeight; + vp9.gof.num_frames_in_gof = 0; + } + + for (const webrtc::CodecBufferUsage & buffer : layerFrame.Buffers()) { + if ((buffer.updated || keyFrame) && buffer.id >= 0 && buffer.id < webrtc::kMaxEncoderBuffers) { + vp9BufferPictures[buffer.id] = vp9Picture; + } + } + + if (keyFrame) { + vp9BufferPictures.fill(0); + } + } + webrtc::VideoEncoder::ScalingSettings EncoderOutputProcessor::GetScalingSettings() const { // H.265 has no thresholds of its own in WebRTC yet and uses those of diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp index ede55634..dc61ffc7 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp @@ -65,21 +65,6 @@ namespace jni // and other state carry over from. constexpr uint8_t kPrimaryRefLast = 0; - // The slots of the two reference buffers L1T3 has, and the frame being - // encoded. - constexpr int kReconstructedSurfaces = 3; - - // The layers of the frames after a key frame, as - // VAEncMiscParameterTemporalLayerStructure takes them, of L1T2 and - // L1T3 as WebRTC's structures make them: L1T3 is 0, 2, 1, 2. - const std::vector kL1T2Pattern = { 1, 0 }; - const std::vector kL1T3Pattern = { 2, 1, 2, 0 }; - - // The share of the bitrate up to each layer until WebRTC gives its - // own, as WebRTC splits it for temporal layers. - constexpr double kL1T2Shares[] = { 0.6, 1.0 }; - constexpr double kL1T3Shares[] = { 0.4, 0.6, 1.0 }; - // What a frame that cannot be encoded returns: libaom takes over the // stream, which stays in AV1, rather than WebRTC changing the codec. constexpr int32_t kEncodeFailed = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; @@ -129,6 +114,22 @@ namespace jni // The level of a stream that fits none. constexpr uint8_t kMaxParametersLevel = 31; + // ref_frame_idx: every reference frame names the first reference of + // the frame, but golden, which names the second where there is one. + std::array RefFrameIdx(const VaapiTemporalLayers::Frame & frame) + { + std::array refFrameIdx = {}; + + if (!frame.references.empty()) { + refFrameIdx.fill(static_cast(frame.references[0])); + } + if (frame.references.size() > 1) { + refFrameIdx[kGoldenIndex] = static_cast(frame.references[1]); + } + + return refFrameIdx; + } + uint32_t Align(uint32_t value, uint32_t alignment) { return (value + alignment - 1) / alignment * alignment; @@ -253,7 +254,6 @@ namespace jni ratesChanged(false), obuSizeLength(0), txMode(0), - temporalLayers(1), referenceValid(false), frameNumber(0), callback(nullptr), @@ -347,18 +347,10 @@ namespace jni bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; framerate = std::max(1u, codecSettings.maxFramerate); ratesChanged = true; - temporalLayers = mode == webrtc::ScalabilityMode::kL1T3 ? 3 : mode == webrtc::ScalabilityMode::kL1T2 ? 2 : 1; - slots.fill(-1); referenceValid = false; frameNumber = 0; outputProcessor.Reset(); - - layerBitratesBps.clear(); - - for (int layer = 0; layer < temporalLayers && temporalLayers > 1; layer++) { - const double share = temporalLayers == 3 ? kL1T3Shares[layer] : kL1T2Shares[layer]; - layerBitratesBps.push_back(static_cast(bitrateBps * share)); - } + layers.Configure(mode, bitrateBps); if (!outputProcessor.SetScalabilityMode(mode)) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; @@ -368,7 +360,7 @@ namespace jni const uint32_t surfaceHeight = Align(codecSettings.height, kSuperblockSize); if (!session.Create(VAProfileAV1Profile0, entrypoint, codecSettings.width, codecSettings.height, surfaceWidth, - surfaceHeight, kPackedHeaders, kReconstructedSurfaces)) + surfaceHeight, kPackedHeaders, VaapiTemporalLayers::kReconstructedSurfaces)) { Release(); return WEBRTC_VIDEO_CODEC_ERROR; @@ -427,10 +419,9 @@ namespace jni return WEBRTC_VIDEO_CODEC_OK; } - FrameReferences references; - const int surface = FreeSurface(); + const auto layerPlan = layers.Plan(keyFrame, *layerFrame); - if (surface < 0 || !PlanReferences(keyFrame, *layerFrame, references)) { + if (!layerPlan) { RTC_LOG(LS_WARNING) << implementationName << " AV1: a frame refers to a slot that holds no frame"; return kEncodeFailed; } @@ -440,7 +431,7 @@ namespace jni // A frame up to the alignment larger is cropped; one of another size, // which raced with a change of size, is scaled. const bool encoded = session.Upload(frame, static_cast(codecSettings.width), - static_cast(codecSettings.height), kSizeAlignment - 1) && Submit(keyFrame, references, surface) + static_cast(codecSettings.height), kSizeAlignment - 1) && Submit(keyFrame, *layerPlan) && session.ReadOutput(output); session.DropBuffers(); @@ -463,12 +454,7 @@ namespace jni return kEncodeFailed; } - // The slots the frame replaces hold it now. - for (int slot = 0; slot < static_cast(slots.size()); slot++) { - if (references.refreshFrameFlags & (1 << slot)) { - slots[slot] = surface; - } - } + layers.Commit(*layerPlan); referenceValid = true; frameNumber++; @@ -479,81 +465,11 @@ namespace jni return WEBRTC_VIDEO_CODEC_OK; } - bool VaapiAv1Encoder::PlanReferences(bool keyFrame, - const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame, FrameReferences & references) const - { - references.temporalId = layerFrame.TemporalId(); - - // A key frame replaces every slot. - if (keyFrame) { - references.refreshFrameFlags = 0xFF; - return true; - } - - std::vector referenced; - - for (const webrtc::CodecBufferUsage & buffer : layerFrame.Buffers()) { - if (buffer.id < 0 || buffer.id >= static_cast(slots.size())) { - return false; - } - if (buffer.referenced) { - if (slots[buffer.id] < 0) { - return false; - } - - referenced.push_back(buffer.id); - } - if (buffer.updated) { - references.refreshFrameFlags |= static_cast(1 << buffer.id); - } - } - - if (referenced.empty()) { - return false; - } - - // Every reference frame names the first reference, but golden, which - // names the second where there is one. - references.refFrameIdx.fill(static_cast(referenced[0])); - - if (referenced.size() > 1) { - references.refFrameIdx[kGoldenIndex] = static_cast(referenced[1]); - references.golden = true; - } - - return true; - } - - bool VaapiAv1Encoder::AddRateControl(bool keyFrame) - { - // No initial quantizer: the rate control picks its own, as FFmpeg - // leaves it. - if (temporalLayers == 1) { - return session.AddRateControl(bitrateBps, framerate, 0); - } - - // Without a rate control of each layer, AMD's driver gives the upper - // layers next to no bits. - return session.AddLayerRateControl(layerBitratesBps, framerate, - temporalLayers == 3 ? kL1T3Pattern : kL1T2Pattern, keyFrame); - } - - int VaapiAv1Encoder::FreeSurface() const - { - for (int surface = 0; surface < kReconstructedSurfaces; surface++) { - if (std::find(slots.begin(), slots.end(), surface) == slots.end()) { - return surface; - } - } - - return -1; - } - - bool VaapiAv1Encoder::Submit(bool keyFrame, const FrameReferences & references, int surface) + bool VaapiAv1Encoder::Submit(bool keyFrame, const VaapiTemporalLayers::Frame & frame) { const uint32_t sbCols = Align(codecSettings.width, kSuperblockSize) / kSuperblockSize; const uint32_t sbRows = Align(codecSettings.height, kSuperblockSize) / kSuperblockSize; - const bool layered = temporalLayers > 1; + const bool layered = layers.Count() > 1; if (keyFrame) { VAEncSequenceParameterBufferAV1 sequence = {}; @@ -574,20 +490,20 @@ namespace jni } } - if ((keyFrame || ratesChanged) && !AddRateControl(keyFrame)) { + if ((keyFrame || ratesChanged) && !layers.AddRateControl(session, bitrateBps, framerate, keyFrame)) { return false; } std::vector frameHeader; FrameHeaderLayout layout; - WriteFrameHeader(frameHeader, layout, keyFrame, references); + WriteFrameHeader(frameHeader, layout, keyFrame, frame); VAEncPictureParameterBufferAV1 picture = {}; picture.frame_width_minus_1 = static_cast(codecSettings.width - 1); picture.frame_height_minus_1 = static_cast(codecSettings.height - 1); - picture.reconstructed_frame = session.Reconstructed(surface); + picture.reconstructed_frame = session.Reconstructed(frame.surface); picture.coded_buf = session.CodedBuffer(); - picture.refresh_frame_flags = references.refreshFrameFlags; + picture.refresh_frame_flags = frame.refreshFlags; for (VASurfaceID & reference : picture.reference_frames) { reference = VA_INVALID_SURFACE; @@ -599,18 +515,19 @@ namespace jni picture.picture_flags.bits.error_resilient_mode = 1; } else { - for (int slot = 0; slot < static_cast(slots.size()); slot++) { - if (slots[slot] >= 0) { - picture.reference_frames[slot] = session.Reconstructed(slots[slot]); + for (int slot = 0; slot < layers.SlotCount(); slot++) { + if (layers.Surface(slot) >= 0) { + picture.reference_frames[slot] = session.Reconstructed(layers.Surface(slot)); } } - std::copy(references.refFrameIdx.begin(), references.refFrameIdx.end(), picture.ref_frame_idx); + const std::array refFrameIdx = RefFrameIdx(frame); + std::copy(refFrameIdx.begin(), refFrameIdx.end(), picture.ref_frame_idx); picture.primary_ref_frame = kPrimaryRefLast; picture.ref_frame_ctrl_l0.fields.search_idx0 = kLastFrame; - if (references.golden) { + if (frame.references.size() > 1) { picture.ref_frame_ctrl_l0.fields.search_idx1 = kGoldenFrame; } @@ -618,10 +535,10 @@ namespace jni } if (layered) { - picture.temporal_id = static_cast(references.temporalId); - picture.hierarchical_level_plus1 = static_cast(references.temporalId + 1); + picture.temporal_id = static_cast(frame.temporalId); + picture.hierarchical_level_plus1 = static_cast(frame.temporalId + 1); picture.tile_group_obu_hdr_info.bits.obu_extension_flag = 1; - picture.tile_group_obu_hdr_info.bits.temporal_id = static_cast(references.temporalId); + picture.tile_group_obu_hdr_info.bits.temporal_id = static_cast(frame.temporalId); } picture.order_hint = static_cast(frameNumber); @@ -678,6 +595,8 @@ namespace jni // point for each, from all layers down to the lowest. Then // operating_point_idc, of spatial layer 0 and those temporal layers, // seq_level_idx, and seq_tier above level 3.3. + const int temporalLayers = layers.Count(); + writer.Put(temporalLayers - 1, 5); for (int point = 0; point < temporalLayers; point++) { @@ -731,11 +650,12 @@ namespace jni } void VaapiAv1Encoder::WriteFrameHeader(std::vector & obu, FrameHeaderLayout & layout, bool keyFrame, - const FrameReferences & references) const + const VaapiTemporalLayers::Frame & frame) const { // uncompressed_header, section 5.9, as FFmpeg's VA-API encoder writes // it, for the sequence header above. - const bool layered = temporalLayers > 1; + const bool layered = layers.Count() > 1; + const std::array refFrameIdx = RefFrameIdx(frame); const uint32_t miCols = 2 * ((codecSettings.width + 7) >> 3); const uint32_t miRows = 2 * ((codecSettings.height + 7) >> 3); const uint32_t sbCols = (miCols + 15) >> 4; @@ -762,12 +682,12 @@ namespace jni if (!keyFrame) { // primary_ref_frame, refresh_frame_flags. writer.Put(kPrimaryRefLast, 3); - writer.Put(references.refreshFrameFlags, 8); + writer.Put(frame.refreshFlags, 8); // frame_refs_short_signaling, then ref_frame_idx. writer.Put(0, 1); for (int i = 0; i < kRefsPerFrame; i++) { - writer.Put(references.refFrameIdx[i], 3); + writer.Put(refFrameIdx[i], 3); } } @@ -849,7 +769,7 @@ namespace jni writer.TrailingBits(); obu = MakeObu(kObuFrameHeader, writer.Bytes(), obuSizeLength, - layered ? std::optional(references.temporalId) : std::nullopt); + layered ? std::optional(frame.temporalId) : std::nullopt); layout.size = static_cast(obu.size() * 8); } @@ -882,16 +802,7 @@ namespace jni bitrateBps = bitrate; outputProcessor.OnRatesUpdated(parameters.bitrate); - - // The bitrate of each temporal layer, counting those below it. - uint32_t sum = 0; - - for (int layer = 0; layer < static_cast(layerBitratesBps.size()); layer++) { - sum += parameters.bitrate.GetBitrate(0, layer); - // A layer WebRTC gives no bitrate is not encoded; it keeps the - // rate of the one below, which the driver takes. - layerBitratesBps[layer] = std::max(sum, 1u); - } + layers.OnRatesUpdated(parameters.bitrate); if (parameters.framerate_fps >= 1.0) { framerate = static_cast(parameters.framerate_fps + 0.5); @@ -911,13 +822,7 @@ namespace jni info.scaling_settings = outputProcessor.GetScalingSettings(); info.requested_resolution_alignment = kSizeAlignment; - // With temporal layers, the share of the frame rate up to each. - if (temporalLayers > 1) { - for (int layer = 0; layer < temporalLayers; layer++) { - info.fps_allocation[0].push_back(static_cast( - EncoderInfo::kMaxFramerateFraction >> (temporalLayers - 1 - layer))); - } - } + layers.FillFpsAllocation(info); return info; } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiTemporalLayers.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiTemporalLayers.cpp new file mode 100644 index 00000000..1e8cfc70 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiTemporalLayers.cpp @@ -0,0 +1,173 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/linux/VaapiTemporalLayers.h" + +#include + +namespace jni +{ + namespace + { + // The layers of the frames after a key frame, as + // VAEncMiscParameterTemporalLayerStructure takes them, of L1T2 and + // L1T3 as WebRTC's structures make them: L1T3 is 0, 2, 1, 2. + const std::vector kL1T2Pattern = { 1, 0 }; + const std::vector kL1T3Pattern = { 2, 1, 2, 0 }; + + // The share of the bitrate up to each layer until WebRTC gives its + // own, as WebRTC splits it for temporal layers. + constexpr double kL1T2Shares[] = { 0.6, 1.0 }; + constexpr double kL1T3Shares[] = { 0.4, 0.6, 1.0 }; + } + + VaapiTemporalLayers::VaapiTemporalLayers() : + count(1) + { + slots.fill(-1); + } + + bool VaapiTemporalLayers::Configure(webrtc::ScalabilityMode mode, uint32_t bitrateBps) + { + switch (mode) { + case webrtc::ScalabilityMode::kL1T1: + count = 1; + break; + case webrtc::ScalabilityMode::kL1T2: + count = 2; + break; + case webrtc::ScalabilityMode::kL1T3: + count = 3; + break; + default: + return false; + } + + slots.fill(-1); + bitratesBps.clear(); + + for (int layer = 0; layer < count && count > 1; layer++) { + const double share = count == 3 ? kL1T3Shares[layer] : kL1T2Shares[layer]; + bitratesBps.push_back(static_cast(bitrateBps * share)); + } + + return true; + } + + int VaapiTemporalLayers::Count() const + { + return count; + } + + std::optional VaapiTemporalLayers::Plan(bool keyFrame, + const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame) const + { + Frame frame; + frame.temporalId = layerFrame.TemporalId(); + + for (int surface = 0; surface < kReconstructedSurfaces && frame.surface < 0; surface++) { + if (std::find(slots.begin(), slots.end(), surface) == slots.end()) { + frame.surface = surface; + } + } + + if (frame.surface < 0) { + return std::nullopt; + } + + // A key frame replaces every slot. + if (keyFrame) { + frame.refreshFlags = 0xFF; + return frame; + } + + for (const webrtc::CodecBufferUsage & buffer : layerFrame.Buffers()) { + if (buffer.id < 0 || buffer.id >= static_cast(slots.size())) { + return std::nullopt; + } + if (buffer.referenced) { + if (slots[buffer.id] < 0) { + return std::nullopt; + } + + frame.references.push_back(buffer.id); + } + if (buffer.updated) { + frame.refreshFlags |= static_cast(1 << buffer.id); + } + } + + if (frame.references.empty()) { + return std::nullopt; + } + + return frame; + } + + void VaapiTemporalLayers::Commit(const Frame & frame) + { + for (int slot = 0; slot < static_cast(slots.size()); slot++) { + if (frame.refreshFlags & (1 << slot)) { + slots[slot] = frame.surface; + } + } + } + + int VaapiTemporalLayers::SlotCount() const + { + return static_cast(slots.size()); + } + + int VaapiTemporalLayers::Surface(int slot) const + { + return slots[slot]; + } + + void VaapiTemporalLayers::OnRatesUpdated(const webrtc::VideoBitrateAllocation & bitrates) + { + uint32_t sum = 0; + + for (int layer = 0; layer < static_cast(bitratesBps.size()); layer++) { + sum += bitrates.GetBitrate(0, layer); + // A layer WebRTC gives no bitrate is not encoded; it keeps the + // rate of the one below, which the driver takes. + bitratesBps[layer] = std::max(sum, 1u); + } + } + + bool VaapiTemporalLayers::AddRateControl(VaapiEncodeSession & session, uint32_t bitrateBps, uint32_t framerate, + bool keyFrame) const + { + if (count == 1) { + return session.AddRateControl(bitrateBps, framerate, 0); + } + + return session.AddLayerRateControl(bitratesBps, framerate, count == 3 ? kL1T3Pattern : kL1T2Pattern, + keyFrame); + } + + void VaapiTemporalLayers::FillFpsAllocation(webrtc::VideoEncoder::EncoderInfo & info) const + { + if (count == 1) { + return; + } + + for (int layer = 0; layer < count; layer++) { + info.fps_allocation[0].push_back(static_cast( + webrtc::VideoEncoder::EncoderInfo::kMaxFramerateFraction >> (count - 1 - layer))); + } + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp index c94b0801..ecc507b9 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp @@ -46,8 +46,13 @@ namespace jni constexpr uint8_t kFilterLevel = 16; constexpr uint8_t kSharpnessLevel = 4; - // The reference slot the next frame refers to as its last frame. - constexpr uint32_t kLastSlot = 0; + // The reference frames LAST and GOLDEN in ref_frame_ctrl_l0. + constexpr uint32_t kLastFrame = 1; + constexpr uint32_t kGoldenFrame = 2; + + // What a frame that cannot be encoded returns: libvpx takes over the + // stream, which stays in VP9, rather than WebRTC changing the codec. + constexpr int32_t kEncodeFailed = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; uint32_t Align(uint32_t value, uint32_t alignment) { @@ -66,7 +71,6 @@ namespace jni bitrateBps(0), framerate(30), ratesChanged(false), - current(0), referenceValid(false), callback(nullptr), outputProcessor(webrtc::kVideoCodecVP9, format) @@ -86,8 +90,19 @@ namespace jni if (settings->numberOfSimulcastStreams > 1) { return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED; } - // One spatial and one temporal layer is all this encoder produces. - if (settings->VP9().numberOfSpatialLayers > 1 || settings->VP9().numberOfTemporalLayers > 1) { + // Temporal layers; spatial ones are left to libvpx. + if (settings->VP9().numberOfSpatialLayers > 1) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + const int temporalLayers = settings->VP9().numberOfTemporalLayers; + const webrtc::ScalabilityMode mode = settings->GetScalabilityMode().value_or(temporalLayers == 3 + ? webrtc::ScalabilityMode::kL1T3 + : temporalLayers == 2 ? webrtc::ScalabilityMode::kL1T2 : webrtc::ScalabilityMode::kL1T1); + + if (mode != webrtc::ScalabilityMode::kL1T1 && mode != webrtc::ScalabilityMode::kL1T2 + && mode != webrtc::ScalabilityMode::kL1T3) + { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } @@ -97,14 +112,19 @@ namespace jni bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; framerate = std::max(1u, codecSettings.maxFramerate); ratesChanged = true; - current = 0; referenceValid = false; outputProcessor.Reset(); + layers.Configure(mode, bitrateBps); + + if (!outputProcessor.SetScalabilityMode(mode)) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } const uint32_t surfaceWidth = Align(codecSettings.width, kSurfaceAlignment); const uint32_t surfaceHeight = Align(codecSettings.height, kSurfaceAlignment); - if (!session.Create(VAProfileVP9Profile0, entrypoint, surfaceWidth, surfaceHeight, surfaceWidth, surfaceHeight)) + if (!session.Create(VAProfileVP9Profile0, entrypoint, surfaceWidth, surfaceHeight, surfaceWidth, surfaceHeight, + VA_ENC_PACKED_HEADER_NONE, VaapiTemporalLayers::kReconstructedSurfaces)) { Release(); return WEBRTC_VIDEO_CODEC_ERROR; @@ -113,7 +133,7 @@ namespace jni initialized = true; RTC_LOG(LS_INFO) << implementationName << " VP9 initialized: " << codecSettings.width << "x" - << codecSettings.height << " at " << bitrateBps << " bps"; + << codecSettings.height << " at " << bitrateBps << " bps, " << webrtc::ScalabilityModeToString(mode); return WEBRTC_VIDEO_CODEC_OK; } @@ -150,17 +170,31 @@ namespace jni }); } + const auto layerFrame = outputProcessor.NextFrameConfig(keyFrame); + + // No layer is encoded now, as when WebRTC gives none a bitrate. + if (!layerFrame) { + return WEBRTC_VIDEO_CODEC_OK; + } + + const auto layerPlan = layers.Plan(keyFrame, *layerFrame); + + if (!layerPlan) { + RTC_LOG(LS_WARNING) << implementationName << " VP9: a frame refers to a slot that holds no frame"; + return kEncodeFailed; + } + std::vector output; // VP9 codes any size, so the frame is taken as it is, and scaled only // if it raced with a change of size. const bool encoded = session.Upload(frame, static_cast(codecSettings.width), - static_cast(codecSettings.height), 0) && Submit(keyFrame) && session.ReadOutput(output); + static_cast(codecSettings.height), 0) && Submit(keyFrame, *layerPlan) && session.ReadOutput(output); session.DropBuffers(); if (!encoded) { - return WEBRTC_VIDEO_CODEC_ERROR; + return kEncodeFailed; } webrtc::EncodedImage image; @@ -173,13 +207,12 @@ namespace jni webrtc::CodecSpecificInfo info; - if (!outputProcessor.Process(output, keyFrame, image, info)) { - return WEBRTC_VIDEO_CODEC_ERROR; + if (!outputProcessor.Process(output, keyFrame, *layerFrame, image, info)) { + return kEncodeFailed; } - // The frame just encoded is the reference of the next one. + layers.Commit(*layerPlan); referenceValid = true; - current = 1 - current; ratesChanged = false; callback->OnEncodedImage(image, &info); @@ -187,7 +220,7 @@ namespace jni return WEBRTC_VIDEO_CODEC_OK; } - bool VaapiVp9Encoder::Submit(bool keyFrame) + bool VaapiVp9Encoder::Submit(bool keyFrame, const VaapiTemporalLayers::Frame & frame) { if (keyFrame) { VAEncSequenceParameterBufferVP9 sequence = {}; @@ -198,14 +231,12 @@ namespace jni } } - // No initial quantizer: VP9 rate control picks its own, as FFmpeg - // leaves it. - if ((keyFrame || ratesChanged) && !session.AddRateControl(bitrateBps, framerate, 0)) { + if ((keyFrame || ratesChanged) && !layers.AddRateControl(session, bitrateBps, framerate, keyFrame)) { return false; } VAEncPictureParameterBufferVP9 picture = {}; - FillPicture(picture, keyFrame); + FillPicture(picture, keyFrame, frame); return session.AddBuffer(VAEncPictureParameterBufferType, &picture, sizeof(picture)) && session.Encode(); } @@ -222,13 +253,14 @@ namespace jni sequence.bits_per_second = bitrateBps; } - void VaapiVp9Encoder::FillPicture(VAEncPictureParameterBufferVP9 & picture, bool keyFrame) const + void VaapiVp9Encoder::FillPicture(VAEncPictureParameterBufferVP9 & picture, bool keyFrame, + const VaapiTemporalLayers::Frame & frame) const { picture.frame_width_src = codecSettings.width; picture.frame_height_src = codecSettings.height; picture.frame_width_dst = codecSettings.width; picture.frame_height_dst = codecSettings.height; - picture.reconstructed_frame = session.Reconstructed(current); + picture.reconstructed_frame = session.Reconstructed(frame.surface); picture.coded_buf = session.CodedBuffer(); for (VASurfaceID & reference : picture.reference_frames) { @@ -240,16 +272,36 @@ namespace jni picture.pic_flags.bits.frame_type = kKeyFrameType; } else { - picture.reference_frames[kLastSlot] = session.Reconstructed(1 - current); - // The last frame is the only reference, in list 0. - picture.ref_flags.bits.ref_frame_ctrl_l0 = 1; - picture.ref_flags.bits.ref_last_idx = kLastSlot; + for (int slot = 0; slot < layers.SlotCount(); slot++) { + if (layers.Surface(slot) >= 0) { + picture.reference_frames[slot] = session.Reconstructed(layers.Surface(slot)); + } + } + + // The first reference is the last frame, a second one golden; both + // in list 0. + picture.ref_flags.bits.ref_frame_ctrl_l0 = kLastFrame; + picture.ref_flags.bits.ref_last_idx = static_cast(frame.references[0]); + + if (frame.references.size() > 1) { + picture.ref_flags.bits.ref_frame_ctrl_l0 |= kGoldenFrame; + picture.ref_flags.bits.ref_gf_idx = static_cast(frame.references[1]); + } + picture.pic_flags.bits.frame_type = kInterFrameType; } + picture.ref_flags.bits.temporal_id = static_cast(frame.temporalId); picture.pic_flags.bits.show_frame = 1; - // Every slot holds this frame afterwards. - picture.refresh_frame_flags = 0xFF; + + // VP9 takes motion vectors from the frame decoded before, and carries + // probabilities from frame to frame, neither of which a receiver + // without the upper layers has; error resilience leaves both out, as + // libvpx does with layers. + if (layers.Count() > 1) { + picture.pic_flags.bits.error_resilient_mode = 1; + } + picture.refresh_frame_flags = frame.refreshFlags; picture.luma_ac_qindex = kQIndex; picture.filter_level = kFilterLevel; @@ -266,6 +318,8 @@ namespace jni } bitrateBps = bitrate; + outputProcessor.OnRatesUpdated(parameters.bitrate); + layers.OnRatesUpdated(parameters.bitrate); if (parameters.framerate_fps >= 1.0) { framerate = static_cast(parameters.framerate_fps + 0.5); @@ -284,6 +338,8 @@ namespace jni info.supports_simulcast = false; info.scaling_settings = outputProcessor.GetScalingSettings(); + layers.FillFpsAllocation(info); + return info; } } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java index 84c517f0..7b9ea334 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java @@ -45,7 +45,8 @@ * Intel's: with VA-API on Linux, and with Media Foundation on Windows only * with the system property {@code webrtc.hardware.vp9Encoder} set to * {@code true} when the factory is created, since that encoder has not been - * tested on such a GPU yet. Otherwise VP9 is encoded with libvpx. On Linux, H.265 is encoded with NVENC on NVIDIA + * tested on such a GPU yet. With VA-API, VP9 is encoded in temporal layers + * too (L1T2, L1T3). Otherwise VP9 is encoded with libvpx. On Linux, H.265 is encoded with NVENC on NVIDIA * GPUs and with VA-API on the GPUs of Intel and AMD whose driver encodes it. * A hardware encoder that fails to start, for example because the GPU has no * encoder sessions left, or fails while encoding, is replaced by the next one diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java index 5486c161..26e8aa61 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java @@ -524,14 +524,31 @@ void av1RestartsCleanly() throws Exception { @Test void av1EncodesTemporalLayers() throws Exception { - // Temporal layers are references between frames of one size, which a - // hardware encoder can make; an SFU can then forward a lower frame - // rate without asking for key frames. + assertEncodesTemporalLayers(AV1, HARDWARE_AV1_REQUIRED); + } + + @Test + void vp9EncodesTemporalLayers() throws Exception { + // VA-API encodes VP9 in temporal layers, Media Foundation does not. + assumeTrue(OS.contains("linux"), "hardware VP9 encodes temporal layers on Linux only"); + + assertEncodesTemporalLayers(VP9, HARDWARE_VP9_REQUIRED); + } + + /** + * Sends the codec in L1T2 and then L1T3, and checks that the hardware + * encodes the layers, rather than handing them to the software encoder. + * Temporal layers are references between frames of one size, which a + * hardware encoder can make; an SFU can then forward a lower frame rate + * without asking for key frames. + */ + private void assertEncodesTemporalLayers(Predicate codec, boolean required) + throws Exception { for (String mode : new String[] { "L1T2", "L1T3" }) { PeerConnectionFactory hardware = hardwareFactory(); CountDownLatch received = new CountDownLatch(30); - try (TestMediaCall call = new TestMediaCall(hardware, true, false, AV1)) { + try (TestMediaCall call = new TestMediaCall(hardware, true, false, codec)) { call.setVideoSize(640, 360); call.negotiate(); @@ -560,7 +577,7 @@ void av1EncodesTemporalLayers() throws Exception { track.removeSink(sink); receiver.dispose(); - assertHardware(String.valueOf(outbound.get("encoderImplementation")), HARDWARE_AV1_REQUIRED); + assertHardware(String.valueOf(outbound.get("encoderImplementation")), required); assertEquals(mode, outbound.get("scalabilityMode")); assertTrue(!(pliCount instanceof Number) || ((Number) pliCount).longValue() <= 1, "key frames requested: " + pliCount); From 57b919689ec46a620a5347111a74294843bafee6 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Thu, 15 Oct 2026 21:37:51 +0200 Subject: [PATCH 3/3] feat: encode H.264 and H.265 temporal layers with VA-API The VA-API H.264 and H.265 encoders take L1T2 and L1T3 too. H.265 carries the temporal id in its NAL units and keeps only lower layer pictures in its reference picture sets; H.264 leaves the top layer unreferenced, allows gaps in frame_num and reorders the reference list. Each layer gets its own rate control only where the driver has one that works. The H.264 encoder now uses the shared VaapiEncodeSession. --- docs/guide/advanced/video-codecs.md | 2 +- .../video/codec/EncoderOutputProcessor.h | 10 +- .../video/codec/linux/VaapiEncodeSession.h | 3 +- .../video/codec/linux/VaapiH264Encoder.h | 77 ++- .../video/codec/linux/VaapiH265Encoder.h | 41 +- .../video/codec/linux/VaapiTemporalLayers.h | 27 +- .../video/codec/EncoderOutputProcessor.cpp | 22 +- .../video/codec/linux/VaapiAv1Encoder.cpp | 4 +- .../video/codec/linux/VaapiH264Encoder.cpp | 614 ++++++------------ .../video/codec/linux/VaapiH265Encoder.cpp | 239 +++++-- .../video/codec/linux/VaapiTemporalLayers.cpp | 43 +- .../codec/linux/VaapiVideoEncoderFactory.cpp | 9 +- .../video/codec/linux/VaapiVp9Encoder.cpp | 4 +- .../codec/HardwareVideoEncoderFactory.java | 13 +- .../HardwareVideoEncoderIntegrationTest.java | 35 +- 15 files changed, 616 insertions(+), 527 deletions(-) diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index a8d739bf..84ab7eb2 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -30,7 +30,7 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder() | Linux | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) | NVENC on NVIDIA GPUs, otherwise the VA-API encoder of the GPU driver (Intel, AMD) where it encodes H.265 | NVENC on NVIDIA RTX 40 and newer, otherwise the VA-API encoder of the GPU driver where it encodes AV1 (e.g. AMD RDNA3 and newer) | | macOS | VideoToolbox, as with `DefaultVideoEncoderFactory` | VideoToolbox, on Macs with a hardware HEVC encoder, such as all with Apple silicon | Software | -Hardware encoders produce a single layer, but for the VA-API encoders of AV1 and VP9, which encode temporal layers too (`L1T2` and `L1T3`), with a rate control of each layer, so that an SFU can forward a lower frame rate. A stream that asks for spatial layers, or for temporal layers from another hardware encoder, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom. +Hardware encoders produce a single layer, but for the VA-API encoders, which encode temporal layers too (`L1T2` and `L1T3`) in every codec, so that an SFU can forward a lower frame rate. Each layer gets a rate control of its own where the driver has one that works (AMD's for H.265 and AV1, Intel's for VP9); otherwise the layers share one, which treats them alike. A stream that asks for spatial layers, or for temporal layers from another hardware encoder, is encoded in software. AMD's VA-API encoder takes AV1 from 320x128 up; smaller streams are encoded by libaom. VP9 can be encoded in hardware too, with the VP9 encoder of the GPU driver, which Intel GPUs have: through VA-API on Linux, and through Media Foundation on Windows. On Windows it is off by default, since it has not been tested on such a GPU yet; turn it on with a system property before the factory is created: diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h b/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h index fbc761fd..81328ac7 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/EncoderOutputProcessor.h @@ -48,9 +48,9 @@ namespace jni // or the sequence header of AV1. Encoders put them in front of the first // key frame, not necessarily in front of every one, so they are kept and // put back where missing. The frame dependencies WebRTC needs for AV1 and - // H.265 are those of a stream without layers, unless an AV1 or VP9 encoder - // that makes temporal layers sets its scalability mode and asks for the - // layer of each frame before it encodes it. VP9 frames go out as they + // H.265 are those of a stream without layers, unless an encoder that + // makes temporal layers sets its scalability mode and asks for the layer + // of each frame before it encodes it. VP9 frames go out as they // come, with the VP9 information of a single layer, as the Java encoders // get it, or with temporal layers in flexible mode, as libvpx gives it. class EncoderOutputProcessor @@ -61,8 +61,8 @@ namespace jni // Forgets the parameter sets, for a stream that starts over. void Reset(); - // Sets the layers of the stream, temporal ones of AV1 and VP9 only, - // as in L1T3. Returns false for a mode WebRTC has no structure of. + // Sets the layers of the stream, temporal ones only, as in L1T3. + // Returns false for a mode WebRTC has no structure of. bool SetScalabilityMode(webrtc::ScalabilityMode mode); // Lets the layers follow the bitrate WebRTC gives each. void OnRatesUpdated(const webrtc::VideoBitrateAllocation & bitrates); diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h index 23a4ec43..a4029fa8 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiEncodeSession.h @@ -35,8 +35,7 @@ namespace jni // parameter buffers of a frame, uploading a frame as NV12, encoding it // and reading what came out. The codec's encoder fills in the parameters. // - // Used by the H.265 and VP9 encoders; the H.264 one keeps its own, older - // copy of the same. + // Used by every VA-API encoder. class VaapiEncodeSession { public: diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH264Encoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH264Encoder.h index a94be3a5..2f7ae31d 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH264Encoder.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH264Encoder.h @@ -19,7 +19,9 @@ #include "media/video/codec/EncoderOutputProcessor.h" #include "media/video/codec/linux/NalUnitWriter.h" +#include "media/video/codec/linux/VaapiEncodeSession.h" #include "media/video/codec/linux/VaapiLibrary.h" +#include "media/video/codec/linux/VaapiTemporalLayers.h" #include "api/video/video_frame.h" #include "api/video_codecs/sdp_video_format.h" @@ -29,6 +31,7 @@ #include #include +#include #include #include #include @@ -38,15 +41,20 @@ namespace jni // Encodes H.264 Constrained Baseline with the GPU driver's VA-API encoder, // as Intel and AMD drivers provide it on Linux. // - // Encoding is synchronous, with one reference frame and P-frames only, so - // each frame is encoded before Encode() returns. The encoder writes the - // parameter sets and slice headers itself and hands them to the driver as - // packed headers, as FFmpeg does, since Intel's iHD driver writes none of - // its own; a driver that takes no packed headers writes them from the - // parameters it is given. A driver that leaves the parameter sets out of a - // key frame, which WebRTC cannot send without them, makes the encoder give - // up, as does anything else that fails, so that the software encoder takes - // over. + // Encoding is synchronous, with P-frames only, so each frame is encoded + // before Encode() returns. The encoder writes the parameter sets and slice + // headers itself and hands them to the driver as packed headers, as FFmpeg + // does, since Intel's iHD driver writes none of its own; a driver that + // takes no packed headers writes them from the parameters it is given. + // Each frame refers to the frame just before it, or, with temporal layers + // (L1T2, L1T3), which need a driver that takes the slice headers, to the + // one of its layer or a lower one that WebRTC's structure of the mode + // names: the top layer is not used for reference, frame_num may skip the + // frames of the layers a receiver goes without, and each slice puts its + // reference first in the list. A driver that leaves the parameter sets out + // of a key frame, which WebRTC cannot send without them, makes the encoder + // give up, as does anything else that fails, so that the software encoder + // takes over. class VaapiH264Encoder : public webrtc::VideoEncoder { public: @@ -61,43 +69,40 @@ namespace jni EncoderInfo GetEncoderInfo() const override; private: - bool CreateSession(); - void DestroySession(); - bool Upload(const webrtc::VideoFrame & frame); - bool Submit(bool idr); - bool ReadOutput(std::vector & output); + // A frame being encoded: its layer, its frame_num and whether it is + // used for reference. + struct Picture + { + VaapiTemporalLayers::Frame frame; + uint32_t frameNum = 0; + bool reference = true; + }; - bool AddBuffer(VABufferType type, void * data, size_t size); - bool AddMiscParameter(VAEncMiscParameterType type, const void * data, size_t size); - bool AddPackedHeader(VAEncPackedHeaderType type, const std::vector & data, size_t bits); - void DestroyFrameBuffers(); + bool Submit(bool idr, const Picture & picture); void FillSequence(VAEncSequenceParameterBufferH264 & sequence) const; - void FillPicture(VAEncPictureParameterBufferH264 & picture, bool idr) const; - void FillSlice(VAEncSliceParameterBufferH264 & slice, bool idr) const; + void FillPicture(VAEncPictureParameterBufferH264 & parameters, bool idr, const Picture & picture) const; + void FillSlice(VAEncSliceParameterBufferH264 & slice, bool idr, const Picture & picture) const; + + // The picture a reference slot holds, as VA-API names it. + VAPictureH264 Reference(int slot) const; + // The order count of a picture, as pic_order_cnt_type 2 derives it. + static int32_t OrderCount(uint32_t frameNum, bool reference); // The SPS and PPS, and the header of the slice, matching the // parameters above. Each returns the bits it appended. size_t WriteParameterSets(std::vector & output) const; - size_t WriteSliceHeader(std::vector & output, bool idr) const; - - bool Check(VAStatus status, const char * operation) const; + size_t WriteSliceHeader(std::vector & output, bool idr, const Picture & picture) const; private: VaapiLibrary & library; - const VaapiFunctions & va; const std::string implementationName; + VaapiEncodeSession session; - VAConfigID config; // The VA_ENC_PACKED_HEADER_* the driver takes from the encoder. uint32_t packedHeaders; - VAContextID context; - VASurfaceID inputSurface; - // The frame being encoded, and the one it refers to. - VASurfaceID reconstructed[2]; - VABufferID codedBuffer; - std::vector frameBuffers; + bool initialized; webrtc::VideoCodec codecSettings; uint32_t widthInMbs; uint32_t heightInMbs; @@ -105,11 +110,13 @@ namespace jni uint32_t framerate; bool ratesChanged; - // The frame number within the current IDR period, and which of - // the reconstructed surfaces holds the reference. - uint32_t frameNum; + // The frame_num of the last reference picture, and the temporal + // layers with the frame_num of the picture each reference slot + // holds. + uint32_t previousReferenceFrameNum; uint32_t idrPicId; - int current; + VaapiTemporalLayers layers; + std::array slotFrameNums; bool referenceValid; webrtc::EncodedImageCallback * callback; diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Encoder.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Encoder.h index 553dd443..c4d15077 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Encoder.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiH265Encoder.h @@ -21,6 +21,7 @@ #include "media/video/codec/linux/NalUnitWriter.h" #include "media/video/codec/linux/VaapiEncodeSession.h" #include "media/video/codec/linux/VaapiLibrary.h" +#include "media/video/codec/linux/VaapiTemporalLayers.h" #include "api/video/video_frame.h" #include "api/video_codecs/sdp_video_format.h" @@ -30,6 +31,7 @@ #include #include +#include #include #include #include @@ -45,7 +47,13 @@ namespace jni // headers for a driver that takes them, as Intel's and AMD's do. A driver // that predicts only with both reference lists filled, as Intel's // low-power encoder, gets each P-frame as a B-frame whose two lists hold - // the same previous picture ("low delay B"), as FFmpeg does. The block + // the same previous picture ("low delay B"), as FFmpeg does. With + // temporal layers (L1T2, L1T3), which need a driver that takes the slice + // headers, a picture refers to the one of its layer or a lower one that + // WebRTC's structure of the mode names, its slice NAL unit carries its + // temporal id, and its reference picture set keeps the pictures of the + // reference slots of its layer and below, so that a receiver without the + // upper layers misses none of them. The block // sizes and coding tools are those the driver says it takes, as FFmpeg's // VA-API encoder chooses them. The frame size is kept to a multiple of the // smallest coding block; the coded picture is that aligned to 16 and at @@ -70,21 +78,31 @@ namespace jni private: // What the driver says about the HEVC encoder it has. void QueryCapabilities(); - bool Submit(bool idr); + bool Submit(bool idr, const VaapiTemporalLayers::Frame & frame); void FillSequence(VAEncSequenceParameterBufferHEVC & sequence) const; - void FillPicture(VAEncPictureParameterBufferHEVC & picture, bool idr) const; - void FillSlice(VAEncSliceParameterBufferHEVC & slice, bool idr) const; + void FillPicture(VAEncPictureParameterBufferHEVC & picture, bool idr, + const VaapiTemporalLayers::Frame & frame) const; + void FillSlice(VAEncSliceParameterBufferHEVC & slice, bool idr, const VaapiTemporalLayers::Frame & frame) const; // The VPS, SPS and PPS, and the slice segment header, matching // the parameters above. Each returns the bits it appended. size_t WriteParameterSets(std::vector & output) const; - size_t WriteSliceHeader(std::vector & output, bool idr) const; + size_t WriteSliceHeader(std::vector & output, bool idr, const VaapiTemporalLayers::Frame & frame) const; void WriteProfileTierLevel(NalUnitWriter & writer) const; // diff_cu_qp_delta_depth. uint32_t CuQpDeltaDepth() const; - // The picture the current one refers to. - VAPictureHEVC Reference() const; + // The pictures the reference slots hold, newest first, each once, + // and whether the current frame refers to it: its reference + // picture set. + struct ReferencePicture + { + int slot; + int32_t pictureOrder; + bool used; + }; + std::vector ReferencePictureSet(const VaapiTemporalLayers::Frame & frame) const; + VAPictureHEVC Picture(const ReferencePicture & reference) const; private: VaapiLibrary & library; @@ -122,10 +140,13 @@ namespace jni uint32_t framerate; bool ratesChanged; - // The picture order count since the last IDR picture, and which of - // the reconstructed surfaces holds the current picture. + // The picture order count since the last IDR picture, the temporal + // layers, and the picture order count and temporal id of the + // picture each of their reference slots holds. int32_t pictureOrder; - int current; + VaapiTemporalLayers layers; + std::array slotOrders; + std::array slotTemporalIds; bool referenceValid; webrtc::EncodedImageCallback * callback; diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiTemporalLayers.h b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiTemporalLayers.h index 769fc7ea..67197f98 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiTemporalLayers.h +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiTemporalLayers.h @@ -18,10 +18,12 @@ #define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_TEMPORAL_LAYERS_H_ #include "media/video/codec/linux/VaapiEncodeSession.h" +#include "media/video/codec/linux/VaapiLibrary.h" #include "api/video/video_bitrate_allocation.h" #include "api/video_codecs/scalability_mode.h" #include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/svc/scalability_mode_util.h" #include "modules/video_coding/svc/scalable_video_controller.h" #include @@ -36,7 +38,10 @@ namespace jni // it replaces, as WebRTC's structure of the mode says, each buffer of the // structure being the slot of the same number; which reconstructed surface // each slot holds; and the rate control of each layer, without which - // AMD's driver gives the upper layers next to no bits. + // AMD's driver gives the upper layers next to no bits, where the driver + // says it has one and the encoder wants it. Intel's H.264 and H.265 + // encoders say not, and refuse the layer structure that goes with it; + // their rate control treats every layer alike. class VaapiTemporalLayers { public: @@ -52,17 +57,28 @@ namespace jni // The slots of its references, the first being its last frame, // the second, if any, its golden frame. std::vector references; + // The slots it replaces, all for a key frame, and of those the + // ones WebRTC's structure goes on to use: a key frame leaves + // the others empty, which nothing refers to then, and which a + // codec would otherwise keep as references for good. uint8_t refreshFlags = 0; + uint8_t keptFlags = 0; // The reconstructed surface it is encoded into. int surface = -1; }; VaapiTemporalLayers(); + // Whether the driver's encoder of the profile has a rate control + // of each temporal layer, for as many layers as given. + static bool LayerRateControl(VaapiLibrary & library, VAProfile profile, VAEntrypoint entrypoint, + int layers); + // Starts over in the mode, splitting the bitrate across the layers - // until WebRTC gives its own split. Returns false for a mode other - // than L1T1, L1T2 and L1T3. - bool Configure(webrtc::ScalabilityMode mode, uint32_t bitrateBps); + // until WebRTC gives its own split, for a rate control of each layer + // where there is one. Returns false for a mode other than L1T1, L1T2 + // and L1T3. + bool Configure(webrtc::ScalabilityMode mode, uint32_t bitrateBps, bool layerRateControl); // The number of temporal layers, one without layers. int Count() const; @@ -74,6 +90,8 @@ namespace jni const webrtc::ScalableVideoController::LayerFrameConfig & layerFrame) const; // Takes the frame, once encoded, into the slots it replaces. void Commit(const Frame & frame); + // Empties a slot whose picture the decoder no longer keeps. + void Forget(int slot); // The slots kept, and the reconstructed surface each holds, -1 for // none. A key frame fills the slots beyond too, but no frame refers @@ -93,6 +111,7 @@ namespace jni private: int count; + bool layerRateControl; std::array slots; // The bitrate of each layer, counting the layers below it. std::vector bitratesBps; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp index c401dbdd..86e40ebe 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/EncoderOutputProcessor.cpp @@ -394,12 +394,28 @@ namespace jni info.codecSpecific.H264.temporal_idx = webrtc::kNoTemporalIdx; info.codecSpecific.H264.base_layer_sync = false; info.codecSpecific.H264.idr_frame = keyFrame; - return; + + if (scalabilityMode == webrtc::ScalabilityMode::kL1T1) { + return; + } + + // With temporal layers, the layer, and whether an upper layer frame + // refers to the base layer only, as OpenH264 gives them; the frame + // dependencies follow below. The base layer is buffer 0 in WebRTC's + // structures. + bool baseOnly = true; + + for (const webrtc::CodecBufferUsage & buffer : layerFrame.Buffers()) { + baseOnly &= !buffer.referenced || buffer.id == 0; + } + + info.codecSpecific.H264.temporal_idx = static_cast(layerFrame.TemporalId()); + info.codecSpecific.H264.base_layer_sync = layerFrame.TemporalId() > 0 && baseOnly; } // AV1 and H.265 have no codec specific information; WebRTC sends - // their frame dependencies, and those of VP9, from these, in the - // dependency descriptor. + // their frame dependencies, and those of VP9 and of H.264 with layers, + // from these, in the dependency descriptor. info.generic_frame_info = svcController->OnEncodeDone(layerFrame); info.scalability_mode = scalabilityMode; diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp index dc61ffc7..eecfb1b1 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiAv1Encoder.cpp @@ -350,7 +350,8 @@ namespace jni referenceValid = false; frameNumber = 0; outputProcessor.Reset(); - layers.Configure(mode, bitrateBps); + layers.Configure(mode, bitrateBps, VaapiTemporalLayers::LayerRateControl(library, VAProfileAV1Profile0, + entrypoint, webrtc::ScalabilityModeToNumTemporalLayers(mode))); if (!outputProcessor.SetScalabilityMode(mode)) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; @@ -436,6 +437,7 @@ namespace jni session.DropBuffers(); + if (!encoded) { return kEncodeFailed; } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Encoder.cpp index 97f9840c..ad8f2437 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH264Encoder.cpp @@ -17,15 +17,11 @@ #include "media/video/codec/linux/VaapiH264Encoder.h" #include "api/video/encoded_image.h" -#include "api/video/i420_buffer.h" #include "modules/video_coding/include/video_codec_interface.h" #include "modules/video_coding/include/video_error_codes.h" #include "rtc_base/logging.h" -#include "third_party/libyuv/include/libyuv/convert_from.h" #include -#include -#include namespace jni { @@ -44,7 +40,7 @@ namespace jni constexpr uint8_t kSliceTypeI = 2; // H.264 NAL unit types (table 7-1), and the nal_ref_idc of everything - // the encoder writes, all of which is used for reference. + // the encoder writes that is used for reference. constexpr uint32_t kNalSlice = 1; constexpr uint32_t kNalIdrSlice = 5; constexpr uint32_t kNalSps = 7; @@ -60,10 +56,10 @@ namespace jni constexpr uint32_t kInitialQp = 26; - // How many seconds of video the rate control may buffer, as a - // multiple of the bitrate. - constexpr uint32_t kHrdBufferNumerator = 3; - constexpr uint32_t kHrdBufferDenominator = 2; + // modification_of_pic_nums_idc: subtract from the predicted picture + // number, or end the list of modifications. + constexpr uint32_t kSubtractPicNum = 0; + constexpr uint32_t kEndModifications = 3; // The lowest H.264 level whose limits a stream fits in, going by its // frame size and macroblock rate, per table A-1 of the standard. @@ -107,27 +103,32 @@ namespace jni VaapiH264Encoder::VaapiH264Encoder(VaapiLibrary & library, const webrtc::SdpVideoFormat & format) : library(library), - va(library.Api()), implementationName("VA-API (" + library.Vendor() + ")"), - config(VA_INVALID_ID), - packedHeaders(0), - context(VA_INVALID_ID), - inputSurface(VA_INVALID_SURFACE), - reconstructed{ VA_INVALID_SURFACE, VA_INVALID_SURFACE }, - codedBuffer(VA_INVALID_ID), + session(library, "VA-API H.264"), + packedHeaders(VA_ENC_PACKED_HEADER_NONE), + initialized(false), codecSettings(), widthInMbs(0), heightInMbs(0), bitrateBps(0), framerate(30), ratesChanged(false), - frameNum(0), + previousReferenceFrameNum(0), idrPicId(0), - current(0), + slotFrameNums{ 0, 0 }, referenceValid(false), callback(nullptr), outputProcessor(webrtc::kVideoCodecH264, format) { + VAConfigAttrib packed = {}; + packed.type = VAConfigAttribEncPackedHeaders; + + if (library.Api().GetConfigAttributes(library.Display(), VAProfileH264ConstrainedBaseline, library.Entrypoint(), + &packed, 1) == VA_STATUS_SUCCESS && packed.value != VA_ATTRIB_NOT_SUPPORTED + && (packed.value & kPackedHeaders) == kPackedHeaders) + { + packedHeaders = kPackedHeaders; + } } VaapiH264Encoder::~VaapiH264Encoder() @@ -135,17 +136,6 @@ namespace jni Release(); } - bool VaapiH264Encoder::Check(VAStatus status, const char * operation) const - { - if (status == VA_STATUS_SUCCESS) { - return true; - } - - RTC_LOG(LS_WARNING) << "VA-API: " << operation << " failed: " << library.ErrorString(status); - - return false; - } - int32_t VaapiH264Encoder::InitEncode(const webrtc::VideoCodec * settings, const Settings & encoderSettings) { if (settings == nullptr || settings->width == 0 || settings->height == 0) { @@ -165,6 +155,16 @@ namespace jni return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } + // Temporal layers, where the encoder writes the slice headers, which + // carry what makes the layers; the rest is left to OpenH264. + const webrtc::ScalabilityMode mode = settings->GetScalabilityMode().value_or(webrtc::ScalabilityMode::kL1T1); + + if (mode != webrtc::ScalabilityMode::kL1T1 && (packedHeaders != kPackedHeaders + || (mode != webrtc::ScalabilityMode::kL1T2 && mode != webrtc::ScalabilityMode::kL1T3))) + { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + Release(); codecSettings = *settings; @@ -175,89 +175,35 @@ namespace jni bitrateBps = std::max(1u, codecSettings.startBitrate) * 1000; framerate = std::max(1u, codecSettings.maxFramerate); ratesChanged = true; - frameNum = 0; - current = 0; + previousReferenceFrameNum = 0; referenceValid = false; outputProcessor.Reset(); + // One rate control for all layers, which then treats them alike: + // Intel's driver has none of each layer for H.264, and AMD's, which + // says it has, overshoots the bitrate by half again with it. + layers.Configure(mode, bitrateBps, false); - if (!CreateSession()) { - Release(); - return WEBRTC_VIDEO_CODEC_ERROR; - } - - RTC_LOG(LS_INFO) << implementationName << " initialized: " << codecSettings.width << "x" - << codecSettings.height << " at " << bitrateBps << " bps"; - - return WEBRTC_VIDEO_CODEC_OK; - } - - bool VaapiH264Encoder::CreateSession() - { - VADisplay display = library.Display(); - - VAConfigAttrib packed = {}; - packed.type = VAConfigAttribEncPackedHeaders; - - packedHeaders = 0; - - if (va.GetConfigAttributes(display, VAProfileH264ConstrainedBaseline, library.Entrypoint(), &packed, 1) == - VA_STATUS_SUCCESS && packed.value != VA_ATTRIB_NOT_SUPPORTED - && (packed.value & kPackedHeaders) == kPackedHeaders) - { - packedHeaders = kPackedHeaders; - } - - VAConfigAttrib attributes[3] = {}; - attributes[0].type = VAConfigAttribRTFormat; - attributes[0].value = VA_RT_FORMAT_YUV420; - attributes[1].type = VAConfigAttribRateControl; - attributes[1].value = VA_RC_CBR; - attributes[2].type = VAConfigAttribEncPackedHeaders; - attributes[2].value = packedHeaders; - - const int attributeCount = packedHeaders != 0 ? 3 : 2; - - if (!Check(va.CreateConfig(display, VAProfileH264ConstrainedBaseline, library.Entrypoint(), attributes, - attributeCount, &config), "creating the configuration")) - { - config = VA_INVALID_ID; - return false; + if (!outputProcessor.SetScalabilityMode(mode)) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } // Whole macroblocks; the stream is cropped to the frame size. - const unsigned int width = widthInMbs * 16; - const unsigned int height = heightInMbs * 16; - - VASurfaceID surfaces[3] = { VA_INVALID_SURFACE, VA_INVALID_SURFACE, VA_INVALID_SURFACE }; + const uint32_t width = widthInMbs * 16; + const uint32_t height = heightInMbs * 16; - if (!Check(va.CreateSurfaces(display, VA_RT_FORMAT_YUV420, width, height, surfaces, 3, nullptr, 0), - "creating surfaces")) + if (!session.Create(VAProfileH264ConstrainedBaseline, library.Entrypoint(), width, height, width, height, + packedHeaders, VaapiTemporalLayers::kReconstructedSurfaces)) { - return false; - } - - inputSurface = surfaces[0]; - reconstructed[0] = surfaces[1]; - reconstructed[1] = surfaces[2]; - - if (!Check(va.CreateContext(display, config, width, height, VA_PROGRESSIVE, surfaces, 3, &context), - "creating the context")) - { - context = VA_INVALID_ID; - return false; + Release(); + return WEBRTC_VIDEO_CODEC_ERROR; } - // Room for a frame that does not compress at all. - const unsigned int codedSize = width * height * 3 / 2 + 64 * 1024; + initialized = true; - if (!Check(va.CreateBuffer(display, context, VAEncCodedBufferType, codedSize, 1, nullptr, &codedBuffer), - "creating the output buffer")) - { - codedBuffer = VA_INVALID_ID; - return false; - } + RTC_LOG(LS_INFO) << implementationName << " initialized: " << codecSettings.width << "x" + << codecSettings.height << " at " << bitrateBps << " bps, " << webrtc::ScalabilityModeToString(mode); - return true; + return WEBRTC_VIDEO_CODEC_OK; } int32_t VaapiH264Encoder::RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback * encodeCallback) @@ -269,48 +215,18 @@ namespace jni int32_t VaapiH264Encoder::Release() { - DestroySession(); - - return WEBRTC_VIDEO_CODEC_OK; - } - - void VaapiH264Encoder::DestroySession() - { - VADisplay display = library.Display(); - - DestroyFrameBuffers(); - - if (codedBuffer != VA_INVALID_ID) { - va.DestroyBuffer(display, codedBuffer); - codedBuffer = VA_INVALID_ID; - } - if (context != VA_INVALID_ID) { - va.DestroyContext(display, context); - context = VA_INVALID_ID; - } - - VASurfaceID surfaces[3] = { inputSurface, reconstructed[0], reconstructed[1] }; - - if (inputSurface != VA_INVALID_SURFACE) { - va.DestroySurfaces(display, surfaces, 3); - } - - inputSurface = VA_INVALID_SURFACE; - reconstructed[0] = VA_INVALID_SURFACE; - reconstructed[1] = VA_INVALID_SURFACE; - - if (config != VA_INVALID_ID) { - va.DestroyConfig(display, config); - config = VA_INVALID_ID; - } + session.Destroy(); + initialized = false; referenceValid = false; + + return WEBRTC_VIDEO_CODEC_OK; } int32_t VaapiH264Encoder::Encode(const webrtc::VideoFrame & frame, const std::vector * frameTypes) { - if (context == VA_INVALID_ID || callback == nullptr) { + if (!initialized || callback == nullptr) { return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } @@ -322,18 +238,42 @@ namespace jni }); } - if (idr) { - frameNum = 0; + const auto layerFrame = outputProcessor.NextFrameConfig(idr); + + // No layer is encoded now, as when WebRTC gives none a bitrate. + if (!layerFrame) { + return WEBRTC_VIDEO_CODEC_OK; } - std::vector output; + const auto layerPlan = layers.Plan(idr, *layerFrame); - if (!Upload(frame) || !Submit(idr) || !ReadOutput(output)) { - DestroyFrameBuffers(); + if (!layerPlan) { + RTC_LOG(LS_WARNING) << implementationName << " H.264: a frame refers to a slot that holds none"; return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; } - DestroyFrameBuffers(); + // frame_num goes up after each reference picture only, so that a + // receiver misses no frame_num for a picture of the top layer it goes + // without. + Picture picture; + picture.frame = *layerPlan; + picture.reference = idr || layerPlan->refreshFlags != 0; + picture.frameNum = idr ? 0 : (previousReferenceFrameNum + 1) % kMaxFrameNum; + + std::vector output; + + // WebRTC initializes the encoder again when the frame size changes, + // so a frame of another size is one that raced with that. An odd frame + // is cropped to the even size the encoder took, which needs no copy. + const bool encoded = session.Upload(frame, static_cast(codecSettings.width), + static_cast(codecSettings.height), 1) && Submit(idr, picture) && session.ReadOutput(output); + + session.DropBuffers(); + + + if (!encoded) { + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } webrtc::EncodedImage image; image._encodedWidth = codecSettings.width; @@ -347,14 +287,24 @@ namespace jni // A driver that writes no parameter sets into key frames makes a // stream nothing can decode; better in software then. - if (!outputProcessor.Process(output, idr, image, info)) { + if (!outputProcessor.Process(output, idr, *layerFrame, image, info)) { return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; } - // The frame just encoded is the reference of the next one. + // The slots the frame replaces hold it now. + layers.Commit(*layerPlan); + + for (int slot = 0; slot < layers.SlotCount(); slot++) { + if (layerPlan->keptFlags & (1 << slot)) { + slotFrameNums[slot] = picture.frameNum; + } + } + + if (picture.reference) { + previousReferenceFrameNum = picture.frameNum; + } + referenceValid = true; - current = 1 - current; - frameNum = (frameNum + 1) % kMaxFrameNum; ratesChanged = false; if (idr) { @@ -366,120 +316,36 @@ namespace jni return WEBRTC_VIDEO_CODEC_OK; } - bool VaapiH264Encoder::Upload(const webrtc::VideoFrame & frame) - { - webrtc::scoped_refptr i420 = frame.video_frame_buffer()->ToI420(); - - if (!i420) { - return false; - } - - const int width = static_cast(codecSettings.width); - const int height = static_cast(codecSettings.height); - - // WebRTC initializes the encoder again when the frame size changes, - // so a frame of another size is one that raced with that. An odd frame - // is cropped to the even size the encoder took, which needs no copy. - const bool cropped = i420->width() - width >= 0 && i420->width() - width <= 1 - && i420->height() - height >= 0 && i420->height() - height <= 1; - - if (!cropped) { - webrtc::scoped_refptr scaled = webrtc::I420Buffer::Create(width, height); - scaled->ScaleFrom(*i420); - i420 = scaled; - } - - VADisplay display = library.Display(); - VAImage image = {}; - image.image_id = VA_INVALID_ID; - - // Writing into the surface directly where the driver allows it, - // otherwise into an image that is copied into the surface. - bool derived = va.DeriveImage(display, inputSurface, &image) == VA_STATUS_SUCCESS - && image.format.fourcc == VA_FOURCC_NV12; - - if (!derived) { - if (image.image_id != VA_INVALID_ID) { - va.DestroyImage(display, image.image_id); - } - - VAImageFormat format = {}; - format.fourcc = VA_FOURCC_NV12; - format.byte_order = VA_LSB_FIRST; - format.bits_per_pixel = 12; - - if (!Check(va.CreateImage(display, &format, width, height, &image), "creating an image")) { - return false; - } - } - - void * data = nullptr; - bool uploaded = Check(va.MapBuffer(display, image.buf, &data), "mapping an image"); - - if (uploaded) { - uint8_t * base = static_cast(data); - - libyuv::I420ToNV12(i420->DataY(), i420->StrideY(), i420->DataU(), i420->StrideU(), - i420->DataV(), i420->StrideV(), - base + image.offsets[0], static_cast(image.pitches[0]), - base + image.offsets[1], static_cast(image.pitches[1]), - width, height); - - va.UnmapBuffer(display, image.buf); - - if (!derived) { - uploaded = Check(va.PutImage(display, inputSurface, image.image_id, 0, 0, width, height, - 0, 0, width, height), "copying an image"); - } - } - - va.DestroyImage(display, image.image_id); - - return uploaded; - } - - bool VaapiH264Encoder::Submit(bool idr) + bool VaapiH264Encoder::Submit(bool idr, const Picture & picture) { if (idr) { VAEncSequenceParameterBufferH264 sequence = {}; FillSequence(sequence); - if (!AddBuffer(VAEncSequenceParameterBufferType, &sequence, sizeof(sequence))) { + if (!session.AddBuffer(VAEncSequenceParameterBufferType, &sequence, sizeof(sequence))) { return false; } } if (idr || ratesChanged) { - VAEncMiscParameterRateControl rateControl = {}; - rateControl.bits_per_second = bitrateBps; - rateControl.target_percentage = 100; - rateControl.window_size = 1000; - rateControl.initial_qp = kInitialQp; - - VAEncMiscParameterFrameRate frameRate = {}; - // The numerator in the low 16 bits, the denominator in the high. - frameRate.framerate = framerate | (1u << 16); - - VAEncMiscParameterHRD hrd = {}; - hrd.buffer_size = static_cast( - static_cast(bitrateBps) * kHrdBufferNumerator / kHrdBufferDenominator); - hrd.initial_buffer_fullness = hrd.buffer_size / 2; - - if (!AddMiscParameter(VAEncMiscParameterTypeRateControl, &rateControl, sizeof(rateControl)) || - !AddMiscParameter(VAEncMiscParameterTypeFrameRate, &frameRate, sizeof(frameRate)) || - !AddMiscParameter(VAEncMiscParameterTypeHRD, &hrd, sizeof(hrd))) - { + // Without layers, from the initial QP the encoder has always + // started from. + const bool added = layers.Count() == 1 + ? session.AddRateControl(bitrateBps, framerate, kInitialQp) + : layers.AddRateControl(session, bitrateBps, framerate, idr); + + if (!added) { return false; } } - VAEncPictureParameterBufferH264 picture = {}; - FillPicture(picture, idr); + VAEncPictureParameterBufferH264 parameters = {}; + FillPicture(parameters, idr, picture); VAEncSliceParameterBufferH264 slice = {}; - FillSlice(slice, idr); + FillSlice(slice, idr, picture); - if (!AddBuffer(VAEncPictureParameterBufferType, &picture, sizeof(picture))) { + if (!session.AddBuffer(VAEncPictureParameterBufferType, ¶meters, sizeof(parameters))) { return false; } @@ -487,117 +353,21 @@ namespace jni std::vector parameterSets; const size_t bits = WriteParameterSets(parameterSets); - if (!AddPackedHeader(VAEncPackedHeaderSequence, parameterSets, bits)) { + if (!session.AddPackedHeader(VAEncPackedHeaderSequence, parameterSets, bits)) { return false; } } if (packedHeaders & VA_ENC_PACKED_HEADER_SLICE) { std::vector sliceHeader; - const size_t bits = WriteSliceHeader(sliceHeader, idr); + const size_t bits = WriteSliceHeader(sliceHeader, idr, picture); - if (!AddPackedHeader(VAEncPackedHeaderSlice, sliceHeader, bits)) { + if (!session.AddPackedHeader(VAEncPackedHeaderSlice, sliceHeader, bits)) { return false; } } - if (!AddBuffer(VAEncSliceParameterBufferType, &slice, sizeof(slice))) { - return false; - } - - VADisplay display = library.Display(); - - if (!Check(va.BeginPicture(display, context, inputSurface), "beginning a picture")) { - return false; - } - - bool rendered = Check(va.RenderPicture(display, context, frameBuffers.data(), - static_cast(frameBuffers.size())), "rendering a picture"); - - // A picture that was begun has to be ended. - bool ended = Check(va.EndPicture(display, context), "ending a picture"); - - return rendered && ended && Check(va.SyncSurface(display, inputSurface), "waiting for a picture"); - } - - bool VaapiH264Encoder::ReadOutput(std::vector & output) - { - VADisplay display = library.Display(); - void * data = nullptr; - - if (!Check(va.MapBuffer(display, codedBuffer, &data), "mapping the output")) { - return false; - } - - bool complete = true; - - for (auto * segment = static_cast(data); segment != nullptr; - segment = static_cast(segment->next)) - { - if (segment->status & VA_CODED_BUF_STATUS_SLICE_OVERFLOW_MASK) { - complete = false; - } - - const uint8_t * bytes = static_cast(segment->buf); - output.insert(output.end(), bytes, bytes + segment->size); - } - - va.UnmapBuffer(display, codedBuffer); - - if (!complete || output.empty()) { - RTC_LOG(LS_WARNING) << "VA-API: a frame did not fit the output buffer"; - return false; - } - - return true; - } - - bool VaapiH264Encoder::AddBuffer(VABufferType type, void * data, size_t size) - { - VABufferID buffer = VA_INVALID_ID; - - if (!Check(va.CreateBuffer(library.Display(), context, type, static_cast(size), 1, data, - &buffer), "creating a parameter buffer")) - { - return false; - } - - frameBuffers.push_back(buffer); - - return true; - } - - bool VaapiH264Encoder::AddMiscParameter(VAEncMiscParameterType type, const void * data, size_t size) - { - // A VAEncMiscParameterBuffer: the type, followed by the parameter. - std::vector buffer(sizeof(VAEncMiscParameterBuffer) + size); - - auto * header = reinterpret_cast(buffer.data()); - header->type = type; - std::memcpy(header->data, data, size); - - return AddBuffer(VAEncMiscParameterBufferType, buffer.data(), buffer.size()); - } - - bool VaapiH264Encoder::AddPackedHeader(VAEncPackedHeaderType type, const std::vector & data, - size_t bits) - { - VAEncPackedHeaderParameterBuffer parameters = {}; - parameters.type = type; - parameters.bit_length = static_cast(bits); - parameters.has_emulation_bytes = 1; - - return AddBuffer(VAEncPackedHeaderParameterBufferType, ¶meters, sizeof(parameters)) - && AddBuffer(VAEncPackedHeaderDataBufferType, const_cast(data.data()), data.size()); - } - - void VaapiH264Encoder::DestroyFrameBuffers() - { - for (VABufferID buffer : frameBuffers) { - va.DestroyBuffer(library.Display(), buffer); - } - - frameBuffers.clear(); + return session.AddBuffer(VAEncSliceParameterBufferType, &slice, sizeof(slice)) && session.Encode(); } void VaapiH264Encoder::FillSequence(VAEncSequenceParameterBufferH264 & sequence) const @@ -608,7 +378,8 @@ namespace jni sequence.intra_idr_period = kIntraPeriod; sequence.ip_period = 1; sequence.bits_per_second = bitrateBps; - sequence.max_num_ref_frames = 1; + // The pictures of both reference slots with layers. + sequence.max_num_ref_frames = layers.Count() > 1 ? 2 : 1; sequence.picture_width_in_mbs = static_cast(widthInMbs); sequence.picture_height_in_mbs = static_cast(heightInMbs); @@ -636,47 +407,75 @@ namespace jni sequence.time_scale = framerate * 2; } - void VaapiH264Encoder::FillPicture(VAEncPictureParameterBufferH264 & picture, bool idr) const + int32_t VaapiH264Encoder::OrderCount(uint32_t frameNum, bool reference) { - picture.CurrPic.picture_id = reconstructed[current]; - picture.CurrPic.frame_idx = frameNum; - picture.CurrPic.flags = 0; - picture.CurrPic.TopFieldOrderCnt = static_cast(frameNum * 2); - picture.CurrPic.BottomFieldOrderCnt = picture.CurrPic.TopFieldOrderCnt; + // Twice frame_num for a reference picture, one less for another. + return static_cast(frameNum * 2) - (reference ? 0 : 1); + } - for (VAPictureH264 & reference : picture.ReferenceFrames) { + VAPictureH264 VaapiH264Encoder::Reference(int slot) const + { + VAPictureH264 reference = {}; + reference.picture_id = session.Reconstructed(layers.Surface(slot)); + reference.frame_idx = slotFrameNums[slot]; + reference.flags = VA_PICTURE_H264_SHORT_TERM_REFERENCE; + reference.TopFieldOrderCnt = OrderCount(slotFrameNums[slot], true); + reference.BottomFieldOrderCnt = reference.TopFieldOrderCnt; + + return reference; + } + + void VaapiH264Encoder::FillPicture(VAEncPictureParameterBufferH264 & parameters, bool idr, + const Picture & picture) const + { + parameters.CurrPic.picture_id = session.Reconstructed(picture.frame.surface); + parameters.CurrPic.frame_idx = picture.frameNum; + parameters.CurrPic.flags = 0; + parameters.CurrPic.TopFieldOrderCnt = OrderCount(picture.frameNum, picture.reference); + parameters.CurrPic.BottomFieldOrderCnt = parameters.CurrPic.TopFieldOrderCnt; + + for (VAPictureH264 & reference : parameters.ReferenceFrames) { InvalidatePicture(reference); } if (!idr) { - const uint32_t previous = (frameNum + kMaxFrameNum - 1) % kMaxFrameNum; - - VAPictureH264 & reference = picture.ReferenceFrames[0]; - reference.picture_id = reconstructed[1 - current]; - reference.frame_idx = previous; - reference.flags = VA_PICTURE_H264_SHORT_TERM_REFERENCE; - reference.TopFieldOrderCnt = static_cast(previous * 2); - reference.BottomFieldOrderCnt = reference.TopFieldOrderCnt; - } - - picture.coded_buf = codedBuffer; - picture.pic_parameter_set_id = 0; - picture.seq_parameter_set_id = 0; - picture.last_picture = 0; - picture.frame_num = static_cast(frameNum); - picture.pic_init_qp = kInitialQp; - picture.num_ref_idx_l0_active_minus1 = 0; - picture.num_ref_idx_l1_active_minus1 = 0; - - picture.pic_fields.bits.idr_pic_flag = idr ? 1 : 0; - picture.pic_fields.bits.reference_pic_flag = 1; + // The pictures the decoder keeps: the one referred to, and with + // layers the other slot's too. + int count = 0; + + for (int slot = 0; slot < layers.SlotCount(); slot++) { + const bool referred = slot == picture.frame.references[0]; + const bool kept = layers.Count() > 1 && layers.Surface(slot) >= 0; + bool listed = false; + + for (int i = 0; i < count; i++) { + listed |= parameters.ReferenceFrames[i].picture_id == session.Reconstructed(layers.Surface(slot)); + } + + if ((referred || kept) && !listed) { + parameters.ReferenceFrames[count++] = Reference(slot); + } + } + } + + parameters.coded_buf = session.CodedBuffer(); + parameters.pic_parameter_set_id = 0; + parameters.seq_parameter_set_id = 0; + parameters.last_picture = 0; + parameters.frame_num = static_cast(picture.frameNum); + parameters.pic_init_qp = kInitialQp; + parameters.num_ref_idx_l0_active_minus1 = 0; + parameters.num_ref_idx_l1_active_minus1 = 0; + + parameters.pic_fields.bits.idr_pic_flag = idr ? 1 : 0; + parameters.pic_fields.bits.reference_pic_flag = picture.reference ? 1 : 0; // CAVLC: Constrained Baseline has no CABAC. - picture.pic_fields.bits.entropy_coding_mode_flag = 0; - picture.pic_fields.bits.transform_8x8_mode_flag = 0; - picture.pic_fields.bits.deblocking_filter_control_present_flag = 1; + parameters.pic_fields.bits.entropy_coding_mode_flag = 0; + parameters.pic_fields.bits.transform_8x8_mode_flag = 0; + parameters.pic_fields.bits.deblocking_filter_control_present_flag = 1; } - void VaapiH264Encoder::FillSlice(VAEncSliceParameterBufferH264 & slice, bool idr) const + void VaapiH264Encoder::FillSlice(VAEncSliceParameterBufferH264 & slice, bool idr, const Picture & picture) const { slice.macroblock_address = 0; slice.num_macroblocks = widthInMbs * heightInMbs; @@ -696,14 +495,7 @@ namespace jni } if (!idr) { - const uint32_t previous = (frameNum + kMaxFrameNum - 1) % kMaxFrameNum; - - VAPictureH264 & reference = slice.RefPicList0[0]; - reference.picture_id = reconstructed[1 - current]; - reference.frame_idx = previous; - reference.flags = VA_PICTURE_H264_SHORT_TERM_REFERENCE; - reference.TopFieldOrderCnt = static_cast(previous * 2); - reference.BottomFieldOrderCnt = reference.TopFieldOrderCnt; + slice.RefPicList0[0] = Reference(picture.frame.references[0]); } slice.slice_qp_delta = 0; @@ -712,6 +504,8 @@ namespace jni size_t VaapiH264Encoder::WriteParameterSets(std::vector & output) const { + const bool layered = layers.Count() > 1; + // The SPS (7.3.2.1.1) of FillSequence(). NalUnitWriter sps; sps.Bits(0, 1); @@ -726,8 +520,10 @@ namespace jni sps.Ue(0); sps.Ue(kLog2MaxFrameNum - 4); sps.Ue(2); - sps.Ue(1); - sps.Flag(false); + // max_num_ref_frames, and gaps_in_frame_num_value_allowed_flag, which + // a receiver without the upper layers needs. + sps.Ue(layered ? 2 : 1); + sps.Flag(layered); sps.Ue(widthInMbs - 1); sps.Ue(heightInMbs - 1); sps.Flag(true); @@ -790,38 +586,54 @@ namespace jni return sps.AppendTo(output) + pps.AppendTo(output); } - size_t VaapiH264Encoder::WriteSliceHeader(std::vector & output, bool idr) const + size_t VaapiH264Encoder::WriteSliceHeader(std::vector & output, bool idr, const Picture & picture) const { // The slice header (7.3.3) of FillSlice(), which the driver goes on // from with the slice data. NalUnitWriter header; header.Bits(0, 1); - header.Bits(kNalRefIdc, 2); + header.Bits(picture.reference ? kNalRefIdc : 0, 2); header.Bits(idr ? kNalIdrSlice : kNalSlice, 5); header.Ue(0); header.Ue(idr ? kSliceTypeI : kSliceTypeP); header.Ue(0); - header.Bits(frameNum, kLog2MaxFrameNum); + header.Bits(picture.frameNum, kLog2MaxFrameNum); if (idr) { header.Ue(idrPicId); } else { - // num_ref_idx_active_override_flag and - // ref_pic_list_modification_flag_l0. - header.Flag(false); + // num_ref_idx_active_override_flag. header.Flag(false); - } - // dec_ref_pic_marking(): no_output_of_prior_pics_flag and - // long_term_reference_flag, or adaptive_ref_pic_marking_mode_flag. - if (idr) { - header.Flag(false); - header.Flag(false); + // ref_pic_list_modification(): with layers, the picture referred + // to goes first in the list, which would otherwise start with the + // newest reference picture. + const bool layered = layers.Count() > 1; + + header.Flag(layered); + + if (layered) { + const uint32_t referenceFrameNum = slotFrameNums[picture.frame.references[0]]; + const uint32_t difference = (picture.frameNum + kMaxFrameNum - referenceFrameNum) % kMaxFrameNum; + + header.Ue(kSubtractPicNum); + header.Ue(difference - 1); + header.Ue(kEndModifications); + } } - else { - header.Flag(false); + + // dec_ref_pic_marking() of a reference picture: no_output_of_prior_pics_flag + // and long_term_reference_flag, or adaptive_ref_pic_marking_mode_flag. + if (picture.reference) { + if (idr) { + header.Flag(false); + header.Flag(false); + } + else { + header.Flag(false); + } } header.Se(0); @@ -844,6 +656,8 @@ namespace jni } bitrateBps = bitrate; + outputProcessor.OnRatesUpdated(parameters.bitrate); + layers.OnRatesUpdated(parameters.bitrate); if (parameters.framerate_fps >= 1.0) { framerate = static_cast(parameters.framerate_fps + 0.5); @@ -864,6 +678,8 @@ namespace jni // NV12 needs even dimensions. info.requested_resolution_alignment = 2; + layers.FillFpsAllocation(info); + return info; } } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Encoder.cpp index 37670174..43a2cf45 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiH265Encoder.cpp @@ -35,6 +35,7 @@ namespace jni constexpr uint8_t kSliceTypeB = 0; constexpr uint8_t kSliceTypeP = 1; constexpr uint8_t kSliceTypeI = 2; + constexpr uint8_t kNalTrailN = 0; constexpr uint8_t kNalTrailR = 1; constexpr uint8_t kNalIdrWRadl = 19; constexpr uint8_t kNalVps = 32; @@ -142,7 +143,8 @@ namespace jni framerate(30), ratesChanged(false), pictureOrder(0), - current(0), + slotOrders{ 0, 0 }, + slotTemporalIds{ 0, 0 }, referenceValid(false), callback(nullptr), outputProcessor(webrtc::kVideoCodecH265, format) @@ -220,6 +222,16 @@ namespace jni return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } + // Temporal layers, where the encoder writes the slice headers, which + // carry the layers; spatial ones are left to the next codec. + const webrtc::ScalabilityMode mode = settings->GetScalabilityMode().value_or(webrtc::ScalabilityMode::kL1T1); + + if (mode != webrtc::ScalabilityMode::kL1T1 && (packedHeaders != kPackedHeaders + || (mode != webrtc::ScalabilityMode::kL1T2 && mode != webrtc::ScalabilityMode::kL1T3))) + { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + Release(); const uint32_t ctbSize = 1u << log2CtbSize; @@ -238,14 +250,19 @@ namespace jni framerate = std::max(1u, codecSettings.maxFramerate); ratesChanged = true; pictureOrder = 0; - current = 0; referenceValid = false; outputProcessor.Reset(); + layers.Configure(mode, bitrateBps, VaapiTemporalLayers::LayerRateControl(library, VAProfileHEVCMain, + entrypoint, webrtc::ScalabilityModeToNumTemporalLayers(mode))); + + if (!outputProcessor.SetScalabilityMode(mode)) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } // Surfaces of whole coding tree blocks, which the driver may write all // of. if (!session.Create(VAProfileHEVCMain, entrypoint, codedWidth, codedHeight, widthInCtbs << log2CtbSize, - heightInCtbs << log2CtbSize, packedHeaders)) + heightInCtbs << log2CtbSize, packedHeaders, VaapiTemporalLayers::kReconstructedSurfaces)) { Release(); return WEBRTC_VIDEO_CODEC_ERROR; @@ -254,7 +271,8 @@ namespace jni initialized = true; RTC_LOG(LS_INFO) << implementationName << " H.265 initialized: " << codecSettings.width << "x" - << codecSettings.height << " at " << bitrateBps << " bps, CTB " << ctbSize << ", min CB " << alignment; + << codecSettings.height << " at " << bitrateBps << " bps, CTB " << ctbSize << ", min CB " << alignment << ", " + << webrtc::ScalabilityModeToString(mode); return WEBRTC_VIDEO_CODEC_OK; } @@ -295,14 +313,30 @@ namespace jni pictureOrder = 0; } + const auto layerFrame = outputProcessor.NextFrameConfig(idr); + + // No layer is encoded now, as when WebRTC gives none a bitrate. + if (!layerFrame) { + return WEBRTC_VIDEO_CODEC_OK; + } + + const auto layerPlan = layers.Plan(idr, *layerFrame); + + if (!layerPlan) { + RTC_LOG(LS_WARNING) << implementationName << " H.265: a picture refers to a slot that holds none"; + return WEBRTC_VIDEO_CODEC_ERROR; + } + const int alignment = 1 << log2MinCbSize; std::vector output; const bool encoded = session.Upload(frame, static_cast(codecSettings.width), - static_cast(codecSettings.height), alignment - 1) && Submit(idr) && session.ReadOutput(output); + static_cast(codecSettings.height), alignment - 1) && Submit(idr, *layerPlan) + && session.ReadOutput(output); session.DropBuffers(); + if (!encoded) { return WEBRTC_VIDEO_CODEC_ERROR; } @@ -319,14 +353,27 @@ namespace jni // A driver that writes no parameter sets into key frames makes a // stream nothing can decode. - if (!outputProcessor.Process(output, idr, image, info)) { + if (!outputProcessor.Process(output, idr, *layerFrame, image, info)) { RTC_LOG(LS_WARNING) << implementationName << " H.265: the driver wrote no parameter sets"; return WEBRTC_VIDEO_CODEC_ERROR; } - // The frame just encoded is the reference of the next one. + // The slots the picture replaces hold it now. + layers.Commit(*layerPlan); + + for (int slot = 0; slot < layers.SlotCount(); slot++) { + if (layerPlan->keptFlags & (1 << slot)) { + slotOrders[slot] = pictureOrder; + slotTemporalIds[slot] = layerPlan->temporalId; + } + else if (slotTemporalIds[slot] > layerPlan->temporalId) { + // Left out of the picture's reference picture set, so gone from + // the decoder's. + layers.Forget(slot); + } + } + referenceValid = true; - current = 1 - current; pictureOrder++; ratesChanged = false; @@ -335,7 +382,7 @@ namespace jni return WEBRTC_VIDEO_CODEC_OK; } - bool VaapiH265Encoder::Submit(bool idr) + bool VaapiH265Encoder::Submit(bool idr, const VaapiTemporalLayers::Frame & frame) { if (idr) { VAEncSequenceParameterBufferHEVC sequence = {}; @@ -346,15 +393,23 @@ namespace jni } } - if ((idr || ratesChanged) && !session.AddRateControl(bitrateBps, framerate, kInitialQp)) { - return false; + if (idr || ratesChanged) { + // Without layers, from the initial QP the encoder has always + // started from. + const bool added = layers.Count() == 1 + ? session.AddRateControl(bitrateBps, framerate, kInitialQp) + : layers.AddRateControl(session, bitrateBps, framerate, idr); + + if (!added) { + return false; + } } VAEncPictureParameterBufferHEVC picture = {}; - FillPicture(picture, idr); + FillPicture(picture, idr, frame); VAEncSliceParameterBufferHEVC slice = {}; - FillSlice(slice, idr); + FillSlice(slice, idr, frame); if (!session.AddBuffer(VAEncPictureParameterBufferType, &picture, sizeof(picture))) { return false; @@ -371,7 +426,7 @@ namespace jni if (packedHeaders & VA_ENC_PACKED_HEADER_SLICE) { std::vector sliceHeader; - const size_t bits = WriteSliceHeader(sliceHeader, idr); + const size_t bits = WriteSliceHeader(sliceHeader, idr, frame); if (!session.AddPackedHeader(VAEncPackedHeaderSlice, sliceHeader, bits)) { return false; @@ -415,20 +470,55 @@ namespace jni sequence.vui_time_scale = framerate; } - VAPictureHEVC VaapiH265Encoder::Reference() const + std::vector VaapiH265Encoder::ReferencePictureSet( + const VaapiTemporalLayers::Frame & frame) const { - VAPictureHEVC reference = {}; - reference.picture_id = session.Reconstructed(1 - current); - reference.pic_order_cnt = pictureOrder - 1; - // A short-term reference before the current picture. - reference.flags = VA_PICTURE_HEVC_RPS_ST_CURR_BEFORE; + std::vector set; + + for (int slot = 0; slot < layers.SlotCount(); slot++) { + // A picture of a higher layer, which a receiver without that layer + // does not have. + if (layers.Surface(slot) < 0 || slotTemporalIds[slot] > frame.temporalId) { + continue; + } + + const bool used = std::find(frame.references.begin(), frame.references.end(), slot) + != frame.references.end(); + auto same = std::find_if(set.begin(), set.end(), [&](const ReferencePicture & picture) { + return layers.Surface(picture.slot) == layers.Surface(slot); + }); - return reference; + if (same != set.end()) { + same->used |= used; + } + else { + set.push_back({ slot, slotOrders[slot], used }); + } + } + + std::sort(set.begin(), set.end(), [](const ReferencePicture & a, const ReferencePicture & b) { + return a.pictureOrder > b.pictureOrder; + }); + + return set; + } + + VAPictureHEVC VaapiH265Encoder::Picture(const ReferencePicture & reference) const + { + VAPictureHEVC picture = {}; + picture.picture_id = session.Reconstructed(layers.Surface(reference.slot)); + picture.pic_order_cnt = reference.pictureOrder; + // A short-term reference before the current picture, which the + // current one refers to or keeps for later. + picture.flags = reference.used ? VA_PICTURE_HEVC_RPS_ST_CURR_BEFORE : 0; + + return picture; } - void VaapiH265Encoder::FillPicture(VAEncPictureParameterBufferHEVC & picture, bool idr) const + void VaapiH265Encoder::FillPicture(VAEncPictureParameterBufferHEVC & picture, bool idr, + const VaapiTemporalLayers::Frame & frame) const { - picture.decoded_curr_pic.picture_id = session.Reconstructed(current); + picture.decoded_curr_pic.picture_id = session.Reconstructed(frame.surface); picture.decoded_curr_pic.pic_order_cnt = pictureOrder; picture.decoded_curr_pic.flags = 0; @@ -437,7 +527,11 @@ namespace jni } if (!idr) { - picture.reference_frames[0] = Reference(); + const std::vector set = ReferencePictureSet(frame); + + for (size_t i = 0; i < set.size(); i++) { + picture.reference_frames[i] = Picture(set[i]); + } } picture.coded_buf = session.CodedBuffer(); @@ -453,18 +547,22 @@ namespace jni picture.num_ref_idx_l0_default_active_minus1 = 0; picture.num_ref_idx_l1_default_active_minus1 = 0; picture.slice_pic_parameter_set_id = 0; - picture.nal_unit_type = idr ? kNalIdrWRadl : kNalTrailR; + // A picture no other refers to, as of the top layer, is a sub-layer + // non-reference picture. + const bool reference = idr || frame.refreshFlags != 0; + picture.nal_unit_type = idr ? kNalIdrWRadl : (reference ? kNalTrailR : kNalTrailN); picture.pic_fields.bits.idr_pic_flag = idr ? 1 : 0; picture.pic_fields.bits.coding_type = idr ? kCodingTypeI : kCodingTypeP; - picture.pic_fields.bits.reference_pic_flag = 1; + picture.pic_fields.bits.reference_pic_flag = reference ? 1 : 0; picture.pic_fields.bits.sign_data_hiding_enabled_flag = signDataHiding ? 1 : 0; picture.pic_fields.bits.transform_skip_enabled_flag = transformSkip ? 1 : 0; // Needed for the rate control to change the QP within a picture. picture.pic_fields.bits.cu_qp_delta_enabled_flag = cuQpDelta ? 1 : 0; } - void VaapiH265Encoder::FillSlice(VAEncSliceParameterBufferHEVC & slice, bool idr) const + void VaapiH265Encoder::FillSlice(VAEncSliceParameterBufferHEVC & slice, bool idr, + const VaapiTemporalLayers::Frame & frame) const { slice.slice_segment_address = 0; slice.num_ctu_in_slice = widthInCtbs * heightInCtbs; @@ -481,10 +579,16 @@ namespace jni } if (!idr) { - slice.ref_pic_list0[0] = Reference(); - - if (lowDelayB) { - slice.ref_pic_list1[0] = Reference(); + // The one picture the current one refers to. + for (const ReferencePicture & reference : ReferencePictureSet(frame)) { + if (reference.used) { + slice.ref_pic_list0[0] = Picture(reference); + + if (lowDelayB) { + slice.ref_pic_list1[0] = Picture(reference); + } + break; + } } } @@ -519,6 +623,20 @@ namespace jni writer.Bits(0, 32); writer.Bits(0, 12); writer.Bits(LevelIdc(codedWidth * codedHeight, framerate), 8); + + // No profile or level of their own for the sub-layers. + const int maxSubLayersMinus1 = layers.Count() - 1; + + for (int i = 0; i < maxSubLayersMinus1; i++) { + writer.Flag(false); + writer.Flag(false); + } + + if (maxSubLayersMinus1 > 0) { + for (int i = maxSubLayersMinus1; i < 8; i++) { + writer.Bits(0, 2); + } + } } size_t VaapiH265Encoder::WriteParameterSets(std::vector & output) const @@ -530,6 +648,15 @@ namespace jni writer.Bits(1, 3); }; + // A sub-layer for each temporal layer. Without layers, the current + // picture and one reference picture are decoded at a time; with + // them, the current one and those of the two reference slots. There + // is no reordering. The temporal id nesting flag is set without + // layers, as the standard has it then, and left clear with them. + const uint32_t maxSubLayersMinus1 = static_cast(layers.Count() - 1); + const uint32_t maxDecPicBufferingMinus1 = layers.Count() > 1 ? 2 : 1; + const bool temporalIdNesting = layers.Count() == 1; + // The VPS (7.3.2.1). NalUnitWriter vps; nalHeader(vps, kNalVps); @@ -537,14 +664,13 @@ namespace jni vps.Flag(true); vps.Flag(true); vps.Bits(0, 6); - vps.Bits(0, 3); - vps.Flag(true); + vps.Bits(maxSubLayersMinus1, 3); + vps.Flag(temporalIdNesting); vps.Bits(0xFFFF, 16); WriteProfileTierLevel(vps); - // One sub-layer, decoding with the current and one reference - // picture, without reordering. + // The ordering information of the highest sub-layer stands for all. vps.Flag(false); - vps.Ue(1); + vps.Ue(maxDecPicBufferingMinus1); vps.Ue(0); vps.Ue(0); vps.Bits(0, 6); @@ -557,8 +683,8 @@ namespace jni NalUnitWriter sps; nalHeader(sps, kNalSps); sps.Bits(0, 4); - sps.Bits(0, 3); - sps.Flag(true); + sps.Bits(maxSubLayersMinus1, 3); + sps.Flag(temporalIdNesting); WriteProfileTierLevel(sps); sps.Ue(0); sps.Ue(1); @@ -583,7 +709,7 @@ namespace jni sps.Ue(0); sps.Ue(kLog2MaxPicOrderCntLsb - 4); sps.Flag(false); - sps.Ue(1); + sps.Ue(maxDecPicBufferingMinus1); sps.Ue(0); sps.Ue(0); sps.Ue(log2MinCbSize - 3); @@ -664,15 +790,19 @@ namespace jni return vps.AppendTo(output) + sps.AppendTo(output) + pps.AppendTo(output); } - size_t VaapiH265Encoder::WriteSliceHeader(std::vector & output, bool idr) const + size_t VaapiH265Encoder::WriteSliceHeader(std::vector & output, bool idr, + const VaapiTemporalLayers::Frame & frame) const { // The slice segment header (7.3.6.1) of FillSlice(), up to the slice - // data, which the driver writes. + // data, which the driver writes, in a NAL unit of the picture's + // temporal layer. + const bool reference = idr || frame.refreshFlags != 0; + NalUnitWriter header; header.Bits(0, 1); - header.Bits(idr ? kNalIdrWRadl : kNalTrailR, 6); + header.Bits(idr ? kNalIdrWRadl : (reference ? kNalTrailR : kNalTrailN), 6); header.Bits(0, 6); - header.Bits(1, 3); + header.Bits(static_cast(frame.temporalId + 1), 3); header.Flag(true); @@ -686,13 +816,22 @@ namespace jni if (!idr) { header.Bits(static_cast(pictureOrder) & ((1u << kLog2MaxPicOrderCntLsb) - 1), kLog2MaxPicOrderCntLsb); - // st_ref_pic_set(0) here, not in the SPS: the previous picture, - // used by this one. + // st_ref_pic_set(0) here, not in the SPS: the pictures of the + // reference slots, before this one, of which it uses one. Those it + // keeps for later may be missing at a receiver without the upper + // layers, which the standard allows. + const std::vector set = ReferencePictureSet(frame); + int32_t previous = pictureOrder; + header.Flag(false); - header.Ue(1); - header.Ue(0); + header.Ue(static_cast(set.size())); header.Ue(0); - header.Flag(true); + + for (const ReferencePicture & reference : set) { + header.Ue(static_cast(previous - reference.pictureOrder - 1)); + header.Flag(reference.used); + previous = reference.pictureOrder; + } } if (sao) { @@ -728,6 +867,8 @@ namespace jni } bitrateBps = bitrate; + outputProcessor.OnRatesUpdated(parameters.bitrate); + layers.OnRatesUpdated(parameters.bitrate); if (parameters.framerate_fps >= 1.0) { framerate = static_cast(parameters.framerate_fps + 0.5); @@ -748,6 +889,8 @@ namespace jni // Whole coding blocks, so that the coded picture is the frame. info.requested_resolution_alignment = 1 << log2MinCbSize; + layers.FillFpsAllocation(info); + return info; } } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiTemporalLayers.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiTemporalLayers.cpp index 1e8cfc70..ff0ec85b 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiTemporalLayers.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiTemporalLayers.cpp @@ -35,12 +35,32 @@ namespace jni } VaapiTemporalLayers::VaapiTemporalLayers() : - count(1) + count(1), + layerRateControl(false) { slots.fill(-1); } - bool VaapiTemporalLayers::Configure(webrtc::ScalabilityMode mode, uint32_t bitrateBps) + bool VaapiTemporalLayers::LayerRateControl(VaapiLibrary & library, VAProfile profile, VAEntrypoint entrypoint, + int layers) + { + VAConfigAttrib attribute = {}; + attribute.type = VAConfigAttribEncRateControlExt; + + if (library.Api().GetConfigAttributes(library.Display(), profile, entrypoint, &attribute, 1) + != VA_STATUS_SUCCESS || attribute.value == VA_ATTRIB_NOT_SUPPORTED) + { + return false; + } + + VAConfigAttribValEncRateControlExt extension; + extension.value = attribute.value; + + return extension.bits.temporal_layer_bitrate_control_flag + && static_cast(extension.bits.max_num_temporal_layers_minus1) + 1 >= layers; + } + + bool VaapiTemporalLayers::Configure(webrtc::ScalabilityMode mode, uint32_t bitrateBps, bool layerRates) { switch (mode) { case webrtc::ScalabilityMode::kL1T1: @@ -57,6 +77,7 @@ namespace jni } slots.fill(-1); + layerRateControl = layerRates; bitratesBps.clear(); for (int layer = 0; layer < count && count > 1; layer++) { @@ -88,6 +109,12 @@ namespace jni return std::nullopt; } + for (const webrtc::CodecBufferUsage & buffer : layerFrame.Buffers()) { + if (buffer.updated && buffer.id >= 0 && buffer.id < static_cast(slots.size())) { + frame.keptFlags |= static_cast(1 << buffer.id); + } + } + // A key frame replaces every slot. if (keyFrame) { frame.refreshFlags = 0xFF; @@ -120,12 +147,20 @@ namespace jni void VaapiTemporalLayers::Commit(const Frame & frame) { for (int slot = 0; slot < static_cast(slots.size()); slot++) { - if (frame.refreshFlags & (1 << slot)) { + if (frame.keptFlags & (1 << slot)) { slots[slot] = frame.surface; } + else if (frame.refreshFlags == 0xFF) { + slots[slot] = -1; + } } } + void VaapiTemporalLayers::Forget(int slot) + { + slots[slot] = -1; + } + int VaapiTemporalLayers::SlotCount() const { return static_cast(slots.size()); @@ -151,7 +186,7 @@ namespace jni bool VaapiTemporalLayers::AddRateControl(VaapiEncodeSession & session, uint32_t bitrateBps, uint32_t framerate, bool keyFrame) const { - if (count == 1) { + if (count == 1 || !layerRateControl) { return session.AddRateControl(bitrateBps, framerate, 0); } diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp index 4bc6a9c1..175a10de 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVideoEncoderFactory.cpp @@ -72,7 +72,14 @@ namespace jni }; if (h265) { - formats.push_back(CreateH265MainFormat()); + // H.265 has no software encoder whose format would say which + // scalability modes WebRTC may ask for: the temporal layers the + // encoder makes. + webrtc::SdpVideoFormat format = CreateH265MainFormat(); + format.scalability_modes = { + webrtc::ScalabilityMode::kL1T1, webrtc::ScalabilityMode::kL1T2, webrtc::ScalabilityMode::kL1T3 + }; + formats.push_back(format); } if (vp9) { formats.push_back(webrtc::SdpVideoFormat::VP9Profile0()); diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp index ecc507b9..af009f2e 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/linux/VaapiVp9Encoder.cpp @@ -114,7 +114,8 @@ namespace jni ratesChanged = true; referenceValid = false; outputProcessor.Reset(); - layers.Configure(mode, bitrateBps); + layers.Configure(mode, bitrateBps, VaapiTemporalLayers::LayerRateControl(library, VAProfileVP9Profile0, + entrypoint, webrtc::ScalabilityModeToNumTemporalLayers(mode))); if (!outputProcessor.SetScalabilityMode(mode)) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; @@ -193,6 +194,7 @@ namespace jni session.DropBuffers(); + if (!encoded) { return kEncodeFailed; } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java index 7b9ea334..50cc4e8e 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/HardwareVideoEncoderFactory.java @@ -37,17 +37,18 @@ * H.264 is encoded on an NVIDIA GPU with NVENC, on Windows and Linux. On * other GPUs it is encoded with the Media Foundation encoder of the driver on * Windows, and with its VA-API encoder on Linux. AV1 is encoded on GPUs that - * have an AV1 encoder, with NVENC, Media Foundation or VA-API, as a single - * layer, or with VA-API in temporal layers too (L1T2, L1T3); a stream that - * asks for other SVC modes is encoded in software. H.265 (Main profile) is - * encoded with NVENC or Media Foundation on GPUs that have an H.265 encoder. + * have an AV1 encoder, with NVENC, Media Foundation or VA-API. H.265 (Main + * profile) is encoded with NVENC or Media Foundation on GPUs that have an + * H.265 encoder. * VP9 (profile 0) is encoded on GPUs that have a VP9 encoder, which are * Intel's: with VA-API on Linux, and with Media Foundation on Windows only * with the system property {@code webrtc.hardware.vp9Encoder} set to * {@code true} when the factory is created, since that encoder has not been - * tested on such a GPU yet. With VA-API, VP9 is encoded in temporal layers - * too (L1T2, L1T3). Otherwise VP9 is encoded with libvpx. On Linux, H.265 is encoded with NVENC on NVIDIA + * tested on such a GPU yet. Otherwise VP9 is encoded with libvpx. On Linux, H.265 is encoded with NVENC on NVIDIA * GPUs and with VA-API on the GPUs of Intel and AMD whose driver encodes it. + * Hardware encoders produce a single layer, but for the VA-API encoders, which + * encode temporal layers (L1T2, L1T3) too; a stream that asks for other SVC + * modes is encoded in software. * A hardware encoder that fails to start, for example because the GPU has no * encoder sessions left, or fails while encoding, is replaced by the next one * in line, and finally by the software encoder of the same codec, so a stream diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java index 26e8aa61..8de5465e 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/HardwareVideoEncoderIntegrationTest.java @@ -54,6 +54,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; +import java.util.function.Supplier; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.parallel.Execution; @@ -524,7 +525,7 @@ void av1RestartsCleanly() throws Exception { @Test void av1EncodesTemporalLayers() throws Exception { - assertEncodesTemporalLayers(AV1, HARDWARE_AV1_REQUIRED); + assertEncodesTemporalLayers(this::hardwareFactory, AV1, HARDWARE_AV1_REQUIRED); } @Test @@ -532,20 +533,40 @@ void vp9EncodesTemporalLayers() throws Exception { // VA-API encodes VP9 in temporal layers, Media Foundation does not. assumeTrue(OS.contains("linux"), "hardware VP9 encodes temporal layers on Linux only"); - assertEncodesTemporalLayers(VP9, HARDWARE_VP9_REQUIRED); + assertEncodesTemporalLayers(this::hardwareFactory, VP9, HARDWARE_VP9_REQUIRED); + } + + @Test + void h264EncodesTemporalLayers() throws Exception { + // VA-API encodes H.264 in temporal layers; NVENC and Media Foundation + // leave them to OpenH264. Received with the hardware decoders, which + // decode the layers too. + assumeTrue(OS.contains("linux"), "hardware H.264 encodes temporal layers on Linux only"); + + assertEncodesTemporalLayers(this::hardwareCallFactory, H264, HARDWARE_REQUIRED); + } + + @Test + void h265EncodesTemporalLayers() throws Exception { + // There is no software H.265 to receive it, so the hardware decoders do. + assumeTrue(OS.contains("linux"), "hardware H.265 encodes temporal layers on Linux only"); + assumeH265(); + + assertEncodesTemporalLayers(this::hardwareCallFactory, H265, true); } /** - * Sends the codec in L1T2 and then L1T3, and checks that the hardware - * encodes the layers, rather than handing them to the software encoder. + * Sends the codec in L1T2 and then L1T3, each on a call of a factory the + * supplier makes, and checks that the hardware encodes the layers, rather + * than handing them to the software encoder. * Temporal layers are references between frames of one size, which a * hardware encoder can make; an SFU can then forward a lower frame rate * without asking for key frames. */ - private void assertEncodesTemporalLayers(Predicate codec, boolean required) - throws Exception { + private void assertEncodesTemporalLayers(Supplier factory, + Predicate codec, boolean required) throws Exception { for (String mode : new String[] { "L1T2", "L1T3" }) { - PeerConnectionFactory hardware = hardwareFactory(); + PeerConnectionFactory hardware = factory.get(); CountDownLatch received = new CountDownLatch(30); try (TestMediaCall call = new TestMediaCall(hardware, true, false, codec)) {