Skip to content

Commit bb97ca0

Browse files
devopvoiddevopvoid
authored
feat: decode H.264 with VA-API on Linux (#375)
Adds an H.264 decoder for Intel and AMD GPUs, with its own parser. The H.264 and H.265 decoders share a new VaapiDecodeSession.
1 parent a21d043 commit bb97ca0

12 files changed

Lines changed: 2563 additions & 198 deletions

File tree

‎docs/guide/advanced/video-codecs.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ PeerConnectionFactory factory = PeerConnectionFactory.builder()
6262
| Platform | Hardware decoding |
6363
|---|---|
6464
| Windows | H.264, H.265 (Main profile), AV1 and VP9 (profile 0), on GPUs that decode them, through the Media Foundation decoders of Windows on Direct3D 11 (DXVA) |
65-
| Linux | H.264, H.265 (Main profile), VP9 and AV1 (profile 0) on NVIDIA GPUs that decode them, through NVDEC; VP9 (profile 0) and H.265 (Main profile) on Intel and AMD GPUs through VA-API |
65+
| Linux | H.264, H.265 (Main profile), VP9 and AV1 (profile 0) on NVIDIA GPUs that decode them, through NVDEC; H.264, VP9 (profile 0) and H.265 (Main profile) on Intel and AMD GPUs through VA-API |
6666
| macOS | H.264 through VideoToolbox, as with `DefaultVideoDecoderFactory`; VP9 (profile 0) and H.265 (Main profile) through the decoders of VideoToolbox, on Macs that have them, and AV1 (profile 0) on Apple M3 and newer |
6767

6868
On Linux, NVDEC needs an NVIDIA driver, which provides `libcuda.so.1` and `libnvcuvid.so.1`; both are loaded when the factory is made, so a machine without the driver is not affected. Streams with spatial layers are decoded in software.

‎webrtc-jni/src/main/cpp/include/media/video/codec/linux/H264Parser.h‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
/*
2+
* Copyright 2026 Alex Andres
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_H264_PARSER_H_
18+
#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_H264_PARSER_H_
19+
20+
#include <array>
21+
#include <cstdint>
22+
#include <map>
23+
#include <optional>
24+
#include <span>
25+
#include <vector>
26+
27+
namespace jni
28+
{
29+
// The scaling lists of a picture (7.4.2.1.1.1), in the order they are
30+
// coded, which is zigzag order.
31+
struct H264ScalingLists
32+
{
33+
std::array<std::array<uint8_t, 16>, 6> list4x4 = {};
34+
std::array<std::array<uint8_t, 64>, 2> list8x8 = {};
35+
};
36+
37+
// What the decoder needs of a sequence parameter set (7.3.2.1.1).
38+
struct H264Sps
39+
{
40+
uint32_t profileIdc = 0;
41+
uint32_t levelIdc = 0;
42+
uint32_t chromaFormatIdc = 1;
43+
uint32_t bitDepthLumaMinus8 = 0;
44+
uint32_t bitDepthChromaMinus8 = 0;
45+
bool scalingMatrixPresent = false;
46+
// The lists given or inferred, where scalingMatrixPresent is set.
47+
H264ScalingLists scalingLists;
48+
uint32_t log2MaxFrameNumMinus4 = 0;
49+
uint32_t pocType = 0;
50+
uint32_t log2MaxPocLsbMinus4 = 0;
51+
bool deltaPicOrderAlwaysZero = false;
52+
int32_t offsetForNonRefPic = 0;
53+
int32_t offsetForTopToBottomField = 0;
54+
std::vector<int32_t> offsetForRefFrame;
55+
uint32_t maxNumRefFrames = 0;
56+
bool gapsInFrameNumAllowed = false;
57+
uint32_t widthInMbs = 0;
58+
uint32_t heightInMbs = 0;
59+
bool frameMbsOnly = true;
60+
bool direct8x8Inference = false;
61+
// The frame cropping, in luma samples.
62+
uint32_t cropLeft = 0;
63+
uint32_t cropRight = 0;
64+
uint32_t cropTop = 0;
65+
uint32_t cropBottom = 0;
66+
// From the VUI, where it says.
67+
std::optional<uint32_t> maxNumReorderFrames;
68+
69+
uint32_t MaxFrameNum() const;
70+
};
71+
72+
// A picture parameter set (7.3.2.2).
73+
struct H264Pps
74+
{
75+
uint32_t spsId = 0;
76+
bool entropyCodingMode = false;
77+
bool bottomFieldPicOrderInFramePresent = false;
78+
uint32_t numSliceGroupsMinus1 = 0;
79+
uint32_t numRefIdxL0DefaultMinus1 = 0;
80+
uint32_t numRefIdxL1DefaultMinus1 = 0;
81+
bool weightedPred = false;
82+
uint32_t weightedBipredIdc = 0;
83+
int32_t picInitQpMinus26 = 0;
84+
int32_t picInitQsMinus26 = 0;
85+
int32_t chromaQpIndexOffset = 0;
86+
bool deblockingFilterControlPresent = false;
87+
bool constrainedIntraPred = false;
88+
bool redundantPicCntPresent = false;
89+
bool transform8x8Mode = false;
90+
int32_t secondChromaQpIndexOffset = 0;
91+
bool scalingMatrixPresent = false;
92+
// The lists as coded, and which were.
93+
std::array<bool, 8> scalingListPresent = {};
94+
std::array<bool, 8> useDefaultScalingList = {};
95+
H264ScalingLists scalingLists;
96+
};
97+
98+
// A reference picture list modification (7.3.3.1), or a memory
99+
// management control operation (7.3.3.3).
100+
struct H264Operation
101+
{
102+
uint32_t operation = 0;
103+
uint32_t value = 0;
104+
// For memory management: long_term_frame_idx of operations 3 and 6.
105+
uint32_t longTermFrameIdx = 0;
106+
};
107+
108+
// A slice header (7.3.3) of a frame.
109+
struct H264SliceHeader
110+
{
111+
static constexpr int kMaxRefs = 32;
112+
113+
uint8_t nalType = 0;
114+
uint8_t nalRefIdc = 0;
115+
uint32_t firstMbInSlice = 0;
116+
// slice_type modulo 5.
117+
uint32_t sliceType = 0;
118+
uint32_t ppsId = 0;
119+
uint32_t frameNum = 0;
120+
uint32_t idrPicId = 0;
121+
uint32_t pocLsb = 0;
122+
int32_t deltaPocBottom = 0;
123+
std::array<int32_t, 2> deltaPoc = {};
124+
uint32_t redundantPicCnt = 0;
125+
bool directSpatialMvPred = false;
126+
uint32_t numRefIdxL0Minus1 = 0;
127+
uint32_t numRefIdxL1Minus1 = 0;
128+
std::array<std::vector<H264Operation>, 2> listModifications;
129+
uint32_t lumaLog2WeightDenom = 0;
130+
uint32_t chromaLog2WeightDenom = 0;
131+
// Whether any weight of a list was given, and the weights and offsets,
132+
// the default ones where none were given.
133+
std::array<bool, 2> lumaWeightsGiven = {};
134+
std::array<bool, 2> chromaWeightsGiven = {};
135+
std::array<std::array<int32_t, kMaxRefs>, 2> lumaWeight = {};
136+
std::array<std::array<int32_t, kMaxRefs>, 2> lumaOffset = {};
137+
std::array<std::array<std::array<int32_t, 2>, kMaxRefs>, 2> chromaWeight = {};
138+
std::array<std::array<std::array<int32_t, 2>, kMaxRefs>, 2> chromaOffset = {};
139+
bool noOutputOfPriorPics = false;
140+
bool longTermReference = false;
141+
bool adaptiveRefPicMarking = false;
142+
std::vector<H264Operation> memoryManagement;
143+
uint32_t cabacInitIdc = 0;
144+
int32_t qpDelta = 0;
145+
uint32_t disableDeblockingFilterIdc = 0;
146+
int32_t alphaOffsetDiv2 = 0;
147+
int32_t betaOffsetDiv2 = 0;
148+
// Where the slice data begins, in bits from the NAL unit header, with
149+
// emulation prevention bytes not counted, as VA-API takes it.
150+
uint32_t dataBitOffset = 0;
151+
152+
bool IsIdr() const;
153+
bool IsIntra() const;
154+
bool IsB() const;
155+
};
156+
157+
// Parses the parameter sets and slice headers of an H.264 stream, as a
158+
// VA-API decoder, which leaves that to the application, needs them. It
159+
// takes progressive 8 bit 4:2:0 streams without slice groups; anything
160+
// else is not parsed, so that the software decoder takes it.
161+
class H264Parser
162+
{
163+
public:
164+
// NAL unit types (table 7-1).
165+
static constexpr uint8_t kNalSlice = 1;
166+
static constexpr uint8_t kNalIdr = 5;
167+
static constexpr uint8_t kNalSps = 7;
168+
static constexpr uint8_t kNalPps = 8;
169+
static constexpr uint8_t kNalEndOfSequence = 10;
170+
171+
// Slice types modulo 5 (table 7-6).
172+
static constexpr uint32_t kSliceP = 0;
173+
static constexpr uint32_t kSliceB = 1;
174+
static constexpr uint32_t kSliceI = 2;
175+
176+
// Each takes a NAL unit from its header on, with emulation
177+
// prevention bytes, and returns false if it is broken or not one
178+
// the parser takes.
179+
bool ParseSps(std::span<const uint8_t> nal);
180+
bool ParsePps(std::span<const uint8_t> nal);
181+
std::optional<H264SliceHeader> ParseSliceHeader(std::span<const uint8_t> nal) const;
182+
183+
const H264Sps * Sps(uint32_t id) const;
184+
const H264Pps * Pps(uint32_t id) const;
185+
186+
// The scaling lists a picture is decoded with: of the PPS, falling
187+
// back to those of the SPS, or flat ones (7.4.2.2).
188+
static H264ScalingLists ScalingLists(const H264Sps & sps, const H264Pps & pps);
189+
190+
void Reset();
191+
192+
private:
193+
std::map<uint32_t, H264Sps> spss;
194+
std::map<uint32_t, H264Pps> ppss;
195+
};
196+
}
197+
198+
#endif

‎webrtc-jni/src/main/cpp/include/media/video/codec/linux/VaapiDecodeSession.h‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/*
2+
* Copyright 2026 Alex Andres
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_DECODE_SESSION_H_
18+
#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VAAPI_DECODE_SESSION_H_
19+
20+
#include "media/video/codec/linux/VaapiLibrary.h"
21+
22+
#include "api/scoped_refptr.h"
23+
#include "api/video/i420_buffer.h"
24+
25+
#include <va/va.h>
26+
27+
#include <cstddef>
28+
#include <cstdint>
29+
#include <string>
30+
#include <vector>
31+
32+
namespace jni
33+
{
34+
// What every VA-API decoder of a codec whose stream the decoder parses
35+
// needs: a configuration and a context with the surfaces the decoded
36+
// pictures go into, decoding a picture from the parameter and data
37+
// buffers the codec's decoder fills in, and copying a decoded picture out
38+
// as I420, which is what WebRTC's frames hold.
39+
//
40+
// Used by the H.264, H.265 and AV1 decoders; the VP9 one keeps its own,
41+
// older copy of the same.
42+
class VaapiDecodeSession
43+
{
44+
public:
45+
// A parameter or data buffer of a picture.
46+
struct Buffer
47+
{
48+
VABufferType type;
49+
const void * data;
50+
size_t size;
51+
};
52+
53+
// The name says whose operations failed, in the log.
54+
VaapiDecodeSession(VaapiLibrary & library, std::string name);
55+
~VaapiDecodeSession();
56+
57+
VaapiDecodeSession(const VaapiDecodeSession &) = delete;
58+
VaapiDecodeSession & operator=(const VaapiDecodeSession &) = delete;
59+
60+
// Creates the session for 4:2:0 pictures of the given size, which
61+
// is the coded size, with the given number of surfaces.
62+
bool Create(VAProfile profile, uint32_t width, uint32_t height, uint32_t surfaceCount);
63+
void Destroy();
64+
65+
bool IsCreated() const;
66+
uint32_t Width() const;
67+
uint32_t Height() const;
68+
int SurfaceCount() const;
69+
VASurfaceID Surface(int index) const;
70+
71+
// Decodes a picture into the surface from the buffers, in the
72+
// order given, and waits for it.
73+
bool Decode(int surface, const std::vector<Buffer> & buffers);
74+
// The region of the decoded picture in the surface, as I420; null
75+
// if it cannot be read.
76+
webrtc::scoped_refptr<webrtc::I420Buffer> Download(int surface, uint32_t x, uint32_t y, uint32_t width,
77+
uint32_t height);
78+
79+
bool Check(VAStatus status, const char * operation) const;
80+
81+
private:
82+
VaapiLibrary & library;
83+
const VaapiFunctions & va;
84+
const std::string name;
85+
86+
VAConfigID config;
87+
VAContextID context;
88+
std::vector<VASurfaceID> surfaces;
89+
uint32_t width;
90+
uint32_t height;
91+
92+
// The image a decoded surface is copied into, where the driver
93+
// copies surfaces at all.
94+
VAImage downloadImage;
95+
};
96+
}
97+
98+
#endif

0 commit comments

Comments
 (0)