diff --git a/README.md b/README.md index 827525c1c..444e2c2fb 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ For more detailed information, check out the documentation: - [Media Files](https://jrtc.dev/guide/media/media-files) - Sending video and audio files with the media module - [Media Recording](https://jrtc.dev/guide/media/media-recording) - Recording calls into media files with the media module - [Encoded Transforms](https://jrtc.dev/guide/advanced/encoded-transforms) - Reading and changing encoded frames, e.g. for end-to-end encryption +- [Video Codecs](https://jrtc.dev/guide/advanced/video-codecs) - Adding video codecs implemented in Java, or limiting the built-in ones - [Build Notes](https://jrtc.dev/guide/build) - Instructions for building the library from source ## License diff --git a/docs/.vitepress/sidebar.ts b/docs/.vitepress/sidebar.ts index 55a77a742..ca87a3799 100644 --- a/docs/.vitepress/sidebar.ts +++ b/docs/.vitepress/sidebar.ts @@ -74,6 +74,7 @@ function sidebarGuide(): DefaultTheme.SidebarItem[] { items: [ { text: 'Field Trials', link: '/advanced/field-trials' }, { text: 'Encoded Transforms', link: '/advanced/encoded-transforms' }, + { text: 'Video Codecs', link: '/advanced/video-codecs' }, ], }, ] diff --git a/docs/guide/advanced/field-trials.md b/docs/guide/advanced/field-trials.md index e92e52cd8..ff9cd8253 100644 --- a/docs/guide/advanced/field-trials.md +++ b/docs/guide/advanced/field-trials.md @@ -24,6 +24,15 @@ PeerConnectionFactory factory = new PeerConnectionFactory(fieldTrials, audioProc PeerConnectionFactory factory = new PeerConnectionFactory(fieldTrials, audioModule, audioProcessing); ``` +`PeerConnectionFactory.builder()` sets them too, next to anything else a factory is created with, such as its [video codecs](/guide/advanced/video-codecs): + +```java +PeerConnectionFactory factory = PeerConnectionFactory.builder() + .setFieldTrials(fieldTrials) + .setAudioDeviceModule(audioModule) + .build(); +``` + ## Constraints Each key and value must be non-null and non-empty. Passing a map that violates this constraint throws an exception when the `PeerConnectionFactory` is created. diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md new file mode 100644 index 000000000..9c88d61eb --- /dev/null +++ b/docs/guide/advanced/video-codecs.md @@ -0,0 +1,168 @@ +# Video Codecs + +A `PeerConnectionFactory` decides which video codecs it can send and receive through a video encoder factory and a video decoder factory. Unless told otherwise it uses the codecs built into the library. This guide explains how to replace or extend them: to add a codec WebRTC does not have, to use a hardware encoder reached through another library, or to limit the codecs a factory offers. + +The API lives in the `dev.onvoid.webrtc.media.video.codec` package. + +## Built-in Codecs + +`DefaultVideoEncoderFactory` and `DefaultVideoDecoderFactory` provide the codecs built into the library. On Windows and Linux these are VP8, VP9, AV1 and H.264 in software; on macOS they are those of WebRTC's default factories, with H.264 through VideoToolbox. Ask a factory for the exact list: + +```java +for (VideoCodecInfo codec : new DefaultVideoEncoderFactory().getSupportedCodecs()) { + System.out.println(codec.getName() + " " + codec.getParameters()); +} +``` + +The encoders and decoders these factories create are `NativeVideoEncoder`s and `NativeVideoDecoder`s. They are placeholders that make WebRTC create the built-in codec, which then runs entirely inside WebRTC, so their methods are not to be called from Java. + +## Setting the Factories + +The codec factories are set with `PeerConnectionFactory.builder()`, which also takes everything the constructors do: + +```java +PeerConnectionFactory factory = PeerConnectionFactory.builder() + .setAudioDeviceModule(audioModule) + .setVideoEncoderFactory(new MyEncoderFactory()) + .setVideoDecoderFactory(new MyDecoderFactory()) + .build(); +``` + +A factory that is not set is the built-in one. The supported codecs are asked for once, when the `PeerConnectionFactory` is built; an exception thrown there fails `build()`. Encoders and decoders are created later, from WebRTC threads, one per stream. + +Both ends have to agree on a codec, so a codec added on the sending side also has to be offered by the decoder factory of the receiving side. + +## Adding a Codec + +A factory of one's own usually hands out its own codec next to the built-in ones, by delegating to a default factory: + +```java +class MyEncoderFactory implements VideoEncoderFactory { + + private final DefaultVideoEncoderFactory builtIn = new DefaultVideoEncoderFactory(); + + @Override + public List getSupportedCodecs() { + List codecs = new ArrayList<>(); + codecs.add(new VideoCodecInfo("X-MYCODEC")); + codecs.addAll(builtIn.getSupportedCodecs()); + return codecs; + } + + @Override + public VideoEncoder createEncoder(VideoCodecInfo info) { + if (info.getName().equalsIgnoreCase("X-MYCODEC")) { + return new MyEncoder(); + } + return builtIn.createEncoder(info); + } +} +``` + +The list is in order of preference. A codec name WebRTC does not know, such as `X-MYCODEC`, is negotiated like any other and sent with the generic RTP packetization. `RTCRtpTransceiver.setCodecPreferences()` picks among the negotiated codecs per transceiver. + +## Implementing an Encoder + +WebRTC calls `initEncode()`, then `encode()` for each frame and `setRates()` whenever the target bitrate changes, and finally `release()`. It may initialize a released encoder again, possibly with another frame size. These calls come from one WebRTC thread at a time. + +```java +class MyEncoder implements VideoEncoder { + + private Callback callback; + + @Override + public VideoCodecStatus initEncode(Settings settings, Callback callback) { + this.callback = callback; + // Set up the encoder for settings.width x settings.height at + // settings.startBitrateKbps. + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus encode(VideoFrame frame, EncodeInfo info) { + I420Buffer pixels = frame.buffer.toI420(); + ByteBuffer payload = encodeSomehow(pixels, info.isKeyFrameRequested()); + + callback.onEncodedFrame(EncodedImage.builder() + .setBuffer(payload) + .setEncodedWidth(pixels.getWidth()) + .setEncodedHeight(pixels.getHeight()) + .setCaptureTimeNs(frame.timestampNs) + .setFrameType(info.isKeyFrameRequested() + ? EncodedImage.FrameType.KEY + : EncodedImage.FrameType.DELTA) + .build()); + + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus setRates(RateControlParameters parameters) { + // Aim for parameters.bitrate.getSum() bps at parameters.framerateFps. + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus release() { + return VideoCodecStatus.OK; + } +} +``` + +Things to keep in mind: + +- The capture time of an `EncodedImage` must be the `timestampNs` of the frame it encodes. That is how WebRTC matches output to input; an image with any other capture time is dropped. +- The frame passed to `encode()` is valid only until the method returns. An encoder that works asynchronously calls `frame.retain()` and later `frame.release()`, and hands its output to the callback from its own thread, in encoding order. The pixels must not be changed, since other consumers may share them. +- `onEncodedFrame()` copies the payload, so the buffer may be reused once it returns. Frames handed over after `release()` are ignored. +- `getScalingSettings()`, `getEncoderInfo()`, `getResolutionBitrateLimits()`, `getImplementationName()` and `isHardwareEncoder()` have defaults and may be overridden. The default scaling settings use WebRTC's quantizer thresholds for VP8, VP9 and H.264; a custom codec without a quantizer should return `ScalingSettings.OFF`. + +## Implementing a Decoder + +A decoder follows the same pattern with `initDecode()`, `decode()` and `release()`: + +```java +class MyDecoder implements VideoDecoder { + + private Callback callback; + + @Override + public VideoCodecStatus initDecode(Settings settings, Callback callback) { + this.callback = callback; + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus decode(EncodedImage image) { + NativeI420Buffer buffer = decodeSomehow(image.getBuffer()); + VideoFrame frame = new VideoFrame(buffer, image.getCaptureTimeNs()); + + callback.onDecodedFrame(frame, null, null); + frame.release(); + + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus release() { + return VideoCodecStatus.OK; + } +} +``` + +- The image and its buffer are read-only and valid only until `decode()` returns; copy the data to keep it longer. +- A decoded frame must carry the capture time of the image it was decoded from as its timestamp. +- `onDecodedFrame()` takes a reference of its own, so the caller still releases the frame it created. Frame buffers other than `NativeI420Buffer` are copied, and have to provide direct byte buffers. + +## Errors + +Whatever an encoder, decoder or factory method throws is logged and treated as `VideoCodecStatus.ERROR`, and a factory that throws from `createEncoder()` or `createDecoder()` creates no codec for that stream. On an error other than `FALLBACK_SOFTWARE` and `UNINITIALIZED`, WebRTC releases the codec and initializes it again. + +## Related API + +- `PeerConnectionFactory.Builder` — sets the video encoder and decoder factories. +- `VideoEncoderFactory`, `VideoDecoderFactory` — create the codecs of a factory. +- `VideoEncoder`, `VideoDecoder`, `EncodedImage` — codecs implemented in Java. +- `DefaultVideoEncoderFactory`, `DefaultVideoDecoderFactory` — the built-in codecs. +- `RTCRtpTransceiver.setCodecPreferences()` — chooses among the negotiated codecs. + +For the full API, see the JavaDoc of the `dev.onvoid.webrtc.media.video.codec` package. diff --git a/docs/guide/index.md b/docs/guide/index.md index c478919ab..84c248ba1 100644 --- a/docs/guide/index.md +++ b/docs/guide/index.md @@ -41,6 +41,7 @@ This section provides detailed guides for various features of the webrtc-java li - [Field Trials](/guide/advanced/field-trials) - Enabling experimental features and tuning WebRTC internals - [Encoded Transforms](/guide/advanced/encoded-transforms) - Reading and changing encoded frames, e.g. for end-to-end encryption +- [Video Codecs](/guide/advanced/video-codecs) - Adding video codecs implemented in Java, or limiting the built-in ones ## Additional Resources diff --git a/docs/index.md b/docs/index.md index 1dd4739c9..12c2f867d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -32,6 +32,8 @@ features: details: Record what a peer connection sends or receives into MKV, WebM or MP4 files. Encoded frames go into the file as they are, without re-encoding, so recording costs next to no CPU and keeps the exact quality of the call. - title: End-to-End Encryption details: Encoded transforms let Java code read, change or drop every encoded frame between encoder and network, like insertable streams in the browser; the building block for end-to-end encryption, frame metadata and stream analysis. + - title: Custom Video Codecs + details: Plug video encoders and decoders written in Java into WebRTC, next to the built-in VP8, VP9, AV1 and H.264, to bring in a codec of your own or a hardware encoder from another library. - title: Screen Sharing details: Share application windows or the full desktop with minimal setup; integrate screen capture streams like any other media stream. - title: Data Channels diff --git a/webrtc-jni/src/main/cpp/CMakeLists.txt b/webrtc-jni/src/main/cpp/CMakeLists.txt index 45f27c818..2c83e39dc 100644 --- a/webrtc-jni/src/main/cpp/CMakeLists.txt +++ b/webrtc-jni/src/main/cpp/CMakeLists.txt @@ -55,6 +55,7 @@ file(GLOB SOURCES_MEDIA "src/media/*.cpp") file(GLOB SOURCES_MEDIA_AUDIO "src/media/audio/*.cpp") file(GLOB SOURCES_MEDIA_AUDIO_OS "src/media/audio/${SOURCE_TARGET}/*.cpp") file(GLOB SOURCES_MEDIA_VIDEO "src/media/video/*.cpp") +file(GLOB SOURCES_MEDIA_VIDEO_CODEC "src/media/video/codec/*.cpp") file(GLOB SOURCES_MEDIA_VIDEO_DESKTOP "src/media/video/desktop/*.cpp") file(GLOB SOURCES_MEDIA_VIDEO_DESKTOP_OS "src/media/video/desktop/${SOURCE_TARGET}/*.cpp") file(GLOB SOURCES_MEDIA_VIDEO_OS "src/media/video/${SOURCE_TARGET}/*.cpp") @@ -69,6 +70,7 @@ list(APPEND SOURCES ${SOURCES_MEDIA_AUDIO} ${SOURCES_MEDIA_AUDIO_OS} ${SOURCES_MEDIA_VIDEO} + ${SOURCES_MEDIA_VIDEO_CODEC} ${SOURCES_MEDIA_VIDEO_DESKTOP} ${SOURCES_MEDIA_VIDEO_DESKTOP_OS} ${SOURCES_MEDIA_VIDEO_OS} diff --git a/webrtc-jni/src/main/cpp/include/JNI_DefaultVideoDecoderFactory.h b/webrtc-jni/src/main/cpp/include/JNI_DefaultVideoDecoderFactory.h new file mode 100644 index 000000000..175e20f30 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/JNI_DefaultVideoDecoderFactory.h @@ -0,0 +1,36 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +/* Header for class dev_onvoid_webrtc_media_video_codec_DefaultVideoDecoderFactory */ + +#ifndef _Included_dev_onvoid_webrtc_media_video_codec_DefaultVideoDecoderFactory +#define _Included_dev_onvoid_webrtc_media_video_codec_DefaultVideoDecoderFactory +#ifdef __cplusplus +extern "C" { +#endif + /* + * Class: dev_onvoid_webrtc_media_video_codec_DefaultVideoDecoderFactory + * Method: getSupportedCodecsInternal + * Signature: ()[Ldev/onvoid/webrtc/media/video/codec/VideoCodecInfo; + */ + JNIEXPORT jobjectArray JNICALL Java_dev_onvoid_webrtc_media_video_codec_DefaultVideoDecoderFactory_getSupportedCodecsInternal + (JNIEnv *, jclass); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/webrtc-jni/src/main/cpp/include/JNI_DefaultVideoEncoderFactory.h b/webrtc-jni/src/main/cpp/include/JNI_DefaultVideoEncoderFactory.h new file mode 100644 index 000000000..b70ee737f --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/JNI_DefaultVideoEncoderFactory.h @@ -0,0 +1,36 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +/* Header for class dev_onvoid_webrtc_media_video_codec_DefaultVideoEncoderFactory */ + +#ifndef _Included_dev_onvoid_webrtc_media_video_codec_DefaultVideoEncoderFactory +#define _Included_dev_onvoid_webrtc_media_video_codec_DefaultVideoEncoderFactory +#ifdef __cplusplus +extern "C" { +#endif + /* + * Class: dev_onvoid_webrtc_media_video_codec_DefaultVideoEncoderFactory + * Method: getSupportedCodecsInternal + * Signature: ()[Ldev/onvoid/webrtc/media/video/codec/VideoCodecInfo; + */ + JNIEXPORT jobjectArray JNICALL Java_dev_onvoid_webrtc_media_video_codec_DefaultVideoEncoderFactory_getSupportedCodecsInternal + (JNIEnv *, jclass); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/webrtc-jni/src/main/cpp/include/JNI_NativeVideoDecoderCallback.h b/webrtc-jni/src/main/cpp/include/JNI_NativeVideoDecoderCallback.h new file mode 100644 index 000000000..827c0c3c3 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/JNI_NativeVideoDecoderCallback.h @@ -0,0 +1,36 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +/* Header for class dev_onvoid_webrtc_media_video_codec_NativeVideoDecoderCallback */ + +#ifndef _Included_dev_onvoid_webrtc_media_video_codec_NativeVideoDecoderCallback +#define _Included_dev_onvoid_webrtc_media_video_codec_NativeVideoDecoderCallback +#ifdef __cplusplus +extern "C" { +#endif + /* + * Class: dev_onvoid_webrtc_media_video_codec_NativeVideoDecoderCallback + * Method: onDecodedFrame + * Signature: (JLdev/onvoid/webrtc/media/video/VideoFrame;II)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_codec_NativeVideoDecoderCallback_onDecodedFrame + (JNIEnv *, jclass, jlong, jobject, jint, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/webrtc-jni/src/main/cpp/include/JNI_NativeVideoEncoderCallback.h b/webrtc-jni/src/main/cpp/include/JNI_NativeVideoEncoderCallback.h new file mode 100644 index 000000000..38d26c6d2 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/JNI_NativeVideoEncoderCallback.h @@ -0,0 +1,36 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +/* Header for class dev_onvoid_webrtc_media_video_codec_NativeVideoEncoderCallback */ + +#ifndef _Included_dev_onvoid_webrtc_media_video_codec_NativeVideoEncoderCallback +#define _Included_dev_onvoid_webrtc_media_video_codec_NativeVideoEncoderCallback +#ifdef __cplusplus +extern "C" { +#endif + /* + * Class: dev_onvoid_webrtc_media_video_codec_NativeVideoEncoderCallback + * Method: onEncodedFrame + * Signature: (JLdev/onvoid/webrtc/media/video/codec/EncodedImage;)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_codec_NativeVideoEncoderCallback_onEncodedFrame + (JNIEnv *, jclass, jlong, jobject); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h b/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h index eeff18380..c5b40c021 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h +++ b/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h @@ -82,10 +82,10 @@ extern "C" { /* * Class: dev_onvoid_webrtc_PeerConnectionFactory * Method: initialize - * Signature: (Ljava/util/Map;Ldev/onvoid/webrtc/media/audio/AudioDeviceModuleBase;Ldev/onvoid/webrtc/media/audio/AudioProcessing;)V + * Signature: (Ljava/util/Map;Ldev/onvoid/webrtc/media/audio/AudioDeviceModuleBase;Ldev/onvoid/webrtc/media/audio/AudioProcessing;Ldev/onvoid/webrtc/media/video/codec/VideoEncoderFactory;Ldev/onvoid/webrtc/media/video/codec/VideoDecoderFactory;)V */ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_initialize - (JNIEnv *, jobject, jobject, jobject, jobject); + (JNIEnv *, jobject, jobject, jobject, jobject, jobject, jobject); #ifdef __cplusplus } diff --git a/webrtc-jni/src/main/cpp/include/JNI_WebRTC.h b/webrtc-jni/src/main/cpp/include/JNI_WebRTC.h index 0eb3888c7..6d5d6b568 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_WebRTC.h +++ b/webrtc-jni/src/main/cpp/include/JNI_WebRTC.h @@ -23,6 +23,7 @@ #define PKG_MEDIA "dev/onvoid/webrtc/media/" #define PKG_AUDIO "dev/onvoid/webrtc/media/audio/" #define PKG_VIDEO "dev/onvoid/webrtc/media/video/" +#define PKG_CODEC "dev/onvoid/webrtc/media/video/codec/" #define PKG_DESKTOP "dev/onvoid/webrtc/media/video/desktop/" #define BOOLEAN_SIG "Ljava/lang/Boolean;" diff --git a/webrtc-jni/src/main/cpp/include/api/VideoFrame.h b/webrtc-jni/src/main/cpp/include/api/VideoFrame.h index 6be8e8847..6b79e2550 100644 --- a/webrtc-jni/src/main/cpp/include/api/VideoFrame.h +++ b/webrtc-jni/src/main/cpp/include/api/VideoFrame.h @@ -30,6 +30,17 @@ namespace jni namespace VideoFrame { webrtc::VideoFrame toNative(JNIEnv * env, const JavaRef & javaFrame); + + // Wraps the frame for Java without copying its pixels, converting + // them to I420 first if they are in another format. The Java frame + // holds a reference to the pixel buffer, which it gives up when it + // is released. Returns null if the pixels cannot be converted. + JavaLocalRef toJava(JNIEnv * env, const webrtc::VideoFrame & frame); + + // Returns the native pixel buffer of a Java VideoFrameBuffer. A + // NativeI420Buffer is shared; any other buffer is converted to I420 + // in Java and copied. Returns null if it has no readable pixels. + webrtc::scoped_refptr toNativeBuffer(JNIEnv * env, const JavaRef & javaBuffer); } namespace I420Buffer @@ -47,6 +58,24 @@ namespace jni jfieldID buffer; jfieldID rotation; jfieldID timestampNs; + jmethodID release; + }; + + class JavaI420BufferClass : public JavaClass + { + public: + explicit JavaI420BufferClass(JNIEnv * env); + + jclass cls; + jmethodID toI420; + jmethodID getWidth; + jmethodID getHeight; + jmethodID getDataY; + jmethodID getDataU; + jmethodID getDataV; + jmethodID getStrideY; + jmethodID getStrideU; + jmethodID getStrideV; }; class JavaNativeI420BufferClass : public JavaClass diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h b/webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h new file mode 100644 index 000000000..c3d43f524 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h @@ -0,0 +1,34 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_DEFAULT_VIDEO_CODEC_FACTORIES_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_DEFAULT_VIDEO_CODEC_FACTORIES_H_ + +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" + +#include + +namespace jni +{ + // The video codecs built into this library: VP8, VP9, AV1 and H.264 in + // software, or on macOS those of WebRTC's default Objective-C factories, + // which use VideoToolbox. + std::unique_ptr CreateDefaultVideoEncoderFactory(); + std::unique_ptr CreateDefaultVideoDecoderFactory(); +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/EncodedImage.h b/webrtc-jni/src/main/cpp/include/media/video/codec/EncodedImage.h new file mode 100644 index 000000000..1299273d7 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/EncodedImage.h @@ -0,0 +1,61 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_ENCODED_IMAGE_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_ENCODED_IMAGE_H_ + +#include "JavaClass.h" +#include "JavaRef.h" + +#include "api/video/encoded_image.h" + +#include + +#include + +namespace jni +{ + namespace EncodedImage + { + // Wraps the image for a Java decoder. The payload is not copied: the + // Java buffer is read-only and points into the image, so it is valid + // only for as long as the image is. + JavaLocalRef toJava(JNIEnv * env, const webrtc::EncodedImage & image); + + // Converts the image a Java encoder produced, copying its payload. + webrtc::EncodedImage toNative(JNIEnv * env, const JavaRef & image); + + int64_t getCaptureTimeNs(JNIEnv * env, const JavaRef & image); + + class JavaEncodedImageClass : public JavaClass + { + public: + explicit JavaEncodedImageClass(JNIEnv * env); + + jclass cls; + jmethodID fromNative; + jmethodID getDirectPayload; + jmethodID getFrameTypeIndex; + jmethodID getQpOrUnknown; + jfieldID encodedWidth; + jfieldID encodedHeight; + jfieldID captureTimeNs; + jfieldID rotation; + }; + } +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/JavaLocalFrame.h b/webrtc-jni/src/main/cpp/include/media/video/codec/JavaLocalFrame.h new file mode 100644 index 000000000..23c46af6b --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/JavaLocalFrame.h @@ -0,0 +1,43 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_JAVA_LOCAL_FRAME_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_JAVA_LOCAL_FRAME_H_ + +#include + +namespace jni +{ + // Scopes the local references made on a thread WebRTC owns. Such a + // thread stays attached to the JVM for as long as it runs, so without a + // frame of their own the references made for one call would pile up + // behind those of the next. + class JavaLocalFrame + { + public: + JavaLocalFrame(JNIEnv * env, jint capacity); + ~JavaLocalFrame(); + + JavaLocalFrame(const JavaLocalFrame &) = delete; + JavaLocalFrame & operator=(const JavaLocalFrame &) = delete; + + private: + JNIEnv * env; + bool pushed; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoCodecInfo.h b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoCodecInfo.h new file mode 100644 index 000000000..dd69f44e6 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoCodecInfo.h @@ -0,0 +1,58 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_CODEC_INFO_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_CODEC_INFO_H_ + +#include "JavaClass.h" +#include "JavaRef.h" + +#include "api/video_codecs/sdp_video_format.h" + +#include + +#include + +namespace jni +{ + namespace VideoCodecInfo + { + JavaLocalRef toJava(JNIEnv * env, const webrtc::SdpVideoFormat & format); + webrtc::SdpVideoFormat toNative(JNIEnv * env, const JavaRef & info); + + JavaLocalRef toJavaArray(JNIEnv * env, const std::vector & formats); + + // Converts a java.util.List of VideoCodecInfos, skipping null + // elements. + std::vector toNativeList(JNIEnv * env, const JavaRef & list); + + class JavaVideoCodecInfoClass : public JavaClass + { + public: + explicit JavaVideoCodecInfoClass(JNIEnv * env); + + jclass cls; + jmethodID ctor; + jfieldID name; + jfieldID parameters; + jfieldID scalabilityModes; + + jmethodID listToArray; + }; + } +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoCodecUtils.h b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoCodecUtils.h new file mode 100644 index 000000000..d9a6f64dc --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoCodecUtils.h @@ -0,0 +1,48 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_CODEC_UTILS_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_CODEC_UTILS_H_ + +#include "JavaClass.h" + +#include + +#include + +namespace jni +{ + // Logs and clears a pending Java exception, which is how the codec + // wrappers deal with whatever a Java codec or factory throws: nothing + // may throw into WebRTC, and no JNI call may be made with an exception + // pending. Returns whether there was one. + bool ClearCodecException(JNIEnv * env, const char * context); + + // Converts a Java VideoCodecStatus to a WEBRTC_VIDEO_CODEC_* code. A + // pending exception, which it clears, and a null status are an error. + int32_t ToNativeCodecStatus(JNIEnv * env, jobject status, const char * context); + + class JavaVideoCodecStatusClass : public JavaClass + { + public: + explicit JavaVideoCodecStatusClass(JNIEnv * env); + + jclass cls; + jmethodID getNumber; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderFactoryWrapper.h b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderFactoryWrapper.h new file mode 100644 index 000000000..7ce079e0d --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderFactoryWrapper.h @@ -0,0 +1,73 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_DECODER_FACTORY_WRAPPER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_DECODER_FACTORY_WRAPPER_H_ + +#include "JavaClass.h" +#include "JavaRef.h" + +#include "api/environment/environment.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" + +#include + +#include +#include + +namespace jni +{ + // Lets WebRTC create its video decoders through a Java + // VideoDecoderFactory. A decoder the factory implements in Java is run + // by a VideoDecoderWrapper; a NativeVideoDecoder it returns becomes the + // built-in decoder for its codec. + class VideoDecoderFactoryWrapper : public webrtc::VideoDecoderFactory + { + public: + // Asks the Java factory for its supported codecs, which it may + // throw a Java exception for. + VideoDecoderFactoryWrapper(JNIEnv * env, jobject factory); + ~VideoDecoderFactoryWrapper() override = default; + + std::vector GetSupportedFormats() const override; + std::unique_ptr Create(const webrtc::Environment & env, + const webrtc::SdpVideoFormat & format) override; + + private: + class JavaVideoDecoderFactoryClass : public JavaClass + { + public: + explicit JavaVideoDecoderFactoryClass(JNIEnv * env); + + jmethodID getSupportedCodecs; + jmethodID createDecoder; + + jclass nativeDecoderClass; + jfieldID nativeDecoderCodecInfo; + }; + + private: + const JavaGlobalRef factory; + const std::shared_ptr javaClass; + const std::unique_ptr defaultFactory; + + std::vector supportedFormats; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderWrapper.h b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderWrapper.h new file mode 100644 index 000000000..8192decb6 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderWrapper.h @@ -0,0 +1,126 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_DECODER_WRAPPER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_DECODER_WRAPPER_H_ + +#include "JavaClass.h" +#include "JavaRef.h" + +#include "api/video/encoded_image.h" +#include "api/video_codecs/video_decoder.h" +#include "common_video/h264/h264_bitstream_parser.h" +#include "rtc_base/synchronization/mutex.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace jni +{ + // Runs a Java VideoDecoder inside WebRTC. + // + // WebRTC calls the decoder from one thread at a time, while the Java + // decoder may hand its frames over from any thread, through a callback + // that reaches this wrapper only until the decoder is released. + class VideoDecoderWrapper : public webrtc::VideoDecoder + { + public: + VideoDecoderWrapper(JNIEnv * env, jobject decoder); + ~VideoDecoderWrapper() override; + + using webrtc::VideoDecoder::Decode; + + bool Configure(const Settings & settings) override; + int32_t Decode(const webrtc::EncodedImage & image, int64_t renderTimeMs) override; + int32_t RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback * callback) override; + int32_t Release() override; + DecoderInfo GetDecoderInfo() const override; + const char * ImplementationName() const override; + + // Takes a frame from the Java decoder. Called through the Java + // callback, which holds its lock meanwhile. + void OnDecodedFrame(JNIEnv * env, jobject frame, std::optional decodeTimeMs, + std::optional qp); + + private: + struct FrameExtraInfo + { + // Identifies the frame. + int64_t timestampNs; + uint32_t rtpTimestamp; + int64_t ntpTimeMs; + std::optional qp; + }; + + bool ConfigureInternal(JNIEnv * env); + int32_t ReleaseInternal(JNIEnv * env); + int32_t HandleReturnCode(JNIEnv * env, int32_t status, const char * method); + void InvalidateCallback(JNIEnv * env); + + std::optional ParseQp(const webrtc::EncodedImage & image); + + private: + class JavaVideoDecoderClass : public JavaClass + { + public: + explicit JavaVideoDecoderClass(JNIEnv * env); + + jmethodID initDecode; + jmethodID release; + jmethodID decode; + jmethodID getImplementationName; + jmethodID isHardwareDecoder; + + jclass settingsClass; + jmethodID settingsCtor; + + jclass callbackClass; + jmethodID callbackCtor; + jmethodID callbackInvalidate; + }; + + private: + const JavaGlobalRef decoder; + const std::shared_ptr javaClass; + + std::string implementationName; + bool hardwareAccelerated; + + // The callback the Java decoder was initialized with. + std::unique_ptr> javaCallback; + + std::atomic callback; + + // Written on the decoder thread and read on the Java decoder's. + webrtc::Mutex frameExtraInfosLock; + std::deque frameExtraInfos; + + Settings settings; + bool initialized; + + // Parsing stops while the decoder provides the QP itself. + bool qpParsingEnabled; + webrtc::H264BitstreamParser h264BitstreamParser; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderFactoryWrapper.h b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderFactoryWrapper.h new file mode 100644 index 000000000..e1b9ef9a3 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderFactoryWrapper.h @@ -0,0 +1,73 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_ENCODER_FACTORY_WRAPPER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_ENCODER_FACTORY_WRAPPER_H_ + +#include "JavaClass.h" +#include "JavaRef.h" + +#include "api/environment/environment.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" + +#include + +#include +#include + +namespace jni +{ + // Lets WebRTC create its video encoders through a Java + // VideoEncoderFactory. An encoder the factory implements in Java is run + // by a VideoEncoderWrapper; a NativeVideoEncoder it returns becomes the + // built-in encoder for its codec. + class VideoEncoderFactoryWrapper : public webrtc::VideoEncoderFactory + { + public: + // Asks the Java factory for its supported codecs, which it may + // throw a Java exception for. + VideoEncoderFactoryWrapper(JNIEnv * env, jobject factory); + ~VideoEncoderFactoryWrapper() override = default; + + std::vector GetSupportedFormats() const override; + std::unique_ptr Create(const webrtc::Environment & env, + const webrtc::SdpVideoFormat & format) override; + + private: + class JavaVideoEncoderFactoryClass : public JavaClass + { + public: + explicit JavaVideoEncoderFactoryClass(JNIEnv * env); + + jmethodID getSupportedCodecs; + jmethodID createEncoder; + + jclass nativeEncoderClass; + jfieldID nativeEncoderCodecInfo; + }; + + private: + const JavaGlobalRef factory; + const std::shared_ptr javaClass; + const std::unique_ptr defaultFactory; + + std::vector supportedFormats; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderWrapper.h b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderWrapper.h new file mode 100644 index 000000000..57df254c3 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderWrapper.h @@ -0,0 +1,164 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_ENCODER_WRAPPER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_ENCODER_WRAPPER_H_ + +#include "JavaClass.h" +#include "JavaRef.h" + +#include "api/video/encoded_image.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" +#include "common_video/h264/h264_bitstream_parser.h" +#include "modules/video_coding/codecs/vp9/include/vp9_globals.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/svc/scalable_video_controller_no_layering.h" +#include "rtc_base/synchronization/mutex.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace jni +{ + // Runs a Java VideoEncoder inside WebRTC. + // + // WebRTC calls the encoder from one thread at a time, while the Java + // encoder may hand its frames over from any thread, through a callback + // that reaches this wrapper only until the encoder is released. + class VideoEncoderWrapper : public webrtc::VideoEncoder + { + public: + VideoEncoderWrapper(JNIEnv * env, jobject encoder, const webrtc::SdpVideoFormat & format); + ~VideoEncoderWrapper() override; + + int32_t InitEncode(const webrtc::VideoCodec * codecSettings, const Settings & settings) override; + int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback * callback) override; + int32_t Release() override; + int32_t Encode(const webrtc::VideoFrame & frame, const std::vector * frameTypes) override; + void SetRates(const RateControlParameters & parameters) override; + EncoderInfo GetEncoderInfo() const override; + + // Takes a frame from the Java encoder. Called through the Java + // callback, which holds its lock meanwhile. + void OnEncodedFrame(JNIEnv * env, jobject image); + + private: + struct FrameExtraInfo + { + // Identifies the frame. + int64_t captureTimeNs; + uint32_t rtpTimestamp; + }; + + int32_t InitEncodeInternal(JNIEnv * env); + int32_t ReleaseInternal(JNIEnv * env); + // Handles a status the Java encoder returned: tries to reset the + // encoder on errors WebRTC does not give up on. + int32_t HandleReturnCode(JNIEnv * env, int32_t status, const char * method); + void InvalidateCallback(JNIEnv * env); + void UpdateEncoderInfo(JNIEnv * env); + + ScalingSettings GetScalingSettings(JNIEnv * env) const; + std::vector GetResolutionBitrateLimits(JNIEnv * env) const; + + int ParseQp(const webrtc::EncodedImage & image); + webrtc::CodecSpecificInfo ParseCodecSpecificInfo(const webrtc::EncodedImage & image); + + private: + class JavaVideoEncoderClass : public JavaClass + { + public: + explicit JavaVideoEncoderClass(JNIEnv * env); + + jmethodID initEncode; + jmethodID release; + jmethodID encode; + jmethodID setRates; + jmethodID getScalingSettings; + jmethodID getResolutionBitrateLimits; + jmethodID getEncoderInfo; + jmethodID getImplementationName; + jmethodID isHardwareEncoder; + + jclass settingsClass; + jmethodID settingsCtor; + + jclass encodeInfoClass; + jmethodID encodeInfoFromNative; + + jclass bitrateAllocationClass; + jmethodID bitrateAllocationCtor; + + jclass rateControlClass; + jmethodID rateControlCtor; + + jfieldID scalingOn; + jfieldID scalingLow; + jfieldID scalingHigh; + + jfieldID limitsFrameSizePixels; + jfieldID limitsMinStartBitrateBps; + jfieldID limitsMinBitrateBps; + jfieldID limitsMaxBitrateBps; + + jfieldID infoRequestedResolutionAlignment; + jfieldID infoApplyAlignmentToAllSimulcastLayers; + + jclass callbackClass; + jmethodID callbackCtor; + jmethodID callbackInvalidate; + + jclass intArrayClass; + }; + + private: + const JavaGlobalRef encoder; + const std::shared_ptr javaClass; + const webrtc::SdpVideoFormat format; + + // The callback the Java encoder was initialized with. + std::unique_ptr> javaCallback; + + std::atomic callback; + + // Written on the encoder thread and read on the Java encoder's. + webrtc::Mutex frameExtraInfosLock; + std::deque frameExtraInfos; + + bool initialized; + std::optional capabilities; + int numberOfCores; + webrtc::VideoCodec codecSettings; + EncoderInfo encoderInfo; + + // Used on the thread the Java encoder hands its frames over on. + webrtc::H264BitstreamParser h264BitstreamParser; + webrtc::ScalableVideoControllerNoLayering svcController; + webrtc::GofInfoVP9 gof; + size_t gofIndex; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/src/JNI_DefaultVideoDecoderFactory.cpp b/webrtc-jni/src/main/cpp/src/JNI_DefaultVideoDecoderFactory.cpp new file mode 100644 index 000000000..3a0f113e4 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/JNI_DefaultVideoDecoderFactory.cpp @@ -0,0 +1,35 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "JNI_DefaultVideoDecoderFactory.h" +#include "media/video/codec/DefaultVideoCodecFactories.h" +#include "media/video/codec/VideoCodecInfo.h" +#include "JavaUtils.h" + +JNIEXPORT jobjectArray JNICALL Java_dev_onvoid_webrtc_media_video_codec_DefaultVideoDecoderFactory_getSupportedCodecsInternal +(JNIEnv * env, jclass caller) +{ + try { + auto factory = jni::CreateDefaultVideoDecoderFactory(); + + return jni::VideoCodecInfo::toJavaArray(env, factory->GetSupportedFormats()).release(); + } + catch (...) { + ThrowCxxJavaException(env); + } + + return nullptr; +} diff --git a/webrtc-jni/src/main/cpp/src/JNI_DefaultVideoEncoderFactory.cpp b/webrtc-jni/src/main/cpp/src/JNI_DefaultVideoEncoderFactory.cpp new file mode 100644 index 000000000..cf0159610 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/JNI_DefaultVideoEncoderFactory.cpp @@ -0,0 +1,35 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "JNI_DefaultVideoEncoderFactory.h" +#include "media/video/codec/DefaultVideoCodecFactories.h" +#include "media/video/codec/VideoCodecInfo.h" +#include "JavaUtils.h" + +JNIEXPORT jobjectArray JNICALL Java_dev_onvoid_webrtc_media_video_codec_DefaultVideoEncoderFactory_getSupportedCodecsInternal +(JNIEnv * env, jclass caller) +{ + try { + auto factory = jni::CreateDefaultVideoEncoderFactory(); + + return jni::VideoCodecInfo::toJavaArray(env, factory->GetSupportedFormats()).release(); + } + catch (...) { + ThrowCxxJavaException(env); + } + + return nullptr; +} diff --git a/webrtc-jni/src/main/cpp/src/JNI_NativeVideoDecoderCallback.cpp b/webrtc-jni/src/main/cpp/src/JNI_NativeVideoDecoderCallback.cpp new file mode 100644 index 000000000..0a0728cd2 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/JNI_NativeVideoDecoderCallback.cpp @@ -0,0 +1,46 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "JNI_NativeVideoDecoderCallback.h" +#include "media/video/codec/VideoDecoderWrapper.h" +#include "JavaUtils.h" + +#include + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_codec_NativeVideoDecoderCallback_onDecodedFrame +(JNIEnv * env, jclass caller, jlong handle, jobject frame, jint decodeTimeMs, jint qp) +{ + // The Java callback holds its lock and has checked the handle, so the + // wrapper cannot be released meanwhile. + auto decoder = reinterpret_cast(handle); + + std::optional decodeTime; + std::optional frameQp; + + if (decodeTimeMs >= 0) { + decodeTime = static_cast(decodeTimeMs); + } + if (qp >= 0 && qp <= 255) { + frameQp = static_cast(qp); + } + + try { + decoder->OnDecodedFrame(env, frame, decodeTime, frameQp); + } + catch (...) { + ThrowCxxJavaException(env); + } +} diff --git a/webrtc-jni/src/main/cpp/src/JNI_NativeVideoEncoderCallback.cpp b/webrtc-jni/src/main/cpp/src/JNI_NativeVideoEncoderCallback.cpp new file mode 100644 index 000000000..b611e0ee6 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/JNI_NativeVideoEncoderCallback.cpp @@ -0,0 +1,34 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "JNI_NativeVideoEncoderCallback.h" +#include "media/video/codec/VideoEncoderWrapper.h" +#include "JavaUtils.h" + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_codec_NativeVideoEncoderCallback_onEncodedFrame +(JNIEnv * env, jclass caller, jlong handle, jobject image) +{ + // The Java callback holds its lock and has checked the handle, so the + // wrapper cannot be released meanwhile. + auto encoder = reinterpret_cast(handle); + + try { + encoder->OnEncodedFrame(env, image); + } + catch (...) { + ThrowCxxJavaException(env); + } +} diff --git a/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp b/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp index 4e3064d8f..b1cc03048 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp @@ -37,31 +37,11 @@ #include "api/audio_codecs/builtin_audio_decoder_factory.h" #include "api/audio_codecs/builtin_audio_encoder_factory.h" -#ifdef __APPLE__ -#include "sdk/objc/components/video_codec/RTCDefaultVideoDecoderFactory.h" -#include "sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.h" -#include "sdk/objc/native/api/video_decoder_factory.h" -#include "sdk/objc/native/api/video_encoder_factory.h" -#else -#include "api/video_codecs/builtin_video_decoder_factory.h" -#include "api/video_codecs/builtin_video_encoder_factory.h" -#include "api/video_codecs/video_decoder_factory_template_dav1d_adapter.h" -#include "api/video_codecs/video_decoder_factory_template_libvpx_vp8_adapter.h" -#include "api/video_codecs/video_decoder_factory_template_libvpx_vp9_adapter.h" -#include "api/video_codecs/video_decoder_factory_template_open_h264_adapter.h" -#include "api/video_codecs/video_encoder_factory_template_libaom_av1_adapter.h" -#include "api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h" -#include "api/video_codecs/video_encoder_factory_template_libvpx_vp9_adapter.h" -#include "api/video_codecs/video_encoder_factory_template_open_h264_adapter.h" -#endif - -#include "api/video_codecs/video_decoder_factory.h" -#include "api/video_codecs/video_decoder_factory_template.h" -#include "api/video_codecs/video_encoder_factory.h" -#include "api/video_codecs/video_encoder_factory_template.h" - #include "api/ProxyAudioDeviceModule.h" #include "media/audio/CustomAudioSource.h" +#include "media/video/codec/DefaultVideoCodecFactories.h" +#include "media/video/codec/VideoDecoderFactoryWrapper.h" +#include "media/video/codec/VideoEncoderFactoryWrapper.h" #include "api/media_stream_interface.h" #include "rtc_base/logging.h" @@ -70,7 +50,8 @@ #include JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_initialize -(JNIEnv * env, jobject caller, jobject jFieldTrials, jobject audioModule, jobject audioProcessing) +(JNIEnv * env, jobject caller, jobject jFieldTrials, jobject audioModule, jobject audioProcessing, + jobject jVideoEncoderFactory, jobject jVideoDecoderFactory) { webrtc::AudioDeviceModule * audioDevModule = (audioModule != nullptr) ? GetHandle(env, audioModule) @@ -96,6 +77,15 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_initialize ? nullptr : std::make_unique(std::move(fieldTrialsMap)); + // Asks the Java factories for their codecs, which is where a broken + // factory surfaces, before anything else is set up. + std::unique_ptr videoEncoderFactory = (jVideoEncoderFactory != nullptr) + ? std::make_unique(env, jVideoEncoderFactory) + : jni::CreateDefaultVideoEncoderFactory(); + std::unique_ptr videoDecoderFactory = (jVideoDecoderFactory != nullptr) + ? std::make_unique(env, jVideoDecoderFactory) + : jni::CreateDefaultVideoDecoderFactory(); + auto networkThread = webrtc::Thread::CreateWithSocketServer(); networkThread->SetName("webrtc_jni_network_thread", nullptr); @@ -153,21 +143,8 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_initialize proxy, webrtc::CreateBuiltinAudioEncoderFactory(), webrtc::CreateBuiltinAudioDecoderFactory(), -#ifdef __APPLE__ - webrtc::ObjCToNativeVideoEncoderFactory([[RTC_OBJC_TYPE(RTCDefaultVideoEncoderFactory) alloc] init]), - webrtc::ObjCToNativeVideoDecoderFactory([[RTC_OBJC_TYPE(RTCDefaultVideoDecoderFactory) alloc] init]), -#else - std::make_unique>(), - std::make_unique>(), -#endif + std::move(videoEncoderFactory), + std::move(videoDecoderFactory), nullptr, apm, nullptr, diff --git a/webrtc-jni/src/main/cpp/src/api/VideoFrame.cpp b/webrtc-jni/src/main/cpp/src/api/VideoFrame.cpp index 26d9b673d..24f3ec502 100644 --- a/webrtc-jni/src/main/cpp/src/api/VideoFrame.cpp +++ b/webrtc-jni/src/main/cpp/src/api/VideoFrame.cpp @@ -71,6 +71,81 @@ namespace jni .set_rotation(static_cast(rotation)) .build(); } + + JavaLocalRef toJava(JNIEnv * env, const webrtc::VideoFrame & frame) + { + webrtc::scoped_refptr i420Buffer = frame.video_frame_buffer()->ToI420(); + + if (i420Buffer == nullptr) { + return JavaLocalRef(env, nullptr); + } + + const auto javaClass = JavaClasses::get(env); + + JavaLocalRef jBuffer = I420Buffer::toJava(env, i420Buffer); + + // The reference the Java buffer gives up when it is released. + i420Buffer->AddRef(); + + jobject jFrame = env->NewObject(javaClass->cls, javaClass->ctor, jBuffer.get(), + static_cast(frame.rotation()), + static_cast(frame.timestamp_us() * webrtc::kNumNanosecsPerMicrosec)); + + if (jFrame == nullptr) { + // The Java buffer never got to a frame that would release it. + i420Buffer->Release(); + ExceptionCheck(env); + } + + return JavaLocalRef(env, jFrame); + } + + webrtc::scoped_refptr toNativeBuffer(JNIEnv * env, const JavaRef & javaBuffer) + { + const auto nativeClass = JavaClasses::get(env); + + if (env->IsInstanceOf(javaBuffer, nativeClass->cls)) { + webrtc::I420BufferInterface * buffer = GetHandle(env, javaBuffer); + + return webrtc::scoped_refptr(buffer); + } + + const auto javaClass = JavaClasses::get(env); + + JavaLocalRef i420(env, env->CallObjectMethod(javaBuffer, javaClass->toI420)); + ExceptionCheck(env); + + if (i420.get() == nullptr) { + return nullptr; + } + + const jint width = env->CallIntMethod(i420, javaClass->getWidth); + const jint height = env->CallIntMethod(i420, javaClass->getHeight); + const jint strideY = env->CallIntMethod(i420, javaClass->getStrideY); + const jint strideU = env->CallIntMethod(i420, javaClass->getStrideU); + const jint strideV = env->CallIntMethod(i420, javaClass->getStrideV); + ExceptionCheck(env); + + JavaLocalRef dataY(env, env->CallObjectMethod(i420, javaClass->getDataY)); + JavaLocalRef dataU(env, env->CallObjectMethod(i420, javaClass->getDataU)); + JavaLocalRef dataV(env, env->CallObjectMethod(i420, javaClass->getDataV)); + ExceptionCheck(env); + + if (dataY.get() == nullptr || dataU.get() == nullptr || dataV.get() == nullptr) { + return nullptr; + } + + // Heap buffers have no address, and only direct ones can be read. + const uint8_t * y = static_cast(env->GetDirectBufferAddress(dataY)); + const uint8_t * u = static_cast(env->GetDirectBufferAddress(dataU)); + const uint8_t * v = static_cast(env->GetDirectBufferAddress(dataV)); + + if (y == nullptr || u == nullptr || v == nullptr || width <= 0 || height <= 0) { + return nullptr; + } + + return webrtc::I420Buffer::Copy(width, height, y, strideY, u, strideU, v, strideV); + } } namespace I420Buffer @@ -105,6 +180,25 @@ namespace jni buffer = GetFieldID(env, cls, "buffer", "L" PKG_VIDEO "VideoFrameBuffer;"); rotation = GetFieldID(env, cls, "rotation", "I"); timestampNs = GetFieldID(env, cls, "timestampNs", "J"); + release = GetMethod(env, cls, "release", "()V"); + } + + JavaI420BufferClass::JavaI420BufferClass(JNIEnv * env) + { + cls = FindClass(env, PKG_VIDEO"I420Buffer"); + + // Called on buffers of any kind, so it is looked up on their type. + jclass bufferClass = FindClass(env, PKG_VIDEO"VideoFrameBuffer"); + + toI420 = GetMethod(env, bufferClass, "toI420", "()L" PKG_VIDEO "I420Buffer;"); + getWidth = GetMethod(env, cls, "getWidth", "()I"); + getHeight = GetMethod(env, cls, "getHeight", "()I"); + getDataY = GetMethod(env, cls, "getDataY", "()" BYTE_BUFFER_SIG); + getDataU = GetMethod(env, cls, "getDataU", "()" BYTE_BUFFER_SIG); + getDataV = GetMethod(env, cls, "getDataV", "()" BYTE_BUFFER_SIG); + getStrideY = GetMethod(env, cls, "getStrideY", "()I"); + getStrideU = GetMethod(env, cls, "getStrideU", "()I"); + getStrideV = GetMethod(env, cls, "getStrideV", "()I"); } JavaNativeI420BufferClass::JavaNativeI420BufferClass(JNIEnv * env) diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp new file mode 100644 index 000000000..9fd32ac5a --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp @@ -0,0 +1,64 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/DefaultVideoCodecFactories.h" + +#ifdef __APPLE__ +#include "sdk/objc/components/video_codec/RTCDefaultVideoDecoderFactory.h" +#include "sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.h" +#include "sdk/objc/native/api/video_decoder_factory.h" +#include "sdk/objc/native/api/video_encoder_factory.h" +#else +#include "api/video_codecs/video_decoder_factory_template.h" +#include "api/video_codecs/video_decoder_factory_template_dav1d_adapter.h" +#include "api/video_codecs/video_decoder_factory_template_libvpx_vp8_adapter.h" +#include "api/video_codecs/video_decoder_factory_template_libvpx_vp9_adapter.h" +#include "api/video_codecs/video_decoder_factory_template_open_h264_adapter.h" +#include "api/video_codecs/video_encoder_factory_template.h" +#include "api/video_codecs/video_encoder_factory_template_libaom_av1_adapter.h" +#include "api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h" +#include "api/video_codecs/video_encoder_factory_template_libvpx_vp9_adapter.h" +#include "api/video_codecs/video_encoder_factory_template_open_h264_adapter.h" +#endif + +namespace jni +{ + std::unique_ptr CreateDefaultVideoEncoderFactory() + { +#ifdef __APPLE__ + return webrtc::ObjCToNativeVideoEncoderFactory([[RTC_OBJC_TYPE(RTCDefaultVideoEncoderFactory) alloc] init]); +#else + return std::make_unique>(); +#endif + } + + std::unique_ptr CreateDefaultVideoDecoderFactory() + { +#ifdef __APPLE__ + return webrtc::ObjCToNativeVideoDecoderFactory([[RTC_OBJC_TYPE(RTCDefaultVideoDecoderFactory) alloc] init]); +#else + return std::make_unique>(); +#endif + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/EncodedImage.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/EncodedImage.cpp new file mode 100644 index 000000000..ef0408e1e --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/EncodedImage.cpp @@ -0,0 +1,115 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/EncodedImage.h" +#include "Exception.h" +#include "JavaClasses.h" +#include "JavaObject.h" +#include "JavaUtils.h" +#include "JNI_WebRTC.h" + +#include "api/video/video_frame_type.h" +#include "api/video/video_rotation.h" +#include "rtc_base/time_utils.h" + +namespace jni +{ + namespace EncodedImage + { + JavaLocalRef toJava(JNIEnv * env, const webrtc::EncodedImage & image) + { + const auto javaClass = JavaClasses::get(env); + + // NewDirectByteBuffer does not accept an empty region at a null + // address, which an empty image may have. + static uint8_t empty = 0; + uint8_t * data = image.size() > 0 ? const_cast(image.data()) : ∅ + + JavaLocalRef buffer(env, env->NewDirectByteBuffer(data, static_cast(image.size()))); + + ExceptionCheck(env); + + jobject jImage = env->CallStaticObjectMethod(javaClass->cls, javaClass->fromNative, buffer.get(), + static_cast(image._encodedWidth), + static_cast(image._encodedHeight), + static_cast(image.capture_time_ms_ * webrtc::kNumNanosecsPerMillisec), + static_cast(image.frame_type()), + static_cast(image.rotation_), + static_cast(image.qp_)); + + ExceptionCheck(env); + + return JavaLocalRef(env, jImage); + } + + webrtc::EncodedImage toNative(JNIEnv * env, const JavaRef & image) + { + const auto javaClass = JavaClasses::get(env); + + JavaLocalRef payload(env, env->CallObjectMethod(image, javaClass->getDirectPayload)); + + ExceptionCheck(env); + + const uint8_t * data = static_cast(env->GetDirectBufferAddress(payload.get())); + const jlong size = env->GetDirectBufferCapacity(payload.get()); + + if (size < 0 || (data == nullptr && size > 0)) { + throw Exception("The EncodedImage buffer cannot be read"); + } + + jint frameType = env->CallIntMethod(image, javaClass->getFrameTypeIndex); + ExceptionCheck(env); + + jint qp = env->CallIntMethod(image, javaClass->getQpOrUnknown); + ExceptionCheck(env); + + JavaObject obj(env, image); + + webrtc::EncodedImage nativeImage; + nativeImage.SetEncodedData(webrtc::EncodedImageBuffer::Create(data, static_cast(size))); + nativeImage._encodedWidth = static_cast(obj.getInt(javaClass->encodedWidth)); + nativeImage._encodedHeight = static_cast(obj.getInt(javaClass->encodedHeight)); + nativeImage.rotation_ = static_cast(obj.getInt(javaClass->rotation)); + nativeImage.qp_ = qp; + nativeImage.set_frame_type(static_cast(frameType)); + + return nativeImage; + } + + int64_t getCaptureTimeNs(JNIEnv * env, const JavaRef & image) + { + const auto javaClass = JavaClasses::get(env); + + return env->GetLongField(image, javaClass->captureTimeNs); + } + + JavaEncodedImageClass::JavaEncodedImageClass(JNIEnv * env) + { + cls = FindClass(env, PKG_CODEC"EncodedImage"); + + fromNative = GetStaticMethod(env, cls, "fromNative", + "(" BYTE_BUFFER_SIG "IIJIII)L" PKG_CODEC "EncodedImage;"); + getDirectPayload = GetMethod(env, cls, "getDirectPayload", "()" BYTE_BUFFER_SIG); + getFrameTypeIndex = GetMethod(env, cls, "getFrameTypeIndex", "()I"); + getQpOrUnknown = GetMethod(env, cls, "getQpOrUnknown", "()I"); + + encodedWidth = GetFieldID(env, cls, "encodedWidth", "I"); + encodedHeight = GetFieldID(env, cls, "encodedHeight", "I"); + captureTimeNs = GetFieldID(env, cls, "captureTimeNs", "J"); + rotation = GetFieldID(env, cls, "rotation", "I"); + } + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/JavaLocalFrame.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/JavaLocalFrame.cpp new file mode 100644 index 000000000..f735831cd --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/JavaLocalFrame.cpp @@ -0,0 +1,37 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/JavaLocalFrame.h" + +namespace jni +{ + JavaLocalFrame::JavaLocalFrame(JNIEnv * env, jint capacity) : + env(env), + pushed(env->PushLocalFrame(capacity) == JNI_OK) + { + if (!pushed) { + // Out of memory: the references then go to the enclosing frame. + env->ExceptionClear(); + } + } + + JavaLocalFrame::~JavaLocalFrame() + { + if (pushed) { + env->PopLocalFrame(nullptr); + } + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoCodecInfo.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoCodecInfo.cpp new file mode 100644 index 000000000..b9c9ad607 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoCodecInfo.cpp @@ -0,0 +1,160 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/VideoCodecInfo.h" +#include "Exception.h" +#include "JavaClasses.h" +#include "JavaHashMap.h" +#include "JavaObject.h" +#include "JavaString.h" +#include "JavaUtils.h" +#include "JNI_WebRTC.h" + +#include "api/video_codecs/scalability_mode.h" +#include "modules/video_coding/svc/scalability_mode_util.h" + +#include + +namespace jni +{ + namespace VideoCodecInfo + { + JavaLocalRef toJava(JNIEnv * env, const webrtc::SdpVideoFormat & format) + { + const auto javaClass = JavaClasses::get(env); + + JavaHashMap parameters(env); + + for (const auto & [key, value] : format.parameters) { + parameters.put(JavaString::toJava(env, key), JavaString::toJava(env, value)); + } + + std::vector modes; + + for (webrtc::ScalabilityMode mode : format.scalability_modes) { + modes.emplace_back(webrtc::ScalabilityModeToString(mode)); + } + + JavaLocalRef name = JavaString::toJava(env, format.name); + JavaLocalRef parameterMap = parameters; + JavaLocalRef modeArray = JavaString::createArray(env, modes); + + jobject info = env->NewObject(javaClass->cls, javaClass->ctor, name.get(), + parameterMap.get(), modeArray.get()); + + ExceptionCheck(env); + + return JavaLocalRef(env, info); + } + + webrtc::SdpVideoFormat toNative(JNIEnv * env, const JavaRef & info) + { + const auto javaClass = JavaClasses::get(env); + + JavaObject obj(env, info); + + webrtc::SdpVideoFormat format(JavaString::toNative(env, obj.getString(javaClass->name))); + + // The field always holds a HashMap. + for (const auto & entry : JavaHashMap(env, obj.getObject(javaClass->parameters))) { + std::string key = JavaString::toNative(env, static_java_ref_cast(env, entry.first)); + std::string value = JavaString::toNative(env, static_java_ref_cast(env, entry.second)); + + format.parameters.emplace(std::move(key), std::move(value)); + } + + JavaLocalRef modes = obj.getObjectArray(javaClass->scalabilityModes); + jsize count = modes.get() != nullptr ? env->GetArrayLength(modes.get()) : 0; + + for (jsize i = 0; i < count; i++) { + JavaLocalRef mode(env, static_cast(env->GetObjectArrayElement(modes.get(), i))); + auto scalabilityMode = webrtc::ScalabilityModeFromString(JavaString::toNative(env, mode)); + + // A mode WebRTC does not know cannot be negotiated anyway. + if (scalabilityMode.has_value()) { + format.scalability_modes.push_back(*scalabilityMode); + } + } + + return format; + } + + JavaLocalRef toJavaArray(JNIEnv * env, const std::vector & formats) + { + const auto javaClass = JavaClasses::get(env); + + jobjectArray array = env->NewObjectArray(static_cast(formats.size()), javaClass->cls, nullptr); + + ExceptionCheck(env); + + for (size_t i = 0; i < formats.size(); i++) { + JavaLocalRef info = toJava(env, formats[i]); + + env->SetObjectArrayElement(array, static_cast(i), info.get()); + } + + return JavaLocalRef(env, array); + } + + std::vector toNativeList(JNIEnv * env, const JavaRef & list) + { + std::vector formats; + + if (list.get() == nullptr) { + return formats; + } + + const auto javaClass = JavaClasses::get(env); + + JavaLocalRef array(env, + static_cast(env->CallObjectMethod(list, javaClass->listToArray))); + + ExceptionCheck(env); + + jsize count = env->GetArrayLength(array.get()); + + for (jsize i = 0; i < count; i++) { + JavaLocalRef info(env, env->GetObjectArrayElement(array.get(), i)); + + if (info.get() == nullptr) { + continue; + } + if (!env->IsInstanceOf(info.get(), javaClass->cls)) { + throw Exception("The list of supported codecs holds an object that is not a VideoCodecInfo"); + } + + formats.push_back(toNative(env, info)); + } + + return formats; + } + + JavaVideoCodecInfoClass::JavaVideoCodecInfoClass(JNIEnv * env) + { + cls = FindClass(env, PKG_CODEC"VideoCodecInfo"); + + ctor = GetMethod(env, cls, "", "(" STRING_SIG MAP_SIG "[" STRING_SIG ")V"); + + name = GetFieldID(env, cls, "name", STRING_SIG); + parameters = GetFieldID(env, cls, "parameters", "Ljava/util/HashMap;"); + scalabilityModes = GetFieldID(env, cls, "scalabilityModes", "[" STRING_SIG); + + jclass listClass = FindClass(env, "java/util/List"); + + listToArray = GetMethod(env, listClass, "toArray", "()[Ljava/lang/Object;"); + } + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoCodecUtils.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoCodecUtils.cpp new file mode 100644 index 000000000..a24366109 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoCodecUtils.cpp @@ -0,0 +1,71 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/VideoCodecUtils.h" +#include "JavaClasses.h" +#include "JavaUtils.h" +#include "JNI_WebRTC.h" + +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" + +namespace jni +{ + bool ClearCodecException(JNIEnv * env, const char * context) + { + if (!env->ExceptionCheck()) { + return false; + } + + // Cleared before logging, since a Java log sink cannot be called + // with an exception pending. + env->ExceptionDescribe(); + env->ExceptionClear(); + + RTC_LOG(LS_WARNING) << "Java video codec threw in " << context; + + return true; + } + + int32_t ToNativeCodecStatus(JNIEnv * env, jobject status, const char * context) + { + if (ClearCodecException(env, context)) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + if (status == nullptr) { + RTC_LOG(LS_WARNING) << "Java video codec returned no status from " << context; + + return WEBRTC_VIDEO_CODEC_ERROR; + } + + const auto javaClass = JavaClasses::get(env); + + jint number = env->CallIntMethod(status, javaClass->getNumber); + + if (ClearCodecException(env, context)) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + return static_cast(number); + } + + JavaVideoCodecStatusClass::JavaVideoCodecStatusClass(JNIEnv * env) + { + cls = FindClass(env, PKG_CODEC"VideoCodecStatus"); + + getNumber = GetMethod(env, cls, "getNumber", "()I"); + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderFactoryWrapper.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderFactoryWrapper.cpp new file mode 100644 index 000000000..9347c227d --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderFactoryWrapper.cpp @@ -0,0 +1,97 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/VideoDecoderFactoryWrapper.h" +#include "media/video/codec/DefaultVideoCodecFactories.h" +#include "media/video/codec/JavaLocalFrame.h" +#include "media/video/codec/VideoCodecInfo.h" +#include "media/video/codec/VideoCodecUtils.h" +#include "media/video/codec/VideoDecoderWrapper.h" +#include "JavaClasses.h" +#include "JavaUtils.h" +#include "JNI_WebRTC.h" + +#include "rtc_base/logging.h" + +namespace jni +{ + VideoDecoderFactoryWrapper::VideoDecoderFactoryWrapper(JNIEnv * env, jobject factory) : + factory(env, factory), + javaClass(JavaClasses::get(env)), + defaultFactory(CreateDefaultVideoDecoderFactory()) + { + JavaLocalRef codecs(env, env->CallObjectMethod(factory, javaClass->getSupportedCodecs)); + + ExceptionCheck(env); + + supportedFormats = VideoCodecInfo::toNativeList(env, codecs); + } + + std::vector VideoDecoderFactoryWrapper::GetSupportedFormats() const + { + return supportedFormats; + } + + std::unique_ptr VideoDecoderFactoryWrapper::Create(const webrtc::Environment & environment, + const webrtc::SdpVideoFormat & format) + { + JNIEnv * env = AttachCurrentThread(); + + if (env == nullptr) { + return nullptr; + } + + JavaLocalFrame localFrame(env, 16); + + try { + JavaLocalRef info = VideoCodecInfo::toJava(env, format); + jobject decoder = env->CallObjectMethod(factory, javaClass->createDecoder, info.get()); + + if (ClearCodecException(env, "createDecoder") || decoder == nullptr) { + RTC_LOG(LS_WARNING) << "Java decoder factory created no decoder for " << format.ToString(); + + return nullptr; + } + + if (env->IsInstanceOf(decoder, javaClass->nativeDecoderClass)) { + JavaLocalRef nativeInfo(env, env->GetObjectField(decoder, javaClass->nativeDecoderCodecInfo)); + + return defaultFactory->Create(environment, VideoCodecInfo::toNative(env, nativeInfo)); + } + + return std::make_unique(env, decoder); + } + catch (...) { + ClearCodecException(env, "createDecoder"); + + RTC_LOG(LS_WARNING) << "Java decoder factory failed to create a decoder for " << format.ToString(); + } + + return nullptr; + } + + VideoDecoderFactoryWrapper::JavaVideoDecoderFactoryClass::JavaVideoDecoderFactoryClass(JNIEnv * env) + { + jclass cls = FindClass(env, PKG_CODEC"VideoDecoderFactory"); + + getSupportedCodecs = GetMethod(env, cls, "getSupportedCodecs", "()" LIST_SIG); + createDecoder = GetMethod(env, cls, "createDecoder", + "(L" PKG_CODEC "VideoCodecInfo;)L" PKG_CODEC "VideoDecoder;"); + + nativeDecoderClass = FindClass(env, PKG_CODEC"NativeVideoDecoder"); + nativeDecoderCodecInfo = GetFieldID(env, nativeDecoderClass, "codecInfo", "L" PKG_CODEC "VideoCodecInfo;"); + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderWrapper.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderWrapper.cpp new file mode 100644 index 000000000..08b352339 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderWrapper.cpp @@ -0,0 +1,401 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/VideoDecoderWrapper.h" +#include "media/video/codec/EncodedImage.h" +#include "media/video/codec/JavaLocalFrame.h" +#include "media/video/codec/VideoCodecUtils.h" +#include "api/VideoFrame.h" +#include "Exception.h" +#include "JavaClasses.h" +#include "JavaString.h" +#include "JavaUtils.h" +#include "JNI_WebRTC.h" + +#include "api/video/render_resolution.h" +#include "api/video/video_frame.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/utility/vp8_header_parser.h" +#include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" +#include "rtc_base/logging.h" +#include "rtc_base/time_utils.h" + +namespace jni +{ + namespace + { + // Enough for the objects of one call into the decoder. + constexpr jint kLocalFrameCapacity = 32; + + // How many frames the decoder may hold back before the oldest records + // are given up, so that one that never produces output does not make + // them grow without bound. Several seconds of video. + constexpr size_t kMaxPendingFrames = 300; + + // RTP video timestamps count at 90 kHz. + constexpr int64_t kRtpTicksPerMillisecond = 90; + } + + VideoDecoderWrapper::VideoDecoderWrapper(JNIEnv * env, jobject decoder) : + decoder(env, decoder), + javaClass(JavaClasses::get(env)), + hardwareAccelerated(false), + callback(nullptr), + initialized(false), + qpParsingEnabled(true) + { + JavaLocalFrame localFrame(env, kLocalFrameCapacity); + + jobject name = env->CallObjectMethod(decoder, javaClass->getImplementationName); + + if (!ClearCodecException(env, "getImplementationName") && name != nullptr) { + try { + implementationName = JavaString::toNative(env, + JavaLocalRef(env, static_cast(env->NewLocalRef(name)))); + } + catch (...) { + ClearCodecException(env, "getImplementationName"); + } + } + + jboolean hardware = env->CallBooleanMethod(decoder, javaClass->isHardwareDecoder); + + if (!ClearCodecException(env, "isHardwareDecoder")) { + hardwareAccelerated = hardware == JNI_TRUE; + } + } + + VideoDecoderWrapper::~VideoDecoderWrapper() + { + JNIEnv * env = AttachCurrentThread(); + + if (env != nullptr) { + InvalidateCallback(env); + } + } + + bool VideoDecoderWrapper::Configure(const Settings & decoderSettings) + { + JNIEnv * env = AttachCurrentThread(); + + if (env == nullptr) { + return false; + } + + settings = decoderSettings; + + return ConfigureInternal(env); + } + + bool VideoDecoderWrapper::ConfigureInternal(JNIEnv * env) + { + JavaLocalFrame localFrame(env, kLocalFrameCapacity); + + webrtc::RenderResolution resolution = settings.max_render_resolution(); + + jobject jSettings = env->NewObject(javaClass->settingsClass, javaClass->settingsCtor, + static_cast(settings.number_of_cores()), + static_cast(resolution.Valid() ? resolution.Width() : 0), + static_cast(resolution.Valid() ? resolution.Height() : 0)); + + if (ClearCodecException(env, "initDecode") || jSettings == nullptr) { + return false; + } + + // A callback of its own for each initialization, so that frames of an + // earlier one cannot reach this one. + InvalidateCallback(env); + + jobject jCallback = env->NewObject(javaClass->callbackClass, javaClass->callbackCtor, + reinterpret_cast(this)); + + if (ClearCodecException(env, "initDecode") || jCallback == nullptr) { + return false; + } + + javaCallback = std::make_unique>(env, jCallback); + + jobject status = env->CallObjectMethod(decoder, javaClass->initDecode, jSettings, jCallback); + int32_t result = ToNativeCodecStatus(env, status, "initDecode"); + + RTC_LOG(LS_INFO) << "Java decoder initDecode: " << result; + + initialized = result == WEBRTC_VIDEO_CODEC_OK; + + // A decoder that stopped providing the QP after a reset gets it + // parsed again. + qpParsingEnabled = true; + + return initialized; + } + + int32_t VideoDecoderWrapper::Decode(const webrtc::EncodedImage & inputImage, int64_t renderTimeMs) + { + if (!initialized) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + + JNIEnv * env = AttachCurrentThread(); + + if (env == nullptr) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + JavaLocalFrame localFrame(env, kLocalFrameCapacity); + + // The capture time identifies the frame, and received images carry + // none, so it is derived from the RTP timestamp. + webrtc::EncodedImage image(inputImage); + image.capture_time_ms_ = image.RtpTimestamp() / kRtpTicksPerMillisecond; + + { + webrtc::MutexLock lock(&frameExtraInfosLock); + + if (frameExtraInfos.size() >= kMaxPendingFrames) { + frameExtraInfos.pop_front(); + } + + frameExtraInfos.push_back(FrameExtraInfo { + image.capture_time_ms_ * webrtc::kNumNanosecsPerMillisec, + image.RtpTimestamp(), + image.ntp_time_ms_, + qpParsingEnabled ? ParseQp(image) : std::nullopt + }); + } + + jobject jImage = nullptr; + + try { + jImage = EncodedImage::toJava(env, image).release(); + } + catch (...) { + ClearCodecException(env, "decode"); + } + + if (jImage == nullptr) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + jobject status = env->CallObjectMethod(decoder, javaClass->decode, jImage); + int32_t result = ToNativeCodecStatus(env, status, "decode"); + + return HandleReturnCode(env, result, "decode"); + } + + int32_t VideoDecoderWrapper::RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback * decodeCallback) + { + callback = decodeCallback; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VideoDecoderWrapper::Release() + { + JNIEnv * env = AttachCurrentThread(); + + if (env == nullptr) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + return ReleaseInternal(env); + } + + int32_t VideoDecoderWrapper::ReleaseInternal(JNIEnv * env) + { + JavaLocalFrame localFrame(env, kLocalFrameCapacity); + + jobject status = env->CallObjectMethod(decoder, javaClass->release); + int32_t result = ToNativeCodecStatus(env, status, "release"); + + RTC_LOG(LS_INFO) << "Java decoder release: " << result; + + InvalidateCallback(env); + + { + webrtc::MutexLock lock(&frameExtraInfosLock); + frameExtraInfos.clear(); + } + + initialized = false; + + return result; + } + + webrtc::VideoDecoder::DecoderInfo VideoDecoderWrapper::GetDecoderInfo() const + { + DecoderInfo info; + info.implementation_name = implementationName; + info.is_hardware_accelerated = hardwareAccelerated; + + return info; + } + + const char * VideoDecoderWrapper::ImplementationName() const + { + return implementationName.c_str(); + } + + void VideoDecoderWrapper::OnDecodedFrame(JNIEnv * env, jobject jFrame, std::optional decodeTimeMs, + std::optional qp) + { + const auto frameClass = JavaClasses::get(env); + + const int64_t timestampNs = env->GetLongField(jFrame, frameClass->timestampNs); + const jint rotation = env->GetIntField(jFrame, frameClass->rotation); + + FrameExtraInfo extraInfo; + + { + webrtc::MutexLock lock(&frameExtraInfosLock); + + // A decoder may drop frames, so the records of those it dropped + // come before the one of this frame. + do { + if (frameExtraInfos.empty()) { + RTC_LOG(LS_WARNING) << "Java decoder produced a frame for no image it was given, timestamp " + << timestampNs << " ns"; + return; + } + + extraInfo = frameExtraInfos.front(); + frameExtraInfos.pop_front(); + } + while (extraInfo.timestampNs != timestampNs); + } + + JavaLocalRef buffer(env, env->GetObjectField(jFrame, frameClass->buffer)); + webrtc::scoped_refptr nativeBuffer = VideoFrame::toNativeBuffer(env, buffer); + + if (nativeBuffer == nullptr) { + throw Exception("The decoded frame has no pixels that can be read: its buffer has to provide " + "direct byte buffers"); + } + + webrtc::VideoFrame frame = webrtc::VideoFrame::Builder() + .set_video_frame_buffer(nativeBuffer) + .set_rtp_timestamp(extraInfo.rtpTimestamp) + .set_timestamp_ms(timestampNs / webrtc::kNumNanosecsPerMillisec) + .set_ntp_time_ms(extraInfo.ntpTimeMs) + .set_rotation(static_cast(rotation)) + .build(); + + // A decoder that provides the QP itself saves parsing the bitstream. + qpParsingEnabled = !qp.has_value(); + + webrtc::DecodedImageCallback * decodeCallback = callback.load(); + + if (decodeCallback != nullptr) { + decodeCallback->Decoded(frame, decodeTimeMs, qp.has_value() ? qp : extraInfo.qp); + } + } + + int32_t VideoDecoderWrapper::HandleReturnCode(JNIEnv * env, int32_t status, const char * method) + { + if (status >= 0) { + // OK or NO_OUTPUT. + return status; + } + + RTC_LOG(LS_WARNING) << "Java decoder " << method << ": " << status; + + if (status == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE || status == WEBRTC_VIDEO_CODEC_UNINITIALIZED) { + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + // Any other error may be one the decoder recovers from. + if (ReleaseInternal(env) == WEBRTC_VIDEO_CODEC_OK && ConfigureInternal(env)) { + RTC_LOG(LS_WARNING) << "Java decoder reset"; + + return WEBRTC_VIDEO_CODEC_ERROR; + } + + RTC_LOG(LS_WARNING) << "Java decoder could not be reset"; + + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + void VideoDecoderWrapper::InvalidateCallback(JNIEnv * env) + { + if (javaCallback == nullptr) { + return; + } + + // Waits for a frame that is being handed over right now. + env->CallVoidMethod(javaCallback->get(), javaClass->callbackInvalidate); + ClearCodecException(env, "invalidate"); + + javaCallback.reset(); + } + + std::optional VideoDecoderWrapper::ParseQp(const webrtc::EncodedImage & image) + { + if (image.qp_ != -1) { + return static_cast(image.qp_); + } + + int qp; + + switch (settings.codec_type()) { + case webrtc::kVideoCodecVP8: + if (webrtc::vp8::GetQp(image.data(), image.size(), &qp)) { + return static_cast(qp); + } + break; + + case webrtc::kVideoCodecVP9: + if (webrtc::vp9::GetQp(image.data(), image.size(), &qp)) { + return static_cast(qp); + } + break; + + case webrtc::kVideoCodecH264: { + h264BitstreamParser.ParseBitstream(image); + + std::optional sliceQp = h264BitstreamParser.GetLastSliceQp(); + + if (sliceQp.has_value()) { + return static_cast(*sliceQp); + } + break; + } + + default: + break; + } + + return std::nullopt; + } + + VideoDecoderWrapper::JavaVideoDecoderClass::JavaVideoDecoderClass(JNIEnv * env) + { + jclass cls = FindClass(env, PKG_CODEC"VideoDecoder"); + + initDecode = GetMethod(env, cls, "initDecode", + "(L" PKG_CODEC "VideoDecoder$Settings;L" PKG_CODEC "VideoDecoder$Callback;)L" PKG_CODEC "VideoCodecStatus;"); + release = GetMethod(env, cls, "release", "()L" PKG_CODEC "VideoCodecStatus;"); + decode = GetMethod(env, cls, "decode", + "(L" PKG_CODEC "EncodedImage;)L" PKG_CODEC "VideoCodecStatus;"); + getImplementationName = GetMethod(env, cls, "getImplementationName", "()" STRING_SIG); + isHardwareDecoder = GetMethod(env, cls, "isHardwareDecoder", "()Z"); + + settingsClass = FindClass(env, PKG_CODEC"VideoDecoder$Settings"); + settingsCtor = GetMethod(env, settingsClass, "", "(III)V"); + + callbackClass = FindClass(env, PKG_CODEC"NativeVideoDecoderCallback"); + callbackCtor = GetMethod(env, callbackClass, "", "(J)V"); + callbackInvalidate = GetMethod(env, callbackClass, "invalidate", "()V"); + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderFactoryWrapper.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderFactoryWrapper.cpp new file mode 100644 index 000000000..3c6a536de --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderFactoryWrapper.cpp @@ -0,0 +1,97 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/VideoEncoderFactoryWrapper.h" +#include "media/video/codec/DefaultVideoCodecFactories.h" +#include "media/video/codec/JavaLocalFrame.h" +#include "media/video/codec/VideoCodecInfo.h" +#include "media/video/codec/VideoCodecUtils.h" +#include "media/video/codec/VideoEncoderWrapper.h" +#include "JavaClasses.h" +#include "JavaUtils.h" +#include "JNI_WebRTC.h" + +#include "rtc_base/logging.h" + +namespace jni +{ + VideoEncoderFactoryWrapper::VideoEncoderFactoryWrapper(JNIEnv * env, jobject factory) : + factory(env, factory), + javaClass(JavaClasses::get(env)), + defaultFactory(CreateDefaultVideoEncoderFactory()) + { + JavaLocalRef codecs(env, env->CallObjectMethod(factory, javaClass->getSupportedCodecs)); + + ExceptionCheck(env); + + supportedFormats = VideoCodecInfo::toNativeList(env, codecs); + } + + std::vector VideoEncoderFactoryWrapper::GetSupportedFormats() const + { + return supportedFormats; + } + + std::unique_ptr VideoEncoderFactoryWrapper::Create(const webrtc::Environment & environment, + const webrtc::SdpVideoFormat & format) + { + JNIEnv * env = AttachCurrentThread(); + + if (env == nullptr) { + return nullptr; + } + + JavaLocalFrame localFrame(env, 16); + + try { + JavaLocalRef info = VideoCodecInfo::toJava(env, format); + jobject encoder = env->CallObjectMethod(factory, javaClass->createEncoder, info.get()); + + if (ClearCodecException(env, "createEncoder") || encoder == nullptr) { + RTC_LOG(LS_WARNING) << "Java encoder factory created no encoder for " << format.ToString(); + + return nullptr; + } + + if (env->IsInstanceOf(encoder, javaClass->nativeEncoderClass)) { + JavaLocalRef nativeInfo(env, env->GetObjectField(encoder, javaClass->nativeEncoderCodecInfo)); + + return defaultFactory->Create(environment, VideoCodecInfo::toNative(env, nativeInfo)); + } + + return std::make_unique(env, encoder, format); + } + catch (...) { + ClearCodecException(env, "createEncoder"); + + RTC_LOG(LS_WARNING) << "Java encoder factory failed to create an encoder for " << format.ToString(); + } + + return nullptr; + } + + VideoEncoderFactoryWrapper::JavaVideoEncoderFactoryClass::JavaVideoEncoderFactoryClass(JNIEnv * env) + { + jclass cls = FindClass(env, PKG_CODEC"VideoEncoderFactory"); + + getSupportedCodecs = GetMethod(env, cls, "getSupportedCodecs", "()" LIST_SIG); + createEncoder = GetMethod(env, cls, "createEncoder", + "(L" PKG_CODEC "VideoCodecInfo;)L" PKG_CODEC "VideoEncoder;"); + + nativeEncoderClass = FindClass(env, PKG_CODEC"NativeVideoEncoder"); + nativeEncoderCodecInfo = GetFieldID(env, nativeEncoderClass, "codecInfo", "L" PKG_CODEC "VideoCodecInfo;"); + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderWrapper.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderWrapper.cpp new file mode 100644 index 000000000..ff500532f --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderWrapper.cpp @@ -0,0 +1,704 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/VideoEncoderWrapper.h" +#include "media/video/codec/EncodedImage.h" +#include "media/video/codec/JavaLocalFrame.h" +#include "media/video/codec/VideoCodecUtils.h" +#include "api/VideoFrame.h" +#include "JavaClasses.h" +#include "JavaPrimitive.h" +#include "JavaString.h" +#include "JavaUtils.h" +#include "JNI_WebRTC.h" + +#include "api/video/render_resolution.h" +#include "api/video/video_codec_constants.h" +#include "modules/video_coding/codecs/interface/common_constants.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/utility/vp8_header_parser.h" +#include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" +#include "rtc_base/logging.h" +#include "rtc_base/time_utils.h" + +#include + +namespace jni +{ + namespace + { + // Enough for the objects of one call into the encoder. + constexpr jint kLocalFrameCapacity = 32; + + // How many frames the encoder may hold back before the oldest records + // are given up, so that one that never produces output does not make + // them grow without bound. Several seconds of video. + constexpr size_t kMaxPendingFrames = 300; + + // The QP thresholds WebRTC's own encoders use for quality scaling. + constexpr int kLowVp8QpThreshold = 29; + constexpr int kHighVp8QpThreshold = 95; + // The QP is parsed from the VP9 bitstream, whose range is 0 to 255. + constexpr int kLowVp9QpThreshold = 96; + constexpr int kHighVp9QpThreshold = 185; + constexpr int kLowH264QpThreshold = 24; + constexpr int kHighH264QpThreshold = 37; + + std::optional OptionalInt(JNIEnv * env, jobject value) + { + if (value == nullptr) { + return std::nullopt; + } + + return Integer::getValue(env, value); + } + } + + VideoEncoderWrapper::VideoEncoderWrapper(JNIEnv * env, jobject encoder, const webrtc::SdpVideoFormat & format) : + encoder(env, encoder), + javaClass(JavaClasses::get(env)), + format(format), + callback(nullptr), + initialized(false), + numberOfCores(1), + codecSettings(), + gofIndex(0) + { + UpdateEncoderInfo(env); + } + + VideoEncoderWrapper::~VideoEncoderWrapper() + { + JNIEnv * env = AttachCurrentThread(); + + if (env != nullptr) { + InvalidateCallback(env); + } + } + + int32_t VideoEncoderWrapper::InitEncode(const webrtc::VideoCodec * settings, const Settings & encoderSettings) + { + JNIEnv * env = AttachCurrentThread(); + + if (env == nullptr || settings == nullptr) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + codecSettings = *settings; + capabilities = encoderSettings.capabilities; + numberOfCores = encoderSettings.number_of_cores; + + return InitEncodeInternal(env); + } + + int32_t VideoEncoderWrapper::InitEncodeInternal(JNIEnv * env) + { + JavaLocalFrame localFrame(env, kLocalFrameCapacity); + + bool automaticResizeOn; + + switch (codecSettings.codecType) { + case webrtc::kVideoCodecVP8: + automaticResizeOn = codecSettings.VP8()->automaticResizeOn; + break; + + case webrtc::kVideoCodecVP9: + automaticResizeOn = codecSettings.VP9()->automaticResizeOn; + gof.SetGofInfoVP9(webrtc::TemporalStructureMode::kTemporalStructureMode1); + gofIndex = 0; + break; + + default: + automaticResizeOn = true; + break; + } + + const bool lossNotification = capabilities.has_value() && capabilities->loss_notification; + const bool screenContent = codecSettings.mode == webrtc::VideoCodecMode::kScreensharing; + + jobject settings = env->NewObject(javaClass->settingsClass, javaClass->settingsCtor, + static_cast(numberOfCores), + static_cast(codecSettings.width), + static_cast(codecSettings.height), + static_cast(codecSettings.startBitrate), + static_cast(codecSettings.minBitrate), + static_cast(codecSettings.maxBitrate), + static_cast(codecSettings.maxFramerate), + static_cast(codecSettings.numberOfSimulcastStreams), + automaticResizeOn ? JNI_TRUE : JNI_FALSE, + screenContent ? JNI_TRUE : JNI_FALSE, + lossNotification ? JNI_TRUE : JNI_FALSE); + + if (ClearCodecException(env, "initEncode") || settings == nullptr) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // A callback of its own for each initialization, so that frames of an + // earlier one cannot reach this one. + InvalidateCallback(env); + + jobject jCallback = env->NewObject(javaClass->callbackClass, javaClass->callbackCtor, + reinterpret_cast(this)); + + if (ClearCodecException(env, "initEncode") || jCallback == nullptr) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + javaCallback = std::make_unique>(env, jCallback); + + jobject status = env->CallObjectMethod(encoder, javaClass->initEncode, settings, jCallback); + int32_t result = ToNativeCodecStatus(env, status, "initEncode"); + + RTC_LOG(LS_INFO) << "Java encoder initEncode: " << result; + + // Some properties of the encoder may depend on its settings. + UpdateEncoderInfo(env); + + initialized = result == WEBRTC_VIDEO_CODEC_OK; + + return result; + } + + int32_t VideoEncoderWrapper::RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback * encodeCallback) + { + callback = encodeCallback; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VideoEncoderWrapper::Release() + { + JNIEnv * env = AttachCurrentThread(); + + if (env == nullptr) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + return ReleaseInternal(env); + } + + int32_t VideoEncoderWrapper::ReleaseInternal(JNIEnv * env) + { + JavaLocalFrame localFrame(env, kLocalFrameCapacity); + + jobject status = env->CallObjectMethod(encoder, javaClass->release); + int32_t result = ToNativeCodecStatus(env, status, "release"); + + RTC_LOG(LS_INFO) << "Java encoder release: " << result; + + InvalidateCallback(env); + + { + webrtc::MutexLock lock(&frameExtraInfosLock); + frameExtraInfos.clear(); + } + + initialized = false; + + return result; + } + + int32_t VideoEncoderWrapper::Encode(const webrtc::VideoFrame & frame, + const std::vector * frameTypes) + { + if (!initialized) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + + JNIEnv * env = AttachCurrentThread(); + + if (env == nullptr) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + JavaLocalFrame localFrame(env, kLocalFrameCapacity); + + const auto javaFrameClass = JavaClasses::get(env); + jobject jFrame = nullptr; + + try { + jFrame = VideoFrame::toJava(env, frame).release(); + } + catch (...) { + ClearCodecException(env, "encode"); + } + + if (jFrame == nullptr) { + RTC_LOG(LS_WARNING) << "Java encoder: dropping a frame that cannot be converted to I420"; + + return WEBRTC_VIDEO_CODEC_OK; + } + + std::vector types; + + if (frameTypes != nullptr) { + for (webrtc::VideoFrameType type : *frameTypes) { + types.push_back(static_cast(type)); + } + } + if (types.empty()) { + types.push_back(static_cast(webrtc::VideoFrameType::kVideoFrameDelta)); + } + + jobject info = nullptr; + jintArray jTypes = env->NewIntArray(static_cast(types.size())); + + if (jTypes != nullptr) { + env->SetIntArrayRegion(jTypes, 0, static_cast(types.size()), types.data()); + + info = env->CallStaticObjectMethod(javaClass->encodeInfoClass, javaClass->encodeInfoFromNative, jTypes); + } + + int32_t result; + + if (ClearCodecException(env, "encode") || info == nullptr) { + result = WEBRTC_VIDEO_CODEC_ERROR; + } + else { + { + webrtc::MutexLock lock(&frameExtraInfosLock); + + if (frameExtraInfos.size() >= kMaxPendingFrames) { + frameExtraInfos.pop_front(); + } + + frameExtraInfos.push_back(FrameExtraInfo { + frame.timestamp_us() * webrtc::kNumNanosecsPerMicrosec, + frame.rtp_timestamp() + }); + } + + jobject status = env->CallObjectMethod(encoder, javaClass->encode, jFrame, info); + result = ToNativeCodecStatus(env, status, "encode"); + } + + // An encoder that needs the frame any longer has retained it. + env->CallVoidMethod(jFrame, javaFrameClass->release); + ClearCodecException(env, "encode"); + + return HandleReturnCode(env, result, "encode"); + } + + void VideoEncoderWrapper::SetRates(const RateControlParameters & parameters) + { + JNIEnv * env = AttachCurrentThread(); + + if (env == nullptr) { + return; + } + + JavaLocalFrame localFrame(env, kLocalFrameCapacity); + + jobjectArray layers = env->NewObjectArray(webrtc::kMaxSpatialLayers, javaClass->intArrayClass, nullptr); + + if (ClearCodecException(env, "setRates") || layers == nullptr) { + return; + } + + for (size_t spatial = 0; spatial < webrtc::kMaxSpatialLayers; spatial++) { + jint bitrates[webrtc::kMaxTemporalStreams]; + + for (size_t temporal = 0; temporal < webrtc::kMaxTemporalStreams; temporal++) { + bitrates[temporal] = static_cast(parameters.bitrate.GetBitrate(spatial, temporal)); + } + + jintArray layer = env->NewIntArray(webrtc::kMaxTemporalStreams); + + if (layer == nullptr) { + ClearCodecException(env, "setRates"); + return; + } + + env->SetIntArrayRegion(layer, 0, webrtc::kMaxTemporalStreams, bitrates); + env->SetObjectArrayElement(layers, static_cast(spatial), layer); + env->DeleteLocalRef(layer); + } + + jobject allocation = env->NewObject(javaClass->bitrateAllocationClass, javaClass->bitrateAllocationCtor, layers); + jobject jParameters = nullptr; + + if (allocation != nullptr) { + jParameters = env->NewObject(javaClass->rateControlClass, javaClass->rateControlCtor, allocation, + static_cast(parameters.framerate_fps)); + } + + if (ClearCodecException(env, "setRates") || jParameters == nullptr) { + return; + } + + jobject status = env->CallObjectMethod(encoder, javaClass->setRates, jParameters); + int32_t result = ToNativeCodecStatus(env, status, "setRates"); + + if (result < 0) { + RTC_LOG(LS_WARNING) << "Java encoder setRates: " << result; + } + } + + webrtc::VideoEncoder::EncoderInfo VideoEncoderWrapper::GetEncoderInfo() const + { + return encoderInfo; + } + + void VideoEncoderWrapper::OnEncodedFrame(JNIEnv * env, jobject jImage) + { + JavaLocalRef imageRef(env, env->NewLocalRef(jImage)); + + webrtc::EncodedImage image = EncodedImage::toNative(env, imageRef); + const int64_t captureTimeNs = EncodedImage::getCaptureTimeNs(env, imageRef); + + FrameExtraInfo extraInfo; + + { + webrtc::MutexLock lock(&frameExtraInfosLock); + + // Frames come out in the order they went in, though an encoder + // may drop some, so the records of older frames are stale. + while (!frameExtraInfos.empty() && frameExtraInfos.front().captureTimeNs < captureTimeNs) { + frameExtraInfos.pop_front(); + } + + if (frameExtraInfos.empty() || frameExtraInfos.front().captureTimeNs != captureTimeNs) { + RTC_LOG(LS_WARNING) << "Java encoder produced a frame for no frame it was given, capture time " + << captureTimeNs << " ns"; + return; + } + + extraInfo = frameExtraInfos.front(); + frameExtraInfos.pop_front(); + } + + image.SetRtpTimestamp(extraInfo.rtpTimestamp); + image.capture_time_ms_ = captureTimeNs / webrtc::kNumNanosecsPerMillisec; + + if (image.qp_ < 0) { + image.qp_ = ParseQp(image); + } + + webrtc::CodecSpecificInfo info = ParseCodecSpecificInfo(image); + webrtc::EncodedImageCallback * encodeCallback = callback.load(); + + if (encodeCallback != nullptr) { + encodeCallback->OnEncodedImage(image, &info); + } + } + + int32_t VideoEncoderWrapper::HandleReturnCode(JNIEnv * env, int32_t status, const char * method) + { + if (status >= 0) { + // OK or NO_OUTPUT, or OK with a request. + return status; + } + + RTC_LOG(LS_WARNING) << "Java encoder " << method << ": " << status; + + if (status == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE || status == WEBRTC_VIDEO_CODEC_UNINITIALIZED) { + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + // Any other error may be one the encoder recovers from. + if (ReleaseInternal(env) == WEBRTC_VIDEO_CODEC_OK && InitEncodeInternal(env) == WEBRTC_VIDEO_CODEC_OK) { + RTC_LOG(LS_WARNING) << "Java encoder reset"; + + return WEBRTC_VIDEO_CODEC_ERROR; + } + + RTC_LOG(LS_WARNING) << "Java encoder could not be reset"; + + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + void VideoEncoderWrapper::InvalidateCallback(JNIEnv * env) + { + if (javaCallback == nullptr) { + return; + } + + // Waits for a frame that is being handed over right now. + env->CallVoidMethod(javaCallback->get(), javaClass->callbackInvalidate); + ClearCodecException(env, "invalidate"); + + javaCallback.reset(); + } + + void VideoEncoderWrapper::UpdateEncoderInfo(JNIEnv * env) + { + JavaLocalFrame localFrame(env, kLocalFrameCapacity); + + encoderInfo.supports_native_handle = false; + encoderInfo.scaling_settings = GetScalingSettings(env); + encoderInfo.resolution_bitrate_limits = GetResolutionBitrateLimits(env); + + jobject name = env->CallObjectMethod(encoder, javaClass->getImplementationName); + + if (!ClearCodecException(env, "getImplementationName") && name != nullptr) { + try { + encoderInfo.implementation_name = JavaString::toNative(env, + JavaLocalRef(env, static_cast(env->NewLocalRef(name)))); + } + catch (...) { + ClearCodecException(env, "getImplementationName"); + } + } + + jboolean hardware = env->CallBooleanMethod(encoder, javaClass->isHardwareEncoder); + + if (!ClearCodecException(env, "isHardwareEncoder")) { + encoderInfo.is_hardware_accelerated = hardware == JNI_TRUE; + } + + jobject info = env->CallObjectMethod(encoder, javaClass->getEncoderInfo); + + if (!ClearCodecException(env, "getEncoderInfo") && info != nullptr) { + encoderInfo.requested_resolution_alignment = std::max(1, + static_cast(env->GetIntField(info, javaClass->infoRequestedResolutionAlignment))); + encoderInfo.apply_alignment_to_all_simulcast_layers = + env->GetBooleanField(info, javaClass->infoApplyAlignmentToAllSimulcastLayers) == JNI_TRUE; + } + } + + webrtc::VideoEncoder::ScalingSettings VideoEncoderWrapper::GetScalingSettings(JNIEnv * env) const + { + jobject settings = env->CallObjectMethod(encoder, javaClass->getScalingSettings); + + if (ClearCodecException(env, "getScalingSettings") || settings == nullptr) { + return ScalingSettings::kOff; + } + if (env->GetBooleanField(settings, javaClass->scalingOn) != JNI_TRUE) { + return ScalingSettings::kOff; + } + + std::optional low; + std::optional high; + + try { + low = OptionalInt(env, env->GetObjectField(settings, javaClass->scalingLow)); + high = OptionalInt(env, env->GetObjectField(settings, javaClass->scalingHigh)); + } + catch (...) { + ClearCodecException(env, "getScalingSettings"); + } + + if (low && high) { + return ScalingSettings(*low, *high); + } + + switch (codecSettings.codecType) { + case webrtc::kVideoCodecVP8: + return ScalingSettings(low.value_or(kLowVp8QpThreshold), high.value_or(kHighVp8QpThreshold)); + + case webrtc::kVideoCodecVP9: + return ScalingSettings(low.value_or(kLowVp9QpThreshold), high.value_or(kHighVp9QpThreshold)); + + case webrtc::kVideoCodecH264: + return ScalingSettings(low.value_or(kLowH264QpThreshold), high.value_or(kHighH264QpThreshold)); + + default: + // Without a QP to go by, quality scaling cannot work. + return ScalingSettings::kOff; + } + } + + std::vector VideoEncoderWrapper::GetResolutionBitrateLimits( + JNIEnv * env) const + { + std::vector limits; + + jobjectArray array = static_cast( + env->CallObjectMethod(encoder, javaClass->getResolutionBitrateLimits)); + + if (ClearCodecException(env, "getResolutionBitrateLimits") || array == nullptr) { + return limits; + } + + const jsize count = env->GetArrayLength(array); + + for (jsize i = 0; i < count; i++) { + jobject entry = env->GetObjectArrayElement(array, i); + + if (entry == nullptr) { + continue; + } + + limits.emplace_back( + env->GetIntField(entry, javaClass->limitsFrameSizePixels), + env->GetIntField(entry, javaClass->limitsMinStartBitrateBps), + env->GetIntField(entry, javaClass->limitsMinBitrateBps), + env->GetIntField(entry, javaClass->limitsMaxBitrateBps)); + + env->DeleteLocalRef(entry); + } + + return limits; + } + + int VideoEncoderWrapper::ParseQp(const webrtc::EncodedImage & image) + { + int qp = -1; + bool success = false; + + switch (codecSettings.codecType) { + case webrtc::kVideoCodecVP8: + success = webrtc::vp8::GetQp(image.data(), image.size(), &qp); + break; + + case webrtc::kVideoCodecVP9: + success = webrtc::vp9::GetQp(image.data(), image.size(), &qp); + break; + + case webrtc::kVideoCodecH264: + h264BitstreamParser.ParseBitstream(image); + qp = h264BitstreamParser.GetLastSliceQp().value_or(-1); + success = qp >= 0; + break; + + default: + break; + } + + return success ? qp : -1; + } + + webrtc::CodecSpecificInfo VideoEncoderWrapper::ParseCodecSpecificInfo(const webrtc::EncodedImage & image) + { + const bool keyFrame = image.frame_type() == webrtc::VideoFrameType::kVideoFrameKey; + + webrtc::CodecSpecificInfo info; + info.codecType = codecSettings.codecType; + + // The encoder produces a single layer, so the frame dependencies are + // those of a stream without layers. + auto layerFrames = svcController.NextFrameConfig(/*restart=*/keyFrame); + info.generic_frame_info = svcController.OnEncodeDone(layerFrames[0]); + + if (keyFrame) { + info.template_structure = svcController.DependencyStructure(); + info.template_structure->resolutions = { + webrtc::RenderResolution(image._encodedWidth, image._encodedHeight) + }; + } + + switch (codecSettings.codecType) { + case webrtc::kVideoCodecVP8: + info.codecSpecific.VP8.nonReference = false; + info.codecSpecific.VP8.temporalIdx = webrtc::kNoTemporalIdx; + info.codecSpecific.VP8.layerSync = false; + info.codecSpecific.VP8.keyIdx = webrtc::kNoKeyIdx; + break; + + case webrtc::kVideoCodecVP9: + if (keyFrame) { + gofIndex = 0; + } + + info.codecSpecific.VP9.inter_pic_predicted = !keyFrame; + info.codecSpecific.VP9.flexible_mode = false; + info.codecSpecific.VP9.ss_data_available = keyFrame; + info.codecSpecific.VP9.temporal_idx = webrtc::kNoTemporalIdx; + info.codecSpecific.VP9.temporal_up_switch = true; + info.codecSpecific.VP9.inter_layer_predicted = false; + info.codecSpecific.VP9.gof_idx = static_cast(gofIndex++ % gof.num_frames_in_gof); + info.codecSpecific.VP9.num_spatial_layers = 1; + info.codecSpecific.VP9.first_frame_in_picture = true; + info.codecSpecific.VP9.spatial_layer_resolution_present = keyFrame; + + if (keyFrame) { + info.codecSpecific.VP9.width[0] = image._encodedWidth; + info.codecSpecific.VP9.height[0] = image._encodedHeight; + info.codecSpecific.VP9.gof.CopyGofInfoVP9(gof); + } + break; + + case webrtc::kVideoCodecH264: { + // Mode 1 may split a NAL unit across packets; mode 0 has to + // send each NAL unit in a packet of its own. + auto mode = format.parameters.find("packetization-mode"); + const bool nonInterleaved = mode != format.parameters.end() && mode->second == "1"; + + info.codecSpecific.H264.packetization_mode = nonInterleaved + ? webrtc::H264PacketizationMode::NonInterleaved + : webrtc::H264PacketizationMode::SingleNalUnit; + info.codecSpecific.H264.temporal_idx = webrtc::kNoTemporalIdx; + info.codecSpecific.H264.base_layer_sync = false; + info.codecSpecific.H264.idr_frame = keyFrame; + break; + } + + default: + break; + } + + return info; + } + + VideoEncoderWrapper::JavaVideoEncoderClass::JavaVideoEncoderClass(JNIEnv * env) + { + jclass cls = FindClass(env, PKG_CODEC"VideoEncoder"); + + initEncode = GetMethod(env, cls, "initEncode", + "(L" PKG_CODEC "VideoEncoder$Settings;L" PKG_CODEC "VideoEncoder$Callback;)L" PKG_CODEC "VideoCodecStatus;"); + release = GetMethod(env, cls, "release", "()L" PKG_CODEC "VideoCodecStatus;"); + encode = GetMethod(env, cls, "encode", + "(L" PKG_VIDEO "VideoFrame;L" PKG_CODEC "VideoEncoder$EncodeInfo;)L" PKG_CODEC "VideoCodecStatus;"); + setRates = GetMethod(env, cls, "setRates", + "(L" PKG_CODEC "VideoEncoder$RateControlParameters;)L" PKG_CODEC "VideoCodecStatus;"); + getScalingSettings = GetMethod(env, cls, "getScalingSettings", "()L" PKG_CODEC "VideoEncoder$ScalingSettings;"); + getResolutionBitrateLimits = GetMethod(env, cls, "getResolutionBitrateLimits", + "()[L" PKG_CODEC "VideoEncoder$ResolutionBitrateLimits;"); + getEncoderInfo = GetMethod(env, cls, "getEncoderInfo", "()L" PKG_CODEC "VideoEncoder$EncoderInfo;"); + getImplementationName = GetMethod(env, cls, "getImplementationName", "()" STRING_SIG); + isHardwareEncoder = GetMethod(env, cls, "isHardwareEncoder", "()Z"); + + settingsClass = FindClass(env, PKG_CODEC"VideoEncoder$Settings"); + settingsCtor = GetMethod(env, settingsClass, "", "(IIIIIIIIZZZ)V"); + + encodeInfoClass = FindClass(env, PKG_CODEC"VideoEncoder$EncodeInfo"); + encodeInfoFromNative = GetStaticMethod(env, encodeInfoClass, "fromNative", + "([I)L" PKG_CODEC "VideoEncoder$EncodeInfo;"); + + bitrateAllocationClass = FindClass(env, PKG_CODEC"VideoEncoder$BitrateAllocation"); + bitrateAllocationCtor = GetMethod(env, bitrateAllocationClass, "", "([[I)V"); + + rateControlClass = FindClass(env, PKG_CODEC"VideoEncoder$RateControlParameters"); + rateControlCtor = GetMethod(env, rateControlClass, "", + "(L" PKG_CODEC "VideoEncoder$BitrateAllocation;D)V"); + + jclass scalingClass = FindClass(env, PKG_CODEC"VideoEncoder$ScalingSettings"); + scalingOn = GetFieldID(env, scalingClass, "on", "Z"); + scalingLow = GetFieldID(env, scalingClass, "low", INTEGER_SIG); + scalingHigh = GetFieldID(env, scalingClass, "high", INTEGER_SIG); + + jclass limitsClass = FindClass(env, PKG_CODEC"VideoEncoder$ResolutionBitrateLimits"); + limitsFrameSizePixels = GetFieldID(env, limitsClass, "frameSizePixels", "I"); + limitsMinStartBitrateBps = GetFieldID(env, limitsClass, "minStartBitrateBps", "I"); + limitsMinBitrateBps = GetFieldID(env, limitsClass, "minBitrateBps", "I"); + limitsMaxBitrateBps = GetFieldID(env, limitsClass, "maxBitrateBps", "I"); + + jclass infoClass = FindClass(env, PKG_CODEC"VideoEncoder$EncoderInfo"); + infoRequestedResolutionAlignment = GetFieldID(env, infoClass, "requestedResolutionAlignment", "I"); + infoApplyAlignmentToAllSimulcastLayers = GetFieldID(env, infoClass, "applyAlignmentToAllSimulcastLayers", "Z"); + + callbackClass = FindClass(env, PKG_CODEC"NativeVideoEncoderCallback"); + callbackCtor = GetMethod(env, callbackClass, "", "(J)V"); + callbackInvalidate = GetMethod(env, callbackClass, "invalidate", "()V"); + + // The class loader FindClass goes through does not resolve array + // descriptors, and the JNI lookup of a primitive array class needs + // no class loader. + jclass intArray = env->FindClass("[I"); + intArrayClass = static_cast(env->NewGlobalRef(intArray)); + env->DeleteLocalRef(intArray); + } +} diff --git a/webrtc-jni/src/main/cpp/src/rtc/LogSink.cpp b/webrtc-jni/src/main/cpp/src/rtc/LogSink.cpp index 16a5c0881..355dca32a 100644 --- a/webrtc-jni/src/main/cpp/src/rtc/LogSink.cpp +++ b/webrtc-jni/src/main/cpp/src/rtc/LogSink.cpp @@ -39,12 +39,39 @@ namespace jni return; } - JavaLocalRef jSeverity = JavaEnums::toJava(env, severity); - JavaLocalRef jMessage = JavaString::toJava(env, message); + // WebRTC holds its global logging lock while it calls a sink, and it + // is built without unwinding, so nothing may throw from here: an + // exception passing through WebRTC's frames would leave that lock + // held and every thread that logs next would block forever. + // + // The code that logged may have a Java exception pending, which no + // JNI call may be made with. It is set aside and restored after. + jthrowable pending = env->ExceptionOccurred(); - env->CallVoidMethod(javaSink, javaClass->onLogMessage, jSeverity.get(), jMessage.get()); + if (pending != nullptr) { + env->ExceptionClear(); + } + + try { + JavaLocalRef jSeverity = JavaEnums::toJava(env, severity); + JavaLocalRef jMessage = JavaString::toJava(env, message); - ExceptionCheck(env); + env->CallVoidMethod(javaSink, javaClass->onLogMessage, jSeverity.get(), jMessage.get()); + + // What the sink threw is reported, but not rethrown. + if (env->ExceptionCheck()) { + env->ExceptionDescribe(); + env->ExceptionClear(); + } + } + catch (...) { + env->ExceptionClear(); + } + + if (pending != nullptr) { + env->Throw(pending); + env->DeleteLocalRef(pending); + } } LogSink::JavaLogSinkClass::JavaLogSinkClass(JNIEnv * env) diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java index 4938e0efd..b18994374 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java @@ -28,6 +28,10 @@ import dev.onvoid.webrtc.media.audio.CustomAudioSource; import dev.onvoid.webrtc.media.video.VideoTrackSource; import dev.onvoid.webrtc.media.video.VideoTrack; +import dev.onvoid.webrtc.media.video.codec.DefaultVideoDecoderFactory; +import dev.onvoid.webrtc.media.video.codec.DefaultVideoEncoderFactory; +import dev.onvoid.webrtc.media.video.codec.VideoDecoderFactory; +import dev.onvoid.webrtc.media.video.codec.VideoEncoderFactory; import java.util.Map; import java.util.Objects; @@ -52,6 +56,9 @@ * {@link #createAudioSource(AudioOptions)} is called, since that is the only * way to send device-captured audio. A factory that sends pushed audio, or no * audio at all, never opens it. Playout of received audio is never affected. + *

+ * The constructors cover the common combinations of modules. A {@link + * #builder() builder} sets any of them, including the video codecs. * * @author Alex Andres */ @@ -103,7 +110,7 @@ private enum AudioInputMode { * Creates an instance of PeerConnectionFactory. */ public PeerConnectionFactory() { - initialize(null, null, null); + initialize(null, null, null, null, null); } /** @@ -113,7 +120,7 @@ public PeerConnectionFactory() { * @param audioProcessing The custom audio processing module. */ public PeerConnectionFactory(AudioProcessing audioProcessing) { - initialize(null, null, audioProcessing); + initialize(null, null, audioProcessing, null, null); } /** @@ -123,7 +130,7 @@ public PeerConnectionFactory(AudioProcessing audioProcessing) { * @param audioModule The custom audio device module. */ public PeerConnectionFactory(AudioDeviceModuleBase audioModule) { - initialize(null, audioModule, null); + initialize(null, audioModule, null, null, null); } /** @@ -135,7 +142,7 @@ public PeerConnectionFactory(AudioDeviceModuleBase audioModule) { */ public PeerConnectionFactory(AudioDeviceModuleBase audioModule, AudioProcessing audioProcessing) { - initialize(null, audioModule, audioProcessing); + initialize(null, audioModule, audioProcessing, null, null); } /** @@ -147,7 +154,7 @@ public PeerConnectionFactory(AudioDeviceModuleBase audioModule, * non-null and non-empty. */ public PeerConnectionFactory(Map fieldTrials) { - initialize(fieldTrials, null, null); + initialize(fieldTrials, null, null, null, null); } /** @@ -161,7 +168,7 @@ public PeerConnectionFactory(Map fieldTrials) { */ public PeerConnectionFactory(Map fieldTrials, AudioProcessing audioProcessing) { - initialize(fieldTrials, null, audioProcessing); + initialize(fieldTrials, null, audioProcessing, null, null); } /** @@ -175,7 +182,7 @@ public PeerConnectionFactory(Map fieldTrials, */ public PeerConnectionFactory(Map fieldTrials, AudioDeviceModuleBase audioModule) { - initialize(fieldTrials, audioModule, null); + initialize(fieldTrials, audioModule, null, null, null); } /** @@ -190,7 +197,37 @@ public PeerConnectionFactory(Map fieldTrials, */ public PeerConnectionFactory(Map fieldTrials, AudioDeviceModuleBase audioModule, AudioProcessing audioProcessing) { - initialize(fieldTrials, audioModule, audioProcessing); + initialize(fieldTrials, audioModule, audioProcessing, null, null); + } + + /** + * Creates an instance of PeerConnectionFactory with what the builder was + * given. + * + * @param builder The builder with the modules to use. + */ + private PeerConnectionFactory(Builder builder) { + // The built-in codecs need no detour through Java. + VideoEncoderFactory encoderFactory = builder.videoEncoderFactory instanceof DefaultVideoEncoderFactory + ? null + : builder.videoEncoderFactory; + VideoDecoderFactory decoderFactory = builder.videoDecoderFactory instanceof DefaultVideoDecoderFactory + ? null + : builder.videoDecoderFactory; + + initialize(builder.fieldTrials, builder.audioModule, builder.audioProcessing, + encoderFactory, decoderFactory); + } + + /** + * Returns a builder for a PeerConnectionFactory, which sets any of the + * modules the factory is created with. Whatever is not set is what the + * {@link #PeerConnectionFactory() default constructor} uses. + * + * @return A new builder. + */ + public static Builder builder() { + return new Builder(); } /** @@ -364,7 +401,9 @@ public RTCPeerConnection createPeerConnection(RTCConfiguration config, public native void dispose(); private native void initialize(Map fieldTrials, - AudioDeviceModuleBase audioModule, AudioProcessing audioProcessing); + AudioDeviceModuleBase audioModule, AudioProcessing audioProcessing, + VideoEncoderFactory videoEncoderFactory, + VideoDecoderFactory videoDecoderFactory); private native AudioTrackSource createAudioSourceInternal(AudioOptions options); @@ -394,4 +433,122 @@ private native RTCPeerConnection createPeerConnectionInternal( */ private native void setDeviceCaptureEnabled(boolean enabled); + + /** + * Builds a {@link PeerConnectionFactory}. Each module that is not set is + * what the {@link PeerConnectionFactory#PeerConnectionFactory() default + * constructor} uses. + *

+ * Example, a factory with a codec of one's own next to the built-in ones: + * 

{@code
+	 * PeerConnectionFactory factory = PeerConnectionFactory.builder()
+	 *     .setAudioDeviceModule(audioModule)
+	 *     .setVideoEncoderFactory(new MyEncoderFactory())
+	 *     .setVideoDecoderFactory(new MyDecoderFactory())
+	 *     .build();
+	 * }
+ */ + public static final class Builder { + + private Map fieldTrials; + + private AudioDeviceModuleBase audioModule; + + private AudioProcessing audioProcessing; + + private VideoEncoderFactory videoEncoderFactory; + + private VideoDecoderFactory videoDecoderFactory; + + + private Builder() { + } + + /** + * Sets the field trials, which allow enabling experimental WebRTC + * features. + * + * @param fieldTrials The field trials to set, e.g. {@code + * {"WebRTC-Bar": "Enabled"}}. Keys and values must + * be non-null and non-empty. + * + * @return This builder. + */ + public Builder setFieldTrials(Map fieldTrials) { + this.fieldTrials = fieldTrials; + return this; + } + + /** + * Sets the audio device module. + * + * @param audioModule The custom audio device module. + * + * @return This builder. + */ + public Builder setAudioDeviceModule(AudioDeviceModuleBase audioModule) { + this.audioModule = audioModule; + return this; + } + + /** + * Sets the audio processing module. + * + * @param audioProcessing The custom audio processing module. + * + * @return This builder. + */ + public Builder setAudioProcessing(AudioProcessing audioProcessing) { + this.audioProcessing = audioProcessing; + return this; + } + + /** + * Sets the factory of the video encoders, which decides what video + * codecs the factory can send. Without one, the factory uses the + * codecs of a {@link DefaultVideoEncoderFactory}. + *

+ * The supported codecs are asked for once, when the factory is built; + * the encoders are created from WebRTC threads while the factory is + * in use. + * + * @param factory The video encoder factory. + * + * @return This builder. + */ + public Builder setVideoEncoderFactory(VideoEncoderFactory factory) { + this.videoEncoderFactory = factory; + return this; + } + + /** + * Sets the factory of the video decoders, which decides what video + * codecs the factory can receive. Without one, the factory uses the + * codecs of a {@link DefaultVideoDecoderFactory}. + *

+ * The supported codecs are asked for once, when the factory is built; + * the decoders are created from WebRTC threads while the factory is + * in use. + * + * @param factory The video decoder factory. + * + * @return This builder. + */ + public Builder setVideoDecoderFactory(VideoDecoderFactory factory) { + this.videoDecoderFactory = factory; + return this; + } + + /** + * Creates the PeerConnectionFactory. The builder may be used again + * afterwards. + * + * @return The new factory. + */ + public PeerConnectionFactory build() { + return new PeerConnectionFactory(this); + } + + } + } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/DefaultVideoDecoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/DefaultVideoDecoderFactory.java new file mode 100644 index 000000000..95ef3cc08 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/DefaultVideoDecoderFactory.java @@ -0,0 +1,92 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import dev.onvoid.webrtc.internal.NativeLoader; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * The video decoders built into this library, which a {@link + * dev.onvoid.webrtc.PeerConnectionFactory PeerConnectionFactory} uses unless + * it is given a {@link VideoDecoderFactory} of its own. On Windows and Linux + * these are VP8, VP9, AV1 and H.264 in software; on macOS they are those of + * WebRTC's default factories, with H.264 through VideoToolbox. + * {@link #getSupportedCodecs()} lists what the platform has. + *

+ * A factory of one's own can hand out these decoders next to its own ones + * by delegating to an instance of this class. The decoders it creates are + * {@link NativeVideoDecoder}s, which run inside WebRTC and are not to be + * called from Java. + * + * @author Alex Andres + */ +public final class DefaultVideoDecoderFactory implements VideoDecoderFactory { + + static { + try { + NativeLoader.loadLibrary("webrtc-java"); + } + catch (Exception e) { + throw new RuntimeException("Load library 'webrtc-java' failed", e); + } + } + + + private final List supportedCodecs; + + + /** + * Creates a factory for the built-in video decoders. + */ + public DefaultVideoDecoderFactory() { + supportedCodecs = Collections.unmodifiableList( + Arrays.asList(getSupportedCodecsInternal())); + } + + @Override + public List getSupportedCodecs() { + return supportedCodecs; + } + + /** + * Returns a built-in decoder for the given codec. + * + * @param info The codec to decode, as negotiated. + * + * @return A native decoder, or {@code null} if there is no built-in + * decoder for the codec. + */ + @Override + public NativeVideoDecoder createDecoder(VideoCodecInfo info) { + Objects.requireNonNull(info, "VideoCodecInfo is null"); + + for (VideoCodecInfo codec : supportedCodecs) { + if (codec.getName().equalsIgnoreCase(info.getName())) { + return new NativeVideoDecoder(info); + } + } + + return null; + } + + private static native VideoCodecInfo[] getSupportedCodecsInternal(); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/DefaultVideoEncoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/DefaultVideoEncoderFactory.java new file mode 100644 index 000000000..3bae9f797 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/DefaultVideoEncoderFactory.java @@ -0,0 +1,92 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import dev.onvoid.webrtc.internal.NativeLoader; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * The video encoders built into this library, which a {@link + * dev.onvoid.webrtc.PeerConnectionFactory PeerConnectionFactory} uses unless + * it is given a {@link VideoEncoderFactory} of its own. On Windows and Linux + * these are VP8, VP9, AV1 and H.264 in software; on macOS they are those of + * WebRTC's default factories, with H.264 through VideoToolbox. + * {@link #getSupportedCodecs()} lists what the platform has. + *

+ * A factory of one's own can hand out these encoders next to its own ones + * by delegating to an instance of this class. The encoders it creates are + * {@link NativeVideoEncoder}s, which run inside WebRTC and are not to be + * called from Java. + * + * @author Alex Andres + */ +public final class DefaultVideoEncoderFactory implements VideoEncoderFactory { + + static { + try { + NativeLoader.loadLibrary("webrtc-java"); + } + catch (Exception e) { + throw new RuntimeException("Load library 'webrtc-java' failed", e); + } + } + + + private final List supportedCodecs; + + + /** + * Creates a factory for the built-in video encoders. + */ + public DefaultVideoEncoderFactory() { + supportedCodecs = Collections.unmodifiableList( + Arrays.asList(getSupportedCodecsInternal())); + } + + @Override + public List getSupportedCodecs() { + return supportedCodecs; + } + + /** + * Returns a built-in encoder for the given codec. + * + * @param info The codec to encode, as negotiated. + * + * @return A native encoder, or {@code null} if there is no built-in + * encoder for the codec. + */ + @Override + public NativeVideoEncoder createEncoder(VideoCodecInfo info) { + Objects.requireNonNull(info, "VideoCodecInfo is null"); + + for (VideoCodecInfo codec : supportedCodecs) { + if (codec.getName().equalsIgnoreCase(info.getName())) { + return new NativeVideoEncoder(info); + } + } + + return null; + } + + private static native VideoCodecInfo[] getSupportedCodecsInternal(); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/EncodedImage.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/EncodedImage.java new file mode 100644 index 000000000..6a9f9a97d --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/EncodedImage.java @@ -0,0 +1,351 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import java.nio.ByteBuffer; +import java.util.Objects; + +/** + * An encoded video frame: what a {@link VideoEncoder} produces and a {@link + * VideoDecoder} consumes. + *

+ * An encoder hands its images to WebRTC through {@link + * VideoEncoder.Callback#onEncodedFrame(EncodedImage)}, which copies the + * payload, so the buffer may be reused as soon as that call returns. The + * {@link #getCaptureTimeNs() capture time} of such an image must be the + * {@link dev.onvoid.webrtc.media.video.VideoFrame#timestampNs timestamp} of + * the frame it encodes: that is how WebRTC matches the output to the input. + *

+ * The images WebRTC hands to a decoder have a read-only buffer that is valid + * only until {@link VideoDecoder#decode(EncodedImage)} returns; a decoder that + * needs the data longer has to copy it. Their capture time identifies the + * frame, and the decoder has to give its decoded frame the same timestamp. + * + * @author Alex Andres + */ +public final class EncodedImage { + + /** + * The type of an encoded frame. + */ + public enum FrameType { + + /** A frame without data, for example one the encoder dropped. */ + EMPTY(0), + + /** A key frame, which decodes without any other frame. */ + KEY(3), + + /** A delta frame, which depends on earlier frames. */ + DELTA(4); + + + private final int nativeIndex; + + + FrameType(int nativeIndex) { + this.nativeIndex = nativeIndex; + } + + int getNativeIndex() { + return nativeIndex; + } + + static FrameType fromNativeIndex(int nativeIndex) { + for (FrameType type : values()) { + if (type.nativeIndex == nativeIndex) { + return type; + } + } + + throw new IllegalArgumentException("Unknown frame type: " + nativeIndex); + } + + } + + + private final ByteBuffer buffer; + + private final int encodedWidth; + + private final int encodedHeight; + + private final long captureTimeNs; + + private final FrameType frameType; + + private final int rotation; + + private final Integer qp; + + + private EncodedImage(ByteBuffer buffer, int encodedWidth, int encodedHeight, + long captureTimeNs, FrameType frameType, int rotation, Integer qp) { + this.buffer = buffer; + this.encodedWidth = encodedWidth; + this.encodedHeight = encodedHeight; + this.captureTimeNs = captureTimeNs; + this.frameType = frameType; + this.rotation = rotation; + this.qp = qp; + } + + /** + * Returns the encoded payload, from its position to its limit. + * + * @return The encoded data. + */ + public ByteBuffer getBuffer() { + return buffer; + } + + /** + * @return The width of the encoded frame in pixels. + */ + public int getEncodedWidth() { + return encodedWidth; + } + + /** + * @return The height of the encoded frame in pixels. + */ + public int getEncodedHeight() { + return encodedHeight; + } + + /** + * @return The capture time that identifies the frame, in nanoseconds. + */ + public long getCaptureTimeNs() { + return captureTimeNs; + } + + /** + * @return Whether this is a key frame, a delta frame or an empty one. + */ + public FrameType getFrameType() { + return frameType; + } + + /** + * @return The rotation of the frame in degrees, a multiple of 90. + */ + public int getRotation() { + return rotation; + } + + /** + * @return The quantizer the frame was encoded with, or {@code null} if it + * is not known. + */ + public Integer getQp() { + return qp; + } + + @Override + public String toString() { + return String.format("%s@%d [size=%d, width=%d, height=%d, captureTimeNs=%d, frameType=%s, rotation=%d, qp=%s]", + EncodedImage.class.getSimpleName(), hashCode(), + buffer.remaining(), encodedWidth, encodedHeight, + captureTimeNs, frameType, rotation, qp); + } + + /** + * @return A new builder for an encoded image. + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Wraps an image WebRTC hands to a decoder. Called from native code. + */ + @SuppressWarnings("unused") + private static EncodedImage fromNative(ByteBuffer buffer, int encodedWidth, + int encodedHeight, long captureTimeNs, int frameType, int rotation, + int qp) { + return new EncodedImage(buffer.asReadOnlyBuffer(), encodedWidth, + encodedHeight, captureTimeNs, FrameType.fromNativeIndex(frameType), + rotation, qp < 0 ? null : qp); + } + + /** + * Returns the payload as a direct buffer whose capacity is its size, which + * native code can address. A heap buffer is copied. Called from native + * code. + */ + @SuppressWarnings("unused") + private ByteBuffer getDirectPayload() { + if (buffer.isDirect()) { + return buffer.slice(); + } + + ByteBuffer direct = ByteBuffer.allocateDirect(buffer.remaining()); + direct.put(buffer.duplicate()); + direct.flip(); + + return direct; + } + + /** + * Returns the native index of the frame type. Called from native code. + */ + @SuppressWarnings("unused") + private int getFrameTypeIndex() { + return frameType.getNativeIndex(); + } + + /** + * Returns the quantizer, or -1 if it is not known. Called from native code. + */ + @SuppressWarnings("unused") + private int getQpOrUnknown() { + return qp == null ? -1 : qp; + } + + + + /** + * Builds an {@link EncodedImage}, which is what a {@link VideoEncoder} + * does for each frame it encodes. + */ + public static final class Builder { + + private ByteBuffer buffer; + + private int encodedWidth; + + private int encodedHeight; + + private long captureTimeNs; + + private FrameType frameType = FrameType.DELTA; + + private int rotation; + + private Integer qp; + + + private Builder() { + } + + /** + * Sets the encoded payload, which is read from its position to its + * limit. Direct and heap buffers are both accepted. + * + * @param buffer The encoded data. + * + * @return This builder. + */ + public Builder setBuffer(ByteBuffer buffer) { + this.buffer = buffer; + return this; + } + + /** + * @param width The width of the encoded frame in pixels. + * + * @return This builder. + */ + public Builder setEncodedWidth(int width) { + this.encodedWidth = width; + return this; + } + + /** + * @param height The height of the encoded frame in pixels. + * + * @return This builder. + */ + public Builder setEncodedHeight(int height) { + this.encodedHeight = height; + return this; + } + + /** + * Sets the capture time, which must be the timestamp of the frame + * that was encoded. + * + * @param captureTimeNs The capture time in nanoseconds. + * + * @return This builder. + */ + public Builder setCaptureTimeNs(long captureTimeNs) { + this.captureTimeNs = captureTimeNs; + return this; + } + + /** + * @param frameType Whether this is a key frame or a delta frame. + * + * @return This builder. + */ + public Builder setFrameType(FrameType frameType) { + this.frameType = frameType; + return this; + } + + /** + * @param rotation The rotation of the frame in degrees, a multiple of + * 90. + * + * @return This builder. + */ + public Builder setRotation(int rotation) { + this.rotation = rotation; + return this; + } + + /** + * Sets the quantizer. Without one, WebRTC parses it from the payload + * of VP8, VP9 and H.264 frames, which quality scaling depends on. + * + * @param qp The quantizer, or {@code null} if it is not known. + * + * @return This builder. + */ + public Builder setQp(Integer qp) { + this.qp = qp; + return this; + } + + /** + * @return The encoded image. + * + * @throws NullPointerException If the buffer or the frame type is + * missing. + * @throws IllegalArgumentException If the rotation is not a multiple + * of 90 or the size is negative. + */ + public EncodedImage build() { + Objects.requireNonNull(buffer, "Buffer is null"); + Objects.requireNonNull(frameType, "Frame type is null"); + + if (rotation % 90 != 0) { + throw new IllegalArgumentException("Rotation must be a multiple of 90"); + } + if (encodedWidth < 0 || encodedHeight < 0) { + throw new IllegalArgumentException("Size must not be negative"); + } + + return new EncodedImage(buffer, encodedWidth, encodedHeight, + captureTimeNs, frameType, rotation, qp); + } + + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoDecoder.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoDecoder.java new file mode 100644 index 000000000..e2b941b0d --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoDecoder.java @@ -0,0 +1,71 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +/** + * A built-in decoder, created by a {@link DefaultVideoDecoderFactory}. It is a + * placeholder: returned from a {@link VideoDecoderFactory}, it makes WebRTC + * create the native decoder for its codec, which then runs entirely inside + * WebRTC. Its methods are therefore not to be called from Java, and throw + * {@link UnsupportedOperationException}. + * + * @author Alex Andres + */ +public final class NativeVideoDecoder implements VideoDecoder { + + /** The codec to create the native decoder for; read by native code. */ + private final VideoCodecInfo codecInfo; + + + NativeVideoDecoder(VideoCodecInfo codecInfo) { + this.codecInfo = codecInfo; + } + + /** + * @return The codec this decoder decodes. + */ + public VideoCodecInfo getCodecInfo() { + return codecInfo; + } + + @Override + public VideoCodecStatus initDecode(Settings settings, Callback callback) { + throw unsupported(); + } + + @Override + public VideoCodecStatus release() { + throw unsupported(); + } + + @Override + public VideoCodecStatus decode(EncodedImage image) { + throw unsupported(); + } + + @Override + public String toString() { + return String.format("%s@%d [codec=%s]", + NativeVideoDecoder.class.getSimpleName(), hashCode(), + codecInfo.getName()); + } + + private static UnsupportedOperationException unsupported() { + return new UnsupportedOperationException("A native decoder runs inside WebRTC only"); + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoDecoderCallback.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoDecoderCallback.java new file mode 100644 index 000000000..8f030bf79 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoDecoderCallback.java @@ -0,0 +1,68 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import dev.onvoid.webrtc.media.video.VideoFrame; + +import java.util.Objects; + +/** + * The {@link VideoDecoder.Callback} WebRTC hands to a Java decoder. It passes + * decoded frames on to the native decoder wrapper until that wrapper + * invalidates it, when the decoder is released or destroyed, so that a + * decoder that delivers late cannot reach a wrapper that no longer exists. + * + * @author Alex Andres + */ +final class NativeVideoDecoderCallback implements VideoDecoder.Callback { + + /** The native decoder wrapper, 0 once invalidated. */ + private long nativeHandle; + + + /** + * Created from native code. + */ + private NativeVideoDecoderCallback(long nativeHandle) { + this.nativeHandle = nativeHandle; + } + + @Override + public synchronized void onDecodedFrame(VideoFrame frame, Integer decodeTimeMs, Integer qp) { + Objects.requireNonNull(frame, "VideoFrame is null"); + + if (nativeHandle != 0) { + onDecodedFrame(nativeHandle, frame, + decodeTimeMs == null ? -1 : decodeTimeMs, + qp == null ? -1 : qp); + } + } + + /** + * Detaches this callback from the native wrapper. Once this returns, no + * frame is passed on any more, nor is one still being passed on. Called + * from native code. + */ + @SuppressWarnings("unused") + private synchronized void invalidate() { + nativeHandle = 0; + } + + private static native void onDecodedFrame(long nativeHandle, VideoFrame frame, + int decodeTimeMs, int qp); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoEncoder.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoEncoder.java new file mode 100644 index 000000000..1bef65836 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoEncoder.java @@ -0,0 +1,78 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import dev.onvoid.webrtc.media.video.VideoFrame; + +/** + * A built-in encoder, created by a {@link DefaultVideoEncoderFactory}. It is a + * placeholder: returned from a {@link VideoEncoderFactory}, it makes WebRTC + * create the native encoder for its codec, which then runs entirely inside + * WebRTC. Its methods are therefore not to be called from Java, and throw + * {@link UnsupportedOperationException}. + * + * @author Alex Andres + */ +public final class NativeVideoEncoder implements VideoEncoder { + + /** The codec to create the native encoder for; read by native code. */ + private final VideoCodecInfo codecInfo; + + + NativeVideoEncoder(VideoCodecInfo codecInfo) { + this.codecInfo = codecInfo; + } + + /** + * @return The codec this encoder encodes. + */ + public VideoCodecInfo getCodecInfo() { + return codecInfo; + } + + @Override + public VideoCodecStatus initEncode(Settings settings, Callback callback) { + throw unsupported(); + } + + @Override + public VideoCodecStatus release() { + throw unsupported(); + } + + @Override + public VideoCodecStatus encode(VideoFrame frame, EncodeInfo info) { + throw unsupported(); + } + + @Override + public VideoCodecStatus setRates(RateControlParameters parameters) { + throw unsupported(); + } + + @Override + public String toString() { + return String.format("%s@%d [codec=%s]", + NativeVideoEncoder.class.getSimpleName(), hashCode(), + codecInfo.getName()); + } + + private static UnsupportedOperationException unsupported() { + return new UnsupportedOperationException("A native encoder runs inside WebRTC only"); + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoEncoderCallback.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoEncoderCallback.java new file mode 100644 index 000000000..7f58e30f8 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoEncoderCallback.java @@ -0,0 +1,63 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import java.util.Objects; + +/** + * The {@link VideoEncoder.Callback} WebRTC hands to a Java encoder. It passes + * encoded frames on to the native encoder wrapper until that wrapper + * invalidates it, when the encoder is released or destroyed, so that an + * encoder that delivers late cannot reach a wrapper that no longer exists. + * + * @author Alex Andres + */ +final class NativeVideoEncoderCallback implements VideoEncoder.Callback { + + /** The native encoder wrapper, 0 once invalidated. */ + private long nativeHandle; + + + /** + * Created from native code. + */ + private NativeVideoEncoderCallback(long nativeHandle) { + this.nativeHandle = nativeHandle; + } + + @Override + public synchronized void onEncodedFrame(EncodedImage image) { + Objects.requireNonNull(image, "EncodedImage is null"); + + if (nativeHandle != 0) { + onEncodedFrame(nativeHandle, image); + } + } + + /** + * Detaches this callback from the native wrapper. Once this returns, no + * frame is passed on any more, nor is one still being passed on. Called + * from native code. + */ + @SuppressWarnings("unused") + private synchronized void invalidate() { + nativeHandle = 0; + } + + private static native void onEncodedFrame(long nativeHandle, EncodedImage image); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoCodecInfo.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoCodecInfo.java new file mode 100644 index 000000000..c8179dee9 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoCodecInfo.java @@ -0,0 +1,159 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Describes a video codec the way it is negotiated in SDP: its name, such as + * "VP8" or "H264", and its format parameters, such as the H.264 profile. A + * codec factory lists the formats it supports as VideoCodecInfos, and is asked + * for an encoder or decoder for one of them. + *

+ * A name WebRTC does not know, for example "X-MYCODEC", is negotiated like any + * other, and its frames are sent with the generic RTP packetization. + * + * @author Alex Andres + */ +public final class VideoCodecInfo { + + private final String name; + + /** A HashMap, which is what the native side reads. */ + private final HashMap parameters; + + private final String[] scalabilityModes; + + + /** + * Creates a codec description without format parameters. + * + * @param name The codec name, such as "VP8". + */ + public VideoCodecInfo(String name) { + this(name, Collections.emptyMap()); + } + + /** + * Creates a codec description. + * + * @param name The codec name, such as "H264". + * @param parameters The SDP format parameters, such as {@code + * profile-level-id}. + */ + public VideoCodecInfo(String name, Map parameters) { + this(name, parameters, Collections.emptyList()); + } + + /** + * Creates a codec description. + * + * @param name The codec name, such as "VP9". + * @param parameters The SDP format parameters. + * @param scalabilityModes The scalability modes the codec supports, such + * as "L1T3", as named in the WebRTC SVC + * specification. + */ + public VideoCodecInfo(String name, Map parameters, + List scalabilityModes) { + Objects.requireNonNull(name, "Codec name is null"); + Objects.requireNonNull(parameters, "Codec parameters are null"); + Objects.requireNonNull(scalabilityModes, "Scalability modes are null"); + + if (name.isEmpty()) { + throw new IllegalArgumentException("Codec name is empty"); + } + + this.name = name; + this.parameters = new HashMap<>(parameters); + this.scalabilityModes = scalabilityModes.toArray(new String[0]); + + for (Map.Entry entry : this.parameters.entrySet()) { + if (entry.getKey() == null || entry.getValue() == null) { + throw new IllegalArgumentException("Codec parameters must not contain null"); + } + } + for (String mode : this.scalabilityModes) { + Objects.requireNonNull(mode, "Scalability mode is null"); + } + } + + /** + * Created from native code. + */ + @SuppressWarnings("unused") + private VideoCodecInfo(String name, Map parameters, + String[] scalabilityModes) { + this(name, parameters, Arrays.asList(scalabilityModes)); + } + + /** + * @return The codec name, such as "VP8". + */ + public String getName() { + return name; + } + + /** + * @return The SDP format parameters, unmodifiable. + */ + public Map getParameters() { + return Collections.unmodifiableMap(parameters); + } + + /** + * @return The scalability modes the codec supports, unmodifiable. + */ + public List getScalabilityModes() { + return Collections.unmodifiableList(Arrays.asList(scalabilityModes)); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + VideoCodecInfo other = (VideoCodecInfo) obj; + + return name.equalsIgnoreCase(other.name) + && parameters.equals(other.parameters) + && Arrays.equals(scalabilityModes, other.scalabilityModes); + } + + @Override + public int hashCode() { + return Objects.hash(name.toUpperCase(java.util.Locale.ROOT), parameters, + Arrays.hashCode(scalabilityModes)); + } + + @Override + public String toString() { + return String.format("%s@%d [name=%s, parameters=%s, scalabilityModes=%s]", + VideoCodecInfo.class.getSimpleName(), hashCode(), name, + parameters, Arrays.toString(scalabilityModes)); + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoCodecStatus.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoCodecStatus.java new file mode 100644 index 000000000..78ea3e3d8 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoCodecStatus.java @@ -0,0 +1,81 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +/** + * The result of a {@link VideoEncoder} or {@link VideoDecoder} operation. + *

+ * Of the errors, {@link #FALLBACK_SOFTWARE} and {@link #UNINITIALIZED} give + * the codec up for the stream. Any other error makes WebRTC release and + * initialize the codec again, and give it up only if that fails. + * + * @author Alex Andres + */ +public enum VideoCodecStatus { + + /** The operation succeeded. */ + OK(0), + + /** The operation succeeded, but produced no output. */ + NO_OUTPUT(1), + + /** The operation succeeded, and the encoder asks for a key frame. */ + OK_REQUEST_KEYFRAME(4), + + /** The encoder overshot its target bitrate. */ + TARGET_BITRATE_OVERSHOOT(5), + + /** A generic error. */ + ERROR(-1), + + /** Memory could not be allocated. */ + MEMORY(-3), + + /** A parameter was not valid. */ + ERR_PARAMETER(-4), + + /** The operation timed out. */ + TIMEOUT(-6), + + /** The codec was not initialized. */ + UNINITIALIZED(-7), + + /** The codec cannot go on, and another implementation is to be used. */ + FALLBACK_SOFTWARE(-13), + + /** The encoder does not support the requested simulcast settings. */ + ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED(-15), + + /** The encoder failed. */ + ENCODER_FAILURE(-16); + + + private final int number; + + + VideoCodecStatus(int number) { + this.number = number; + } + + /** + * @return The WebRTC status code, a {@code WEBRTC_VIDEO_CODEC_*} value. + */ + public int getNumber() { + return number; + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoDecoder.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoDecoder.java new file mode 100644 index 000000000..3834990fb --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoDecoder.java @@ -0,0 +1,151 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import dev.onvoid.webrtc.media.video.VideoFrame; + +/** + * A video decoder implemented in Java, which a {@link VideoDecoderFactory} + * hands to WebRTC. + *

+ * WebRTC calls {@link #initDecode initDecode}, then {@link #decode decode} + * for each encoded frame, and finally {@link #release release}. It may + * initialize a released decoder again. These calls come from one WebRTC thread + * at a time, never concurrently. The decoder hands each decoded frame to the + * {@link Callback} it was initialized with, either from within {@code decode} + * or later from a thread of its own. + *

+ * Whatever a method throws is logged and treated as {@link + * VideoCodecStatus#ERROR}. + * + * @author Alex Andres + * + * @see VideoDecoderFactory + */ +public interface VideoDecoder { + + /** + * The settings a decoder is initialized with. + */ + class Settings { + + /** The number of CPU cores the decoder may use. */ + public final int numberOfCores; + + /** The largest frame width to expect, 0 if not known. */ + public final int width; + + /** The largest frame height to expect, 0 if not known. */ + public final int height; + + + /** + * Creates decoder settings. + * + * @param numberOfCores The number of CPU cores to use. + * @param width The largest frame width to expect. + * @param height The largest frame height to expect. + */ + public Settings(int numberOfCores, int width, int height) { + this.numberOfCores = numberOfCores; + this.width = width; + this.height = height; + } + + @Override + public String toString() { + return String.format("%s [cores=%d, width=%d, height=%d]", + Settings.class.getSimpleName(), numberOfCores, width, height); + } + + } + + + + /** + * Receives what a decoder produces. + */ + interface Callback { + + /** + * Hands a decoded frame to WebRTC. The frame's timestamp must be the + * {@link EncodedImage#getCaptureTimeNs() capture time} of the image it + * was decoded from. WebRTC takes a reference of its own, so the caller + * still releases the frame it created. May be called from any thread, + * but frames have to be handed over in decoding order. Frames handed + * over after the decoder was released are ignored. + * + * @param frame The decoded frame. + * @param decodeTimeMs How long decoding took in milliseconds, or + * {@code null} if not known. + * @param qp The quantizer of the frame, or {@code null} if + * not known. + */ + void onDecodedFrame(VideoFrame frame, Integer decodeTimeMs, Integer qp); + + } + + + /** + * Initializes the decoder, or initializes it again after a {@link + * #release()}. + * + * @param settings The decoder settings. + * @param callback Receives the decoded frames until the decoder is + * released. + * + * @return {@link VideoCodecStatus#OK} on success. + */ + VideoCodecStatus initDecode(Settings settings, Callback callback); + + /** + * Releases the decoder. It may be initialized again afterwards. + * + * @return {@link VideoCodecStatus#OK} on success. + */ + VideoCodecStatus release(); + + /** + * Decodes an encoded frame. The image and its buffer are valid only until + * this method returns. + * + * @param image The encoded frame. + * + * @return {@link VideoCodecStatus#OK} if the frame was taken, also if the + * decoded frame is delivered later. + */ + VideoCodecStatus decode(EncodedImage image); + + /** + * Returns the name of the implementation, which shows up in statistics. + * + * @return The simple class name unless overridden. + */ + default String getImplementationName() { + return getClass().getSimpleName(); + } + + /** + * Returns whether the decoder runs on dedicated hardware. + * + * @return {@code false} unless overridden. + */ + default boolean isHardwareDecoder() { + return false; + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoDecoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoDecoderFactory.java new file mode 100644 index 000000000..f9b62435f --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoDecoderFactory.java @@ -0,0 +1,56 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import java.util.List; + +/** + * Creates the video decoders of a {@link dev.onvoid.webrtc.PeerConnectionFactory + * PeerConnectionFactory}, which decide what video codecs it can receive. + *

+ * A factory may return its own {@link VideoDecoder}s as well as the built-in + * ones of a {@link DefaultVideoDecoderFactory}, the same way a {@link + * VideoEncoderFactory} does. + *

+ * {@link #createDecoder(VideoCodecInfo)} is called from a WebRTC thread. + * + * @author Alex Andres + * + * @see dev.onvoid.webrtc.PeerConnectionFactory.Builder#setVideoDecoderFactory(VideoDecoderFactory) + */ +public interface VideoDecoderFactory { + + /** + * Returns the codecs this factory can create decoders for, in order of + * preference. Asked for once, when the peer connection factory is + * created. + * + * @return The supported codecs. + */ + List getSupportedCodecs(); + + /** + * Creates a decoder for one of the supported codecs. Each call has to + * return a new decoder, since WebRTC uses one per stream. + * + * @param info The codec to decode, as negotiated. + * + * @return A new decoder, or {@code null} if the codec is not supported. + */ + VideoDecoder createDecoder(VideoCodecInfo info); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoEncoder.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoEncoder.java new file mode 100644 index 000000000..c76bf9793 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoEncoder.java @@ -0,0 +1,510 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import dev.onvoid.webrtc.media.video.VideoFrame; + +import java.util.Arrays; +import java.util.Objects; + +/** + * A video encoder implemented in Java, which a {@link VideoEncoderFactory} + * hands to WebRTC. This is the way to bring in a codec WebRTC does not have, + * or a hardware encoder reached through another library. + *

+ * WebRTC calls {@link #initEncode initEncode}, then {@link #encode encode} + * for each frame and {@link #setRates setRates} whenever the target bitrate + * changes, and finally {@link #release release}. It may initialize a released + * encoder again. These calls come from one WebRTC thread at a time, never + * concurrently. The encoder hands each encoded frame to the {@link Callback} + * it was initialized with, either from within {@code encode} or later from a + * thread of its own. + *

+ * Whatever a method throws is logged and treated as {@link + * VideoCodecStatus#ERROR}. + * + * @author Alex Andres + * + * @see VideoEncoderFactory + */ +public interface VideoEncoder { + + /** + * The settings an encoder is initialized with. + */ + class Settings { + + /** The number of CPU cores the encoder may use. */ + public final int numberOfCores; + + /** The frame width in pixels. */ + public final int width; + + /** The frame height in pixels. */ + public final int height; + + /** The initial target bitrate in kbps. */ + public final int startBitrateKbps; + + /** The minimum bitrate in kbps. */ + public final int minBitrateKbps; + + /** The maximum bitrate in kbps. */ + public final int maxBitrateKbps; + + /** The maximum frame rate in frames per second. */ + public final int maxFramerate; + + /** The number of simulcast streams, 0 or 1 without simulcast. */ + public final int numberOfSimulcastStreams; + + /** Whether the encoder may change the resolution on its own. */ + public final boolean automaticResizeOn; + + /** Whether the content is a screen share rather than camera video. */ + public final boolean screenContent; + + /** Whether the encoder is to react to loss notifications. */ + public final boolean lossNotification; + + + /** + * Creates encoder settings. + * + * @param numberOfCores The number of CPU cores to use. + * @param width The frame width in pixels. + * @param height The frame height in pixels. + * @param startBitrateKbps The initial bitrate in kbps. + * @param minBitrateKbps The minimum bitrate in kbps. + * @param maxBitrateKbps The maximum bitrate in kbps. + * @param maxFramerate The maximum frame rate. + * @param numberOfSimulcastStreams The number of simulcast streams. + * @param automaticResizeOn Whether the encoder may resize. + * @param screenContent Whether the content is a screen. + * @param lossNotification Whether to react to loss + * notifications. + */ + public Settings(int numberOfCores, int width, int height, + int startBitrateKbps, int minBitrateKbps, int maxBitrateKbps, + int maxFramerate, int numberOfSimulcastStreams, + boolean automaticResizeOn, boolean screenContent, + boolean lossNotification) { + this.numberOfCores = numberOfCores; + this.width = width; + this.height = height; + this.startBitrateKbps = startBitrateKbps; + this.minBitrateKbps = minBitrateKbps; + this.maxBitrateKbps = maxBitrateKbps; + this.maxFramerate = maxFramerate; + this.numberOfSimulcastStreams = numberOfSimulcastStreams; + this.automaticResizeOn = automaticResizeOn; + this.screenContent = screenContent; + this.lossNotification = lossNotification; + } + + @Override + public String toString() { + return String.format("%s [cores=%d, width=%d, height=%d, startBitrateKbps=%d, minBitrateKbps=%d, " + + "maxBitrateKbps=%d, maxFramerate=%d, simulcastStreams=%d, automaticResize=%s, " + + "screenContent=%s, lossNotification=%s]", + Settings.class.getSimpleName(), numberOfCores, width, height, + startBitrateKbps, minBitrateKbps, maxBitrateKbps, maxFramerate, + numberOfSimulcastStreams, automaticResizeOn, screenContent, + lossNotification); + } + + } + + + + /** + * Additional information about a frame to encode. + */ + class EncodeInfo { + + /** + * The frame type WebRTC asks for, one per simulcast stream. {@link + * EncodedImage.FrameType#KEY} asks for a key frame. + */ + public final EncodedImage.FrameType[] frameTypes; + + + /** + * Creates the information about a frame to encode. + * + * @param frameTypes The frame type asked for, per simulcast stream. + */ + public EncodeInfo(EncodedImage.FrameType[] frameTypes) { + this.frameTypes = Objects.requireNonNull(frameTypes, "Frame types are null"); + } + + /** + * @return Whether WebRTC asks for a key frame in any stream. + */ + public boolean isKeyFrameRequested() { + for (EncodedImage.FrameType type : frameTypes) { + if (type == EncodedImage.FrameType.KEY) { + return true; + } + } + + return false; + } + + @SuppressWarnings("unused") + private static EncodeInfo fromNative(int[] frameTypes) { + EncodedImage.FrameType[] types = new EncodedImage.FrameType[frameTypes.length]; + + for (int i = 0; i < frameTypes.length; i++) { + types[i] = EncodedImage.FrameType.fromNativeIndex(frameTypes[i]); + } + + return new EncodeInfo(types); + } + + @Override + public String toString() { + return String.format("%s [frameTypes=%s]", + EncodeInfo.class.getSimpleName(), Arrays.toString(frameTypes)); + } + + } + + + + /** + * The bitrate WebRTC allocates to each spatial and temporal layer. + */ + class BitrateAllocation { + + /** + * The bitrate in bps, indexed by spatial layer, then temporal layer. + */ + public final int[][] bitratesBps; + + + /** + * Creates a bitrate allocation. + * + * @param bitratesBps The bitrate in bps per spatial, then temporal + * layer. + */ + public BitrateAllocation(int[][] bitratesBps) { + this.bitratesBps = Objects.requireNonNull(bitratesBps, "Bitrates are null"); + } + + /** + * @return The sum of all layers in bps. + */ + public int getSum() { + int sum = 0; + + for (int[] spatial : bitratesBps) { + for (int bitrate : spatial) { + sum += bitrate; + } + } + + return sum; + } + + @Override + public String toString() { + return String.format("%s [sum=%d, bitratesBps=%s]", + BitrateAllocation.class.getSimpleName(), getSum(), + Arrays.deepToString(bitratesBps)); + } + + } + + + + /** + * The rates an encoder is to aim for. + */ + class RateControlParameters { + + /** The target bitrate per layer. */ + public final BitrateAllocation bitrate; + + /** The target frame rate in frames per second. */ + public final double framerateFps; + + + /** + * Creates rate control parameters. + * + * @param bitrate The target bitrate per layer. + * @param framerateFps The target frame rate. + */ + public RateControlParameters(BitrateAllocation bitrate, double framerateFps) { + this.bitrate = Objects.requireNonNull(bitrate, "Bitrate is null"); + this.framerateFps = framerateFps; + } + + @Override + public String toString() { + return String.format("%s [bitrate=%s, framerateFps=%s]", + RateControlParameters.class.getSimpleName(), bitrate, + framerateFps); + } + + } + + + + /** + * The quantizer thresholds WebRTC scales the resolution by: above the high + * threshold it lowers the resolution, below the low one it raises it. + */ + class ScalingSettings { + + /** No quality scaling. */ + public static final ScalingSettings OFF = new ScalingSettings(false, null, null); + + /** + * Quality scaling with the thresholds WebRTC uses for VP8, VP9 and + * H.264, and none for any other codec. + */ + public static final ScalingSettings DEFAULT = new ScalingSettings(true, null, null); + + /** Whether quality scaling is on. */ + public final boolean on; + + /** The low quantizer threshold, or {@code null} for the default. */ + public final Integer low; + + /** The high quantizer threshold, or {@code null} for the default. */ + public final Integer high; + + + /** + * Creates scaling settings that turn quality scaling on with the given + * thresholds. + * + * @param low The low quantizer threshold. + * @param high The high quantizer threshold. + */ + public ScalingSettings(int low, int high) { + this(true, low, high); + } + + private ScalingSettings(boolean on, Integer low, Integer high) { + this.on = on; + this.low = low; + this.high = high; + } + + @Override + public String toString() { + return on + ? String.format("[low=%s, high=%s]", low, high) + : "OFF"; + } + + } + + + + /** + * The bitrate limits of an encoder at one resolution. + */ + class ResolutionBitrateLimits { + + /** The resolution, as width times height. */ + public final int frameSizePixels; + + /** The lowest bitrate to start encoding this resolution at, in bps. */ + public final int minStartBitrateBps; + + /** The lowest bitrate for this resolution, in bps. */ + public final int minBitrateBps; + + /** The highest bitrate for this resolution, in bps. */ + public final int maxBitrateBps; + + + /** + * Creates bitrate limits for one resolution. + * + * @param frameSizePixels The resolution, as width times height. + * @param minStartBitrateBps The lowest start bitrate in bps. + * @param minBitrateBps The lowest bitrate in bps. + * @param maxBitrateBps The highest bitrate in bps. + */ + public ResolutionBitrateLimits(int frameSizePixels, int minStartBitrateBps, + int minBitrateBps, int maxBitrateBps) { + this.frameSizePixels = frameSizePixels; + this.minStartBitrateBps = minStartBitrateBps; + this.minBitrateBps = minBitrateBps; + this.maxBitrateBps = maxBitrateBps; + } + + } + + + + /** + * Properties of an encoder WebRTC takes into account when feeding it. + */ + class EncoderInfo { + + /** + * The number both frame dimensions have to be divisible by. WebRTC + * crops or scales frames to fit. + */ + public final int requestedResolutionAlignment; + + /** + * Whether the alignment applies to every simulcast layer, rather than + * only to the highest. + */ + public final boolean applyAlignmentToAllSimulcastLayers; + + + /** + * Creates encoder properties. + * + * @param requestedResolutionAlignment The alignment of frame + * dimensions, 1 for none. + * @param applyAlignmentToAllSimulcastLayers Whether the alignment + * applies to every layer. + */ + public EncoderInfo(int requestedResolutionAlignment, + boolean applyAlignmentToAllSimulcastLayers) { + if (requestedResolutionAlignment < 1) { + throw new IllegalArgumentException("Alignment must be at least 1"); + } + + this.requestedResolutionAlignment = requestedResolutionAlignment; + this.applyAlignmentToAllSimulcastLayers = applyAlignmentToAllSimulcastLayers; + } + + } + + + + /** + * Receives what an encoder produces. + */ + interface Callback { + + /** + * Hands an encoded frame to WebRTC, which copies the payload before + * returning. May be called from any thread, but frames have to be + * handed over in the order they were encoded in. Frames handed over + * after the encoder was released are ignored. + * + * @param image The encoded frame, whose capture time is the timestamp + * of the frame it encodes. + */ + void onEncodedFrame(EncodedImage image); + + } + + + /** + * Initializes the encoder, or initializes it again after a {@link + * #release()}, possibly with a different frame size. + * + * @param settings The encoder settings. + * @param callback Receives the encoded frames until the encoder is + * released. + * + * @return {@link VideoCodecStatus#OK} on success. + */ + VideoCodecStatus initEncode(Settings settings, Callback callback); + + /** + * Releases the encoder. It may be initialized again afterwards. + * + * @return {@link VideoCodecStatus#OK} on success. + */ + VideoCodecStatus release(); + + /** + * Encodes a frame. The frame is valid only until this method returns; to + * encode it later, {@link VideoFrame#retain() retain} it and {@link + * VideoFrame#release() release} it once done. Its pixel data must not be + * changed, since other consumers may share it. + * + * @param frame The frame to encode. + * @param info Additional information, such as a key frame request. + * + * @return {@link VideoCodecStatus#OK} if the frame was taken, also if it + * is delivered later or dropped. + */ + VideoCodecStatus encode(VideoFrame frame, EncodeInfo info); + + /** + * Sets the bitrate and frame rate to aim for. + * + * @param parameters The target rates. + * + * @return {@link VideoCodecStatus#OK} on success. + */ + VideoCodecStatus setRates(RateControlParameters parameters); + + /** + * Returns the quality scaling settings. Read after construction and after + * each initialization. + * + * @return {@link ScalingSettings#DEFAULT} unless overridden. + */ + default ScalingSettings getScalingSettings() { + return ScalingSettings.DEFAULT; + } + + /** + * Returns the bitrate limits of the encoder per resolution. Read after + * construction and after each initialization. + * + * @return No limits unless overridden. + */ + default ResolutionBitrateLimits[] getResolutionBitrateLimits() { + return new ResolutionBitrateLimits[0]; + } + + /** + * Returns properties of the encoder. Read after construction and after + * each initialization. + * + * @return No alignment unless overridden. + */ + default EncoderInfo getEncoderInfo() { + return new EncoderInfo(1, false); + } + + /** + * Returns the name of the implementation, which shows up in statistics. + * + * @return The simple class name unless overridden. + */ + default String getImplementationName() { + return getClass().getSimpleName(); + } + + /** + * Returns whether the encoder runs on dedicated hardware. + * + * @return {@code false} unless overridden. + */ + default boolean isHardwareEncoder() { + return false; + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoEncoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoEncoderFactory.java new file mode 100644 index 000000000..62ca8b76b --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoEncoderFactory.java @@ -0,0 +1,75 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import java.util.List; + +/** + * Creates the video encoders of a {@link dev.onvoid.webrtc.PeerConnectionFactory + * PeerConnectionFactory}, which decide what video codecs it can send. + *

+ * A factory may return its own {@link VideoEncoder}s as well as the built-in + * ones of a {@link DefaultVideoEncoderFactory}, for example to add a codec + * while keeping the built-in ones: + * 

{@code
+ * class MyEncoderFactory implements VideoEncoderFactory {
+ *
+ *     private final DefaultVideoEncoderFactory builtIn = new DefaultVideoEncoderFactory();
+ *
+ *     public List getSupportedCodecs() {
+ *         List codecs = new ArrayList<>(builtIn.getSupportedCodecs());
+ *         codecs.add(new VideoCodecInfo("X-MYCODEC"));
+ *         return codecs;
+ *     }
+ *
+ *     public VideoEncoder createEncoder(VideoCodecInfo info) {
+ *         if (info.getName().equals("X-MYCODEC")) {
+ *             return new MyEncoder();
+ *         }
+ *         return builtIn.createEncoder(info);
+ *     }
+ * }
+ * }
+ *

+ * {@link #createEncoder(VideoCodecInfo)} is called from a WebRTC thread. + * + * @author Alex Andres + * + * @see dev.onvoid.webrtc.PeerConnectionFactory.Builder#setVideoEncoderFactory(VideoEncoderFactory) + */ +public interface VideoEncoderFactory { + + /** + * Returns the codecs this factory can create encoders for, in order of + * preference. Asked for once, when the peer connection factory is + * created. + * + * @return The supported codecs. + */ + List getSupportedCodecs(); + + /** + * Creates an encoder for one of the supported codecs. Each call has to + * return a new encoder, since WebRTC uses one per stream. + * + * @param info The codec to encode, as negotiated. + * + * @return A new encoder, or {@code null} if the codec is not supported. + */ + VideoEncoder createEncoder(VideoCodecInfo info); + +} diff --git a/webrtc/src/main/java/module-info.java b/webrtc/src/main/java/module-info.java index 743fff91e..778d8892d 100644 --- a/webrtc/src/main/java/module-info.java +++ b/webrtc/src/main/java/module-info.java @@ -10,6 +10,7 @@ exports dev.onvoid.webrtc.media; exports dev.onvoid.webrtc.media.audio; exports dev.onvoid.webrtc.media.video; + exports dev.onvoid.webrtc.media.video.codec; exports dev.onvoid.webrtc.media.video.desktop; // Not API for applications. A native extension module, such as the FFmpeg diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java b/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java index a96fe56ad..9db911955 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java @@ -60,6 +60,15 @@ class TestMediaCall implements AutoCloseable { TestMediaCall(PeerConnectionFactory factory, boolean video, boolean audio) { + this(factory, video, audio, "VP8"); + } + + /** + * Creates a call whose video prefers the given codec. + * + * @param videoCodec The name of the video codec to prefer. + */ + TestMediaCall(PeerConnectionFactory factory, boolean video, boolean audio, String videoCodec) { caller = new TestPeerConnection(factory); callee = new TestPeerConnection(factory); @@ -72,7 +81,7 @@ class TestMediaCall implements AutoCloseable { videoSender = caller.getPeerConnection().addTrack(videoTrack, Collections.singletonList("stream")); - preferVp8(factory, caller.getPeerConnection(), videoSender); + preferCodec(factory, caller.getPeerConnection(), videoSender, videoCodec); } else { videoSource = null; @@ -191,18 +200,19 @@ public void close() throws InterruptedException { } /** - * Makes VP8 the preferred video codec. Platforms prefer different codecs by - * default, macOS H.264 through VideoToolbox, and the tests should not - * depend on which: VP8 is available everywhere, and it is the codec that - * survives a transform changing the whole payload, which H.264 does not, - * since its packetizer splits frames at start codes in the payload. + * Makes the given video codec the preferred one. Platforms prefer different + * codecs by default, macOS H.264 through VideoToolbox, and the tests should + * not depend on which. Unless a test needs another, they prefer VP8: it is + * available everywhere, and it is the codec that survives a transform + * changing the whole payload, which H.264 does not, since its packetizer + * splits frames at start codes in the payload. */ - private static void preferVp8(PeerConnectionFactory factory, RTCPeerConnection connection, - RTCRtpSender sender) { + private static void preferCodec(PeerConnectionFactory factory, RTCPeerConnection connection, + RTCRtpSender sender, String name) { List codecs = new ArrayList<>( factory.getRtpSenderCapabilities(MediaType.VIDEO).getCodecs()); - codecs.sort(Comparator.comparing(codec -> !"VP8".equalsIgnoreCase(codec.getName()))); + codecs.sort(Comparator.comparing(codec -> !name.equalsIgnoreCase(codec.getName()))); for (RTCRtpTransceiver transceiver : connection.getTransceivers()) { RTCRtpSender transceiverSender = transceiver.getSender(); diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java new file mode 100644 index 000000000..c45141b6e --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java @@ -0,0 +1,685 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import dev.onvoid.webrtc.logging.Logging; +import dev.onvoid.webrtc.media.MediaType; +import dev.onvoid.webrtc.media.video.I420Buffer; +import dev.onvoid.webrtc.media.video.NativeI420Buffer; +import dev.onvoid.webrtc.media.video.VideoFrame; +import dev.onvoid.webrtc.media.video.VideoFrameBuffer; +import dev.onvoid.webrtc.media.video.VideoTrack; +import dev.onvoid.webrtc.media.video.VideoTrackSink; +import dev.onvoid.webrtc.media.video.codec.DefaultVideoDecoderFactory; +import dev.onvoid.webrtc.media.video.codec.DefaultVideoEncoderFactory; +import dev.onvoid.webrtc.media.video.codec.EncodedImage; +import dev.onvoid.webrtc.media.video.codec.NativeVideoEncoder; +import dev.onvoid.webrtc.media.video.codec.VideoCodecInfo; +import dev.onvoid.webrtc.media.video.codec.VideoCodecStatus; +import dev.onvoid.webrtc.media.video.codec.VideoDecoder; +import dev.onvoid.webrtc.media.video.codec.VideoDecoderFactory; +import dev.onvoid.webrtc.media.video.codec.VideoEncoder; +import dev.onvoid.webrtc.media.video.codec.VideoEncoderFactory; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +/** + * Tests video codec factories, built-in and implemented in Java, on calls + * between two local peer connections. + */ +@Execution(ExecutionMode.SAME_THREAD) +class VideoCodecFactoryTests extends TestBase { + + private static final long TIMEOUT_SECONDS = 10; + + /** The codec of {@link TestEncoder} and {@link TestDecoder}. */ + private static final String TEST_CODEC = "X-TEST"; + + + @Test + void defaultFactoriesListBuiltInCodecs() { + List encoders = names(new DefaultVideoEncoderFactory().getSupportedCodecs()); + List decoders = names(new DefaultVideoDecoderFactory().getSupportedCodecs()); + + assertTrue(encoders.contains("VP8"), encoders.toString()); + assertTrue(encoders.contains("VP9"), encoders.toString()); + assertTrue(decoders.contains("VP8"), decoders.toString()); + assertTrue(decoders.contains("VP9"), decoders.toString()); + } + + @Test + void defaultFactoryCreatesNativeCodecs() { + DefaultVideoEncoderFactory factory = new DefaultVideoEncoderFactory(); + + VideoEncoder encoder = factory.createEncoder(new VideoCodecInfo("VP8")); + + assertInstanceOf(NativeVideoEncoder.class, encoder); + assertEquals("VP8", ((NativeVideoEncoder) encoder).getCodecInfo().getName()); + assertThrows(UnsupportedOperationException.class, encoder::release); + + assertNull(factory.createEncoder(new VideoCodecInfo(TEST_CODEC))); + } + + @Test + void builtFactoryUsesDefaultCodecs() { + PeerConnectionFactory built = PeerConnectionFactory.builder() + .setAudioDeviceModule(audioDevModule) + .setVideoEncoderFactory(new DefaultVideoEncoderFactory()) + .setVideoDecoderFactory(new DefaultVideoDecoderFactory()) + .build(); + + try { + assertEquals( + names(factory.getRtpSenderCapabilities(MediaType.VIDEO).getCodecs()), + names(built.getRtpSenderCapabilities(MediaType.VIDEO).getCodecs())); + } + finally { + built.dispose(); + } + } + + @Test + void customCodecIsAdvertised() { + PeerConnectionFactory built = testCodecFactory(new TestEncoderFactory(), new TestDecoderFactory()); + + try { + List send = names(built.getRtpSenderCapabilities(MediaType.VIDEO).getCodecs()); + List receive = names(built.getRtpReceiverCapabilities(MediaType.VIDEO).getCodecs()); + + assertTrue(send.contains(TEST_CODEC), send.toString()); + assertTrue(send.contains("VP8"), send.toString()); + assertTrue(receive.contains(TEST_CODEC), receive.toString()); + } + finally { + built.dispose(); + } + } + + @Test + void javaCodecRoundTrips() throws Exception { + assertRoundTrip(false); + } + + @Test + void javaFrameBufferRoundTrips() throws Exception { + // The decoder hands back frames whose buffer is no NativeI420Buffer, + // which WebRTC has to copy. + assertRoundTrip(true); + } + + /** + * Sends a call through the test codec and checks that every frame + * received is one that was sent, and how WebRTC used the codecs. + */ + private void assertRoundTrip(boolean javaBuffers) throws Exception { + TestEncoderFactory encoderFactory = new TestEncoderFactory(); + TestDecoderFactory decoderFactory = new TestDecoderFactory(); + decoderFactory.javaBuffers = javaBuffers; + PeerConnectionFactory built = testCodecFactory(encoderFactory, decoderFactory); + + Set encodedSizes = ConcurrentHashMap.newKeySet(); + Set encodedLuma = ConcurrentHashMap.newKeySet(); + AtomicReference mismatch = new AtomicReference<>(); + CountDownLatch received = new CountDownLatch(10); + + encoderFactory.onEncode = (frame, luma) -> { + encodedSizes.add(frame.buffer.getWidth() + "x" + frame.buffer.getHeight()); + encodedLuma.add(luma); + }; + + try (TestMediaCall call = new TestMediaCall(built, true, false, TEST_CODEC)) { + call.negotiate(); + + RTCRtpReceiver receiver = call.getReceiver("video"); + VideoTrack track = (VideoTrack) receiver.getTrack(); + VideoTrackSink sink = frame -> { + String size = frame.buffer.getWidth() + "x" + frame.buffer.getHeight(); + int luma = frame.buffer.toI420().getDataY().get(0) & 0xFF; + + if (!encodedSizes.contains(size) || !encodedLuma.contains(luma)) { + mismatch.compareAndSet(null, size + ", luma " + luma + " not in " + encodedSizes + + " " + encodedLuma + ", encoders " + encoderFactory.created.get() + + ", decoders " + decoderFactory.created.get()); + } + + frame.release(); + received.countDown(); + }; + track.addSink(sink); + + call.awaitConnected(); + call.startMedia(); + + assertTrue(received.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), + "too few frames received: encoders " + encoderFactory.created.get() + + ", decoders " + decoderFactory.created.get()); + + track.removeSink(sink); + receiver.dispose(); + } + finally { + built.dispose(); + } + + assertNull(mismatch.get(), "a received frame was never sent: " + mismatch.get()); + + TestEncoder encoder = encoderFactory.last.get(); + + assertEquals(320, encoder.settings.width); + assertEquals(240, encoder.settings.height); + assertTrue(encoder.rates.get() > 0, "rates were never set"); + assertTrue(encoder.released.get(), "encoder was never released"); + assertTrue(decoderFactory.last.get().released.get(), "decoder was never released"); + } + + @Test + void factoryDelegatesToBuiltInCodec() throws Exception { + AtomicInteger encoders = new AtomicInteger(); + AtomicInteger decoders = new AtomicInteger(); + DefaultVideoEncoderFactory builtInEncoders = new DefaultVideoEncoderFactory(); + DefaultVideoDecoderFactory builtInDecoders = new DefaultVideoDecoderFactory(); + + PeerConnectionFactory built = PeerConnectionFactory.builder() + .setAudioDeviceModule(audioDevModule) + .setVideoEncoderFactory(new VideoEncoderFactory() { + + @Override + public List getSupportedCodecs() { + return builtInEncoders.getSupportedCodecs(); + } + + @Override + public VideoEncoder createEncoder(VideoCodecInfo info) { + encoders.incrementAndGet(); + return builtInEncoders.createEncoder(info); + } + }) + .setVideoDecoderFactory(new VideoDecoderFactory() { + + @Override + public List getSupportedCodecs() { + return builtInDecoders.getSupportedCodecs(); + } + + @Override + public VideoDecoder createDecoder(VideoCodecInfo info) { + decoders.incrementAndGet(); + return builtInDecoders.createDecoder(info); + } + }) + .build(); + + try { + assertFramesReceived(built, "VP8"); + } + finally { + built.dispose(); + } + + assertTrue(encoders.get() > 0); + assertTrue(decoders.get() > 0); + } + + @Test + void failingEncoderFactoryDoesNotCrash() throws Exception { + TestEncoderFactory encoderFactory = new TestEncoderFactory() { + + @Override + public VideoEncoder createEncoder(VideoCodecInfo info) { + super.createEncoder(info); + throw new IllegalStateException("createEncoder fails on purpose"); + } + }; + // A Java log sink receives the warning about the failure, which is + // logged while WebRTC holds its logging lock. This once deadlocked + // the call as it closed: the warning went out while the Java + // exception was still pending. + Logging.addLogSink(Logging.Severity.WARNING, (severity, message) -> { }); + + PeerConnectionFactory built = testCodecFactory(encoderFactory, new TestDecoderFactory()); + + try (TestMediaCall call = new TestMediaCall(built, true, false, TEST_CODEC)) { + call.negotiate(); + call.awaitConnected(); + call.startMedia(); + + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + + while (encoderFactory.created.get() == 0 && System.nanoTime() < deadline) { + Thread.sleep(20); + } + + assertTrue(encoderFactory.created.get() > 0, "the encoder factory was never asked"); + } + finally { + built.dispose(); + } + } + + @Test + void failingSupportedCodecsFailsBuild() { + VideoEncoderFactory broken = new TestEncoderFactory() { + + @Override + public List getSupportedCodecs() { + throw new IllegalStateException("getSupportedCodecs fails on purpose"); + } + }; + + assertThrows(Throwable.class, () -> PeerConnectionFactory.builder() + .setAudioDeviceModule(audioDevModule) + .setVideoEncoderFactory(broken) + .build()); + } + + @Test + void encodedImageBuilderValidates() { + assertThrows(NullPointerException.class, () -> EncodedImage.builder().build()); + assertThrows(IllegalArgumentException.class, () -> EncodedImage.builder() + .setBuffer(ByteBuffer.allocate(1)) + .setRotation(45) + .build()); + + EncodedImage image = EncodedImage.builder() + .setBuffer(ByteBuffer.allocate(3)) + .setEncodedWidth(2) + .setEncodedHeight(4) + .setFrameType(EncodedImage.FrameType.KEY) + .setQp(7) + .build(); + + assertEquals(3, image.getBuffer().remaining()); + assertEquals(EncodedImage.FrameType.KEY, image.getFrameType()); + assertEquals(7, (int) image.getQp()); + } + + private void assertFramesReceived(PeerConnectionFactory built, String codec) throws Exception { + CountDownLatch received = new CountDownLatch(10); + + try (TestMediaCall call = new TestMediaCall(built, true, false, codec)) { + call.negotiate(); + + RTCRtpReceiver receiver = call.getReceiver("video"); + VideoTrack track = (VideoTrack) receiver.getTrack(); + VideoTrackSink sink = frame -> { + frame.release(); + received.countDown(); + }; + track.addSink(sink); + + call.awaitConnected(); + call.startMedia(); + + assertTrue(received.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), "too few frames received"); + + track.removeSink(sink); + receiver.dispose(); + } + } + + private PeerConnectionFactory testCodecFactory(VideoEncoderFactory encoderFactory, + VideoDecoderFactory decoderFactory) { + return PeerConnectionFactory.builder() + .setAudioDeviceModule(audioDevModule) + .setVideoEncoderFactory(encoderFactory) + .setVideoDecoderFactory(decoderFactory) + .build(); + } + + private static List names(List codecs) { + return codecs.stream() + .map(codec -> codec instanceof VideoCodecInfo + ? ((VideoCodecInfo) codec).getName() + : ((RTCRtpCodecCapability) codec).getName()) + .collect(Collectors.toList()); + } + + + + interface EncodeListener { + + void onEncode(VideoFrame frame, int luma); + + } + + + + /** + * Offers the test codec next to the built-in ones. + */ + static class TestEncoderFactory implements VideoEncoderFactory { + + final DefaultVideoEncoderFactory builtIn = new DefaultVideoEncoderFactory(); + final AtomicInteger created = new AtomicInteger(); + final AtomicReference last = new AtomicReference<>(); + + volatile EncodeListener onEncode; + + + @Override + public List getSupportedCodecs() { + List codecs = new ArrayList<>(); + codecs.add(new VideoCodecInfo(TEST_CODEC)); + codecs.addAll(builtIn.getSupportedCodecs()); + return codecs; + } + + @Override + public VideoEncoder createEncoder(VideoCodecInfo info) { + created.incrementAndGet(); + + if (!TEST_CODEC.equalsIgnoreCase(info.getName())) { + return builtIn.createEncoder(info); + } + + TestEncoder encoder = new TestEncoder(this); + last.set(encoder); + return encoder; + } + + } + + + + /** + * Offers the test codec next to the built-in ones. + */ + static class TestDecoderFactory implements VideoDecoderFactory { + + final DefaultVideoDecoderFactory builtIn = new DefaultVideoDecoderFactory(); + final AtomicInteger created = new AtomicInteger(); + final AtomicReference last = new AtomicReference<>(); + + volatile boolean javaBuffers; + + + @Override + public List getSupportedCodecs() { + List codecs = new ArrayList<>(); + codecs.add(new VideoCodecInfo(TEST_CODEC, Collections.emptyMap())); + codecs.addAll(builtIn.getSupportedCodecs()); + return codecs; + } + + @Override + public VideoDecoder createDecoder(VideoCodecInfo info) { + created.incrementAndGet(); + + if (!TEST_CODEC.equalsIgnoreCase(info.getName())) { + return builtIn.createDecoder(info); + } + + TestDecoder decoder = new TestDecoder(javaBuffers); + last.set(decoder); + return decoder; + } + + } + + + + /** + * "Encodes" a frame into its size and its first luma sample, every frame + * a key frame, so that the receiver can tell which frame it got. + */ + static class TestEncoder implements VideoEncoder { + + private final TestEncoderFactory factory; + + final AtomicInteger rates = new AtomicInteger(); + final AtomicReference released = new AtomicReference<>(false); + + volatile Settings settings; + + private Callback callback; + + + TestEncoder(TestEncoderFactory factory) { + this.factory = factory; + } + + @Override + public VideoCodecStatus initEncode(Settings settings, Callback callback) { + this.settings = settings; + this.callback = callback; + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus release() { + released.set(true); + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus encode(VideoFrame frame, EncodeInfo info) { + int width = frame.buffer.getWidth(); + int height = frame.buffer.getHeight(); + int luma = frame.buffer.toI420().getDataY().get(0) & 0xFF; + + EncodeListener listener = factory.onEncode; + + if (listener != null) { + listener.onEncode(frame, luma); + } + + ByteBuffer payload = ByteBuffer.allocateDirect(9); + payload.putInt(width).putInt(height).put((byte) luma); + payload.flip(); + + callback.onEncodedFrame(EncodedImage.builder() + .setBuffer(payload) + .setEncodedWidth(width) + .setEncodedHeight(height) + .setCaptureTimeNs(frame.timestampNs) + .setFrameType(EncodedImage.FrameType.KEY) + .build()); + + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus setRates(RateControlParameters parameters) { + if (parameters.bitrate.getSum() > 0) { + rates.incrementAndGet(); + } + return VideoCodecStatus.OK; + } + + @Override + public ScalingSettings getScalingSettings() { + // The payload has no QP to scale by. + return ScalingSettings.OFF; + } + + } + + + + /** + * Decodes what {@link TestEncoder} produces into a frame of the encoded + * size, filled with the encoded luma sample. + */ + static class TestDecoder implements VideoDecoder { + + final AtomicReference released = new AtomicReference<>(false); + + private final boolean javaBuffers; + + private Callback callback; + + + TestDecoder(boolean javaBuffers) { + this.javaBuffers = javaBuffers; + } + + @Override + public VideoCodecStatus initDecode(Settings settings, Callback callback) { + this.callback = callback; + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus release() { + released.set(true); + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus decode(EncodedImage image) { + ByteBuffer payload = image.getBuffer(); + + if (payload.remaining() != 9) { + return VideoCodecStatus.ERR_PARAMETER; + } + + int width = payload.getInt(); + int height = payload.getInt(); + byte luma = payload.get(); + + I420Buffer buffer = javaBuffers + ? new JavaI420Buffer(width, height) + : NativeI420Buffer.allocate(width, height); + ByteBuffer y = buffer.getDataY(); + ByteBuffer u = buffer.getDataU(); + ByteBuffer v = buffer.getDataV(); + + for (int i = 0; i < y.capacity(); i++) { + y.put(i, luma); + } + for (int i = 0; i < u.capacity(); i++) { + u.put(i, (byte) 128); + v.put(i, (byte) 128); + } + + VideoFrame frame = new VideoFrame(buffer, image.getCaptureTimeNs()); + callback.onDecodedFrame(frame, null, null); + frame.release(); + + return VideoCodecStatus.OK; + } + + } + + + + /** + * A frame buffer that lives in Java alone. + */ + static class JavaI420Buffer implements I420Buffer { + + private final int width; + private final int height; + private final ByteBuffer dataY; + private final ByteBuffer dataU; + private final ByteBuffer dataV; + + + JavaI420Buffer(int width, int height) { + int chromaWidth = (width + 1) / 2; + int chromaHeight = (height + 1) / 2; + + this.width = width; + this.height = height; + this.dataY = ByteBuffer.allocateDirect(width * height); + this.dataU = ByteBuffer.allocateDirect(chromaWidth * chromaHeight); + this.dataV = ByteBuffer.allocateDirect(chromaWidth * chromaHeight); + } + + @Override + public ByteBuffer getDataY() { + return dataY.slice(); + } + + @Override + public ByteBuffer getDataU() { + return dataU.slice(); + } + + @Override + public ByteBuffer getDataV() { + return dataV.slice(); + } + + @Override + public int getStrideY() { + return width; + } + + @Override + public int getStrideU() { + return (width + 1) / 2; + } + + @Override + public int getStrideV() { + return (width + 1) / 2; + } + + @Override + public int getWidth() { + return width; + } + + @Override + public int getHeight() { + return height; + } + + @Override + public I420Buffer toI420() { + return this; + } + + @Override + public VideoFrameBuffer cropAndScale(int cropX, int cropY, int cropWidth, + int cropHeight, int scaleWidth, int scaleHeight) { + throw new UnsupportedOperationException(); + } + + @Override + public void retain() { + } + + @Override + public void release() { + } + + } + +}