From 1d531c588c44635ac34cad68803573413c22a969 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 28 Sep 2026 23:13:35 +0200 Subject: [PATCH 1/2] fix: keep a Java log sink from throwing into WebRTC WebRTC calls log sinks while it holds its global logging lock, and it is built without unwinding. The Java log sink bridge ended with ExceptionCheck(), which throws a C++ exception when the sink throws, and also when the code that logged had a Java exception pending, since no JNI call may be made then. That exception passed through WebRTC's frames without releasing the lock, and the next thread to log blocked forever, typically hanging RTCPeerConnection.close(). The bridge now sets a pending exception aside and restores it afterwards, reports what the sink throws, and never throws itself. --- webrtc-jni/src/main/cpp/src/rtc/LogSink.cpp | 35 ++++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/webrtc-jni/src/main/cpp/src/rtc/LogSink.cpp b/webrtc-jni/src/main/cpp/src/rtc/LogSink.cpp index 16a5c0881..355dca32a 100644 --- a/webrtc-jni/src/main/cpp/src/rtc/LogSink.cpp +++ b/webrtc-jni/src/main/cpp/src/rtc/LogSink.cpp @@ -39,12 +39,39 @@ namespace jni return; } - JavaLocalRef jSeverity = JavaEnums::toJava(env, severity); - JavaLocalRef jMessage = JavaString::toJava(env, message); + // WebRTC holds its global logging lock while it calls a sink, and it + // is built without unwinding, so nothing may throw from here: an + // exception passing through WebRTC's frames would leave that lock + // held and every thread that logs next would block forever. + // + // The code that logged may have a Java exception pending, which no + // JNI call may be made with. It is set aside and restored after. + jthrowable pending = env->ExceptionOccurred(); - env->CallVoidMethod(javaSink, javaClass->onLogMessage, jSeverity.get(), jMessage.get()); + if (pending != nullptr) { + env->ExceptionClear(); + } + + try { + JavaLocalRef jSeverity = JavaEnums::toJava(env, severity); + JavaLocalRef jMessage = JavaString::toJava(env, message); - ExceptionCheck(env); + env->CallVoidMethod(javaSink, javaClass->onLogMessage, jSeverity.get(), jMessage.get()); + + // What the sink threw is reported, but not rethrown. + if (env->ExceptionCheck()) { + env->ExceptionDescribe(); + env->ExceptionClear(); + } + } + catch (...) { + env->ExceptionClear(); + } + + if (pending != nullptr) { + env->Throw(pending); + env->DeleteLocalRef(pending); + } } LogSink::JavaLogSinkClass::JavaLogSinkClass(JNIEnv * env) From e56f7a79a5e987f60881b477ebb530078ab43db4 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Mon, 28 Sep 2026 23:13:57 +0200 Subject: [PATCH 2/2] feat: add custom video codec factories A PeerConnectionFactory can now be given its own video encoder and decoder factories, which decide what video codecs it sends and receives. PeerConnectionFactory.builder() sets them, along with everything the constructors take. The new package dev.onvoid.webrtc.media.video.codec has VideoEncoder and VideoDecoder, for codecs implemented in Java, their factories, and EncodedImage and VideoCodecInfo. DefaultVideoEncoderFactory and DefaultVideoDecoderFactory provide the built-in codecs, so a factory can add its own codec while keeping them; the NativeVideoEncoder and NativeVideoDecoder they hand out make WebRTC create the built-in codec natively. A codec name WebRTC does not know is negotiated and sent with the generic packetization. Native wrappers, ported from the Android SDK, run the Java codecs. Frames reach a Java encoder without a copy, encoded images a Java decoder as a read-only view; the callbacks a codec hands its output to reach the wrapper only until the codec is released. Whatever a Java codec or factory throws is logged and treated as an error, and never reaches WebRTC. Decoded frames may use any I420Buffer with direct buffers. --- README.md | 1 + docs/.vitepress/sidebar.ts | 1 + docs/guide/advanced/field-trials.md | 9 + docs/guide/advanced/video-codecs.md | 168 +++++ docs/guide/index.md | 1 + docs/index.md | 2 + webrtc-jni/src/main/cpp/CMakeLists.txt | 2 + .../include/JNI_DefaultVideoDecoderFactory.h | 36 + .../include/JNI_DefaultVideoEncoderFactory.h | 36 + .../include/JNI_NativeVideoDecoderCallback.h | 36 + .../include/JNI_NativeVideoEncoderCallback.h | 36 + .../cpp/include/JNI_PeerConnectionFactory.h | 4 +- webrtc-jni/src/main/cpp/include/JNI_WebRTC.h | 1 + .../src/main/cpp/include/api/VideoFrame.h | 29 + .../video/codec/DefaultVideoCodecFactories.h | 34 + .../include/media/video/codec/EncodedImage.h | 61 ++ .../media/video/codec/JavaLocalFrame.h | 43 ++ .../media/video/codec/VideoCodecInfo.h | 58 ++ .../media/video/codec/VideoCodecUtils.h | 48 ++ .../video/codec/VideoDecoderFactoryWrapper.h | 73 ++ .../media/video/codec/VideoDecoderWrapper.h | 126 ++++ .../video/codec/VideoEncoderFactoryWrapper.h | 73 ++ .../media/video/codec/VideoEncoderWrapper.h | 164 ++++ .../src/JNI_DefaultVideoDecoderFactory.cpp | 35 + .../src/JNI_DefaultVideoEncoderFactory.cpp | 35 + .../src/JNI_NativeVideoDecoderCallback.cpp | 46 ++ .../src/JNI_NativeVideoEncoderCallback.cpp | 34 + .../cpp/src/JNI_PeerConnectionFactory.cpp | 55 +- .../src/main/cpp/src/api/VideoFrame.cpp | 94 +++ .../codec/DefaultVideoCodecFactories.cpp | 64 ++ .../src/media/video/codec/EncodedImage.cpp | 115 +++ .../src/media/video/codec/JavaLocalFrame.cpp | 37 + .../src/media/video/codec/VideoCodecInfo.cpp | 160 ++++ .../src/media/video/codec/VideoCodecUtils.cpp | 71 ++ .../codec/VideoDecoderFactoryWrapper.cpp | 97 +++ .../media/video/codec/VideoDecoderWrapper.cpp | 401 ++++++++++ .../codec/VideoEncoderFactoryWrapper.cpp | 97 +++ .../media/video/codec/VideoEncoderWrapper.cpp | 704 ++++++++++++++++++ .../onvoid/webrtc/PeerConnectionFactory.java | 175 ++++- .../codec/DefaultVideoDecoderFactory.java | 92 +++ .../codec/DefaultVideoEncoderFactory.java | 92 +++ .../media/video/codec/EncodedImage.java | 351 +++++++++ .../media/video/codec/NativeVideoDecoder.java | 71 ++ .../codec/NativeVideoDecoderCallback.java | 68 ++ .../media/video/codec/NativeVideoEncoder.java | 78 ++ .../codec/NativeVideoEncoderCallback.java | 63 ++ .../media/video/codec/VideoCodecInfo.java | 159 ++++ .../media/video/codec/VideoCodecStatus.java | 81 ++ .../media/video/codec/VideoDecoder.java | 151 ++++ .../video/codec/VideoDecoderFactory.java | 56 ++ .../media/video/codec/VideoEncoder.java | 510 +++++++++++++ .../video/codec/VideoEncoderFactory.java | 75 ++ webrtc/src/main/java/module-info.java | 1 + .../java/dev/onvoid/webrtc/TestMediaCall.java | 28 +- .../onvoid/webrtc/VideoCodecFactoryTests.java | 685 +++++++++++++++++ 55 files changed, 5764 insertions(+), 59 deletions(-) create mode 100644 docs/guide/advanced/video-codecs.md create mode 100644 webrtc-jni/src/main/cpp/include/JNI_DefaultVideoDecoderFactory.h create mode 100644 webrtc-jni/src/main/cpp/include/JNI_DefaultVideoEncoderFactory.h create mode 100644 webrtc-jni/src/main/cpp/include/JNI_NativeVideoDecoderCallback.h create mode 100644 webrtc-jni/src/main/cpp/include/JNI_NativeVideoEncoderCallback.h create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/EncodedImage.h create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/JavaLocalFrame.h create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/VideoCodecInfo.h create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/VideoCodecUtils.h create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderFactoryWrapper.h create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderWrapper.h create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderFactoryWrapper.h create mode 100644 webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderWrapper.h create mode 100644 webrtc-jni/src/main/cpp/src/JNI_DefaultVideoDecoderFactory.cpp create mode 100644 webrtc-jni/src/main/cpp/src/JNI_DefaultVideoEncoderFactory.cpp create mode 100644 webrtc-jni/src/main/cpp/src/JNI_NativeVideoDecoderCallback.cpp create mode 100644 webrtc-jni/src/main/cpp/src/JNI_NativeVideoEncoderCallback.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/EncodedImage.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/JavaLocalFrame.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/VideoCodecInfo.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/VideoCodecUtils.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderFactoryWrapper.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderWrapper.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderFactoryWrapper.cpp create mode 100644 webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderWrapper.cpp create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/DefaultVideoDecoderFactory.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/DefaultVideoEncoderFactory.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/EncodedImage.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoDecoder.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoDecoderCallback.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoEncoder.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoEncoderCallback.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoCodecInfo.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoCodecStatus.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoDecoder.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoDecoderFactory.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoEncoder.java create mode 100644 webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoEncoderFactory.java create mode 100644 webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java diff --git a/README.md b/README.md index 827525c1c..444e2c2fb 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ For more detailed information, check out the documentation: - [Media Files](https://jrtc.dev/guide/media/media-files) - Sending video and audio files with the media module - [Media Recording](https://jrtc.dev/guide/media/media-recording) - Recording calls into media files with the media module - [Encoded Transforms](https://jrtc.dev/guide/advanced/encoded-transforms) - Reading and changing encoded frames, e.g. for end-to-end encryption +- [Video Codecs](https://jrtc.dev/guide/advanced/video-codecs) - Adding video codecs implemented in Java, or limiting the built-in ones - [Build Notes](https://jrtc.dev/guide/build) - Instructions for building the library from source ## License diff --git a/docs/.vitepress/sidebar.ts b/docs/.vitepress/sidebar.ts index 55a77a742..ca87a3799 100644 --- a/docs/.vitepress/sidebar.ts +++ b/docs/.vitepress/sidebar.ts @@ -74,6 +74,7 @@ function sidebarGuide(): DefaultTheme.SidebarItem[] { items: [ { text: 'Field Trials', link: '/advanced/field-trials' }, { text: 'Encoded Transforms', link: '/advanced/encoded-transforms' }, + { text: 'Video Codecs', link: '/advanced/video-codecs' }, ], }, ] diff --git a/docs/guide/advanced/field-trials.md b/docs/guide/advanced/field-trials.md index e92e52cd8..ff9cd8253 100644 --- a/docs/guide/advanced/field-trials.md +++ b/docs/guide/advanced/field-trials.md @@ -24,6 +24,15 @@ PeerConnectionFactory factory = new PeerConnectionFactory(fieldTrials, audioProc PeerConnectionFactory factory = new PeerConnectionFactory(fieldTrials, audioModule, audioProcessing); ``` +`PeerConnectionFactory.builder()` sets them too, next to anything else a factory is created with, such as its [video codecs](/guide/advanced/video-codecs): + +```java +PeerConnectionFactory factory = PeerConnectionFactory.builder() + .setFieldTrials(fieldTrials) + .setAudioDeviceModule(audioModule) + .build(); +``` + ## Constraints Each key and value must be non-null and non-empty. Passing a map that violates this constraint throws an exception when the `PeerConnectionFactory` is created. diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md new file mode 100644 index 000000000..9c88d61eb --- /dev/null +++ b/docs/guide/advanced/video-codecs.md @@ -0,0 +1,168 @@ +# Video Codecs + +A `PeerConnectionFactory` decides which video codecs it can send and receive through a video encoder factory and a video decoder factory. Unless told otherwise it uses the codecs built into the library. This guide explains how to replace or extend them: to add a codec WebRTC does not have, to use a hardware encoder reached through another library, or to limit the codecs a factory offers. + +The API lives in the `dev.onvoid.webrtc.media.video.codec` package. + +## Built-in Codecs + +`DefaultVideoEncoderFactory` and `DefaultVideoDecoderFactory` provide the codecs built into the library. On Windows and Linux these are VP8, VP9, AV1 and H.264 in software; on macOS they are those of WebRTC's default factories, with H.264 through VideoToolbox. Ask a factory for the exact list: + +```java +for (VideoCodecInfo codec : new DefaultVideoEncoderFactory().getSupportedCodecs()) { + System.out.println(codec.getName() + " " + codec.getParameters()); +} +``` + +The encoders and decoders these factories create are `NativeVideoEncoder`s and `NativeVideoDecoder`s. They are placeholders that make WebRTC create the built-in codec, which then runs entirely inside WebRTC, so their methods are not to be called from Java. + +## Setting the Factories + +The codec factories are set with `PeerConnectionFactory.builder()`, which also takes everything the constructors do: + +```java +PeerConnectionFactory factory = PeerConnectionFactory.builder() + .setAudioDeviceModule(audioModule) + .setVideoEncoderFactory(new MyEncoderFactory()) + .setVideoDecoderFactory(new MyDecoderFactory()) + .build(); +``` + +A factory that is not set is the built-in one. The supported codecs are asked for once, when the `PeerConnectionFactory` is built; an exception thrown there fails `build()`. Encoders and decoders are created later, from WebRTC threads, one per stream. + +Both ends have to agree on a codec, so a codec added on the sending side also has to be offered by the decoder factory of the receiving side. + +## Adding a Codec + +A factory of one's own usually hands out its own codec next to the built-in ones, by delegating to a default factory: + +```java +class MyEncoderFactory implements VideoEncoderFactory { + + private final DefaultVideoEncoderFactory builtIn = new DefaultVideoEncoderFactory(); + + @Override + public List getSupportedCodecs() { + List codecs = new ArrayList<>(); + codecs.add(new VideoCodecInfo("X-MYCODEC")); + codecs.addAll(builtIn.getSupportedCodecs()); + return codecs; + } + + @Override + public VideoEncoder createEncoder(VideoCodecInfo info) { + if (info.getName().equalsIgnoreCase("X-MYCODEC")) { + return new MyEncoder(); + } + return builtIn.createEncoder(info); + } +} +``` + +The list is in order of preference. A codec name WebRTC does not know, such as `X-MYCODEC`, is negotiated like any other and sent with the generic RTP packetization. `RTCRtpTransceiver.setCodecPreferences()` picks among the negotiated codecs per transceiver. + +## Implementing an Encoder + +WebRTC calls `initEncode()`, then `encode()` for each frame and `setRates()` whenever the target bitrate changes, and finally `release()`. It may initialize a released encoder again, possibly with another frame size. These calls come from one WebRTC thread at a time. + +```java +class MyEncoder implements VideoEncoder { + + private Callback callback; + + @Override + public VideoCodecStatus initEncode(Settings settings, Callback callback) { + this.callback = callback; + // Set up the encoder for settings.width x settings.height at + // settings.startBitrateKbps. + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus encode(VideoFrame frame, EncodeInfo info) { + I420Buffer pixels = frame.buffer.toI420(); + ByteBuffer payload = encodeSomehow(pixels, info.isKeyFrameRequested()); + + callback.onEncodedFrame(EncodedImage.builder() + .setBuffer(payload) + .setEncodedWidth(pixels.getWidth()) + .setEncodedHeight(pixels.getHeight()) + .setCaptureTimeNs(frame.timestampNs) + .setFrameType(info.isKeyFrameRequested() + ? EncodedImage.FrameType.KEY + : EncodedImage.FrameType.DELTA) + .build()); + + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus setRates(RateControlParameters parameters) { + // Aim for parameters.bitrate.getSum() bps at parameters.framerateFps. + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus release() { + return VideoCodecStatus.OK; + } +} +``` + +Things to keep in mind: + +- The capture time of an `EncodedImage` must be the `timestampNs` of the frame it encodes. That is how WebRTC matches output to input; an image with any other capture time is dropped. +- The frame passed to `encode()` is valid only until the method returns. An encoder that works asynchronously calls `frame.retain()` and later `frame.release()`, and hands its output to the callback from its own thread, in encoding order. The pixels must not be changed, since other consumers may share them. +- `onEncodedFrame()` copies the payload, so the buffer may be reused once it returns. Frames handed over after `release()` are ignored. +- `getScalingSettings()`, `getEncoderInfo()`, `getResolutionBitrateLimits()`, `getImplementationName()` and `isHardwareEncoder()` have defaults and may be overridden. The default scaling settings use WebRTC's quantizer thresholds for VP8, VP9 and H.264; a custom codec without a quantizer should return `ScalingSettings.OFF`. + +## Implementing a Decoder + +A decoder follows the same pattern with `initDecode()`, `decode()` and `release()`: + +```java +class MyDecoder implements VideoDecoder { + + private Callback callback; + + @Override + public VideoCodecStatus initDecode(Settings settings, Callback callback) { + this.callback = callback; + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus decode(EncodedImage image) { + NativeI420Buffer buffer = decodeSomehow(image.getBuffer()); + VideoFrame frame = new VideoFrame(buffer, image.getCaptureTimeNs()); + + callback.onDecodedFrame(frame, null, null); + frame.release(); + + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus release() { + return VideoCodecStatus.OK; + } +} +``` + +- The image and its buffer are read-only and valid only until `decode()` returns; copy the data to keep it longer. +- A decoded frame must carry the capture time of the image it was decoded from as its timestamp. +- `onDecodedFrame()` takes a reference of its own, so the caller still releases the frame it created. Frame buffers other than `NativeI420Buffer` are copied, and have to provide direct byte buffers. + +## Errors + +Whatever an encoder, decoder or factory method throws is logged and treated as `VideoCodecStatus.ERROR`, and a factory that throws from `createEncoder()` or `createDecoder()` creates no codec for that stream. On an error other than `FALLBACK_SOFTWARE` and `UNINITIALIZED`, WebRTC releases the codec and initializes it again. + +## Related API + +- `PeerConnectionFactory.Builder` — sets the video encoder and decoder factories. +- `VideoEncoderFactory`, `VideoDecoderFactory` — create the codecs of a factory. +- `VideoEncoder`, `VideoDecoder`, `EncodedImage` — codecs implemented in Java. +- `DefaultVideoEncoderFactory`, `DefaultVideoDecoderFactory` — the built-in codecs. +- `RTCRtpTransceiver.setCodecPreferences()` — chooses among the negotiated codecs. + +For the full API, see the JavaDoc of the `dev.onvoid.webrtc.media.video.codec` package. diff --git a/docs/guide/index.md b/docs/guide/index.md index c478919ab..84c248ba1 100644 --- a/docs/guide/index.md +++ b/docs/guide/index.md @@ -41,6 +41,7 @@ This section provides detailed guides for various features of the webrtc-java li - [Field Trials](/guide/advanced/field-trials) - Enabling experimental features and tuning WebRTC internals - [Encoded Transforms](/guide/advanced/encoded-transforms) - Reading and changing encoded frames, e.g. for end-to-end encryption +- [Video Codecs](/guide/advanced/video-codecs) - Adding video codecs implemented in Java, or limiting the built-in ones ## Additional Resources diff --git a/docs/index.md b/docs/index.md index 1dd4739c9..12c2f867d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -32,6 +32,8 @@ features: details: Record what a peer connection sends or receives into MKV, WebM or MP4 files. Encoded frames go into the file as they are, without re-encoding, so recording costs next to no CPU and keeps the exact quality of the call. - title: End-to-End Encryption details: Encoded transforms let Java code read, change or drop every encoded frame between encoder and network, like insertable streams in the browser; the building block for end-to-end encryption, frame metadata and stream analysis. + - title: Custom Video Codecs + details: Plug video encoders and decoders written in Java into WebRTC, next to the built-in VP8, VP9, AV1 and H.264, to bring in a codec of your own or a hardware encoder from another library. - title: Screen Sharing details: Share application windows or the full desktop with minimal setup; integrate screen capture streams like any other media stream. - title: Data Channels diff --git a/webrtc-jni/src/main/cpp/CMakeLists.txt b/webrtc-jni/src/main/cpp/CMakeLists.txt index 45f27c818..2c83e39dc 100644 --- a/webrtc-jni/src/main/cpp/CMakeLists.txt +++ b/webrtc-jni/src/main/cpp/CMakeLists.txt @@ -55,6 +55,7 @@ file(GLOB SOURCES_MEDIA "src/media/*.cpp") file(GLOB SOURCES_MEDIA_AUDIO "src/media/audio/*.cpp") file(GLOB SOURCES_MEDIA_AUDIO_OS "src/media/audio/${SOURCE_TARGET}/*.cpp") file(GLOB SOURCES_MEDIA_VIDEO "src/media/video/*.cpp") +file(GLOB SOURCES_MEDIA_VIDEO_CODEC "src/media/video/codec/*.cpp") file(GLOB SOURCES_MEDIA_VIDEO_DESKTOP "src/media/video/desktop/*.cpp") file(GLOB SOURCES_MEDIA_VIDEO_DESKTOP_OS "src/media/video/desktop/${SOURCE_TARGET}/*.cpp") file(GLOB SOURCES_MEDIA_VIDEO_OS "src/media/video/${SOURCE_TARGET}/*.cpp") @@ -69,6 +70,7 @@ list(APPEND SOURCES ${SOURCES_MEDIA_AUDIO} ${SOURCES_MEDIA_AUDIO_OS} ${SOURCES_MEDIA_VIDEO} + ${SOURCES_MEDIA_VIDEO_CODEC} ${SOURCES_MEDIA_VIDEO_DESKTOP} ${SOURCES_MEDIA_VIDEO_DESKTOP_OS} ${SOURCES_MEDIA_VIDEO_OS} diff --git a/webrtc-jni/src/main/cpp/include/JNI_DefaultVideoDecoderFactory.h b/webrtc-jni/src/main/cpp/include/JNI_DefaultVideoDecoderFactory.h new file mode 100644 index 000000000..175e20f30 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/JNI_DefaultVideoDecoderFactory.h @@ -0,0 +1,36 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +/* Header for class dev_onvoid_webrtc_media_video_codec_DefaultVideoDecoderFactory */ + +#ifndef _Included_dev_onvoid_webrtc_media_video_codec_DefaultVideoDecoderFactory +#define _Included_dev_onvoid_webrtc_media_video_codec_DefaultVideoDecoderFactory +#ifdef __cplusplus +extern "C" { +#endif + /* + * Class: dev_onvoid_webrtc_media_video_codec_DefaultVideoDecoderFactory + * Method: getSupportedCodecsInternal + * Signature: ()[Ldev/onvoid/webrtc/media/video/codec/VideoCodecInfo; + */ + JNIEXPORT jobjectArray JNICALL Java_dev_onvoid_webrtc_media_video_codec_DefaultVideoDecoderFactory_getSupportedCodecsInternal + (JNIEnv *, jclass); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/webrtc-jni/src/main/cpp/include/JNI_DefaultVideoEncoderFactory.h b/webrtc-jni/src/main/cpp/include/JNI_DefaultVideoEncoderFactory.h new file mode 100644 index 000000000..b70ee737f --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/JNI_DefaultVideoEncoderFactory.h @@ -0,0 +1,36 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +/* Header for class dev_onvoid_webrtc_media_video_codec_DefaultVideoEncoderFactory */ + +#ifndef _Included_dev_onvoid_webrtc_media_video_codec_DefaultVideoEncoderFactory +#define _Included_dev_onvoid_webrtc_media_video_codec_DefaultVideoEncoderFactory +#ifdef __cplusplus +extern "C" { +#endif + /* + * Class: dev_onvoid_webrtc_media_video_codec_DefaultVideoEncoderFactory + * Method: getSupportedCodecsInternal + * Signature: ()[Ldev/onvoid/webrtc/media/video/codec/VideoCodecInfo; + */ + JNIEXPORT jobjectArray JNICALL Java_dev_onvoid_webrtc_media_video_codec_DefaultVideoEncoderFactory_getSupportedCodecsInternal + (JNIEnv *, jclass); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/webrtc-jni/src/main/cpp/include/JNI_NativeVideoDecoderCallback.h b/webrtc-jni/src/main/cpp/include/JNI_NativeVideoDecoderCallback.h new file mode 100644 index 000000000..827c0c3c3 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/JNI_NativeVideoDecoderCallback.h @@ -0,0 +1,36 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +/* Header for class dev_onvoid_webrtc_media_video_codec_NativeVideoDecoderCallback */ + +#ifndef _Included_dev_onvoid_webrtc_media_video_codec_NativeVideoDecoderCallback +#define _Included_dev_onvoid_webrtc_media_video_codec_NativeVideoDecoderCallback +#ifdef __cplusplus +extern "C" { +#endif + /* + * Class: dev_onvoid_webrtc_media_video_codec_NativeVideoDecoderCallback + * Method: onDecodedFrame + * Signature: (JLdev/onvoid/webrtc/media/video/VideoFrame;II)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_codec_NativeVideoDecoderCallback_onDecodedFrame + (JNIEnv *, jclass, jlong, jobject, jint, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/webrtc-jni/src/main/cpp/include/JNI_NativeVideoEncoderCallback.h b/webrtc-jni/src/main/cpp/include/JNI_NativeVideoEncoderCallback.h new file mode 100644 index 000000000..38d26c6d2 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/JNI_NativeVideoEncoderCallback.h @@ -0,0 +1,36 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +/* Header for class dev_onvoid_webrtc_media_video_codec_NativeVideoEncoderCallback */ + +#ifndef _Included_dev_onvoid_webrtc_media_video_codec_NativeVideoEncoderCallback +#define _Included_dev_onvoid_webrtc_media_video_codec_NativeVideoEncoderCallback +#ifdef __cplusplus +extern "C" { +#endif + /* + * Class: dev_onvoid_webrtc_media_video_codec_NativeVideoEncoderCallback + * Method: onEncodedFrame + * Signature: (JLdev/onvoid/webrtc/media/video/codec/EncodedImage;)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_codec_NativeVideoEncoderCallback_onEncodedFrame + (JNIEnv *, jclass, jlong, jobject); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h b/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h index eeff18380..c5b40c021 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h +++ b/webrtc-jni/src/main/cpp/include/JNI_PeerConnectionFactory.h @@ -82,10 +82,10 @@ extern "C" { /* * Class: dev_onvoid_webrtc_PeerConnectionFactory * Method: initialize - * Signature: (Ljava/util/Map;Ldev/onvoid/webrtc/media/audio/AudioDeviceModuleBase;Ldev/onvoid/webrtc/media/audio/AudioProcessing;)V + * Signature: (Ljava/util/Map;Ldev/onvoid/webrtc/media/audio/AudioDeviceModuleBase;Ldev/onvoid/webrtc/media/audio/AudioProcessing;Ldev/onvoid/webrtc/media/video/codec/VideoEncoderFactory;Ldev/onvoid/webrtc/media/video/codec/VideoDecoderFactory;)V */ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_initialize - (JNIEnv *, jobject, jobject, jobject, jobject); + (JNIEnv *, jobject, jobject, jobject, jobject, jobject, jobject); #ifdef __cplusplus } diff --git a/webrtc-jni/src/main/cpp/include/JNI_WebRTC.h b/webrtc-jni/src/main/cpp/include/JNI_WebRTC.h index 0eb3888c7..6d5d6b568 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_WebRTC.h +++ b/webrtc-jni/src/main/cpp/include/JNI_WebRTC.h @@ -23,6 +23,7 @@ #define PKG_MEDIA "dev/onvoid/webrtc/media/" #define PKG_AUDIO "dev/onvoid/webrtc/media/audio/" #define PKG_VIDEO "dev/onvoid/webrtc/media/video/" +#define PKG_CODEC "dev/onvoid/webrtc/media/video/codec/" #define PKG_DESKTOP "dev/onvoid/webrtc/media/video/desktop/" #define BOOLEAN_SIG "Ljava/lang/Boolean;" diff --git a/webrtc-jni/src/main/cpp/include/api/VideoFrame.h b/webrtc-jni/src/main/cpp/include/api/VideoFrame.h index 6be8e8847..6b79e2550 100644 --- a/webrtc-jni/src/main/cpp/include/api/VideoFrame.h +++ b/webrtc-jni/src/main/cpp/include/api/VideoFrame.h @@ -30,6 +30,17 @@ namespace jni namespace VideoFrame { webrtc::VideoFrame toNative(JNIEnv * env, const JavaRef & javaFrame); + + // Wraps the frame for Java without copying its pixels, converting + // them to I420 first if they are in another format. The Java frame + // holds a reference to the pixel buffer, which it gives up when it + // is released. Returns null if the pixels cannot be converted. + JavaLocalRef toJava(JNIEnv * env, const webrtc::VideoFrame & frame); + + // Returns the native pixel buffer of a Java VideoFrameBuffer. A + // NativeI420Buffer is shared; any other buffer is converted to I420 + // in Java and copied. Returns null if it has no readable pixels. + webrtc::scoped_refptr toNativeBuffer(JNIEnv * env, const JavaRef & javaBuffer); } namespace I420Buffer @@ -47,6 +58,24 @@ namespace jni jfieldID buffer; jfieldID rotation; jfieldID timestampNs; + jmethodID release; + }; + + class JavaI420BufferClass : public JavaClass + { + public: + explicit JavaI420BufferClass(JNIEnv * env); + + jclass cls; + jmethodID toI420; + jmethodID getWidth; + jmethodID getHeight; + jmethodID getDataY; + jmethodID getDataU; + jmethodID getDataV; + jmethodID getStrideY; + jmethodID getStrideU; + jmethodID getStrideV; }; class JavaNativeI420BufferClass : public JavaClass diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h b/webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h new file mode 100644 index 000000000..c3d43f524 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/DefaultVideoCodecFactories.h @@ -0,0 +1,34 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_DEFAULT_VIDEO_CODEC_FACTORIES_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_DEFAULT_VIDEO_CODEC_FACTORIES_H_ + +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" + +#include + +namespace jni +{ + // The video codecs built into this library: VP8, VP9, AV1 and H.264 in + // software, or on macOS those of WebRTC's default Objective-C factories, + // which use VideoToolbox. + std::unique_ptr CreateDefaultVideoEncoderFactory(); + std::unique_ptr CreateDefaultVideoDecoderFactory(); +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/EncodedImage.h b/webrtc-jni/src/main/cpp/include/media/video/codec/EncodedImage.h new file mode 100644 index 000000000..1299273d7 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/EncodedImage.h @@ -0,0 +1,61 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_ENCODED_IMAGE_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_ENCODED_IMAGE_H_ + +#include "JavaClass.h" +#include "JavaRef.h" + +#include "api/video/encoded_image.h" + +#include + +#include + +namespace jni +{ + namespace EncodedImage + { + // Wraps the image for a Java decoder. The payload is not copied: the + // Java buffer is read-only and points into the image, so it is valid + // only for as long as the image is. + JavaLocalRef toJava(JNIEnv * env, const webrtc::EncodedImage & image); + + // Converts the image a Java encoder produced, copying its payload. + webrtc::EncodedImage toNative(JNIEnv * env, const JavaRef & image); + + int64_t getCaptureTimeNs(JNIEnv * env, const JavaRef & image); + + class JavaEncodedImageClass : public JavaClass + { + public: + explicit JavaEncodedImageClass(JNIEnv * env); + + jclass cls; + jmethodID fromNative; + jmethodID getDirectPayload; + jmethodID getFrameTypeIndex; + jmethodID getQpOrUnknown; + jfieldID encodedWidth; + jfieldID encodedHeight; + jfieldID captureTimeNs; + jfieldID rotation; + }; + } +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/JavaLocalFrame.h b/webrtc-jni/src/main/cpp/include/media/video/codec/JavaLocalFrame.h new file mode 100644 index 000000000..23c46af6b --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/JavaLocalFrame.h @@ -0,0 +1,43 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_JAVA_LOCAL_FRAME_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_JAVA_LOCAL_FRAME_H_ + +#include + +namespace jni +{ + // Scopes the local references made on a thread WebRTC owns. Such a + // thread stays attached to the JVM for as long as it runs, so without a + // frame of their own the references made for one call would pile up + // behind those of the next. + class JavaLocalFrame + { + public: + JavaLocalFrame(JNIEnv * env, jint capacity); + ~JavaLocalFrame(); + + JavaLocalFrame(const JavaLocalFrame &) = delete; + JavaLocalFrame & operator=(const JavaLocalFrame &) = delete; + + private: + JNIEnv * env; + bool pushed; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoCodecInfo.h b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoCodecInfo.h new file mode 100644 index 000000000..dd69f44e6 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoCodecInfo.h @@ -0,0 +1,58 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_CODEC_INFO_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_CODEC_INFO_H_ + +#include "JavaClass.h" +#include "JavaRef.h" + +#include "api/video_codecs/sdp_video_format.h" + +#include + +#include + +namespace jni +{ + namespace VideoCodecInfo + { + JavaLocalRef toJava(JNIEnv * env, const webrtc::SdpVideoFormat & format); + webrtc::SdpVideoFormat toNative(JNIEnv * env, const JavaRef & info); + + JavaLocalRef toJavaArray(JNIEnv * env, const std::vector & formats); + + // Converts a java.util.List of VideoCodecInfos, skipping null + // elements. + std::vector toNativeList(JNIEnv * env, const JavaRef & list); + + class JavaVideoCodecInfoClass : public JavaClass + { + public: + explicit JavaVideoCodecInfoClass(JNIEnv * env); + + jclass cls; + jmethodID ctor; + jfieldID name; + jfieldID parameters; + jfieldID scalabilityModes; + + jmethodID listToArray; + }; + } +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoCodecUtils.h b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoCodecUtils.h new file mode 100644 index 000000000..d9a6f64dc --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoCodecUtils.h @@ -0,0 +1,48 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_CODEC_UTILS_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_CODEC_UTILS_H_ + +#include "JavaClass.h" + +#include + +#include + +namespace jni +{ + // Logs and clears a pending Java exception, which is how the codec + // wrappers deal with whatever a Java codec or factory throws: nothing + // may throw into WebRTC, and no JNI call may be made with an exception + // pending. Returns whether there was one. + bool ClearCodecException(JNIEnv * env, const char * context); + + // Converts a Java VideoCodecStatus to a WEBRTC_VIDEO_CODEC_* code. A + // pending exception, which it clears, and a null status are an error. + int32_t ToNativeCodecStatus(JNIEnv * env, jobject status, const char * context); + + class JavaVideoCodecStatusClass : public JavaClass + { + public: + explicit JavaVideoCodecStatusClass(JNIEnv * env); + + jclass cls; + jmethodID getNumber; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderFactoryWrapper.h b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderFactoryWrapper.h new file mode 100644 index 000000000..7ce079e0d --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderFactoryWrapper.h @@ -0,0 +1,73 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_DECODER_FACTORY_WRAPPER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_DECODER_FACTORY_WRAPPER_H_ + +#include "JavaClass.h" +#include "JavaRef.h" + +#include "api/environment/environment.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" + +#include + +#include +#include + +namespace jni +{ + // Lets WebRTC create its video decoders through a Java + // VideoDecoderFactory. A decoder the factory implements in Java is run + // by a VideoDecoderWrapper; a NativeVideoDecoder it returns becomes the + // built-in decoder for its codec. + class VideoDecoderFactoryWrapper : public webrtc::VideoDecoderFactory + { + public: + // Asks the Java factory for its supported codecs, which it may + // throw a Java exception for. + VideoDecoderFactoryWrapper(JNIEnv * env, jobject factory); + ~VideoDecoderFactoryWrapper() override = default; + + std::vector GetSupportedFormats() const override; + std::unique_ptr Create(const webrtc::Environment & env, + const webrtc::SdpVideoFormat & format) override; + + private: + class JavaVideoDecoderFactoryClass : public JavaClass + { + public: + explicit JavaVideoDecoderFactoryClass(JNIEnv * env); + + jmethodID getSupportedCodecs; + jmethodID createDecoder; + + jclass nativeDecoderClass; + jfieldID nativeDecoderCodecInfo; + }; + + private: + const JavaGlobalRef factory; + const std::shared_ptr javaClass; + const std::unique_ptr defaultFactory; + + std::vector supportedFormats; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderWrapper.h b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderWrapper.h new file mode 100644 index 000000000..8192decb6 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderWrapper.h @@ -0,0 +1,126 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_DECODER_WRAPPER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_DECODER_WRAPPER_H_ + +#include "JavaClass.h" +#include "JavaRef.h" + +#include "api/video/encoded_image.h" +#include "api/video_codecs/video_decoder.h" +#include "common_video/h264/h264_bitstream_parser.h" +#include "rtc_base/synchronization/mutex.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace jni +{ + // Runs a Java VideoDecoder inside WebRTC. + // + // WebRTC calls the decoder from one thread at a time, while the Java + // decoder may hand its frames over from any thread, through a callback + // that reaches this wrapper only until the decoder is released. + class VideoDecoderWrapper : public webrtc::VideoDecoder + { + public: + VideoDecoderWrapper(JNIEnv * env, jobject decoder); + ~VideoDecoderWrapper() override; + + using webrtc::VideoDecoder::Decode; + + bool Configure(const Settings & settings) override; + int32_t Decode(const webrtc::EncodedImage & image, int64_t renderTimeMs) override; + int32_t RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback * callback) override; + int32_t Release() override; + DecoderInfo GetDecoderInfo() const override; + const char * ImplementationName() const override; + + // Takes a frame from the Java decoder. Called through the Java + // callback, which holds its lock meanwhile. + void OnDecodedFrame(JNIEnv * env, jobject frame, std::optional decodeTimeMs, + std::optional qp); + + private: + struct FrameExtraInfo + { + // Identifies the frame. + int64_t timestampNs; + uint32_t rtpTimestamp; + int64_t ntpTimeMs; + std::optional qp; + }; + + bool ConfigureInternal(JNIEnv * env); + int32_t ReleaseInternal(JNIEnv * env); + int32_t HandleReturnCode(JNIEnv * env, int32_t status, const char * method); + void InvalidateCallback(JNIEnv * env); + + std::optional ParseQp(const webrtc::EncodedImage & image); + + private: + class JavaVideoDecoderClass : public JavaClass + { + public: + explicit JavaVideoDecoderClass(JNIEnv * env); + + jmethodID initDecode; + jmethodID release; + jmethodID decode; + jmethodID getImplementationName; + jmethodID isHardwareDecoder; + + jclass settingsClass; + jmethodID settingsCtor; + + jclass callbackClass; + jmethodID callbackCtor; + jmethodID callbackInvalidate; + }; + + private: + const JavaGlobalRef decoder; + const std::shared_ptr javaClass; + + std::string implementationName; + bool hardwareAccelerated; + + // The callback the Java decoder was initialized with. + std::unique_ptr> javaCallback; + + std::atomic callback; + + // Written on the decoder thread and read on the Java decoder's. + webrtc::Mutex frameExtraInfosLock; + std::deque frameExtraInfos; + + Settings settings; + bool initialized; + + // Parsing stops while the decoder provides the QP itself. + bool qpParsingEnabled; + webrtc::H264BitstreamParser h264BitstreamParser; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderFactoryWrapper.h b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderFactoryWrapper.h new file mode 100644 index 000000000..e1b9ef9a3 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderFactoryWrapper.h @@ -0,0 +1,73 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_ENCODER_FACTORY_WRAPPER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_ENCODER_FACTORY_WRAPPER_H_ + +#include "JavaClass.h" +#include "JavaRef.h" + +#include "api/environment/environment.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" + +#include + +#include +#include + +namespace jni +{ + // Lets WebRTC create its video encoders through a Java + // VideoEncoderFactory. An encoder the factory implements in Java is run + // by a VideoEncoderWrapper; a NativeVideoEncoder it returns becomes the + // built-in encoder for its codec. + class VideoEncoderFactoryWrapper : public webrtc::VideoEncoderFactory + { + public: + // Asks the Java factory for its supported codecs, which it may + // throw a Java exception for. + VideoEncoderFactoryWrapper(JNIEnv * env, jobject factory); + ~VideoEncoderFactoryWrapper() override = default; + + std::vector GetSupportedFormats() const override; + std::unique_ptr Create(const webrtc::Environment & env, + const webrtc::SdpVideoFormat & format) override; + + private: + class JavaVideoEncoderFactoryClass : public JavaClass + { + public: + explicit JavaVideoEncoderFactoryClass(JNIEnv * env); + + jmethodID getSupportedCodecs; + jmethodID createEncoder; + + jclass nativeEncoderClass; + jfieldID nativeEncoderCodecInfo; + }; + + private: + const JavaGlobalRef factory; + const std::shared_ptr javaClass; + const std::unique_ptr defaultFactory; + + std::vector supportedFormats; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderWrapper.h b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderWrapper.h new file mode 100644 index 000000000..57df254c3 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/media/video/codec/VideoEncoderWrapper.h @@ -0,0 +1,164 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_ENCODER_WRAPPER_H_ +#define JNI_WEBRTC_MEDIA_VIDEO_CODEC_VIDEO_ENCODER_WRAPPER_H_ + +#include "JavaClass.h" +#include "JavaRef.h" + +#include "api/video/encoded_image.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" +#include "common_video/h264/h264_bitstream_parser.h" +#include "modules/video_coding/codecs/vp9/include/vp9_globals.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/svc/scalable_video_controller_no_layering.h" +#include "rtc_base/synchronization/mutex.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace jni +{ + // Runs a Java VideoEncoder inside WebRTC. + // + // WebRTC calls the encoder from one thread at a time, while the Java + // encoder may hand its frames over from any thread, through a callback + // that reaches this wrapper only until the encoder is released. + class VideoEncoderWrapper : public webrtc::VideoEncoder + { + public: + VideoEncoderWrapper(JNIEnv * env, jobject encoder, const webrtc::SdpVideoFormat & format); + ~VideoEncoderWrapper() override; + + int32_t InitEncode(const webrtc::VideoCodec * codecSettings, const Settings & settings) override; + int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback * callback) override; + int32_t Release() override; + int32_t Encode(const webrtc::VideoFrame & frame, const std::vector * frameTypes) override; + void SetRates(const RateControlParameters & parameters) override; + EncoderInfo GetEncoderInfo() const override; + + // Takes a frame from the Java encoder. Called through the Java + // callback, which holds its lock meanwhile. + void OnEncodedFrame(JNIEnv * env, jobject image); + + private: + struct FrameExtraInfo + { + // Identifies the frame. + int64_t captureTimeNs; + uint32_t rtpTimestamp; + }; + + int32_t InitEncodeInternal(JNIEnv * env); + int32_t ReleaseInternal(JNIEnv * env); + // Handles a status the Java encoder returned: tries to reset the + // encoder on errors WebRTC does not give up on. + int32_t HandleReturnCode(JNIEnv * env, int32_t status, const char * method); + void InvalidateCallback(JNIEnv * env); + void UpdateEncoderInfo(JNIEnv * env); + + ScalingSettings GetScalingSettings(JNIEnv * env) const; + std::vector GetResolutionBitrateLimits(JNIEnv * env) const; + + int ParseQp(const webrtc::EncodedImage & image); + webrtc::CodecSpecificInfo ParseCodecSpecificInfo(const webrtc::EncodedImage & image); + + private: + class JavaVideoEncoderClass : public JavaClass + { + public: + explicit JavaVideoEncoderClass(JNIEnv * env); + + jmethodID initEncode; + jmethodID release; + jmethodID encode; + jmethodID setRates; + jmethodID getScalingSettings; + jmethodID getResolutionBitrateLimits; + jmethodID getEncoderInfo; + jmethodID getImplementationName; + jmethodID isHardwareEncoder; + + jclass settingsClass; + jmethodID settingsCtor; + + jclass encodeInfoClass; + jmethodID encodeInfoFromNative; + + jclass bitrateAllocationClass; + jmethodID bitrateAllocationCtor; + + jclass rateControlClass; + jmethodID rateControlCtor; + + jfieldID scalingOn; + jfieldID scalingLow; + jfieldID scalingHigh; + + jfieldID limitsFrameSizePixels; + jfieldID limitsMinStartBitrateBps; + jfieldID limitsMinBitrateBps; + jfieldID limitsMaxBitrateBps; + + jfieldID infoRequestedResolutionAlignment; + jfieldID infoApplyAlignmentToAllSimulcastLayers; + + jclass callbackClass; + jmethodID callbackCtor; + jmethodID callbackInvalidate; + + jclass intArrayClass; + }; + + private: + const JavaGlobalRef encoder; + const std::shared_ptr javaClass; + const webrtc::SdpVideoFormat format; + + // The callback the Java encoder was initialized with. + std::unique_ptr> javaCallback; + + std::atomic callback; + + // Written on the encoder thread and read on the Java encoder's. + webrtc::Mutex frameExtraInfosLock; + std::deque frameExtraInfos; + + bool initialized; + std::optional capabilities; + int numberOfCores; + webrtc::VideoCodec codecSettings; + EncoderInfo encoderInfo; + + // Used on the thread the Java encoder hands its frames over on. + webrtc::H264BitstreamParser h264BitstreamParser; + webrtc::ScalableVideoControllerNoLayering svcController; + webrtc::GofInfoVP9 gof; + size_t gofIndex; + }; +} + +#endif diff --git a/webrtc-jni/src/main/cpp/src/JNI_DefaultVideoDecoderFactory.cpp b/webrtc-jni/src/main/cpp/src/JNI_DefaultVideoDecoderFactory.cpp new file mode 100644 index 000000000..3a0f113e4 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/JNI_DefaultVideoDecoderFactory.cpp @@ -0,0 +1,35 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "JNI_DefaultVideoDecoderFactory.h" +#include "media/video/codec/DefaultVideoCodecFactories.h" +#include "media/video/codec/VideoCodecInfo.h" +#include "JavaUtils.h" + +JNIEXPORT jobjectArray JNICALL Java_dev_onvoid_webrtc_media_video_codec_DefaultVideoDecoderFactory_getSupportedCodecsInternal +(JNIEnv * env, jclass caller) +{ + try { + auto factory = jni::CreateDefaultVideoDecoderFactory(); + + return jni::VideoCodecInfo::toJavaArray(env, factory->GetSupportedFormats()).release(); + } + catch (...) { + ThrowCxxJavaException(env); + } + + return nullptr; +} diff --git a/webrtc-jni/src/main/cpp/src/JNI_DefaultVideoEncoderFactory.cpp b/webrtc-jni/src/main/cpp/src/JNI_DefaultVideoEncoderFactory.cpp new file mode 100644 index 000000000..cf0159610 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/JNI_DefaultVideoEncoderFactory.cpp @@ -0,0 +1,35 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "JNI_DefaultVideoEncoderFactory.h" +#include "media/video/codec/DefaultVideoCodecFactories.h" +#include "media/video/codec/VideoCodecInfo.h" +#include "JavaUtils.h" + +JNIEXPORT jobjectArray JNICALL Java_dev_onvoid_webrtc_media_video_codec_DefaultVideoEncoderFactory_getSupportedCodecsInternal +(JNIEnv * env, jclass caller) +{ + try { + auto factory = jni::CreateDefaultVideoEncoderFactory(); + + return jni::VideoCodecInfo::toJavaArray(env, factory->GetSupportedFormats()).release(); + } + catch (...) { + ThrowCxxJavaException(env); + } + + return nullptr; +} diff --git a/webrtc-jni/src/main/cpp/src/JNI_NativeVideoDecoderCallback.cpp b/webrtc-jni/src/main/cpp/src/JNI_NativeVideoDecoderCallback.cpp new file mode 100644 index 000000000..0a0728cd2 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/JNI_NativeVideoDecoderCallback.cpp @@ -0,0 +1,46 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "JNI_NativeVideoDecoderCallback.h" +#include "media/video/codec/VideoDecoderWrapper.h" +#include "JavaUtils.h" + +#include + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_codec_NativeVideoDecoderCallback_onDecodedFrame +(JNIEnv * env, jclass caller, jlong handle, jobject frame, jint decodeTimeMs, jint qp) +{ + // The Java callback holds its lock and has checked the handle, so the + // wrapper cannot be released meanwhile. + auto decoder = reinterpret_cast(handle); + + std::optional decodeTime; + std::optional frameQp; + + if (decodeTimeMs >= 0) { + decodeTime = static_cast(decodeTimeMs); + } + if (qp >= 0 && qp <= 255) { + frameQp = static_cast(qp); + } + + try { + decoder->OnDecodedFrame(env, frame, decodeTime, frameQp); + } + catch (...) { + ThrowCxxJavaException(env); + } +} diff --git a/webrtc-jni/src/main/cpp/src/JNI_NativeVideoEncoderCallback.cpp b/webrtc-jni/src/main/cpp/src/JNI_NativeVideoEncoderCallback.cpp new file mode 100644 index 000000000..b611e0ee6 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/JNI_NativeVideoEncoderCallback.cpp @@ -0,0 +1,34 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "JNI_NativeVideoEncoderCallback.h" +#include "media/video/codec/VideoEncoderWrapper.h" +#include "JavaUtils.h" + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_codec_NativeVideoEncoderCallback_onEncodedFrame +(JNIEnv * env, jclass caller, jlong handle, jobject image) +{ + // The Java callback holds its lock and has checked the handle, so the + // wrapper cannot be released meanwhile. + auto encoder = reinterpret_cast(handle); + + try { + encoder->OnEncodedFrame(env, image); + } + catch (...) { + ThrowCxxJavaException(env); + } +} diff --git a/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp b/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp index 4e3064d8f..b1cc03048 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_PeerConnectionFactory.cpp @@ -37,31 +37,11 @@ #include "api/audio_codecs/builtin_audio_decoder_factory.h" #include "api/audio_codecs/builtin_audio_encoder_factory.h" -#ifdef __APPLE__ -#include "sdk/objc/components/video_codec/RTCDefaultVideoDecoderFactory.h" -#include "sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.h" -#include "sdk/objc/native/api/video_decoder_factory.h" -#include "sdk/objc/native/api/video_encoder_factory.h" -#else -#include "api/video_codecs/builtin_video_decoder_factory.h" -#include "api/video_codecs/builtin_video_encoder_factory.h" -#include "api/video_codecs/video_decoder_factory_template_dav1d_adapter.h" -#include "api/video_codecs/video_decoder_factory_template_libvpx_vp8_adapter.h" -#include "api/video_codecs/video_decoder_factory_template_libvpx_vp9_adapter.h" -#include "api/video_codecs/video_decoder_factory_template_open_h264_adapter.h" -#include "api/video_codecs/video_encoder_factory_template_libaom_av1_adapter.h" -#include "api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h" -#include "api/video_codecs/video_encoder_factory_template_libvpx_vp9_adapter.h" -#include "api/video_codecs/video_encoder_factory_template_open_h264_adapter.h" -#endif - -#include "api/video_codecs/video_decoder_factory.h" -#include "api/video_codecs/video_decoder_factory_template.h" -#include "api/video_codecs/video_encoder_factory.h" -#include "api/video_codecs/video_encoder_factory_template.h" - #include "api/ProxyAudioDeviceModule.h" #include "media/audio/CustomAudioSource.h" +#include "media/video/codec/DefaultVideoCodecFactories.h" +#include "media/video/codec/VideoDecoderFactoryWrapper.h" +#include "media/video/codec/VideoEncoderFactoryWrapper.h" #include "api/media_stream_interface.h" #include "rtc_base/logging.h" @@ -70,7 +50,8 @@ #include JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_initialize -(JNIEnv * env, jobject caller, jobject jFieldTrials, jobject audioModule, jobject audioProcessing) +(JNIEnv * env, jobject caller, jobject jFieldTrials, jobject audioModule, jobject audioProcessing, + jobject jVideoEncoderFactory, jobject jVideoDecoderFactory) { webrtc::AudioDeviceModule * audioDevModule = (audioModule != nullptr) ? GetHandle(env, audioModule) @@ -96,6 +77,15 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_initialize ? nullptr : std::make_unique(std::move(fieldTrialsMap)); + // Asks the Java factories for their codecs, which is where a broken + // factory surfaces, before anything else is set up. + std::unique_ptr videoEncoderFactory = (jVideoEncoderFactory != nullptr) + ? std::make_unique(env, jVideoEncoderFactory) + : jni::CreateDefaultVideoEncoderFactory(); + std::unique_ptr videoDecoderFactory = (jVideoDecoderFactory != nullptr) + ? std::make_unique(env, jVideoDecoderFactory) + : jni::CreateDefaultVideoDecoderFactory(); + auto networkThread = webrtc::Thread::CreateWithSocketServer(); networkThread->SetName("webrtc_jni_network_thread", nullptr); @@ -153,21 +143,8 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_PeerConnectionFactory_initialize proxy, webrtc::CreateBuiltinAudioEncoderFactory(), webrtc::CreateBuiltinAudioDecoderFactory(), -#ifdef __APPLE__ - webrtc::ObjCToNativeVideoEncoderFactory([[RTC_OBJC_TYPE(RTCDefaultVideoEncoderFactory) alloc] init]), - webrtc::ObjCToNativeVideoDecoderFactory([[RTC_OBJC_TYPE(RTCDefaultVideoDecoderFactory) alloc] init]), -#else - std::make_unique>(), - std::make_unique>(), -#endif + std::move(videoEncoderFactory), + std::move(videoDecoderFactory), nullptr, apm, nullptr, diff --git a/webrtc-jni/src/main/cpp/src/api/VideoFrame.cpp b/webrtc-jni/src/main/cpp/src/api/VideoFrame.cpp index 26d9b673d..24f3ec502 100644 --- a/webrtc-jni/src/main/cpp/src/api/VideoFrame.cpp +++ b/webrtc-jni/src/main/cpp/src/api/VideoFrame.cpp @@ -71,6 +71,81 @@ namespace jni .set_rotation(static_cast(rotation)) .build(); } + + JavaLocalRef toJava(JNIEnv * env, const webrtc::VideoFrame & frame) + { + webrtc::scoped_refptr i420Buffer = frame.video_frame_buffer()->ToI420(); + + if (i420Buffer == nullptr) { + return JavaLocalRef(env, nullptr); + } + + const auto javaClass = JavaClasses::get(env); + + JavaLocalRef jBuffer = I420Buffer::toJava(env, i420Buffer); + + // The reference the Java buffer gives up when it is released. + i420Buffer->AddRef(); + + jobject jFrame = env->NewObject(javaClass->cls, javaClass->ctor, jBuffer.get(), + static_cast(frame.rotation()), + static_cast(frame.timestamp_us() * webrtc::kNumNanosecsPerMicrosec)); + + if (jFrame == nullptr) { + // The Java buffer never got to a frame that would release it. + i420Buffer->Release(); + ExceptionCheck(env); + } + + return JavaLocalRef(env, jFrame); + } + + webrtc::scoped_refptr toNativeBuffer(JNIEnv * env, const JavaRef & javaBuffer) + { + const auto nativeClass = JavaClasses::get(env); + + if (env->IsInstanceOf(javaBuffer, nativeClass->cls)) { + webrtc::I420BufferInterface * buffer = GetHandle(env, javaBuffer); + + return webrtc::scoped_refptr(buffer); + } + + const auto javaClass = JavaClasses::get(env); + + JavaLocalRef i420(env, env->CallObjectMethod(javaBuffer, javaClass->toI420)); + ExceptionCheck(env); + + if (i420.get() == nullptr) { + return nullptr; + } + + const jint width = env->CallIntMethod(i420, javaClass->getWidth); + const jint height = env->CallIntMethod(i420, javaClass->getHeight); + const jint strideY = env->CallIntMethod(i420, javaClass->getStrideY); + const jint strideU = env->CallIntMethod(i420, javaClass->getStrideU); + const jint strideV = env->CallIntMethod(i420, javaClass->getStrideV); + ExceptionCheck(env); + + JavaLocalRef dataY(env, env->CallObjectMethod(i420, javaClass->getDataY)); + JavaLocalRef dataU(env, env->CallObjectMethod(i420, javaClass->getDataU)); + JavaLocalRef dataV(env, env->CallObjectMethod(i420, javaClass->getDataV)); + ExceptionCheck(env); + + if (dataY.get() == nullptr || dataU.get() == nullptr || dataV.get() == nullptr) { + return nullptr; + } + + // Heap buffers have no address, and only direct ones can be read. + const uint8_t * y = static_cast(env->GetDirectBufferAddress(dataY)); + const uint8_t * u = static_cast(env->GetDirectBufferAddress(dataU)); + const uint8_t * v = static_cast(env->GetDirectBufferAddress(dataV)); + + if (y == nullptr || u == nullptr || v == nullptr || width <= 0 || height <= 0) { + return nullptr; + } + + return webrtc::I420Buffer::Copy(width, height, y, strideY, u, strideU, v, strideV); + } } namespace I420Buffer @@ -105,6 +180,25 @@ namespace jni buffer = GetFieldID(env, cls, "buffer", "L" PKG_VIDEO "VideoFrameBuffer;"); rotation = GetFieldID(env, cls, "rotation", "I"); timestampNs = GetFieldID(env, cls, "timestampNs", "J"); + release = GetMethod(env, cls, "release", "()V"); + } + + JavaI420BufferClass::JavaI420BufferClass(JNIEnv * env) + { + cls = FindClass(env, PKG_VIDEO"I420Buffer"); + + // Called on buffers of any kind, so it is looked up on their type. + jclass bufferClass = FindClass(env, PKG_VIDEO"VideoFrameBuffer"); + + toI420 = GetMethod(env, bufferClass, "toI420", "()L" PKG_VIDEO "I420Buffer;"); + getWidth = GetMethod(env, cls, "getWidth", "()I"); + getHeight = GetMethod(env, cls, "getHeight", "()I"); + getDataY = GetMethod(env, cls, "getDataY", "()" BYTE_BUFFER_SIG); + getDataU = GetMethod(env, cls, "getDataU", "()" BYTE_BUFFER_SIG); + getDataV = GetMethod(env, cls, "getDataV", "()" BYTE_BUFFER_SIG); + getStrideY = GetMethod(env, cls, "getStrideY", "()I"); + getStrideU = GetMethod(env, cls, "getStrideU", "()I"); + getStrideV = GetMethod(env, cls, "getStrideV", "()I"); } JavaNativeI420BufferClass::JavaNativeI420BufferClass(JNIEnv * env) diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp new file mode 100644 index 000000000..9fd32ac5a --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/DefaultVideoCodecFactories.cpp @@ -0,0 +1,64 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/DefaultVideoCodecFactories.h" + +#ifdef __APPLE__ +#include "sdk/objc/components/video_codec/RTCDefaultVideoDecoderFactory.h" +#include "sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.h" +#include "sdk/objc/native/api/video_decoder_factory.h" +#include "sdk/objc/native/api/video_encoder_factory.h" +#else +#include "api/video_codecs/video_decoder_factory_template.h" +#include "api/video_codecs/video_decoder_factory_template_dav1d_adapter.h" +#include "api/video_codecs/video_decoder_factory_template_libvpx_vp8_adapter.h" +#include "api/video_codecs/video_decoder_factory_template_libvpx_vp9_adapter.h" +#include "api/video_codecs/video_decoder_factory_template_open_h264_adapter.h" +#include "api/video_codecs/video_encoder_factory_template.h" +#include "api/video_codecs/video_encoder_factory_template_libaom_av1_adapter.h" +#include "api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h" +#include "api/video_codecs/video_encoder_factory_template_libvpx_vp9_adapter.h" +#include "api/video_codecs/video_encoder_factory_template_open_h264_adapter.h" +#endif + +namespace jni +{ + std::unique_ptr CreateDefaultVideoEncoderFactory() + { +#ifdef __APPLE__ + return webrtc::ObjCToNativeVideoEncoderFactory([[RTC_OBJC_TYPE(RTCDefaultVideoEncoderFactory) alloc] init]); +#else + return std::make_unique>(); +#endif + } + + std::unique_ptr CreateDefaultVideoDecoderFactory() + { +#ifdef __APPLE__ + return webrtc::ObjCToNativeVideoDecoderFactory([[RTC_OBJC_TYPE(RTCDefaultVideoDecoderFactory) alloc] init]); +#else + return std::make_unique>(); +#endif + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/EncodedImage.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/EncodedImage.cpp new file mode 100644 index 000000000..ef0408e1e --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/EncodedImage.cpp @@ -0,0 +1,115 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/EncodedImage.h" +#include "Exception.h" +#include "JavaClasses.h" +#include "JavaObject.h" +#include "JavaUtils.h" +#include "JNI_WebRTC.h" + +#include "api/video/video_frame_type.h" +#include "api/video/video_rotation.h" +#include "rtc_base/time_utils.h" + +namespace jni +{ + namespace EncodedImage + { + JavaLocalRef toJava(JNIEnv * env, const webrtc::EncodedImage & image) + { + const auto javaClass = JavaClasses::get(env); + + // NewDirectByteBuffer does not accept an empty region at a null + // address, which an empty image may have. + static uint8_t empty = 0; + uint8_t * data = image.size() > 0 ? const_cast(image.data()) : ∅ + + JavaLocalRef buffer(env, env->NewDirectByteBuffer(data, static_cast(image.size()))); + + ExceptionCheck(env); + + jobject jImage = env->CallStaticObjectMethod(javaClass->cls, javaClass->fromNative, buffer.get(), + static_cast(image._encodedWidth), + static_cast(image._encodedHeight), + static_cast(image.capture_time_ms_ * webrtc::kNumNanosecsPerMillisec), + static_cast(image.frame_type()), + static_cast(image.rotation_), + static_cast(image.qp_)); + + ExceptionCheck(env); + + return JavaLocalRef(env, jImage); + } + + webrtc::EncodedImage toNative(JNIEnv * env, const JavaRef & image) + { + const auto javaClass = JavaClasses::get(env); + + JavaLocalRef payload(env, env->CallObjectMethod(image, javaClass->getDirectPayload)); + + ExceptionCheck(env); + + const uint8_t * data = static_cast(env->GetDirectBufferAddress(payload.get())); + const jlong size = env->GetDirectBufferCapacity(payload.get()); + + if (size < 0 || (data == nullptr && size > 0)) { + throw Exception("The EncodedImage buffer cannot be read"); + } + + jint frameType = env->CallIntMethod(image, javaClass->getFrameTypeIndex); + ExceptionCheck(env); + + jint qp = env->CallIntMethod(image, javaClass->getQpOrUnknown); + ExceptionCheck(env); + + JavaObject obj(env, image); + + webrtc::EncodedImage nativeImage; + nativeImage.SetEncodedData(webrtc::EncodedImageBuffer::Create(data, static_cast(size))); + nativeImage._encodedWidth = static_cast(obj.getInt(javaClass->encodedWidth)); + nativeImage._encodedHeight = static_cast(obj.getInt(javaClass->encodedHeight)); + nativeImage.rotation_ = static_cast(obj.getInt(javaClass->rotation)); + nativeImage.qp_ = qp; + nativeImage.set_frame_type(static_cast(frameType)); + + return nativeImage; + } + + int64_t getCaptureTimeNs(JNIEnv * env, const JavaRef & image) + { + const auto javaClass = JavaClasses::get(env); + + return env->GetLongField(image, javaClass->captureTimeNs); + } + + JavaEncodedImageClass::JavaEncodedImageClass(JNIEnv * env) + { + cls = FindClass(env, PKG_CODEC"EncodedImage"); + + fromNative = GetStaticMethod(env, cls, "fromNative", + "(" BYTE_BUFFER_SIG "IIJIII)L" PKG_CODEC "EncodedImage;"); + getDirectPayload = GetMethod(env, cls, "getDirectPayload", "()" BYTE_BUFFER_SIG); + getFrameTypeIndex = GetMethod(env, cls, "getFrameTypeIndex", "()I"); + getQpOrUnknown = GetMethod(env, cls, "getQpOrUnknown", "()I"); + + encodedWidth = GetFieldID(env, cls, "encodedWidth", "I"); + encodedHeight = GetFieldID(env, cls, "encodedHeight", "I"); + captureTimeNs = GetFieldID(env, cls, "captureTimeNs", "J"); + rotation = GetFieldID(env, cls, "rotation", "I"); + } + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/JavaLocalFrame.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/JavaLocalFrame.cpp new file mode 100644 index 000000000..f735831cd --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/JavaLocalFrame.cpp @@ -0,0 +1,37 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/JavaLocalFrame.h" + +namespace jni +{ + JavaLocalFrame::JavaLocalFrame(JNIEnv * env, jint capacity) : + env(env), + pushed(env->PushLocalFrame(capacity) == JNI_OK) + { + if (!pushed) { + // Out of memory: the references then go to the enclosing frame. + env->ExceptionClear(); + } + } + + JavaLocalFrame::~JavaLocalFrame() + { + if (pushed) { + env->PopLocalFrame(nullptr); + } + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoCodecInfo.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoCodecInfo.cpp new file mode 100644 index 000000000..b9c9ad607 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoCodecInfo.cpp @@ -0,0 +1,160 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/VideoCodecInfo.h" +#include "Exception.h" +#include "JavaClasses.h" +#include "JavaHashMap.h" +#include "JavaObject.h" +#include "JavaString.h" +#include "JavaUtils.h" +#include "JNI_WebRTC.h" + +#include "api/video_codecs/scalability_mode.h" +#include "modules/video_coding/svc/scalability_mode_util.h" + +#include + +namespace jni +{ + namespace VideoCodecInfo + { + JavaLocalRef toJava(JNIEnv * env, const webrtc::SdpVideoFormat & format) + { + const auto javaClass = JavaClasses::get(env); + + JavaHashMap parameters(env); + + for (const auto & [key, value] : format.parameters) { + parameters.put(JavaString::toJava(env, key), JavaString::toJava(env, value)); + } + + std::vector modes; + + for (webrtc::ScalabilityMode mode : format.scalability_modes) { + modes.emplace_back(webrtc::ScalabilityModeToString(mode)); + } + + JavaLocalRef name = JavaString::toJava(env, format.name); + JavaLocalRef parameterMap = parameters; + JavaLocalRef modeArray = JavaString::createArray(env, modes); + + jobject info = env->NewObject(javaClass->cls, javaClass->ctor, name.get(), + parameterMap.get(), modeArray.get()); + + ExceptionCheck(env); + + return JavaLocalRef(env, info); + } + + webrtc::SdpVideoFormat toNative(JNIEnv * env, const JavaRef & info) + { + const auto javaClass = JavaClasses::get(env); + + JavaObject obj(env, info); + + webrtc::SdpVideoFormat format(JavaString::toNative(env, obj.getString(javaClass->name))); + + // The field always holds a HashMap. + for (const auto & entry : JavaHashMap(env, obj.getObject(javaClass->parameters))) { + std::string key = JavaString::toNative(env, static_java_ref_cast(env, entry.first)); + std::string value = JavaString::toNative(env, static_java_ref_cast(env, entry.second)); + + format.parameters.emplace(std::move(key), std::move(value)); + } + + JavaLocalRef modes = obj.getObjectArray(javaClass->scalabilityModes); + jsize count = modes.get() != nullptr ? env->GetArrayLength(modes.get()) : 0; + + for (jsize i = 0; i < count; i++) { + JavaLocalRef mode(env, static_cast(env->GetObjectArrayElement(modes.get(), i))); + auto scalabilityMode = webrtc::ScalabilityModeFromString(JavaString::toNative(env, mode)); + + // A mode WebRTC does not know cannot be negotiated anyway. + if (scalabilityMode.has_value()) { + format.scalability_modes.push_back(*scalabilityMode); + } + } + + return format; + } + + JavaLocalRef toJavaArray(JNIEnv * env, const std::vector & formats) + { + const auto javaClass = JavaClasses::get(env); + + jobjectArray array = env->NewObjectArray(static_cast(formats.size()), javaClass->cls, nullptr); + + ExceptionCheck(env); + + for (size_t i = 0; i < formats.size(); i++) { + JavaLocalRef info = toJava(env, formats[i]); + + env->SetObjectArrayElement(array, static_cast(i), info.get()); + } + + return JavaLocalRef(env, array); + } + + std::vector toNativeList(JNIEnv * env, const JavaRef & list) + { + std::vector formats; + + if (list.get() == nullptr) { + return formats; + } + + const auto javaClass = JavaClasses::get(env); + + JavaLocalRef array(env, + static_cast(env->CallObjectMethod(list, javaClass->listToArray))); + + ExceptionCheck(env); + + jsize count = env->GetArrayLength(array.get()); + + for (jsize i = 0; i < count; i++) { + JavaLocalRef info(env, env->GetObjectArrayElement(array.get(), i)); + + if (info.get() == nullptr) { + continue; + } + if (!env->IsInstanceOf(info.get(), javaClass->cls)) { + throw Exception("The list of supported codecs holds an object that is not a VideoCodecInfo"); + } + + formats.push_back(toNative(env, info)); + } + + return formats; + } + + JavaVideoCodecInfoClass::JavaVideoCodecInfoClass(JNIEnv * env) + { + cls = FindClass(env, PKG_CODEC"VideoCodecInfo"); + + ctor = GetMethod(env, cls, "", "(" STRING_SIG MAP_SIG "[" STRING_SIG ")V"); + + name = GetFieldID(env, cls, "name", STRING_SIG); + parameters = GetFieldID(env, cls, "parameters", "Ljava/util/HashMap;"); + scalabilityModes = GetFieldID(env, cls, "scalabilityModes", "[" STRING_SIG); + + jclass listClass = FindClass(env, "java/util/List"); + + listToArray = GetMethod(env, listClass, "toArray", "()[Ljava/lang/Object;"); + } + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoCodecUtils.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoCodecUtils.cpp new file mode 100644 index 000000000..a24366109 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoCodecUtils.cpp @@ -0,0 +1,71 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/VideoCodecUtils.h" +#include "JavaClasses.h" +#include "JavaUtils.h" +#include "JNI_WebRTC.h" + +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" + +namespace jni +{ + bool ClearCodecException(JNIEnv * env, const char * context) + { + if (!env->ExceptionCheck()) { + return false; + } + + // Cleared before logging, since a Java log sink cannot be called + // with an exception pending. + env->ExceptionDescribe(); + env->ExceptionClear(); + + RTC_LOG(LS_WARNING) << "Java video codec threw in " << context; + + return true; + } + + int32_t ToNativeCodecStatus(JNIEnv * env, jobject status, const char * context) + { + if (ClearCodecException(env, context)) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + if (status == nullptr) { + RTC_LOG(LS_WARNING) << "Java video codec returned no status from " << context; + + return WEBRTC_VIDEO_CODEC_ERROR; + } + + const auto javaClass = JavaClasses::get(env); + + jint number = env->CallIntMethod(status, javaClass->getNumber); + + if (ClearCodecException(env, context)) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + return static_cast(number); + } + + JavaVideoCodecStatusClass::JavaVideoCodecStatusClass(JNIEnv * env) + { + cls = FindClass(env, PKG_CODEC"VideoCodecStatus"); + + getNumber = GetMethod(env, cls, "getNumber", "()I"); + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderFactoryWrapper.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderFactoryWrapper.cpp new file mode 100644 index 000000000..9347c227d --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderFactoryWrapper.cpp @@ -0,0 +1,97 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/VideoDecoderFactoryWrapper.h" +#include "media/video/codec/DefaultVideoCodecFactories.h" +#include "media/video/codec/JavaLocalFrame.h" +#include "media/video/codec/VideoCodecInfo.h" +#include "media/video/codec/VideoCodecUtils.h" +#include "media/video/codec/VideoDecoderWrapper.h" +#include "JavaClasses.h" +#include "JavaUtils.h" +#include "JNI_WebRTC.h" + +#include "rtc_base/logging.h" + +namespace jni +{ + VideoDecoderFactoryWrapper::VideoDecoderFactoryWrapper(JNIEnv * env, jobject factory) : + factory(env, factory), + javaClass(JavaClasses::get(env)), + defaultFactory(CreateDefaultVideoDecoderFactory()) + { + JavaLocalRef codecs(env, env->CallObjectMethod(factory, javaClass->getSupportedCodecs)); + + ExceptionCheck(env); + + supportedFormats = VideoCodecInfo::toNativeList(env, codecs); + } + + std::vector VideoDecoderFactoryWrapper::GetSupportedFormats() const + { + return supportedFormats; + } + + std::unique_ptr VideoDecoderFactoryWrapper::Create(const webrtc::Environment & environment, + const webrtc::SdpVideoFormat & format) + { + JNIEnv * env = AttachCurrentThread(); + + if (env == nullptr) { + return nullptr; + } + + JavaLocalFrame localFrame(env, 16); + + try { + JavaLocalRef info = VideoCodecInfo::toJava(env, format); + jobject decoder = env->CallObjectMethod(factory, javaClass->createDecoder, info.get()); + + if (ClearCodecException(env, "createDecoder") || decoder == nullptr) { + RTC_LOG(LS_WARNING) << "Java decoder factory created no decoder for " << format.ToString(); + + return nullptr; + } + + if (env->IsInstanceOf(decoder, javaClass->nativeDecoderClass)) { + JavaLocalRef nativeInfo(env, env->GetObjectField(decoder, javaClass->nativeDecoderCodecInfo)); + + return defaultFactory->Create(environment, VideoCodecInfo::toNative(env, nativeInfo)); + } + + return std::make_unique(env, decoder); + } + catch (...) { + ClearCodecException(env, "createDecoder"); + + RTC_LOG(LS_WARNING) << "Java decoder factory failed to create a decoder for " << format.ToString(); + } + + return nullptr; + } + + VideoDecoderFactoryWrapper::JavaVideoDecoderFactoryClass::JavaVideoDecoderFactoryClass(JNIEnv * env) + { + jclass cls = FindClass(env, PKG_CODEC"VideoDecoderFactory"); + + getSupportedCodecs = GetMethod(env, cls, "getSupportedCodecs", "()" LIST_SIG); + createDecoder = GetMethod(env, cls, "createDecoder", + "(L" PKG_CODEC "VideoCodecInfo;)L" PKG_CODEC "VideoDecoder;"); + + nativeDecoderClass = FindClass(env, PKG_CODEC"NativeVideoDecoder"); + nativeDecoderCodecInfo = GetFieldID(env, nativeDecoderClass, "codecInfo", "L" PKG_CODEC "VideoCodecInfo;"); + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderWrapper.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderWrapper.cpp new file mode 100644 index 000000000..08b352339 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderWrapper.cpp @@ -0,0 +1,401 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/VideoDecoderWrapper.h" +#include "media/video/codec/EncodedImage.h" +#include "media/video/codec/JavaLocalFrame.h" +#include "media/video/codec/VideoCodecUtils.h" +#include "api/VideoFrame.h" +#include "Exception.h" +#include "JavaClasses.h" +#include "JavaString.h" +#include "JavaUtils.h" +#include "JNI_WebRTC.h" + +#include "api/video/render_resolution.h" +#include "api/video/video_frame.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/utility/vp8_header_parser.h" +#include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" +#include "rtc_base/logging.h" +#include "rtc_base/time_utils.h" + +namespace jni +{ + namespace + { + // Enough for the objects of one call into the decoder. + constexpr jint kLocalFrameCapacity = 32; + + // How many frames the decoder may hold back before the oldest records + // are given up, so that one that never produces output does not make + // them grow without bound. Several seconds of video. + constexpr size_t kMaxPendingFrames = 300; + + // RTP video timestamps count at 90 kHz. + constexpr int64_t kRtpTicksPerMillisecond = 90; + } + + VideoDecoderWrapper::VideoDecoderWrapper(JNIEnv * env, jobject decoder) : + decoder(env, decoder), + javaClass(JavaClasses::get(env)), + hardwareAccelerated(false), + callback(nullptr), + initialized(false), + qpParsingEnabled(true) + { + JavaLocalFrame localFrame(env, kLocalFrameCapacity); + + jobject name = env->CallObjectMethod(decoder, javaClass->getImplementationName); + + if (!ClearCodecException(env, "getImplementationName") && name != nullptr) { + try { + implementationName = JavaString::toNative(env, + JavaLocalRef(env, static_cast(env->NewLocalRef(name)))); + } + catch (...) { + ClearCodecException(env, "getImplementationName"); + } + } + + jboolean hardware = env->CallBooleanMethod(decoder, javaClass->isHardwareDecoder); + + if (!ClearCodecException(env, "isHardwareDecoder")) { + hardwareAccelerated = hardware == JNI_TRUE; + } + } + + VideoDecoderWrapper::~VideoDecoderWrapper() + { + JNIEnv * env = AttachCurrentThread(); + + if (env != nullptr) { + InvalidateCallback(env); + } + } + + bool VideoDecoderWrapper::Configure(const Settings & decoderSettings) + { + JNIEnv * env = AttachCurrentThread(); + + if (env == nullptr) { + return false; + } + + settings = decoderSettings; + + return ConfigureInternal(env); + } + + bool VideoDecoderWrapper::ConfigureInternal(JNIEnv * env) + { + JavaLocalFrame localFrame(env, kLocalFrameCapacity); + + webrtc::RenderResolution resolution = settings.max_render_resolution(); + + jobject jSettings = env->NewObject(javaClass->settingsClass, javaClass->settingsCtor, + static_cast(settings.number_of_cores()), + static_cast(resolution.Valid() ? resolution.Width() : 0), + static_cast(resolution.Valid() ? resolution.Height() : 0)); + + if (ClearCodecException(env, "initDecode") || jSettings == nullptr) { + return false; + } + + // A callback of its own for each initialization, so that frames of an + // earlier one cannot reach this one. + InvalidateCallback(env); + + jobject jCallback = env->NewObject(javaClass->callbackClass, javaClass->callbackCtor, + reinterpret_cast(this)); + + if (ClearCodecException(env, "initDecode") || jCallback == nullptr) { + return false; + } + + javaCallback = std::make_unique>(env, jCallback); + + jobject status = env->CallObjectMethod(decoder, javaClass->initDecode, jSettings, jCallback); + int32_t result = ToNativeCodecStatus(env, status, "initDecode"); + + RTC_LOG(LS_INFO) << "Java decoder initDecode: " << result; + + initialized = result == WEBRTC_VIDEO_CODEC_OK; + + // A decoder that stopped providing the QP after a reset gets it + // parsed again. + qpParsingEnabled = true; + + return initialized; + } + + int32_t VideoDecoderWrapper::Decode(const webrtc::EncodedImage & inputImage, int64_t renderTimeMs) + { + if (!initialized) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + + JNIEnv * env = AttachCurrentThread(); + + if (env == nullptr) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + JavaLocalFrame localFrame(env, kLocalFrameCapacity); + + // The capture time identifies the frame, and received images carry + // none, so it is derived from the RTP timestamp. + webrtc::EncodedImage image(inputImage); + image.capture_time_ms_ = image.RtpTimestamp() / kRtpTicksPerMillisecond; + + { + webrtc::MutexLock lock(&frameExtraInfosLock); + + if (frameExtraInfos.size() >= kMaxPendingFrames) { + frameExtraInfos.pop_front(); + } + + frameExtraInfos.push_back(FrameExtraInfo { + image.capture_time_ms_ * webrtc::kNumNanosecsPerMillisec, + image.RtpTimestamp(), + image.ntp_time_ms_, + qpParsingEnabled ? ParseQp(image) : std::nullopt + }); + } + + jobject jImage = nullptr; + + try { + jImage = EncodedImage::toJava(env, image).release(); + } + catch (...) { + ClearCodecException(env, "decode"); + } + + if (jImage == nullptr) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + jobject status = env->CallObjectMethod(decoder, javaClass->decode, jImage); + int32_t result = ToNativeCodecStatus(env, status, "decode"); + + return HandleReturnCode(env, result, "decode"); + } + + int32_t VideoDecoderWrapper::RegisterDecodeCompleteCallback(webrtc::DecodedImageCallback * decodeCallback) + { + callback = decodeCallback; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VideoDecoderWrapper::Release() + { + JNIEnv * env = AttachCurrentThread(); + + if (env == nullptr) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + return ReleaseInternal(env); + } + + int32_t VideoDecoderWrapper::ReleaseInternal(JNIEnv * env) + { + JavaLocalFrame localFrame(env, kLocalFrameCapacity); + + jobject status = env->CallObjectMethod(decoder, javaClass->release); + int32_t result = ToNativeCodecStatus(env, status, "release"); + + RTC_LOG(LS_INFO) << "Java decoder release: " << result; + + InvalidateCallback(env); + + { + webrtc::MutexLock lock(&frameExtraInfosLock); + frameExtraInfos.clear(); + } + + initialized = false; + + return result; + } + + webrtc::VideoDecoder::DecoderInfo VideoDecoderWrapper::GetDecoderInfo() const + { + DecoderInfo info; + info.implementation_name = implementationName; + info.is_hardware_accelerated = hardwareAccelerated; + + return info; + } + + const char * VideoDecoderWrapper::ImplementationName() const + { + return implementationName.c_str(); + } + + void VideoDecoderWrapper::OnDecodedFrame(JNIEnv * env, jobject jFrame, std::optional decodeTimeMs, + std::optional qp) + { + const auto frameClass = JavaClasses::get(env); + + const int64_t timestampNs = env->GetLongField(jFrame, frameClass->timestampNs); + const jint rotation = env->GetIntField(jFrame, frameClass->rotation); + + FrameExtraInfo extraInfo; + + { + webrtc::MutexLock lock(&frameExtraInfosLock); + + // A decoder may drop frames, so the records of those it dropped + // come before the one of this frame. + do { + if (frameExtraInfos.empty()) { + RTC_LOG(LS_WARNING) << "Java decoder produced a frame for no image it was given, timestamp " + << timestampNs << " ns"; + return; + } + + extraInfo = frameExtraInfos.front(); + frameExtraInfos.pop_front(); + } + while (extraInfo.timestampNs != timestampNs); + } + + JavaLocalRef buffer(env, env->GetObjectField(jFrame, frameClass->buffer)); + webrtc::scoped_refptr nativeBuffer = VideoFrame::toNativeBuffer(env, buffer); + + if (nativeBuffer == nullptr) { + throw Exception("The decoded frame has no pixels that can be read: its buffer has to provide " + "direct byte buffers"); + } + + webrtc::VideoFrame frame = webrtc::VideoFrame::Builder() + .set_video_frame_buffer(nativeBuffer) + .set_rtp_timestamp(extraInfo.rtpTimestamp) + .set_timestamp_ms(timestampNs / webrtc::kNumNanosecsPerMillisec) + .set_ntp_time_ms(extraInfo.ntpTimeMs) + .set_rotation(static_cast(rotation)) + .build(); + + // A decoder that provides the QP itself saves parsing the bitstream. + qpParsingEnabled = !qp.has_value(); + + webrtc::DecodedImageCallback * decodeCallback = callback.load(); + + if (decodeCallback != nullptr) { + decodeCallback->Decoded(frame, decodeTimeMs, qp.has_value() ? qp : extraInfo.qp); + } + } + + int32_t VideoDecoderWrapper::HandleReturnCode(JNIEnv * env, int32_t status, const char * method) + { + if (status >= 0) { + // OK or NO_OUTPUT. + return status; + } + + RTC_LOG(LS_WARNING) << "Java decoder " << method << ": " << status; + + if (status == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE || status == WEBRTC_VIDEO_CODEC_UNINITIALIZED) { + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + // Any other error may be one the decoder recovers from. + if (ReleaseInternal(env) == WEBRTC_VIDEO_CODEC_OK && ConfigureInternal(env)) { + RTC_LOG(LS_WARNING) << "Java decoder reset"; + + return WEBRTC_VIDEO_CODEC_ERROR; + } + + RTC_LOG(LS_WARNING) << "Java decoder could not be reset"; + + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + void VideoDecoderWrapper::InvalidateCallback(JNIEnv * env) + { + if (javaCallback == nullptr) { + return; + } + + // Waits for a frame that is being handed over right now. + env->CallVoidMethod(javaCallback->get(), javaClass->callbackInvalidate); + ClearCodecException(env, "invalidate"); + + javaCallback.reset(); + } + + std::optional VideoDecoderWrapper::ParseQp(const webrtc::EncodedImage & image) + { + if (image.qp_ != -1) { + return static_cast(image.qp_); + } + + int qp; + + switch (settings.codec_type()) { + case webrtc::kVideoCodecVP8: + if (webrtc::vp8::GetQp(image.data(), image.size(), &qp)) { + return static_cast(qp); + } + break; + + case webrtc::kVideoCodecVP9: + if (webrtc::vp9::GetQp(image.data(), image.size(), &qp)) { + return static_cast(qp); + } + break; + + case webrtc::kVideoCodecH264: { + h264BitstreamParser.ParseBitstream(image); + + std::optional sliceQp = h264BitstreamParser.GetLastSliceQp(); + + if (sliceQp.has_value()) { + return static_cast(*sliceQp); + } + break; + } + + default: + break; + } + + return std::nullopt; + } + + VideoDecoderWrapper::JavaVideoDecoderClass::JavaVideoDecoderClass(JNIEnv * env) + { + jclass cls = FindClass(env, PKG_CODEC"VideoDecoder"); + + initDecode = GetMethod(env, cls, "initDecode", + "(L" PKG_CODEC "VideoDecoder$Settings;L" PKG_CODEC "VideoDecoder$Callback;)L" PKG_CODEC "VideoCodecStatus;"); + release = GetMethod(env, cls, "release", "()L" PKG_CODEC "VideoCodecStatus;"); + decode = GetMethod(env, cls, "decode", + "(L" PKG_CODEC "EncodedImage;)L" PKG_CODEC "VideoCodecStatus;"); + getImplementationName = GetMethod(env, cls, "getImplementationName", "()" STRING_SIG); + isHardwareDecoder = GetMethod(env, cls, "isHardwareDecoder", "()Z"); + + settingsClass = FindClass(env, PKG_CODEC"VideoDecoder$Settings"); + settingsCtor = GetMethod(env, settingsClass, "", "(III)V"); + + callbackClass = FindClass(env, PKG_CODEC"NativeVideoDecoderCallback"); + callbackCtor = GetMethod(env, callbackClass, "", "(J)V"); + callbackInvalidate = GetMethod(env, callbackClass, "invalidate", "()V"); + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderFactoryWrapper.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderFactoryWrapper.cpp new file mode 100644 index 000000000..3c6a536de --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderFactoryWrapper.cpp @@ -0,0 +1,97 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/VideoEncoderFactoryWrapper.h" +#include "media/video/codec/DefaultVideoCodecFactories.h" +#include "media/video/codec/JavaLocalFrame.h" +#include "media/video/codec/VideoCodecInfo.h" +#include "media/video/codec/VideoCodecUtils.h" +#include "media/video/codec/VideoEncoderWrapper.h" +#include "JavaClasses.h" +#include "JavaUtils.h" +#include "JNI_WebRTC.h" + +#include "rtc_base/logging.h" + +namespace jni +{ + VideoEncoderFactoryWrapper::VideoEncoderFactoryWrapper(JNIEnv * env, jobject factory) : + factory(env, factory), + javaClass(JavaClasses::get(env)), + defaultFactory(CreateDefaultVideoEncoderFactory()) + { + JavaLocalRef codecs(env, env->CallObjectMethod(factory, javaClass->getSupportedCodecs)); + + ExceptionCheck(env); + + supportedFormats = VideoCodecInfo::toNativeList(env, codecs); + } + + std::vector VideoEncoderFactoryWrapper::GetSupportedFormats() const + { + return supportedFormats; + } + + std::unique_ptr VideoEncoderFactoryWrapper::Create(const webrtc::Environment & environment, + const webrtc::SdpVideoFormat & format) + { + JNIEnv * env = AttachCurrentThread(); + + if (env == nullptr) { + return nullptr; + } + + JavaLocalFrame localFrame(env, 16); + + try { + JavaLocalRef info = VideoCodecInfo::toJava(env, format); + jobject encoder = env->CallObjectMethod(factory, javaClass->createEncoder, info.get()); + + if (ClearCodecException(env, "createEncoder") || encoder == nullptr) { + RTC_LOG(LS_WARNING) << "Java encoder factory created no encoder for " << format.ToString(); + + return nullptr; + } + + if (env->IsInstanceOf(encoder, javaClass->nativeEncoderClass)) { + JavaLocalRef nativeInfo(env, env->GetObjectField(encoder, javaClass->nativeEncoderCodecInfo)); + + return defaultFactory->Create(environment, VideoCodecInfo::toNative(env, nativeInfo)); + } + + return std::make_unique(env, encoder, format); + } + catch (...) { + ClearCodecException(env, "createEncoder"); + + RTC_LOG(LS_WARNING) << "Java encoder factory failed to create an encoder for " << format.ToString(); + } + + return nullptr; + } + + VideoEncoderFactoryWrapper::JavaVideoEncoderFactoryClass::JavaVideoEncoderFactoryClass(JNIEnv * env) + { + jclass cls = FindClass(env, PKG_CODEC"VideoEncoderFactory"); + + getSupportedCodecs = GetMethod(env, cls, "getSupportedCodecs", "()" LIST_SIG); + createEncoder = GetMethod(env, cls, "createEncoder", + "(L" PKG_CODEC "VideoCodecInfo;)L" PKG_CODEC "VideoEncoder;"); + + nativeEncoderClass = FindClass(env, PKG_CODEC"NativeVideoEncoder"); + nativeEncoderCodecInfo = GetFieldID(env, nativeEncoderClass, "codecInfo", "L" PKG_CODEC "VideoCodecInfo;"); + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderWrapper.cpp b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderWrapper.cpp new file mode 100644 index 000000000..ff500532f --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderWrapper.cpp @@ -0,0 +1,704 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/video/codec/VideoEncoderWrapper.h" +#include "media/video/codec/EncodedImage.h" +#include "media/video/codec/JavaLocalFrame.h" +#include "media/video/codec/VideoCodecUtils.h" +#include "api/VideoFrame.h" +#include "JavaClasses.h" +#include "JavaPrimitive.h" +#include "JavaString.h" +#include "JavaUtils.h" +#include "JNI_WebRTC.h" + +#include "api/video/render_resolution.h" +#include "api/video/video_codec_constants.h" +#include "modules/video_coding/codecs/interface/common_constants.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/utility/vp8_header_parser.h" +#include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" +#include "rtc_base/logging.h" +#include "rtc_base/time_utils.h" + +#include + +namespace jni +{ + namespace + { + // Enough for the objects of one call into the encoder. + constexpr jint kLocalFrameCapacity = 32; + + // How many frames the encoder may hold back before the oldest records + // are given up, so that one that never produces output does not make + // them grow without bound. Several seconds of video. + constexpr size_t kMaxPendingFrames = 300; + + // The QP thresholds WebRTC's own encoders use for quality scaling. + constexpr int kLowVp8QpThreshold = 29; + constexpr int kHighVp8QpThreshold = 95; + // The QP is parsed from the VP9 bitstream, whose range is 0 to 255. + constexpr int kLowVp9QpThreshold = 96; + constexpr int kHighVp9QpThreshold = 185; + constexpr int kLowH264QpThreshold = 24; + constexpr int kHighH264QpThreshold = 37; + + std::optional OptionalInt(JNIEnv * env, jobject value) + { + if (value == nullptr) { + return std::nullopt; + } + + return Integer::getValue(env, value); + } + } + + VideoEncoderWrapper::VideoEncoderWrapper(JNIEnv * env, jobject encoder, const webrtc::SdpVideoFormat & format) : + encoder(env, encoder), + javaClass(JavaClasses::get(env)), + format(format), + callback(nullptr), + initialized(false), + numberOfCores(1), + codecSettings(), + gofIndex(0) + { + UpdateEncoderInfo(env); + } + + VideoEncoderWrapper::~VideoEncoderWrapper() + { + JNIEnv * env = AttachCurrentThread(); + + if (env != nullptr) { + InvalidateCallback(env); + } + } + + int32_t VideoEncoderWrapper::InitEncode(const webrtc::VideoCodec * settings, const Settings & encoderSettings) + { + JNIEnv * env = AttachCurrentThread(); + + if (env == nullptr || settings == nullptr) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + codecSettings = *settings; + capabilities = encoderSettings.capabilities; + numberOfCores = encoderSettings.number_of_cores; + + return InitEncodeInternal(env); + } + + int32_t VideoEncoderWrapper::InitEncodeInternal(JNIEnv * env) + { + JavaLocalFrame localFrame(env, kLocalFrameCapacity); + + bool automaticResizeOn; + + switch (codecSettings.codecType) { + case webrtc::kVideoCodecVP8: + automaticResizeOn = codecSettings.VP8()->automaticResizeOn; + break; + + case webrtc::kVideoCodecVP9: + automaticResizeOn = codecSettings.VP9()->automaticResizeOn; + gof.SetGofInfoVP9(webrtc::TemporalStructureMode::kTemporalStructureMode1); + gofIndex = 0; + break; + + default: + automaticResizeOn = true; + break; + } + + const bool lossNotification = capabilities.has_value() && capabilities->loss_notification; + const bool screenContent = codecSettings.mode == webrtc::VideoCodecMode::kScreensharing; + + jobject settings = env->NewObject(javaClass->settingsClass, javaClass->settingsCtor, + static_cast(numberOfCores), + static_cast(codecSettings.width), + static_cast(codecSettings.height), + static_cast(codecSettings.startBitrate), + static_cast(codecSettings.minBitrate), + static_cast(codecSettings.maxBitrate), + static_cast(codecSettings.maxFramerate), + static_cast(codecSettings.numberOfSimulcastStreams), + automaticResizeOn ? JNI_TRUE : JNI_FALSE, + screenContent ? JNI_TRUE : JNI_FALSE, + lossNotification ? JNI_TRUE : JNI_FALSE); + + if (ClearCodecException(env, "initEncode") || settings == nullptr) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // A callback of its own for each initialization, so that frames of an + // earlier one cannot reach this one. + InvalidateCallback(env); + + jobject jCallback = env->NewObject(javaClass->callbackClass, javaClass->callbackCtor, + reinterpret_cast(this)); + + if (ClearCodecException(env, "initEncode") || jCallback == nullptr) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + javaCallback = std::make_unique>(env, jCallback); + + jobject status = env->CallObjectMethod(encoder, javaClass->initEncode, settings, jCallback); + int32_t result = ToNativeCodecStatus(env, status, "initEncode"); + + RTC_LOG(LS_INFO) << "Java encoder initEncode: " << result; + + // Some properties of the encoder may depend on its settings. + UpdateEncoderInfo(env); + + initialized = result == WEBRTC_VIDEO_CODEC_OK; + + return result; + } + + int32_t VideoEncoderWrapper::RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback * encodeCallback) + { + callback = encodeCallback; + + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t VideoEncoderWrapper::Release() + { + JNIEnv * env = AttachCurrentThread(); + + if (env == nullptr) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + return ReleaseInternal(env); + } + + int32_t VideoEncoderWrapper::ReleaseInternal(JNIEnv * env) + { + JavaLocalFrame localFrame(env, kLocalFrameCapacity); + + jobject status = env->CallObjectMethod(encoder, javaClass->release); + int32_t result = ToNativeCodecStatus(env, status, "release"); + + RTC_LOG(LS_INFO) << "Java encoder release: " << result; + + InvalidateCallback(env); + + { + webrtc::MutexLock lock(&frameExtraInfosLock); + frameExtraInfos.clear(); + } + + initialized = false; + + return result; + } + + int32_t VideoEncoderWrapper::Encode(const webrtc::VideoFrame & frame, + const std::vector * frameTypes) + { + if (!initialized) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + + JNIEnv * env = AttachCurrentThread(); + + if (env == nullptr) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + + JavaLocalFrame localFrame(env, kLocalFrameCapacity); + + const auto javaFrameClass = JavaClasses::get(env); + jobject jFrame = nullptr; + + try { + jFrame = VideoFrame::toJava(env, frame).release(); + } + catch (...) { + ClearCodecException(env, "encode"); + } + + if (jFrame == nullptr) { + RTC_LOG(LS_WARNING) << "Java encoder: dropping a frame that cannot be converted to I420"; + + return WEBRTC_VIDEO_CODEC_OK; + } + + std::vector types; + + if (frameTypes != nullptr) { + for (webrtc::VideoFrameType type : *frameTypes) { + types.push_back(static_cast(type)); + } + } + if (types.empty()) { + types.push_back(static_cast(webrtc::VideoFrameType::kVideoFrameDelta)); + } + + jobject info = nullptr; + jintArray jTypes = env->NewIntArray(static_cast(types.size())); + + if (jTypes != nullptr) { + env->SetIntArrayRegion(jTypes, 0, static_cast(types.size()), types.data()); + + info = env->CallStaticObjectMethod(javaClass->encodeInfoClass, javaClass->encodeInfoFromNative, jTypes); + } + + int32_t result; + + if (ClearCodecException(env, "encode") || info == nullptr) { + result = WEBRTC_VIDEO_CODEC_ERROR; + } + else { + { + webrtc::MutexLock lock(&frameExtraInfosLock); + + if (frameExtraInfos.size() >= kMaxPendingFrames) { + frameExtraInfos.pop_front(); + } + + frameExtraInfos.push_back(FrameExtraInfo { + frame.timestamp_us() * webrtc::kNumNanosecsPerMicrosec, + frame.rtp_timestamp() + }); + } + + jobject status = env->CallObjectMethod(encoder, javaClass->encode, jFrame, info); + result = ToNativeCodecStatus(env, status, "encode"); + } + + // An encoder that needs the frame any longer has retained it. + env->CallVoidMethod(jFrame, javaFrameClass->release); + ClearCodecException(env, "encode"); + + return HandleReturnCode(env, result, "encode"); + } + + void VideoEncoderWrapper::SetRates(const RateControlParameters & parameters) + { + JNIEnv * env = AttachCurrentThread(); + + if (env == nullptr) { + return; + } + + JavaLocalFrame localFrame(env, kLocalFrameCapacity); + + jobjectArray layers = env->NewObjectArray(webrtc::kMaxSpatialLayers, javaClass->intArrayClass, nullptr); + + if (ClearCodecException(env, "setRates") || layers == nullptr) { + return; + } + + for (size_t spatial = 0; spatial < webrtc::kMaxSpatialLayers; spatial++) { + jint bitrates[webrtc::kMaxTemporalStreams]; + + for (size_t temporal = 0; temporal < webrtc::kMaxTemporalStreams; temporal++) { + bitrates[temporal] = static_cast(parameters.bitrate.GetBitrate(spatial, temporal)); + } + + jintArray layer = env->NewIntArray(webrtc::kMaxTemporalStreams); + + if (layer == nullptr) { + ClearCodecException(env, "setRates"); + return; + } + + env->SetIntArrayRegion(layer, 0, webrtc::kMaxTemporalStreams, bitrates); + env->SetObjectArrayElement(layers, static_cast(spatial), layer); + env->DeleteLocalRef(layer); + } + + jobject allocation = env->NewObject(javaClass->bitrateAllocationClass, javaClass->bitrateAllocationCtor, layers); + jobject jParameters = nullptr; + + if (allocation != nullptr) { + jParameters = env->NewObject(javaClass->rateControlClass, javaClass->rateControlCtor, allocation, + static_cast(parameters.framerate_fps)); + } + + if (ClearCodecException(env, "setRates") || jParameters == nullptr) { + return; + } + + jobject status = env->CallObjectMethod(encoder, javaClass->setRates, jParameters); + int32_t result = ToNativeCodecStatus(env, status, "setRates"); + + if (result < 0) { + RTC_LOG(LS_WARNING) << "Java encoder setRates: " << result; + } + } + + webrtc::VideoEncoder::EncoderInfo VideoEncoderWrapper::GetEncoderInfo() const + { + return encoderInfo; + } + + void VideoEncoderWrapper::OnEncodedFrame(JNIEnv * env, jobject jImage) + { + JavaLocalRef imageRef(env, env->NewLocalRef(jImage)); + + webrtc::EncodedImage image = EncodedImage::toNative(env, imageRef); + const int64_t captureTimeNs = EncodedImage::getCaptureTimeNs(env, imageRef); + + FrameExtraInfo extraInfo; + + { + webrtc::MutexLock lock(&frameExtraInfosLock); + + // Frames come out in the order they went in, though an encoder + // may drop some, so the records of older frames are stale. + while (!frameExtraInfos.empty() && frameExtraInfos.front().captureTimeNs < captureTimeNs) { + frameExtraInfos.pop_front(); + } + + if (frameExtraInfos.empty() || frameExtraInfos.front().captureTimeNs != captureTimeNs) { + RTC_LOG(LS_WARNING) << "Java encoder produced a frame for no frame it was given, capture time " + << captureTimeNs << " ns"; + return; + } + + extraInfo = frameExtraInfos.front(); + frameExtraInfos.pop_front(); + } + + image.SetRtpTimestamp(extraInfo.rtpTimestamp); + image.capture_time_ms_ = captureTimeNs / webrtc::kNumNanosecsPerMillisec; + + if (image.qp_ < 0) { + image.qp_ = ParseQp(image); + } + + webrtc::CodecSpecificInfo info = ParseCodecSpecificInfo(image); + webrtc::EncodedImageCallback * encodeCallback = callback.load(); + + if (encodeCallback != nullptr) { + encodeCallback->OnEncodedImage(image, &info); + } + } + + int32_t VideoEncoderWrapper::HandleReturnCode(JNIEnv * env, int32_t status, const char * method) + { + if (status >= 0) { + // OK or NO_OUTPUT, or OK with a request. + return status; + } + + RTC_LOG(LS_WARNING) << "Java encoder " << method << ": " << status; + + if (status == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE || status == WEBRTC_VIDEO_CODEC_UNINITIALIZED) { + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + // Any other error may be one the encoder recovers from. + if (ReleaseInternal(env) == WEBRTC_VIDEO_CODEC_OK && InitEncodeInternal(env) == WEBRTC_VIDEO_CODEC_OK) { + RTC_LOG(LS_WARNING) << "Java encoder reset"; + + return WEBRTC_VIDEO_CODEC_ERROR; + } + + RTC_LOG(LS_WARNING) << "Java encoder could not be reset"; + + return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + } + + void VideoEncoderWrapper::InvalidateCallback(JNIEnv * env) + { + if (javaCallback == nullptr) { + return; + } + + // Waits for a frame that is being handed over right now. + env->CallVoidMethod(javaCallback->get(), javaClass->callbackInvalidate); + ClearCodecException(env, "invalidate"); + + javaCallback.reset(); + } + + void VideoEncoderWrapper::UpdateEncoderInfo(JNIEnv * env) + { + JavaLocalFrame localFrame(env, kLocalFrameCapacity); + + encoderInfo.supports_native_handle = false; + encoderInfo.scaling_settings = GetScalingSettings(env); + encoderInfo.resolution_bitrate_limits = GetResolutionBitrateLimits(env); + + jobject name = env->CallObjectMethod(encoder, javaClass->getImplementationName); + + if (!ClearCodecException(env, "getImplementationName") && name != nullptr) { + try { + encoderInfo.implementation_name = JavaString::toNative(env, + JavaLocalRef(env, static_cast(env->NewLocalRef(name)))); + } + catch (...) { + ClearCodecException(env, "getImplementationName"); + } + } + + jboolean hardware = env->CallBooleanMethod(encoder, javaClass->isHardwareEncoder); + + if (!ClearCodecException(env, "isHardwareEncoder")) { + encoderInfo.is_hardware_accelerated = hardware == JNI_TRUE; + } + + jobject info = env->CallObjectMethod(encoder, javaClass->getEncoderInfo); + + if (!ClearCodecException(env, "getEncoderInfo") && info != nullptr) { + encoderInfo.requested_resolution_alignment = std::max(1, + static_cast(env->GetIntField(info, javaClass->infoRequestedResolutionAlignment))); + encoderInfo.apply_alignment_to_all_simulcast_layers = + env->GetBooleanField(info, javaClass->infoApplyAlignmentToAllSimulcastLayers) == JNI_TRUE; + } + } + + webrtc::VideoEncoder::ScalingSettings VideoEncoderWrapper::GetScalingSettings(JNIEnv * env) const + { + jobject settings = env->CallObjectMethod(encoder, javaClass->getScalingSettings); + + if (ClearCodecException(env, "getScalingSettings") || settings == nullptr) { + return ScalingSettings::kOff; + } + if (env->GetBooleanField(settings, javaClass->scalingOn) != JNI_TRUE) { + return ScalingSettings::kOff; + } + + std::optional low; + std::optional high; + + try { + low = OptionalInt(env, env->GetObjectField(settings, javaClass->scalingLow)); + high = OptionalInt(env, env->GetObjectField(settings, javaClass->scalingHigh)); + } + catch (...) { + ClearCodecException(env, "getScalingSettings"); + } + + if (low && high) { + return ScalingSettings(*low, *high); + } + + switch (codecSettings.codecType) { + case webrtc::kVideoCodecVP8: + return ScalingSettings(low.value_or(kLowVp8QpThreshold), high.value_or(kHighVp8QpThreshold)); + + case webrtc::kVideoCodecVP9: + return ScalingSettings(low.value_or(kLowVp9QpThreshold), high.value_or(kHighVp9QpThreshold)); + + case webrtc::kVideoCodecH264: + return ScalingSettings(low.value_or(kLowH264QpThreshold), high.value_or(kHighH264QpThreshold)); + + default: + // Without a QP to go by, quality scaling cannot work. + return ScalingSettings::kOff; + } + } + + std::vector VideoEncoderWrapper::GetResolutionBitrateLimits( + JNIEnv * env) const + { + std::vector limits; + + jobjectArray array = static_cast( + env->CallObjectMethod(encoder, javaClass->getResolutionBitrateLimits)); + + if (ClearCodecException(env, "getResolutionBitrateLimits") || array == nullptr) { + return limits; + } + + const jsize count = env->GetArrayLength(array); + + for (jsize i = 0; i < count; i++) { + jobject entry = env->GetObjectArrayElement(array, i); + + if (entry == nullptr) { + continue; + } + + limits.emplace_back( + env->GetIntField(entry, javaClass->limitsFrameSizePixels), + env->GetIntField(entry, javaClass->limitsMinStartBitrateBps), + env->GetIntField(entry, javaClass->limitsMinBitrateBps), + env->GetIntField(entry, javaClass->limitsMaxBitrateBps)); + + env->DeleteLocalRef(entry); + } + + return limits; + } + + int VideoEncoderWrapper::ParseQp(const webrtc::EncodedImage & image) + { + int qp = -1; + bool success = false; + + switch (codecSettings.codecType) { + case webrtc::kVideoCodecVP8: + success = webrtc::vp8::GetQp(image.data(), image.size(), &qp); + break; + + case webrtc::kVideoCodecVP9: + success = webrtc::vp9::GetQp(image.data(), image.size(), &qp); + break; + + case webrtc::kVideoCodecH264: + h264BitstreamParser.ParseBitstream(image); + qp = h264BitstreamParser.GetLastSliceQp().value_or(-1); + success = qp >= 0; + break; + + default: + break; + } + + return success ? qp : -1; + } + + webrtc::CodecSpecificInfo VideoEncoderWrapper::ParseCodecSpecificInfo(const webrtc::EncodedImage & image) + { + const bool keyFrame = image.frame_type() == webrtc::VideoFrameType::kVideoFrameKey; + + webrtc::CodecSpecificInfo info; + info.codecType = codecSettings.codecType; + + // The encoder produces a single layer, so the frame dependencies are + // those of a stream without layers. + auto layerFrames = svcController.NextFrameConfig(/*restart=*/keyFrame); + info.generic_frame_info = svcController.OnEncodeDone(layerFrames[0]); + + if (keyFrame) { + info.template_structure = svcController.DependencyStructure(); + info.template_structure->resolutions = { + webrtc::RenderResolution(image._encodedWidth, image._encodedHeight) + }; + } + + switch (codecSettings.codecType) { + case webrtc::kVideoCodecVP8: + info.codecSpecific.VP8.nonReference = false; + info.codecSpecific.VP8.temporalIdx = webrtc::kNoTemporalIdx; + info.codecSpecific.VP8.layerSync = false; + info.codecSpecific.VP8.keyIdx = webrtc::kNoKeyIdx; + break; + + case webrtc::kVideoCodecVP9: + if (keyFrame) { + gofIndex = 0; + } + + info.codecSpecific.VP9.inter_pic_predicted = !keyFrame; + info.codecSpecific.VP9.flexible_mode = false; + info.codecSpecific.VP9.ss_data_available = keyFrame; + info.codecSpecific.VP9.temporal_idx = webrtc::kNoTemporalIdx; + info.codecSpecific.VP9.temporal_up_switch = true; + info.codecSpecific.VP9.inter_layer_predicted = false; + info.codecSpecific.VP9.gof_idx = static_cast(gofIndex++ % gof.num_frames_in_gof); + info.codecSpecific.VP9.num_spatial_layers = 1; + info.codecSpecific.VP9.first_frame_in_picture = true; + info.codecSpecific.VP9.spatial_layer_resolution_present = keyFrame; + + if (keyFrame) { + info.codecSpecific.VP9.width[0] = image._encodedWidth; + info.codecSpecific.VP9.height[0] = image._encodedHeight; + info.codecSpecific.VP9.gof.CopyGofInfoVP9(gof); + } + break; + + case webrtc::kVideoCodecH264: { + // Mode 1 may split a NAL unit across packets; mode 0 has to + // send each NAL unit in a packet of its own. + auto mode = format.parameters.find("packetization-mode"); + const bool nonInterleaved = mode != format.parameters.end() && mode->second == "1"; + + info.codecSpecific.H264.packetization_mode = nonInterleaved + ? webrtc::H264PacketizationMode::NonInterleaved + : webrtc::H264PacketizationMode::SingleNalUnit; + info.codecSpecific.H264.temporal_idx = webrtc::kNoTemporalIdx; + info.codecSpecific.H264.base_layer_sync = false; + info.codecSpecific.H264.idr_frame = keyFrame; + break; + } + + default: + break; + } + + return info; + } + + VideoEncoderWrapper::JavaVideoEncoderClass::JavaVideoEncoderClass(JNIEnv * env) + { + jclass cls = FindClass(env, PKG_CODEC"VideoEncoder"); + + initEncode = GetMethod(env, cls, "initEncode", + "(L" PKG_CODEC "VideoEncoder$Settings;L" PKG_CODEC "VideoEncoder$Callback;)L" PKG_CODEC "VideoCodecStatus;"); + release = GetMethod(env, cls, "release", "()L" PKG_CODEC "VideoCodecStatus;"); + encode = GetMethod(env, cls, "encode", + "(L" PKG_VIDEO "VideoFrame;L" PKG_CODEC "VideoEncoder$EncodeInfo;)L" PKG_CODEC "VideoCodecStatus;"); + setRates = GetMethod(env, cls, "setRates", + "(L" PKG_CODEC "VideoEncoder$RateControlParameters;)L" PKG_CODEC "VideoCodecStatus;"); + getScalingSettings = GetMethod(env, cls, "getScalingSettings", "()L" PKG_CODEC "VideoEncoder$ScalingSettings;"); + getResolutionBitrateLimits = GetMethod(env, cls, "getResolutionBitrateLimits", + "()[L" PKG_CODEC "VideoEncoder$ResolutionBitrateLimits;"); + getEncoderInfo = GetMethod(env, cls, "getEncoderInfo", "()L" PKG_CODEC "VideoEncoder$EncoderInfo;"); + getImplementationName = GetMethod(env, cls, "getImplementationName", "()" STRING_SIG); + isHardwareEncoder = GetMethod(env, cls, "isHardwareEncoder", "()Z"); + + settingsClass = FindClass(env, PKG_CODEC"VideoEncoder$Settings"); + settingsCtor = GetMethod(env, settingsClass, "", "(IIIIIIIIZZZ)V"); + + encodeInfoClass = FindClass(env, PKG_CODEC"VideoEncoder$EncodeInfo"); + encodeInfoFromNative = GetStaticMethod(env, encodeInfoClass, "fromNative", + "([I)L" PKG_CODEC "VideoEncoder$EncodeInfo;"); + + bitrateAllocationClass = FindClass(env, PKG_CODEC"VideoEncoder$BitrateAllocation"); + bitrateAllocationCtor = GetMethod(env, bitrateAllocationClass, "", "([[I)V"); + + rateControlClass = FindClass(env, PKG_CODEC"VideoEncoder$RateControlParameters"); + rateControlCtor = GetMethod(env, rateControlClass, "", + "(L" PKG_CODEC "VideoEncoder$BitrateAllocation;D)V"); + + jclass scalingClass = FindClass(env, PKG_CODEC"VideoEncoder$ScalingSettings"); + scalingOn = GetFieldID(env, scalingClass, "on", "Z"); + scalingLow = GetFieldID(env, scalingClass, "low", INTEGER_SIG); + scalingHigh = GetFieldID(env, scalingClass, "high", INTEGER_SIG); + + jclass limitsClass = FindClass(env, PKG_CODEC"VideoEncoder$ResolutionBitrateLimits"); + limitsFrameSizePixels = GetFieldID(env, limitsClass, "frameSizePixels", "I"); + limitsMinStartBitrateBps = GetFieldID(env, limitsClass, "minStartBitrateBps", "I"); + limitsMinBitrateBps = GetFieldID(env, limitsClass, "minBitrateBps", "I"); + limitsMaxBitrateBps = GetFieldID(env, limitsClass, "maxBitrateBps", "I"); + + jclass infoClass = FindClass(env, PKG_CODEC"VideoEncoder$EncoderInfo"); + infoRequestedResolutionAlignment = GetFieldID(env, infoClass, "requestedResolutionAlignment", "I"); + infoApplyAlignmentToAllSimulcastLayers = GetFieldID(env, infoClass, "applyAlignmentToAllSimulcastLayers", "Z"); + + callbackClass = FindClass(env, PKG_CODEC"NativeVideoEncoderCallback"); + callbackCtor = GetMethod(env, callbackClass, "", "(J)V"); + callbackInvalidate = GetMethod(env, callbackClass, "invalidate", "()V"); + + // The class loader FindClass goes through does not resolve array + // descriptors, and the JNI lookup of a primitive array class needs + // no class loader. + jclass intArray = env->FindClass("[I"); + intArrayClass = static_cast(env->NewGlobalRef(intArray)); + env->DeleteLocalRef(intArray); + } +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java index 4938e0efd..b18994374 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/PeerConnectionFactory.java @@ -28,6 +28,10 @@ import dev.onvoid.webrtc.media.audio.CustomAudioSource; import dev.onvoid.webrtc.media.video.VideoTrackSource; import dev.onvoid.webrtc.media.video.VideoTrack; +import dev.onvoid.webrtc.media.video.codec.DefaultVideoDecoderFactory; +import dev.onvoid.webrtc.media.video.codec.DefaultVideoEncoderFactory; +import dev.onvoid.webrtc.media.video.codec.VideoDecoderFactory; +import dev.onvoid.webrtc.media.video.codec.VideoEncoderFactory; import java.util.Map; import java.util.Objects; @@ -52,6 +56,9 @@ * {@link #createAudioSource(AudioOptions)} is called, since that is the only * way to send device-captured audio. A factory that sends pushed audio, or no * audio at all, never opens it. Playout of received audio is never affected. + *

+ * The constructors cover the common combinations of modules. A {@link + * #builder() builder} sets any of them, including the video codecs. * * @author Alex Andres */ @@ -103,7 +110,7 @@ private enum AudioInputMode { * Creates an instance of PeerConnectionFactory. */ public PeerConnectionFactory() { - initialize(null, null, null); + initialize(null, null, null, null, null); } /** @@ -113,7 +120,7 @@ public PeerConnectionFactory() { * @param audioProcessing The custom audio processing module. */ public PeerConnectionFactory(AudioProcessing audioProcessing) { - initialize(null, null, audioProcessing); + initialize(null, null, audioProcessing, null, null); } /** @@ -123,7 +130,7 @@ public PeerConnectionFactory(AudioProcessing audioProcessing) { * @param audioModule The custom audio device module. */ public PeerConnectionFactory(AudioDeviceModuleBase audioModule) { - initialize(null, audioModule, null); + initialize(null, audioModule, null, null, null); } /** @@ -135,7 +142,7 @@ public PeerConnectionFactory(AudioDeviceModuleBase audioModule) { */ public PeerConnectionFactory(AudioDeviceModuleBase audioModule, AudioProcessing audioProcessing) { - initialize(null, audioModule, audioProcessing); + initialize(null, audioModule, audioProcessing, null, null); } /** @@ -147,7 +154,7 @@ public PeerConnectionFactory(AudioDeviceModuleBase audioModule, * non-null and non-empty. */ public PeerConnectionFactory(Map fieldTrials) { - initialize(fieldTrials, null, null); + initialize(fieldTrials, null, null, null, null); } /** @@ -161,7 +168,7 @@ public PeerConnectionFactory(Map fieldTrials) { */ public PeerConnectionFactory(Map fieldTrials, AudioProcessing audioProcessing) { - initialize(fieldTrials, null, audioProcessing); + initialize(fieldTrials, null, audioProcessing, null, null); } /** @@ -175,7 +182,7 @@ public PeerConnectionFactory(Map fieldTrials, */ public PeerConnectionFactory(Map fieldTrials, AudioDeviceModuleBase audioModule) { - initialize(fieldTrials, audioModule, null); + initialize(fieldTrials, audioModule, null, null, null); } /** @@ -190,7 +197,37 @@ public PeerConnectionFactory(Map fieldTrials, */ public PeerConnectionFactory(Map fieldTrials, AudioDeviceModuleBase audioModule, AudioProcessing audioProcessing) { - initialize(fieldTrials, audioModule, audioProcessing); + initialize(fieldTrials, audioModule, audioProcessing, null, null); + } + + /** + * Creates an instance of PeerConnectionFactory with what the builder was + * given. + * + * @param builder The builder with the modules to use. + */ + private PeerConnectionFactory(Builder builder) { + // The built-in codecs need no detour through Java. + VideoEncoderFactory encoderFactory = builder.videoEncoderFactory instanceof DefaultVideoEncoderFactory + ? null + : builder.videoEncoderFactory; + VideoDecoderFactory decoderFactory = builder.videoDecoderFactory instanceof DefaultVideoDecoderFactory + ? null + : builder.videoDecoderFactory; + + initialize(builder.fieldTrials, builder.audioModule, builder.audioProcessing, + encoderFactory, decoderFactory); + } + + /** + * Returns a builder for a PeerConnectionFactory, which sets any of the + * modules the factory is created with. Whatever is not set is what the + * {@link #PeerConnectionFactory() default constructor} uses. + * + * @return A new builder. + */ + public static Builder builder() { + return new Builder(); } /** @@ -364,7 +401,9 @@ public RTCPeerConnection createPeerConnection(RTCConfiguration config, public native void dispose(); private native void initialize(Map fieldTrials, - AudioDeviceModuleBase audioModule, AudioProcessing audioProcessing); + AudioDeviceModuleBase audioModule, AudioProcessing audioProcessing, + VideoEncoderFactory videoEncoderFactory, + VideoDecoderFactory videoDecoderFactory); private native AudioTrackSource createAudioSourceInternal(AudioOptions options); @@ -394,4 +433,122 @@ private native RTCPeerConnection createPeerConnectionInternal( */ private native void setDeviceCaptureEnabled(boolean enabled); + + /** + * Builds a {@link PeerConnectionFactory}. Each module that is not set is + * what the {@link PeerConnectionFactory#PeerConnectionFactory() default + * constructor} uses. + *

+ * Example, a factory with a codec of one's own next to the built-in ones: + * 

{@code
+	 * PeerConnectionFactory factory = PeerConnectionFactory.builder()
+	 *     .setAudioDeviceModule(audioModule)
+	 *     .setVideoEncoderFactory(new MyEncoderFactory())
+	 *     .setVideoDecoderFactory(new MyDecoderFactory())
+	 *     .build();
+	 * }
+ */ + public static final class Builder { + + private Map fieldTrials; + + private AudioDeviceModuleBase audioModule; + + private AudioProcessing audioProcessing; + + private VideoEncoderFactory videoEncoderFactory; + + private VideoDecoderFactory videoDecoderFactory; + + + private Builder() { + } + + /** + * Sets the field trials, which allow enabling experimental WebRTC + * features. + * + * @param fieldTrials The field trials to set, e.g. {@code + * {"WebRTC-Bar": "Enabled"}}. Keys and values must + * be non-null and non-empty. + * + * @return This builder. + */ + public Builder setFieldTrials(Map fieldTrials) { + this.fieldTrials = fieldTrials; + return this; + } + + /** + * Sets the audio device module. + * + * @param audioModule The custom audio device module. + * + * @return This builder. + */ + public Builder setAudioDeviceModule(AudioDeviceModuleBase audioModule) { + this.audioModule = audioModule; + return this; + } + + /** + * Sets the audio processing module. + * + * @param audioProcessing The custom audio processing module. + * + * @return This builder. + */ + public Builder setAudioProcessing(AudioProcessing audioProcessing) { + this.audioProcessing = audioProcessing; + return this; + } + + /** + * Sets the factory of the video encoders, which decides what video + * codecs the factory can send. Without one, the factory uses the + * codecs of a {@link DefaultVideoEncoderFactory}. + *

+ * The supported codecs are asked for once, when the factory is built; + * the encoders are created from WebRTC threads while the factory is + * in use. + * + * @param factory The video encoder factory. + * + * @return This builder. + */ + public Builder setVideoEncoderFactory(VideoEncoderFactory factory) { + this.videoEncoderFactory = factory; + return this; + } + + /** + * Sets the factory of the video decoders, which decides what video + * codecs the factory can receive. Without one, the factory uses the + * codecs of a {@link DefaultVideoDecoderFactory}. + *

+ * The supported codecs are asked for once, when the factory is built; + * the decoders are created from WebRTC threads while the factory is + * in use. + * + * @param factory The video decoder factory. + * + * @return This builder. + */ + public Builder setVideoDecoderFactory(VideoDecoderFactory factory) { + this.videoDecoderFactory = factory; + return this; + } + + /** + * Creates the PeerConnectionFactory. The builder may be used again + * afterwards. + * + * @return The new factory. + */ + public PeerConnectionFactory build() { + return new PeerConnectionFactory(this); + } + + } + } diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/DefaultVideoDecoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/DefaultVideoDecoderFactory.java new file mode 100644 index 000000000..95ef3cc08 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/DefaultVideoDecoderFactory.java @@ -0,0 +1,92 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import dev.onvoid.webrtc.internal.NativeLoader; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * The video decoders built into this library, which a {@link + * dev.onvoid.webrtc.PeerConnectionFactory PeerConnectionFactory} uses unless + * it is given a {@link VideoDecoderFactory} of its own. On Windows and Linux + * these are VP8, VP9, AV1 and H.264 in software; on macOS they are those of + * WebRTC's default factories, with H.264 through VideoToolbox. + * {@link #getSupportedCodecs()} lists what the platform has. + *

+ * A factory of one's own can hand out these decoders next to its own ones + * by delegating to an instance of this class. The decoders it creates are + * {@link NativeVideoDecoder}s, which run inside WebRTC and are not to be + * called from Java. + * + * @author Alex Andres + */ +public final class DefaultVideoDecoderFactory implements VideoDecoderFactory { + + static { + try { + NativeLoader.loadLibrary("webrtc-java"); + } + catch (Exception e) { + throw new RuntimeException("Load library 'webrtc-java' failed", e); + } + } + + + private final List supportedCodecs; + + + /** + * Creates a factory for the built-in video decoders. + */ + public DefaultVideoDecoderFactory() { + supportedCodecs = Collections.unmodifiableList( + Arrays.asList(getSupportedCodecsInternal())); + } + + @Override + public List getSupportedCodecs() { + return supportedCodecs; + } + + /** + * Returns a built-in decoder for the given codec. + * + * @param info The codec to decode, as negotiated. + * + * @return A native decoder, or {@code null} if there is no built-in + * decoder for the codec. + */ + @Override + public NativeVideoDecoder createDecoder(VideoCodecInfo info) { + Objects.requireNonNull(info, "VideoCodecInfo is null"); + + for (VideoCodecInfo codec : supportedCodecs) { + if (codec.getName().equalsIgnoreCase(info.getName())) { + return new NativeVideoDecoder(info); + } + } + + return null; + } + + private static native VideoCodecInfo[] getSupportedCodecsInternal(); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/DefaultVideoEncoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/DefaultVideoEncoderFactory.java new file mode 100644 index 000000000..3bae9f797 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/DefaultVideoEncoderFactory.java @@ -0,0 +1,92 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import dev.onvoid.webrtc.internal.NativeLoader; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * The video encoders built into this library, which a {@link + * dev.onvoid.webrtc.PeerConnectionFactory PeerConnectionFactory} uses unless + * it is given a {@link VideoEncoderFactory} of its own. On Windows and Linux + * these are VP8, VP9, AV1 and H.264 in software; on macOS they are those of + * WebRTC's default factories, with H.264 through VideoToolbox. + * {@link #getSupportedCodecs()} lists what the platform has. + *

+ * A factory of one's own can hand out these encoders next to its own ones + * by delegating to an instance of this class. The encoders it creates are + * {@link NativeVideoEncoder}s, which run inside WebRTC and are not to be + * called from Java. + * + * @author Alex Andres + */ +public final class DefaultVideoEncoderFactory implements VideoEncoderFactory { + + static { + try { + NativeLoader.loadLibrary("webrtc-java"); + } + catch (Exception e) { + throw new RuntimeException("Load library 'webrtc-java' failed", e); + } + } + + + private final List supportedCodecs; + + + /** + * Creates a factory for the built-in video encoders. + */ + public DefaultVideoEncoderFactory() { + supportedCodecs = Collections.unmodifiableList( + Arrays.asList(getSupportedCodecsInternal())); + } + + @Override + public List getSupportedCodecs() { + return supportedCodecs; + } + + /** + * Returns a built-in encoder for the given codec. + * + * @param info The codec to encode, as negotiated. + * + * @return A native encoder, or {@code null} if there is no built-in + * encoder for the codec. + */ + @Override + public NativeVideoEncoder createEncoder(VideoCodecInfo info) { + Objects.requireNonNull(info, "VideoCodecInfo is null"); + + for (VideoCodecInfo codec : supportedCodecs) { + if (codec.getName().equalsIgnoreCase(info.getName())) { + return new NativeVideoEncoder(info); + } + } + + return null; + } + + private static native VideoCodecInfo[] getSupportedCodecsInternal(); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/EncodedImage.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/EncodedImage.java new file mode 100644 index 000000000..6a9f9a97d --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/EncodedImage.java @@ -0,0 +1,351 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import java.nio.ByteBuffer; +import java.util.Objects; + +/** + * An encoded video frame: what a {@link VideoEncoder} produces and a {@link + * VideoDecoder} consumes. + *

+ * An encoder hands its images to WebRTC through {@link + * VideoEncoder.Callback#onEncodedFrame(EncodedImage)}, which copies the + * payload, so the buffer may be reused as soon as that call returns. The + * {@link #getCaptureTimeNs() capture time} of such an image must be the + * {@link dev.onvoid.webrtc.media.video.VideoFrame#timestampNs timestamp} of + * the frame it encodes: that is how WebRTC matches the output to the input. + *

+ * The images WebRTC hands to a decoder have a read-only buffer that is valid + * only until {@link VideoDecoder#decode(EncodedImage)} returns; a decoder that + * needs the data longer has to copy it. Their capture time identifies the + * frame, and the decoder has to give its decoded frame the same timestamp. + * + * @author Alex Andres + */ +public final class EncodedImage { + + /** + * The type of an encoded frame. + */ + public enum FrameType { + + /** A frame without data, for example one the encoder dropped. */ + EMPTY(0), + + /** A key frame, which decodes without any other frame. */ + KEY(3), + + /** A delta frame, which depends on earlier frames. */ + DELTA(4); + + + private final int nativeIndex; + + + FrameType(int nativeIndex) { + this.nativeIndex = nativeIndex; + } + + int getNativeIndex() { + return nativeIndex; + } + + static FrameType fromNativeIndex(int nativeIndex) { + for (FrameType type : values()) { + if (type.nativeIndex == nativeIndex) { + return type; + } + } + + throw new IllegalArgumentException("Unknown frame type: " + nativeIndex); + } + + } + + + private final ByteBuffer buffer; + + private final int encodedWidth; + + private final int encodedHeight; + + private final long captureTimeNs; + + private final FrameType frameType; + + private final int rotation; + + private final Integer qp; + + + private EncodedImage(ByteBuffer buffer, int encodedWidth, int encodedHeight, + long captureTimeNs, FrameType frameType, int rotation, Integer qp) { + this.buffer = buffer; + this.encodedWidth = encodedWidth; + this.encodedHeight = encodedHeight; + this.captureTimeNs = captureTimeNs; + this.frameType = frameType; + this.rotation = rotation; + this.qp = qp; + } + + /** + * Returns the encoded payload, from its position to its limit. + * + * @return The encoded data. + */ + public ByteBuffer getBuffer() { + return buffer; + } + + /** + * @return The width of the encoded frame in pixels. + */ + public int getEncodedWidth() { + return encodedWidth; + } + + /** + * @return The height of the encoded frame in pixels. + */ + public int getEncodedHeight() { + return encodedHeight; + } + + /** + * @return The capture time that identifies the frame, in nanoseconds. + */ + public long getCaptureTimeNs() { + return captureTimeNs; + } + + /** + * @return Whether this is a key frame, a delta frame or an empty one. + */ + public FrameType getFrameType() { + return frameType; + } + + /** + * @return The rotation of the frame in degrees, a multiple of 90. + */ + public int getRotation() { + return rotation; + } + + /** + * @return The quantizer the frame was encoded with, or {@code null} if it + * is not known. + */ + public Integer getQp() { + return qp; + } + + @Override + public String toString() { + return String.format("%s@%d [size=%d, width=%d, height=%d, captureTimeNs=%d, frameType=%s, rotation=%d, qp=%s]", + EncodedImage.class.getSimpleName(), hashCode(), + buffer.remaining(), encodedWidth, encodedHeight, + captureTimeNs, frameType, rotation, qp); + } + + /** + * @return A new builder for an encoded image. + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Wraps an image WebRTC hands to a decoder. Called from native code. + */ + @SuppressWarnings("unused") + private static EncodedImage fromNative(ByteBuffer buffer, int encodedWidth, + int encodedHeight, long captureTimeNs, int frameType, int rotation, + int qp) { + return new EncodedImage(buffer.asReadOnlyBuffer(), encodedWidth, + encodedHeight, captureTimeNs, FrameType.fromNativeIndex(frameType), + rotation, qp < 0 ? null : qp); + } + + /** + * Returns the payload as a direct buffer whose capacity is its size, which + * native code can address. A heap buffer is copied. Called from native + * code. + */ + @SuppressWarnings("unused") + private ByteBuffer getDirectPayload() { + if (buffer.isDirect()) { + return buffer.slice(); + } + + ByteBuffer direct = ByteBuffer.allocateDirect(buffer.remaining()); + direct.put(buffer.duplicate()); + direct.flip(); + + return direct; + } + + /** + * Returns the native index of the frame type. Called from native code. + */ + @SuppressWarnings("unused") + private int getFrameTypeIndex() { + return frameType.getNativeIndex(); + } + + /** + * Returns the quantizer, or -1 if it is not known. Called from native code. + */ + @SuppressWarnings("unused") + private int getQpOrUnknown() { + return qp == null ? -1 : qp; + } + + + + /** + * Builds an {@link EncodedImage}, which is what a {@link VideoEncoder} + * does for each frame it encodes. + */ + public static final class Builder { + + private ByteBuffer buffer; + + private int encodedWidth; + + private int encodedHeight; + + private long captureTimeNs; + + private FrameType frameType = FrameType.DELTA; + + private int rotation; + + private Integer qp; + + + private Builder() { + } + + /** + * Sets the encoded payload, which is read from its position to its + * limit. Direct and heap buffers are both accepted. + * + * @param buffer The encoded data. + * + * @return This builder. + */ + public Builder setBuffer(ByteBuffer buffer) { + this.buffer = buffer; + return this; + } + + /** + * @param width The width of the encoded frame in pixels. + * + * @return This builder. + */ + public Builder setEncodedWidth(int width) { + this.encodedWidth = width; + return this; + } + + /** + * @param height The height of the encoded frame in pixels. + * + * @return This builder. + */ + public Builder setEncodedHeight(int height) { + this.encodedHeight = height; + return this; + } + + /** + * Sets the capture time, which must be the timestamp of the frame + * that was encoded. + * + * @param captureTimeNs The capture time in nanoseconds. + * + * @return This builder. + */ + public Builder setCaptureTimeNs(long captureTimeNs) { + this.captureTimeNs = captureTimeNs; + return this; + } + + /** + * @param frameType Whether this is a key frame or a delta frame. + * + * @return This builder. + */ + public Builder setFrameType(FrameType frameType) { + this.frameType = frameType; + return this; + } + + /** + * @param rotation The rotation of the frame in degrees, a multiple of + * 90. + * + * @return This builder. + */ + public Builder setRotation(int rotation) { + this.rotation = rotation; + return this; + } + + /** + * Sets the quantizer. Without one, WebRTC parses it from the payload + * of VP8, VP9 and H.264 frames, which quality scaling depends on. + * + * @param qp The quantizer, or {@code null} if it is not known. + * + * @return This builder. + */ + public Builder setQp(Integer qp) { + this.qp = qp; + return this; + } + + /** + * @return The encoded image. + * + * @throws NullPointerException If the buffer or the frame type is + * missing. + * @throws IllegalArgumentException If the rotation is not a multiple + * of 90 or the size is negative. + */ + public EncodedImage build() { + Objects.requireNonNull(buffer, "Buffer is null"); + Objects.requireNonNull(frameType, "Frame type is null"); + + if (rotation % 90 != 0) { + throw new IllegalArgumentException("Rotation must be a multiple of 90"); + } + if (encodedWidth < 0 || encodedHeight < 0) { + throw new IllegalArgumentException("Size must not be negative"); + } + + return new EncodedImage(buffer, encodedWidth, encodedHeight, + captureTimeNs, frameType, rotation, qp); + } + + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoDecoder.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoDecoder.java new file mode 100644 index 000000000..e2b941b0d --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoDecoder.java @@ -0,0 +1,71 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +/** + * A built-in decoder, created by a {@link DefaultVideoDecoderFactory}. It is a + * placeholder: returned from a {@link VideoDecoderFactory}, it makes WebRTC + * create the native decoder for its codec, which then runs entirely inside + * WebRTC. Its methods are therefore not to be called from Java, and throw + * {@link UnsupportedOperationException}. + * + * @author Alex Andres + */ +public final class NativeVideoDecoder implements VideoDecoder { + + /** The codec to create the native decoder for; read by native code. */ + private final VideoCodecInfo codecInfo; + + + NativeVideoDecoder(VideoCodecInfo codecInfo) { + this.codecInfo = codecInfo; + } + + /** + * @return The codec this decoder decodes. + */ + public VideoCodecInfo getCodecInfo() { + return codecInfo; + } + + @Override + public VideoCodecStatus initDecode(Settings settings, Callback callback) { + throw unsupported(); + } + + @Override + public VideoCodecStatus release() { + throw unsupported(); + } + + @Override + public VideoCodecStatus decode(EncodedImage image) { + throw unsupported(); + } + + @Override + public String toString() { + return String.format("%s@%d [codec=%s]", + NativeVideoDecoder.class.getSimpleName(), hashCode(), + codecInfo.getName()); + } + + private static UnsupportedOperationException unsupported() { + return new UnsupportedOperationException("A native decoder runs inside WebRTC only"); + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoDecoderCallback.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoDecoderCallback.java new file mode 100644 index 000000000..8f030bf79 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoDecoderCallback.java @@ -0,0 +1,68 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import dev.onvoid.webrtc.media.video.VideoFrame; + +import java.util.Objects; + +/** + * The {@link VideoDecoder.Callback} WebRTC hands to a Java decoder. It passes + * decoded frames on to the native decoder wrapper until that wrapper + * invalidates it, when the decoder is released or destroyed, so that a + * decoder that delivers late cannot reach a wrapper that no longer exists. + * + * @author Alex Andres + */ +final class NativeVideoDecoderCallback implements VideoDecoder.Callback { + + /** The native decoder wrapper, 0 once invalidated. */ + private long nativeHandle; + + + /** + * Created from native code. + */ + private NativeVideoDecoderCallback(long nativeHandle) { + this.nativeHandle = nativeHandle; + } + + @Override + public synchronized void onDecodedFrame(VideoFrame frame, Integer decodeTimeMs, Integer qp) { + Objects.requireNonNull(frame, "VideoFrame is null"); + + if (nativeHandle != 0) { + onDecodedFrame(nativeHandle, frame, + decodeTimeMs == null ? -1 : decodeTimeMs, + qp == null ? -1 : qp); + } + } + + /** + * Detaches this callback from the native wrapper. Once this returns, no + * frame is passed on any more, nor is one still being passed on. Called + * from native code. + */ + @SuppressWarnings("unused") + private synchronized void invalidate() { + nativeHandle = 0; + } + + private static native void onDecodedFrame(long nativeHandle, VideoFrame frame, + int decodeTimeMs, int qp); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoEncoder.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoEncoder.java new file mode 100644 index 000000000..1bef65836 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoEncoder.java @@ -0,0 +1,78 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import dev.onvoid.webrtc.media.video.VideoFrame; + +/** + * A built-in encoder, created by a {@link DefaultVideoEncoderFactory}. It is a + * placeholder: returned from a {@link VideoEncoderFactory}, it makes WebRTC + * create the native encoder for its codec, which then runs entirely inside + * WebRTC. Its methods are therefore not to be called from Java, and throw + * {@link UnsupportedOperationException}. + * + * @author Alex Andres + */ +public final class NativeVideoEncoder implements VideoEncoder { + + /** The codec to create the native encoder for; read by native code. */ + private final VideoCodecInfo codecInfo; + + + NativeVideoEncoder(VideoCodecInfo codecInfo) { + this.codecInfo = codecInfo; + } + + /** + * @return The codec this encoder encodes. + */ + public VideoCodecInfo getCodecInfo() { + return codecInfo; + } + + @Override + public VideoCodecStatus initEncode(Settings settings, Callback callback) { + throw unsupported(); + } + + @Override + public VideoCodecStatus release() { + throw unsupported(); + } + + @Override + public VideoCodecStatus encode(VideoFrame frame, EncodeInfo info) { + throw unsupported(); + } + + @Override + public VideoCodecStatus setRates(RateControlParameters parameters) { + throw unsupported(); + } + + @Override + public String toString() { + return String.format("%s@%d [codec=%s]", + NativeVideoEncoder.class.getSimpleName(), hashCode(), + codecInfo.getName()); + } + + private static UnsupportedOperationException unsupported() { + return new UnsupportedOperationException("A native encoder runs inside WebRTC only"); + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoEncoderCallback.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoEncoderCallback.java new file mode 100644 index 000000000..7f58e30f8 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/NativeVideoEncoderCallback.java @@ -0,0 +1,63 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import java.util.Objects; + +/** + * The {@link VideoEncoder.Callback} WebRTC hands to a Java encoder. It passes + * encoded frames on to the native encoder wrapper until that wrapper + * invalidates it, when the encoder is released or destroyed, so that an + * encoder that delivers late cannot reach a wrapper that no longer exists. + * + * @author Alex Andres + */ +final class NativeVideoEncoderCallback implements VideoEncoder.Callback { + + /** The native encoder wrapper, 0 once invalidated. */ + private long nativeHandle; + + + /** + * Created from native code. + */ + private NativeVideoEncoderCallback(long nativeHandle) { + this.nativeHandle = nativeHandle; + } + + @Override + public synchronized void onEncodedFrame(EncodedImage image) { + Objects.requireNonNull(image, "EncodedImage is null"); + + if (nativeHandle != 0) { + onEncodedFrame(nativeHandle, image); + } + } + + /** + * Detaches this callback from the native wrapper. Once this returns, no + * frame is passed on any more, nor is one still being passed on. Called + * from native code. + */ + @SuppressWarnings("unused") + private synchronized void invalidate() { + nativeHandle = 0; + } + + private static native void onEncodedFrame(long nativeHandle, EncodedImage image); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoCodecInfo.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoCodecInfo.java new file mode 100644 index 000000000..c8179dee9 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoCodecInfo.java @@ -0,0 +1,159 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Describes a video codec the way it is negotiated in SDP: its name, such as + * "VP8" or "H264", and its format parameters, such as the H.264 profile. A + * codec factory lists the formats it supports as VideoCodecInfos, and is asked + * for an encoder or decoder for one of them. + *

+ * A name WebRTC does not know, for example "X-MYCODEC", is negotiated like any + * other, and its frames are sent with the generic RTP packetization. + * + * @author Alex Andres + */ +public final class VideoCodecInfo { + + private final String name; + + /** A HashMap, which is what the native side reads. */ + private final HashMap parameters; + + private final String[] scalabilityModes; + + + /** + * Creates a codec description without format parameters. + * + * @param name The codec name, such as "VP8". + */ + public VideoCodecInfo(String name) { + this(name, Collections.emptyMap()); + } + + /** + * Creates a codec description. + * + * @param name The codec name, such as "H264". + * @param parameters The SDP format parameters, such as {@code + * profile-level-id}. + */ + public VideoCodecInfo(String name, Map parameters) { + this(name, parameters, Collections.emptyList()); + } + + /** + * Creates a codec description. + * + * @param name The codec name, such as "VP9". + * @param parameters The SDP format parameters. + * @param scalabilityModes The scalability modes the codec supports, such + * as "L1T3", as named in the WebRTC SVC + * specification. + */ + public VideoCodecInfo(String name, Map parameters, + List scalabilityModes) { + Objects.requireNonNull(name, "Codec name is null"); + Objects.requireNonNull(parameters, "Codec parameters are null"); + Objects.requireNonNull(scalabilityModes, "Scalability modes are null"); + + if (name.isEmpty()) { + throw new IllegalArgumentException("Codec name is empty"); + } + + this.name = name; + this.parameters = new HashMap<>(parameters); + this.scalabilityModes = scalabilityModes.toArray(new String[0]); + + for (Map.Entry entry : this.parameters.entrySet()) { + if (entry.getKey() == null || entry.getValue() == null) { + throw new IllegalArgumentException("Codec parameters must not contain null"); + } + } + for (String mode : this.scalabilityModes) { + Objects.requireNonNull(mode, "Scalability mode is null"); + } + } + + /** + * Created from native code. + */ + @SuppressWarnings("unused") + private VideoCodecInfo(String name, Map parameters, + String[] scalabilityModes) { + this(name, parameters, Arrays.asList(scalabilityModes)); + } + + /** + * @return The codec name, such as "VP8". + */ + public String getName() { + return name; + } + + /** + * @return The SDP format parameters, unmodifiable. + */ + public Map getParameters() { + return Collections.unmodifiableMap(parameters); + } + + /** + * @return The scalability modes the codec supports, unmodifiable. + */ + public List getScalabilityModes() { + return Collections.unmodifiableList(Arrays.asList(scalabilityModes)); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + VideoCodecInfo other = (VideoCodecInfo) obj; + + return name.equalsIgnoreCase(other.name) + && parameters.equals(other.parameters) + && Arrays.equals(scalabilityModes, other.scalabilityModes); + } + + @Override + public int hashCode() { + return Objects.hash(name.toUpperCase(java.util.Locale.ROOT), parameters, + Arrays.hashCode(scalabilityModes)); + } + + @Override + public String toString() { + return String.format("%s@%d [name=%s, parameters=%s, scalabilityModes=%s]", + VideoCodecInfo.class.getSimpleName(), hashCode(), name, + parameters, Arrays.toString(scalabilityModes)); + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoCodecStatus.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoCodecStatus.java new file mode 100644 index 000000000..78ea3e3d8 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoCodecStatus.java @@ -0,0 +1,81 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +/** + * The result of a {@link VideoEncoder} or {@link VideoDecoder} operation. + *

+ * Of the errors, {@link #FALLBACK_SOFTWARE} and {@link #UNINITIALIZED} give + * the codec up for the stream. Any other error makes WebRTC release and + * initialize the codec again, and give it up only if that fails. + * + * @author Alex Andres + */ +public enum VideoCodecStatus { + + /** The operation succeeded. */ + OK(0), + + /** The operation succeeded, but produced no output. */ + NO_OUTPUT(1), + + /** The operation succeeded, and the encoder asks for a key frame. */ + OK_REQUEST_KEYFRAME(4), + + /** The encoder overshot its target bitrate. */ + TARGET_BITRATE_OVERSHOOT(5), + + /** A generic error. */ + ERROR(-1), + + /** Memory could not be allocated. */ + MEMORY(-3), + + /** A parameter was not valid. */ + ERR_PARAMETER(-4), + + /** The operation timed out. */ + TIMEOUT(-6), + + /** The codec was not initialized. */ + UNINITIALIZED(-7), + + /** The codec cannot go on, and another implementation is to be used. */ + FALLBACK_SOFTWARE(-13), + + /** The encoder does not support the requested simulcast settings. */ + ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED(-15), + + /** The encoder failed. */ + ENCODER_FAILURE(-16); + + + private final int number; + + + VideoCodecStatus(int number) { + this.number = number; + } + + /** + * @return The WebRTC status code, a {@code WEBRTC_VIDEO_CODEC_*} value. + */ + public int getNumber() { + return number; + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoDecoder.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoDecoder.java new file mode 100644 index 000000000..3834990fb --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoDecoder.java @@ -0,0 +1,151 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import dev.onvoid.webrtc.media.video.VideoFrame; + +/** + * A video decoder implemented in Java, which a {@link VideoDecoderFactory} + * hands to WebRTC. + *

+ * WebRTC calls {@link #initDecode initDecode}, then {@link #decode decode} + * for each encoded frame, and finally {@link #release release}. It may + * initialize a released decoder again. These calls come from one WebRTC thread + * at a time, never concurrently. The decoder hands each decoded frame to the + * {@link Callback} it was initialized with, either from within {@code decode} + * or later from a thread of its own. + *

+ * Whatever a method throws is logged and treated as {@link + * VideoCodecStatus#ERROR}. + * + * @author Alex Andres + * + * @see VideoDecoderFactory + */ +public interface VideoDecoder { + + /** + * The settings a decoder is initialized with. + */ + class Settings { + + /** The number of CPU cores the decoder may use. */ + public final int numberOfCores; + + /** The largest frame width to expect, 0 if not known. */ + public final int width; + + /** The largest frame height to expect, 0 if not known. */ + public final int height; + + + /** + * Creates decoder settings. + * + * @param numberOfCores The number of CPU cores to use. + * @param width The largest frame width to expect. + * @param height The largest frame height to expect. + */ + public Settings(int numberOfCores, int width, int height) { + this.numberOfCores = numberOfCores; + this.width = width; + this.height = height; + } + + @Override + public String toString() { + return String.format("%s [cores=%d, width=%d, height=%d]", + Settings.class.getSimpleName(), numberOfCores, width, height); + } + + } + + + + /** + * Receives what a decoder produces. + */ + interface Callback { + + /** + * Hands a decoded frame to WebRTC. The frame's timestamp must be the + * {@link EncodedImage#getCaptureTimeNs() capture time} of the image it + * was decoded from. WebRTC takes a reference of its own, so the caller + * still releases the frame it created. May be called from any thread, + * but frames have to be handed over in decoding order. Frames handed + * over after the decoder was released are ignored. + * + * @param frame The decoded frame. + * @param decodeTimeMs How long decoding took in milliseconds, or + * {@code null} if not known. + * @param qp The quantizer of the frame, or {@code null} if + * not known. + */ + void onDecodedFrame(VideoFrame frame, Integer decodeTimeMs, Integer qp); + + } + + + /** + * Initializes the decoder, or initializes it again after a {@link + * #release()}. + * + * @param settings The decoder settings. + * @param callback Receives the decoded frames until the decoder is + * released. + * + * @return {@link VideoCodecStatus#OK} on success. + */ + VideoCodecStatus initDecode(Settings settings, Callback callback); + + /** + * Releases the decoder. It may be initialized again afterwards. + * + * @return {@link VideoCodecStatus#OK} on success. + */ + VideoCodecStatus release(); + + /** + * Decodes an encoded frame. The image and its buffer are valid only until + * this method returns. + * + * @param image The encoded frame. + * + * @return {@link VideoCodecStatus#OK} if the frame was taken, also if the + * decoded frame is delivered later. + */ + VideoCodecStatus decode(EncodedImage image); + + /** + * Returns the name of the implementation, which shows up in statistics. + * + * @return The simple class name unless overridden. + */ + default String getImplementationName() { + return getClass().getSimpleName(); + } + + /** + * Returns whether the decoder runs on dedicated hardware. + * + * @return {@code false} unless overridden. + */ + default boolean isHardwareDecoder() { + return false; + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoDecoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoDecoderFactory.java new file mode 100644 index 000000000..f9b62435f --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoDecoderFactory.java @@ -0,0 +1,56 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import java.util.List; + +/** + * Creates the video decoders of a {@link dev.onvoid.webrtc.PeerConnectionFactory + * PeerConnectionFactory}, which decide what video codecs it can receive. + *

+ * A factory may return its own {@link VideoDecoder}s as well as the built-in + * ones of a {@link DefaultVideoDecoderFactory}, the same way a {@link + * VideoEncoderFactory} does. + *

+ * {@link #createDecoder(VideoCodecInfo)} is called from a WebRTC thread. + * + * @author Alex Andres + * + * @see dev.onvoid.webrtc.PeerConnectionFactory.Builder#setVideoDecoderFactory(VideoDecoderFactory) + */ +public interface VideoDecoderFactory { + + /** + * Returns the codecs this factory can create decoders for, in order of + * preference. Asked for once, when the peer connection factory is + * created. + * + * @return The supported codecs. + */ + List getSupportedCodecs(); + + /** + * Creates a decoder for one of the supported codecs. Each call has to + * return a new decoder, since WebRTC uses one per stream. + * + * @param info The codec to decode, as negotiated. + * + * @return A new decoder, or {@code null} if the codec is not supported. + */ + VideoDecoder createDecoder(VideoCodecInfo info); + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoEncoder.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoEncoder.java new file mode 100644 index 000000000..c76bf9793 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoEncoder.java @@ -0,0 +1,510 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import dev.onvoid.webrtc.media.video.VideoFrame; + +import java.util.Arrays; +import java.util.Objects; + +/** + * A video encoder implemented in Java, which a {@link VideoEncoderFactory} + * hands to WebRTC. This is the way to bring in a codec WebRTC does not have, + * or a hardware encoder reached through another library. + *

+ * WebRTC calls {@link #initEncode initEncode}, then {@link #encode encode} + * for each frame and {@link #setRates setRates} whenever the target bitrate + * changes, and finally {@link #release release}. It may initialize a released + * encoder again. These calls come from one WebRTC thread at a time, never + * concurrently. The encoder hands each encoded frame to the {@link Callback} + * it was initialized with, either from within {@code encode} or later from a + * thread of its own. + *

+ * Whatever a method throws is logged and treated as {@link + * VideoCodecStatus#ERROR}. + * + * @author Alex Andres + * + * @see VideoEncoderFactory + */ +public interface VideoEncoder { + + /** + * The settings an encoder is initialized with. + */ + class Settings { + + /** The number of CPU cores the encoder may use. */ + public final int numberOfCores; + + /** The frame width in pixels. */ + public final int width; + + /** The frame height in pixels. */ + public final int height; + + /** The initial target bitrate in kbps. */ + public final int startBitrateKbps; + + /** The minimum bitrate in kbps. */ + public final int minBitrateKbps; + + /** The maximum bitrate in kbps. */ + public final int maxBitrateKbps; + + /** The maximum frame rate in frames per second. */ + public final int maxFramerate; + + /** The number of simulcast streams, 0 or 1 without simulcast. */ + public final int numberOfSimulcastStreams; + + /** Whether the encoder may change the resolution on its own. */ + public final boolean automaticResizeOn; + + /** Whether the content is a screen share rather than camera video. */ + public final boolean screenContent; + + /** Whether the encoder is to react to loss notifications. */ + public final boolean lossNotification; + + + /** + * Creates encoder settings. + * + * @param numberOfCores The number of CPU cores to use. + * @param width The frame width in pixels. + * @param height The frame height in pixels. + * @param startBitrateKbps The initial bitrate in kbps. + * @param minBitrateKbps The minimum bitrate in kbps. + * @param maxBitrateKbps The maximum bitrate in kbps. + * @param maxFramerate The maximum frame rate. + * @param numberOfSimulcastStreams The number of simulcast streams. + * @param automaticResizeOn Whether the encoder may resize. + * @param screenContent Whether the content is a screen. + * @param lossNotification Whether to react to loss + * notifications. + */ + public Settings(int numberOfCores, int width, int height, + int startBitrateKbps, int minBitrateKbps, int maxBitrateKbps, + int maxFramerate, int numberOfSimulcastStreams, + boolean automaticResizeOn, boolean screenContent, + boolean lossNotification) { + this.numberOfCores = numberOfCores; + this.width = width; + this.height = height; + this.startBitrateKbps = startBitrateKbps; + this.minBitrateKbps = minBitrateKbps; + this.maxBitrateKbps = maxBitrateKbps; + this.maxFramerate = maxFramerate; + this.numberOfSimulcastStreams = numberOfSimulcastStreams; + this.automaticResizeOn = automaticResizeOn; + this.screenContent = screenContent; + this.lossNotification = lossNotification; + } + + @Override + public String toString() { + return String.format("%s [cores=%d, width=%d, height=%d, startBitrateKbps=%d, minBitrateKbps=%d, " + + "maxBitrateKbps=%d, maxFramerate=%d, simulcastStreams=%d, automaticResize=%s, " + + "screenContent=%s, lossNotification=%s]", + Settings.class.getSimpleName(), numberOfCores, width, height, + startBitrateKbps, minBitrateKbps, maxBitrateKbps, maxFramerate, + numberOfSimulcastStreams, automaticResizeOn, screenContent, + lossNotification); + } + + } + + + + /** + * Additional information about a frame to encode. + */ + class EncodeInfo { + + /** + * The frame type WebRTC asks for, one per simulcast stream. {@link + * EncodedImage.FrameType#KEY} asks for a key frame. + */ + public final EncodedImage.FrameType[] frameTypes; + + + /** + * Creates the information about a frame to encode. + * + * @param frameTypes The frame type asked for, per simulcast stream. + */ + public EncodeInfo(EncodedImage.FrameType[] frameTypes) { + this.frameTypes = Objects.requireNonNull(frameTypes, "Frame types are null"); + } + + /** + * @return Whether WebRTC asks for a key frame in any stream. + */ + public boolean isKeyFrameRequested() { + for (EncodedImage.FrameType type : frameTypes) { + if (type == EncodedImage.FrameType.KEY) { + return true; + } + } + + return false; + } + + @SuppressWarnings("unused") + private static EncodeInfo fromNative(int[] frameTypes) { + EncodedImage.FrameType[] types = new EncodedImage.FrameType[frameTypes.length]; + + for (int i = 0; i < frameTypes.length; i++) { + types[i] = EncodedImage.FrameType.fromNativeIndex(frameTypes[i]); + } + + return new EncodeInfo(types); + } + + @Override + public String toString() { + return String.format("%s [frameTypes=%s]", + EncodeInfo.class.getSimpleName(), Arrays.toString(frameTypes)); + } + + } + + + + /** + * The bitrate WebRTC allocates to each spatial and temporal layer. + */ + class BitrateAllocation { + + /** + * The bitrate in bps, indexed by spatial layer, then temporal layer. + */ + public final int[][] bitratesBps; + + + /** + * Creates a bitrate allocation. + * + * @param bitratesBps The bitrate in bps per spatial, then temporal + * layer. + */ + public BitrateAllocation(int[][] bitratesBps) { + this.bitratesBps = Objects.requireNonNull(bitratesBps, "Bitrates are null"); + } + + /** + * @return The sum of all layers in bps. + */ + public int getSum() { + int sum = 0; + + for (int[] spatial : bitratesBps) { + for (int bitrate : spatial) { + sum += bitrate; + } + } + + return sum; + } + + @Override + public String toString() { + return String.format("%s [sum=%d, bitratesBps=%s]", + BitrateAllocation.class.getSimpleName(), getSum(), + Arrays.deepToString(bitratesBps)); + } + + } + + + + /** + * The rates an encoder is to aim for. + */ + class RateControlParameters { + + /** The target bitrate per layer. */ + public final BitrateAllocation bitrate; + + /** The target frame rate in frames per second. */ + public final double framerateFps; + + + /** + * Creates rate control parameters. + * + * @param bitrate The target bitrate per layer. + * @param framerateFps The target frame rate. + */ + public RateControlParameters(BitrateAllocation bitrate, double framerateFps) { + this.bitrate = Objects.requireNonNull(bitrate, "Bitrate is null"); + this.framerateFps = framerateFps; + } + + @Override + public String toString() { + return String.format("%s [bitrate=%s, framerateFps=%s]", + RateControlParameters.class.getSimpleName(), bitrate, + framerateFps); + } + + } + + + + /** + * The quantizer thresholds WebRTC scales the resolution by: above the high + * threshold it lowers the resolution, below the low one it raises it. + */ + class ScalingSettings { + + /** No quality scaling. */ + public static final ScalingSettings OFF = new ScalingSettings(false, null, null); + + /** + * Quality scaling with the thresholds WebRTC uses for VP8, VP9 and + * H.264, and none for any other codec. + */ + public static final ScalingSettings DEFAULT = new ScalingSettings(true, null, null); + + /** Whether quality scaling is on. */ + public final boolean on; + + /** The low quantizer threshold, or {@code null} for the default. */ + public final Integer low; + + /** The high quantizer threshold, or {@code null} for the default. */ + public final Integer high; + + + /** + * Creates scaling settings that turn quality scaling on with the given + * thresholds. + * + * @param low The low quantizer threshold. + * @param high The high quantizer threshold. + */ + public ScalingSettings(int low, int high) { + this(true, low, high); + } + + private ScalingSettings(boolean on, Integer low, Integer high) { + this.on = on; + this.low = low; + this.high = high; + } + + @Override + public String toString() { + return on + ? String.format("[low=%s, high=%s]", low, high) + : "OFF"; + } + + } + + + + /** + * The bitrate limits of an encoder at one resolution. + */ + class ResolutionBitrateLimits { + + /** The resolution, as width times height. */ + public final int frameSizePixels; + + /** The lowest bitrate to start encoding this resolution at, in bps. */ + public final int minStartBitrateBps; + + /** The lowest bitrate for this resolution, in bps. */ + public final int minBitrateBps; + + /** The highest bitrate for this resolution, in bps. */ + public final int maxBitrateBps; + + + /** + * Creates bitrate limits for one resolution. + * + * @param frameSizePixels The resolution, as width times height. + * @param minStartBitrateBps The lowest start bitrate in bps. + * @param minBitrateBps The lowest bitrate in bps. + * @param maxBitrateBps The highest bitrate in bps. + */ + public ResolutionBitrateLimits(int frameSizePixels, int minStartBitrateBps, + int minBitrateBps, int maxBitrateBps) { + this.frameSizePixels = frameSizePixels; + this.minStartBitrateBps = minStartBitrateBps; + this.minBitrateBps = minBitrateBps; + this.maxBitrateBps = maxBitrateBps; + } + + } + + + + /** + * Properties of an encoder WebRTC takes into account when feeding it. + */ + class EncoderInfo { + + /** + * The number both frame dimensions have to be divisible by. WebRTC + * crops or scales frames to fit. + */ + public final int requestedResolutionAlignment; + + /** + * Whether the alignment applies to every simulcast layer, rather than + * only to the highest. + */ + public final boolean applyAlignmentToAllSimulcastLayers; + + + /** + * Creates encoder properties. + * + * @param requestedResolutionAlignment The alignment of frame + * dimensions, 1 for none. + * @param applyAlignmentToAllSimulcastLayers Whether the alignment + * applies to every layer. + */ + public EncoderInfo(int requestedResolutionAlignment, + boolean applyAlignmentToAllSimulcastLayers) { + if (requestedResolutionAlignment < 1) { + throw new IllegalArgumentException("Alignment must be at least 1"); + } + + this.requestedResolutionAlignment = requestedResolutionAlignment; + this.applyAlignmentToAllSimulcastLayers = applyAlignmentToAllSimulcastLayers; + } + + } + + + + /** + * Receives what an encoder produces. + */ + interface Callback { + + /** + * Hands an encoded frame to WebRTC, which copies the payload before + * returning. May be called from any thread, but frames have to be + * handed over in the order they were encoded in. Frames handed over + * after the encoder was released are ignored. + * + * @param image The encoded frame, whose capture time is the timestamp + * of the frame it encodes. + */ + void onEncodedFrame(EncodedImage image); + + } + + + /** + * Initializes the encoder, or initializes it again after a {@link + * #release()}, possibly with a different frame size. + * + * @param settings The encoder settings. + * @param callback Receives the encoded frames until the encoder is + * released. + * + * @return {@link VideoCodecStatus#OK} on success. + */ + VideoCodecStatus initEncode(Settings settings, Callback callback); + + /** + * Releases the encoder. It may be initialized again afterwards. + * + * @return {@link VideoCodecStatus#OK} on success. + */ + VideoCodecStatus release(); + + /** + * Encodes a frame. The frame is valid only until this method returns; to + * encode it later, {@link VideoFrame#retain() retain} it and {@link + * VideoFrame#release() release} it once done. Its pixel data must not be + * changed, since other consumers may share it. + * + * @param frame The frame to encode. + * @param info Additional information, such as a key frame request. + * + * @return {@link VideoCodecStatus#OK} if the frame was taken, also if it + * is delivered later or dropped. + */ + VideoCodecStatus encode(VideoFrame frame, EncodeInfo info); + + /** + * Sets the bitrate and frame rate to aim for. + * + * @param parameters The target rates. + * + * @return {@link VideoCodecStatus#OK} on success. + */ + VideoCodecStatus setRates(RateControlParameters parameters); + + /** + * Returns the quality scaling settings. Read after construction and after + * each initialization. + * + * @return {@link ScalingSettings#DEFAULT} unless overridden. + */ + default ScalingSettings getScalingSettings() { + return ScalingSettings.DEFAULT; + } + + /** + * Returns the bitrate limits of the encoder per resolution. Read after + * construction and after each initialization. + * + * @return No limits unless overridden. + */ + default ResolutionBitrateLimits[] getResolutionBitrateLimits() { + return new ResolutionBitrateLimits[0]; + } + + /** + * Returns properties of the encoder. Read after construction and after + * each initialization. + * + * @return No alignment unless overridden. + */ + default EncoderInfo getEncoderInfo() { + return new EncoderInfo(1, false); + } + + /** + * Returns the name of the implementation, which shows up in statistics. + * + * @return The simple class name unless overridden. + */ + default String getImplementationName() { + return getClass().getSimpleName(); + } + + /** + * Returns whether the encoder runs on dedicated hardware. + * + * @return {@code false} unless overridden. + */ + default boolean isHardwareEncoder() { + return false; + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoEncoderFactory.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoEncoderFactory.java new file mode 100644 index 000000000..62ca8b76b --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/codec/VideoEncoderFactory.java @@ -0,0 +1,75 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.video.codec; + +import java.util.List; + +/** + * Creates the video encoders of a {@link dev.onvoid.webrtc.PeerConnectionFactory + * PeerConnectionFactory}, which decide what video codecs it can send. + *

+ * A factory may return its own {@link VideoEncoder}s as well as the built-in + * ones of a {@link DefaultVideoEncoderFactory}, for example to add a codec + * while keeping the built-in ones: + * 

{@code
+ * class MyEncoderFactory implements VideoEncoderFactory {
+ *
+ *     private final DefaultVideoEncoderFactory builtIn = new DefaultVideoEncoderFactory();
+ *
+ *     public List getSupportedCodecs() {
+ *         List codecs = new ArrayList<>(builtIn.getSupportedCodecs());
+ *         codecs.add(new VideoCodecInfo("X-MYCODEC"));
+ *         return codecs;
+ *     }
+ *
+ *     public VideoEncoder createEncoder(VideoCodecInfo info) {
+ *         if (info.getName().equals("X-MYCODEC")) {
+ *             return new MyEncoder();
+ *         }
+ *         return builtIn.createEncoder(info);
+ *     }
+ * }
+ * }
+ *

+ * {@link #createEncoder(VideoCodecInfo)} is called from a WebRTC thread. + * + * @author Alex Andres + * + * @see dev.onvoid.webrtc.PeerConnectionFactory.Builder#setVideoEncoderFactory(VideoEncoderFactory) + */ +public interface VideoEncoderFactory { + + /** + * Returns the codecs this factory can create encoders for, in order of + * preference. Asked for once, when the peer connection factory is + * created. + * + * @return The supported codecs. + */ + List getSupportedCodecs(); + + /** + * Creates an encoder for one of the supported codecs. Each call has to + * return a new encoder, since WebRTC uses one per stream. + * + * @param info The codec to encode, as negotiated. + * + * @return A new encoder, or {@code null} if the codec is not supported. + */ + VideoEncoder createEncoder(VideoCodecInfo info); + +} diff --git a/webrtc/src/main/java/module-info.java b/webrtc/src/main/java/module-info.java index 743fff91e..778d8892d 100644 --- a/webrtc/src/main/java/module-info.java +++ b/webrtc/src/main/java/module-info.java @@ -10,6 +10,7 @@ exports dev.onvoid.webrtc.media; exports dev.onvoid.webrtc.media.audio; exports dev.onvoid.webrtc.media.video; + exports dev.onvoid.webrtc.media.video.codec; exports dev.onvoid.webrtc.media.video.desktop; // Not API for applications. A native extension module, such as the FFmpeg diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java b/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java index a96fe56ad..9db911955 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/TestMediaCall.java @@ -60,6 +60,15 @@ class TestMediaCall implements AutoCloseable { TestMediaCall(PeerConnectionFactory factory, boolean video, boolean audio) { + this(factory, video, audio, "VP8"); + } + + /** + * Creates a call whose video prefers the given codec. + * + * @param videoCodec The name of the video codec to prefer. + */ + TestMediaCall(PeerConnectionFactory factory, boolean video, boolean audio, String videoCodec) { caller = new TestPeerConnection(factory); callee = new TestPeerConnection(factory); @@ -72,7 +81,7 @@ class TestMediaCall implements AutoCloseable { videoSender = caller.getPeerConnection().addTrack(videoTrack, Collections.singletonList("stream")); - preferVp8(factory, caller.getPeerConnection(), videoSender); + preferCodec(factory, caller.getPeerConnection(), videoSender, videoCodec); } else { videoSource = null; @@ -191,18 +200,19 @@ public void close() throws InterruptedException { } /** - * Makes VP8 the preferred video codec. Platforms prefer different codecs by - * default, macOS H.264 through VideoToolbox, and the tests should not - * depend on which: VP8 is available everywhere, and it is the codec that - * survives a transform changing the whole payload, which H.264 does not, - * since its packetizer splits frames at start codes in the payload. + * Makes the given video codec the preferred one. Platforms prefer different + * codecs by default, macOS H.264 through VideoToolbox, and the tests should + * not depend on which. Unless a test needs another, they prefer VP8: it is + * available everywhere, and it is the codec that survives a transform + * changing the whole payload, which H.264 does not, since its packetizer + * splits frames at start codes in the payload. */ - private static void preferVp8(PeerConnectionFactory factory, RTCPeerConnection connection, - RTCRtpSender sender) { + private static void preferCodec(PeerConnectionFactory factory, RTCPeerConnection connection, + RTCRtpSender sender, String name) { List codecs = new ArrayList<>( factory.getRtpSenderCapabilities(MediaType.VIDEO).getCodecs()); - codecs.sort(Comparator.comparing(codec -> !"VP8".equalsIgnoreCase(codec.getName()))); + codecs.sort(Comparator.comparing(codec -> !name.equalsIgnoreCase(codec.getName()))); for (RTCRtpTransceiver transceiver : connection.getTransceivers()) { RTCRtpSender transceiverSender = transceiver.getSender(); diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java new file mode 100644 index 000000000..c45141b6e --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/VideoCodecFactoryTests.java @@ -0,0 +1,685 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import dev.onvoid.webrtc.logging.Logging; +import dev.onvoid.webrtc.media.MediaType; +import dev.onvoid.webrtc.media.video.I420Buffer; +import dev.onvoid.webrtc.media.video.NativeI420Buffer; +import dev.onvoid.webrtc.media.video.VideoFrame; +import dev.onvoid.webrtc.media.video.VideoFrameBuffer; +import dev.onvoid.webrtc.media.video.VideoTrack; +import dev.onvoid.webrtc.media.video.VideoTrackSink; +import dev.onvoid.webrtc.media.video.codec.DefaultVideoDecoderFactory; +import dev.onvoid.webrtc.media.video.codec.DefaultVideoEncoderFactory; +import dev.onvoid.webrtc.media.video.codec.EncodedImage; +import dev.onvoid.webrtc.media.video.codec.NativeVideoEncoder; +import dev.onvoid.webrtc.media.video.codec.VideoCodecInfo; +import dev.onvoid.webrtc.media.video.codec.VideoCodecStatus; +import dev.onvoid.webrtc.media.video.codec.VideoDecoder; +import dev.onvoid.webrtc.media.video.codec.VideoDecoderFactory; +import dev.onvoid.webrtc.media.video.codec.VideoEncoder; +import dev.onvoid.webrtc.media.video.codec.VideoEncoderFactory; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +/** + * Tests video codec factories, built-in and implemented in Java, on calls + * between two local peer connections. + */ +@Execution(ExecutionMode.SAME_THREAD) +class VideoCodecFactoryTests extends TestBase { + + private static final long TIMEOUT_SECONDS = 10; + + /** The codec of {@link TestEncoder} and {@link TestDecoder}. */ + private static final String TEST_CODEC = "X-TEST"; + + + @Test + void defaultFactoriesListBuiltInCodecs() { + List encoders = names(new DefaultVideoEncoderFactory().getSupportedCodecs()); + List decoders = names(new DefaultVideoDecoderFactory().getSupportedCodecs()); + + assertTrue(encoders.contains("VP8"), encoders.toString()); + assertTrue(encoders.contains("VP9"), encoders.toString()); + assertTrue(decoders.contains("VP8"), decoders.toString()); + assertTrue(decoders.contains("VP9"), decoders.toString()); + } + + @Test + void defaultFactoryCreatesNativeCodecs() { + DefaultVideoEncoderFactory factory = new DefaultVideoEncoderFactory(); + + VideoEncoder encoder = factory.createEncoder(new VideoCodecInfo("VP8")); + + assertInstanceOf(NativeVideoEncoder.class, encoder); + assertEquals("VP8", ((NativeVideoEncoder) encoder).getCodecInfo().getName()); + assertThrows(UnsupportedOperationException.class, encoder::release); + + assertNull(factory.createEncoder(new VideoCodecInfo(TEST_CODEC))); + } + + @Test + void builtFactoryUsesDefaultCodecs() { + PeerConnectionFactory built = PeerConnectionFactory.builder() + .setAudioDeviceModule(audioDevModule) + .setVideoEncoderFactory(new DefaultVideoEncoderFactory()) + .setVideoDecoderFactory(new DefaultVideoDecoderFactory()) + .build(); + + try { + assertEquals( + names(factory.getRtpSenderCapabilities(MediaType.VIDEO).getCodecs()), + names(built.getRtpSenderCapabilities(MediaType.VIDEO).getCodecs())); + } + finally { + built.dispose(); + } + } + + @Test + void customCodecIsAdvertised() { + PeerConnectionFactory built = testCodecFactory(new TestEncoderFactory(), new TestDecoderFactory()); + + try { + List send = names(built.getRtpSenderCapabilities(MediaType.VIDEO).getCodecs()); + List receive = names(built.getRtpReceiverCapabilities(MediaType.VIDEO).getCodecs()); + + assertTrue(send.contains(TEST_CODEC), send.toString()); + assertTrue(send.contains("VP8"), send.toString()); + assertTrue(receive.contains(TEST_CODEC), receive.toString()); + } + finally { + built.dispose(); + } + } + + @Test + void javaCodecRoundTrips() throws Exception { + assertRoundTrip(false); + } + + @Test + void javaFrameBufferRoundTrips() throws Exception { + // The decoder hands back frames whose buffer is no NativeI420Buffer, + // which WebRTC has to copy. + assertRoundTrip(true); + } + + /** + * Sends a call through the test codec and checks that every frame + * received is one that was sent, and how WebRTC used the codecs. + */ + private void assertRoundTrip(boolean javaBuffers) throws Exception { + TestEncoderFactory encoderFactory = new TestEncoderFactory(); + TestDecoderFactory decoderFactory = new TestDecoderFactory(); + decoderFactory.javaBuffers = javaBuffers; + PeerConnectionFactory built = testCodecFactory(encoderFactory, decoderFactory); + + Set encodedSizes = ConcurrentHashMap.newKeySet(); + Set encodedLuma = ConcurrentHashMap.newKeySet(); + AtomicReference mismatch = new AtomicReference<>(); + CountDownLatch received = new CountDownLatch(10); + + encoderFactory.onEncode = (frame, luma) -> { + encodedSizes.add(frame.buffer.getWidth() + "x" + frame.buffer.getHeight()); + encodedLuma.add(luma); + }; + + try (TestMediaCall call = new TestMediaCall(built, true, false, TEST_CODEC)) { + call.negotiate(); + + RTCRtpReceiver receiver = call.getReceiver("video"); + VideoTrack track = (VideoTrack) receiver.getTrack(); + VideoTrackSink sink = frame -> { + String size = frame.buffer.getWidth() + "x" + frame.buffer.getHeight(); + int luma = frame.buffer.toI420().getDataY().get(0) & 0xFF; + + if (!encodedSizes.contains(size) || !encodedLuma.contains(luma)) { + mismatch.compareAndSet(null, size + ", luma " + luma + " not in " + encodedSizes + + " " + encodedLuma + ", encoders " + encoderFactory.created.get() + + ", decoders " + decoderFactory.created.get()); + } + + frame.release(); + received.countDown(); + }; + track.addSink(sink); + + call.awaitConnected(); + call.startMedia(); + + assertTrue(received.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), + "too few frames received: encoders " + encoderFactory.created.get() + + ", decoders " + decoderFactory.created.get()); + + track.removeSink(sink); + receiver.dispose(); + } + finally { + built.dispose(); + } + + assertNull(mismatch.get(), "a received frame was never sent: " + mismatch.get()); + + TestEncoder encoder = encoderFactory.last.get(); + + assertEquals(320, encoder.settings.width); + assertEquals(240, encoder.settings.height); + assertTrue(encoder.rates.get() > 0, "rates were never set"); + assertTrue(encoder.released.get(), "encoder was never released"); + assertTrue(decoderFactory.last.get().released.get(), "decoder was never released"); + } + + @Test + void factoryDelegatesToBuiltInCodec() throws Exception { + AtomicInteger encoders = new AtomicInteger(); + AtomicInteger decoders = new AtomicInteger(); + DefaultVideoEncoderFactory builtInEncoders = new DefaultVideoEncoderFactory(); + DefaultVideoDecoderFactory builtInDecoders = new DefaultVideoDecoderFactory(); + + PeerConnectionFactory built = PeerConnectionFactory.builder() + .setAudioDeviceModule(audioDevModule) + .setVideoEncoderFactory(new VideoEncoderFactory() { + + @Override + public List getSupportedCodecs() { + return builtInEncoders.getSupportedCodecs(); + } + + @Override + public VideoEncoder createEncoder(VideoCodecInfo info) { + encoders.incrementAndGet(); + return builtInEncoders.createEncoder(info); + } + }) + .setVideoDecoderFactory(new VideoDecoderFactory() { + + @Override + public List getSupportedCodecs() { + return builtInDecoders.getSupportedCodecs(); + } + + @Override + public VideoDecoder createDecoder(VideoCodecInfo info) { + decoders.incrementAndGet(); + return builtInDecoders.createDecoder(info); + } + }) + .build(); + + try { + assertFramesReceived(built, "VP8"); + } + finally { + built.dispose(); + } + + assertTrue(encoders.get() > 0); + assertTrue(decoders.get() > 0); + } + + @Test + void failingEncoderFactoryDoesNotCrash() throws Exception { + TestEncoderFactory encoderFactory = new TestEncoderFactory() { + + @Override + public VideoEncoder createEncoder(VideoCodecInfo info) { + super.createEncoder(info); + throw new IllegalStateException("createEncoder fails on purpose"); + } + }; + // A Java log sink receives the warning about the failure, which is + // logged while WebRTC holds its logging lock. This once deadlocked + // the call as it closed: the warning went out while the Java + // exception was still pending. + Logging.addLogSink(Logging.Severity.WARNING, (severity, message) -> { }); + + PeerConnectionFactory built = testCodecFactory(encoderFactory, new TestDecoderFactory()); + + try (TestMediaCall call = new TestMediaCall(built, true, false, TEST_CODEC)) { + call.negotiate(); + call.awaitConnected(); + call.startMedia(); + + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + + while (encoderFactory.created.get() == 0 && System.nanoTime() < deadline) { + Thread.sleep(20); + } + + assertTrue(encoderFactory.created.get() > 0, "the encoder factory was never asked"); + } + finally { + built.dispose(); + } + } + + @Test + void failingSupportedCodecsFailsBuild() { + VideoEncoderFactory broken = new TestEncoderFactory() { + + @Override + public List getSupportedCodecs() { + throw new IllegalStateException("getSupportedCodecs fails on purpose"); + } + }; + + assertThrows(Throwable.class, () -> PeerConnectionFactory.builder() + .setAudioDeviceModule(audioDevModule) + .setVideoEncoderFactory(broken) + .build()); + } + + @Test + void encodedImageBuilderValidates() { + assertThrows(NullPointerException.class, () -> EncodedImage.builder().build()); + assertThrows(IllegalArgumentException.class, () -> EncodedImage.builder() + .setBuffer(ByteBuffer.allocate(1)) + .setRotation(45) + .build()); + + EncodedImage image = EncodedImage.builder() + .setBuffer(ByteBuffer.allocate(3)) + .setEncodedWidth(2) + .setEncodedHeight(4) + .setFrameType(EncodedImage.FrameType.KEY) + .setQp(7) + .build(); + + assertEquals(3, image.getBuffer().remaining()); + assertEquals(EncodedImage.FrameType.KEY, image.getFrameType()); + assertEquals(7, (int) image.getQp()); + } + + private void assertFramesReceived(PeerConnectionFactory built, String codec) throws Exception { + CountDownLatch received = new CountDownLatch(10); + + try (TestMediaCall call = new TestMediaCall(built, true, false, codec)) { + call.negotiate(); + + RTCRtpReceiver receiver = call.getReceiver("video"); + VideoTrack track = (VideoTrack) receiver.getTrack(); + VideoTrackSink sink = frame -> { + frame.release(); + received.countDown(); + }; + track.addSink(sink); + + call.awaitConnected(); + call.startMedia(); + + assertTrue(received.await(TIMEOUT_SECONDS, TimeUnit.SECONDS), "too few frames received"); + + track.removeSink(sink); + receiver.dispose(); + } + } + + private PeerConnectionFactory testCodecFactory(VideoEncoderFactory encoderFactory, + VideoDecoderFactory decoderFactory) { + return PeerConnectionFactory.builder() + .setAudioDeviceModule(audioDevModule) + .setVideoEncoderFactory(encoderFactory) + .setVideoDecoderFactory(decoderFactory) + .build(); + } + + private static List names(List codecs) { + return codecs.stream() + .map(codec -> codec instanceof VideoCodecInfo + ? ((VideoCodecInfo) codec).getName() + : ((RTCRtpCodecCapability) codec).getName()) + .collect(Collectors.toList()); + } + + + + interface EncodeListener { + + void onEncode(VideoFrame frame, int luma); + + } + + + + /** + * Offers the test codec next to the built-in ones. + */ + static class TestEncoderFactory implements VideoEncoderFactory { + + final DefaultVideoEncoderFactory builtIn = new DefaultVideoEncoderFactory(); + final AtomicInteger created = new AtomicInteger(); + final AtomicReference last = new AtomicReference<>(); + + volatile EncodeListener onEncode; + + + @Override + public List getSupportedCodecs() { + List codecs = new ArrayList<>(); + codecs.add(new VideoCodecInfo(TEST_CODEC)); + codecs.addAll(builtIn.getSupportedCodecs()); + return codecs; + } + + @Override + public VideoEncoder createEncoder(VideoCodecInfo info) { + created.incrementAndGet(); + + if (!TEST_CODEC.equalsIgnoreCase(info.getName())) { + return builtIn.createEncoder(info); + } + + TestEncoder encoder = new TestEncoder(this); + last.set(encoder); + return encoder; + } + + } + + + + /** + * Offers the test codec next to the built-in ones. + */ + static class TestDecoderFactory implements VideoDecoderFactory { + + final DefaultVideoDecoderFactory builtIn = new DefaultVideoDecoderFactory(); + final AtomicInteger created = new AtomicInteger(); + final AtomicReference last = new AtomicReference<>(); + + volatile boolean javaBuffers; + + + @Override + public List getSupportedCodecs() { + List codecs = new ArrayList<>(); + codecs.add(new VideoCodecInfo(TEST_CODEC, Collections.emptyMap())); + codecs.addAll(builtIn.getSupportedCodecs()); + return codecs; + } + + @Override + public VideoDecoder createDecoder(VideoCodecInfo info) { + created.incrementAndGet(); + + if (!TEST_CODEC.equalsIgnoreCase(info.getName())) { + return builtIn.createDecoder(info); + } + + TestDecoder decoder = new TestDecoder(javaBuffers); + last.set(decoder); + return decoder; + } + + } + + + + /** + * "Encodes" a frame into its size and its first luma sample, every frame + * a key frame, so that the receiver can tell which frame it got. + */ + static class TestEncoder implements VideoEncoder { + + private final TestEncoderFactory factory; + + final AtomicInteger rates = new AtomicInteger(); + final AtomicReference released = new AtomicReference<>(false); + + volatile Settings settings; + + private Callback callback; + + + TestEncoder(TestEncoderFactory factory) { + this.factory = factory; + } + + @Override + public VideoCodecStatus initEncode(Settings settings, Callback callback) { + this.settings = settings; + this.callback = callback; + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus release() { + released.set(true); + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus encode(VideoFrame frame, EncodeInfo info) { + int width = frame.buffer.getWidth(); + int height = frame.buffer.getHeight(); + int luma = frame.buffer.toI420().getDataY().get(0) & 0xFF; + + EncodeListener listener = factory.onEncode; + + if (listener != null) { + listener.onEncode(frame, luma); + } + + ByteBuffer payload = ByteBuffer.allocateDirect(9); + payload.putInt(width).putInt(height).put((byte) luma); + payload.flip(); + + callback.onEncodedFrame(EncodedImage.builder() + .setBuffer(payload) + .setEncodedWidth(width) + .setEncodedHeight(height) + .setCaptureTimeNs(frame.timestampNs) + .setFrameType(EncodedImage.FrameType.KEY) + .build()); + + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus setRates(RateControlParameters parameters) { + if (parameters.bitrate.getSum() > 0) { + rates.incrementAndGet(); + } + return VideoCodecStatus.OK; + } + + @Override + public ScalingSettings getScalingSettings() { + // The payload has no QP to scale by. + return ScalingSettings.OFF; + } + + } + + + + /** + * Decodes what {@link TestEncoder} produces into a frame of the encoded + * size, filled with the encoded luma sample. + */ + static class TestDecoder implements VideoDecoder { + + final AtomicReference released = new AtomicReference<>(false); + + private final boolean javaBuffers; + + private Callback callback; + + + TestDecoder(boolean javaBuffers) { + this.javaBuffers = javaBuffers; + } + + @Override + public VideoCodecStatus initDecode(Settings settings, Callback callback) { + this.callback = callback; + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus release() { + released.set(true); + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus decode(EncodedImage image) { + ByteBuffer payload = image.getBuffer(); + + if (payload.remaining() != 9) { + return VideoCodecStatus.ERR_PARAMETER; + } + + int width = payload.getInt(); + int height = payload.getInt(); + byte luma = payload.get(); + + I420Buffer buffer = javaBuffers + ? new JavaI420Buffer(width, height) + : NativeI420Buffer.allocate(width, height); + ByteBuffer y = buffer.getDataY(); + ByteBuffer u = buffer.getDataU(); + ByteBuffer v = buffer.getDataV(); + + for (int i = 0; i < y.capacity(); i++) { + y.put(i, luma); + } + for (int i = 0; i < u.capacity(); i++) { + u.put(i, (byte) 128); + v.put(i, (byte) 128); + } + + VideoFrame frame = new VideoFrame(buffer, image.getCaptureTimeNs()); + callback.onDecodedFrame(frame, null, null); + frame.release(); + + return VideoCodecStatus.OK; + } + + } + + + + /** + * A frame buffer that lives in Java alone. + */ + static class JavaI420Buffer implements I420Buffer { + + private final int width; + private final int height; + private final ByteBuffer dataY; + private final ByteBuffer dataU; + private final ByteBuffer dataV; + + + JavaI420Buffer(int width, int height) { + int chromaWidth = (width + 1) / 2; + int chromaHeight = (height + 1) / 2; + + this.width = width; + this.height = height; + this.dataY = ByteBuffer.allocateDirect(width * height); + this.dataU = ByteBuffer.allocateDirect(chromaWidth * chromaHeight); + this.dataV = ByteBuffer.allocateDirect(chromaWidth * chromaHeight); + } + + @Override + public ByteBuffer getDataY() { + return dataY.slice(); + } + + @Override + public ByteBuffer getDataU() { + return dataU.slice(); + } + + @Override + public ByteBuffer getDataV() { + return dataV.slice(); + } + + @Override + public int getStrideY() { + return width; + } + + @Override + public int getStrideU() { + return (width + 1) / 2; + } + + @Override + public int getStrideV() { + return (width + 1) / 2; + } + + @Override + public int getWidth() { + return width; + } + + @Override + public int getHeight() { + return height; + } + + @Override + public I420Buffer toI420() { + return this; + } + + @Override + public VideoFrameBuffer cropAndScale(int cropX, int cropY, int cropWidth, + int cropHeight, int scaleWidth, int scaleHeight) { + throw new UnsupportedOperationException(); + } + + @Override + public void retain() { + } + + @Override + public void release() { + } + + } + +}