diff --git a/.github/actions/publish/action.yml b/.github/actions/publish/action.yml index fb237ef0..ae5f2613 100644 --- a/.github/actions/publish/action.yml +++ b/.github/actions/publish/action.yml @@ -83,7 +83,7 @@ runs: PROFILES: ${{ inputs.profiles }} SKIP_UPLOAD: ${{ inputs.skip-upload }} run: > - mvn -B deploy -pl .,webrtc,webrtc-java-media -DskipTests + mvn -B deploy -pl .,webrtc,webrtc-java-media,webrtc-java-javafx -DskipTests -Dmaven.repository.credentialScope=id ${PROFILES:+-P$PROFILES} -Dnatives.dir="$PWD/natives" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a11047be..85bfdba3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -224,6 +224,7 @@ jobs: echo '```' ls -1 webrtc/target/*.jar webrtc/target/*.asc \ webrtc-java-media/target/*.jar webrtc-java-media/target/*.asc \ + webrtc-java-javafx/target/*.jar webrtc-java-javafx/target/*.asc \ natives/*.asc 2>/dev/null || true echo '```' } >> "$GITHUB_STEP_SUMMARY" @@ -240,6 +241,9 @@ jobs: webrtc-java-media/target/central-*/** webrtc-java-media/target/*.jar webrtc-java-media/target/*.asc + webrtc-java-javafx/target/central-*/** + webrtc-java-javafx/target/*.jar + webrtc-java-javafx/target/*.asc natives/*.asc if-no-files-found: error retention-days: 7 diff --git a/README.md b/README.md index 18487848..dfbb534b 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ The library provides a comprehensive set of Java classes that map to the WebRTC ## Features - **Complete WebRTC API implementation** - Includes peer connections, media devices, data channels, and more -- **Client API** - A high-level layer that negotiates by itself, keeps your code off WebRTC's threads, and closes everything with one call; with WHIP and WHEP sessions and a Swing video view +- **Client API** - A high-level layer that negotiates by itself, keeps your code off WebRTC's threads, and closes everything with one call; with WHIP and WHEP sessions, and video views for Swing and, with the `webrtc-java-javafx` module, JavaFX - **Cross-platform support** - Works on Windows, macOS, and Linux (x64, ARM, ARM64) - **Media capabilities** - Audio and video capture from cameras and microphones - **Desktop capture** - Screen and application window sharing diff --git a/design/client-api.md b/design/client-api.md index 6ab39233..1665aa10 100644 --- a/design/client-api.md +++ b/design/client-api.md @@ -808,6 +808,9 @@ Next: takes client API senders and remote tracks: `dev.onvoid.webrtc.client.internal.EncodedFrames`, exported to the media module only, hands it their encoded frames, taken on the peer's control executor, where key frame requests run too, and do nothing once the sender or receiver is gone. +- `VideoView` for JavaFX lives in the `webrtc-java-javafx` module, package + `dev.onvoid.webrtc.client.javafx`, compiled for Java 11 since OpenJFX is; JavaFX is an optional + dependency the application brings. - phase 2 is done: `Whip.to(...).publish(rtc)` and `Whep.from(...).play(rtc)` return sessions over `HttpURLConnection` (Java 8), and `VideoView` is in `dev.onvoid.webrtc.client.swing`; diff --git a/docs/guide/client-api.md b/docs/guide/client-api.md index 47bbe1de..1f52220c 100644 --- a/docs/guide/client-api.md +++ b/docs/guide/client-api.md @@ -238,6 +238,35 @@ frame.add(remoteView, BorderLayout.CENTER); `snapshot()` returns a copy of the picture shown, for saving a still or for tests. +For JavaFX, the `webrtc-java-javafx` module has a `VideoView` node in `dev.onvoid.webrtc.client.javafx` that works the same way. It needs Java 11 and the JavaFX your application already uses: + +```xml + + dev.onvoid.webrtc + webrtc-java-javafx + {{ VERSION }} + +``` + +```java +import dev.onvoid.webrtc.client.javafx.VideoView; + +WebRtc rtc = WebRtc.builder() + .eventExecutor(Platform::runLater) // handlers on the JavaFX Application Thread + .build(); + +VideoView remoteView = new VideoView(); +peer.onTrack(track -> { + if (track.isVideo()) { + track.asVideo().onFrame(remoteView); + } +}); + +stage.setScene(new Scene(new StackPane(remoteView), 960, 540)); +``` + +The view converts frames on their own thread and paints at most once per JavaFX pulse. `setMirrored(true)`, or its `mirroredProperty()`, flips a camera preview, and `snapshot()`, on the JavaFX Application Thread, returns a copy of the picture. + ### Don't Wait in a Handler A future of the client API completes on the event thread of its peer. A handler that waits for one would wait for its own thread, so `join()` and `get()` there throw a `BlockingCallException` instead of hanging. Chain the next step instead: diff --git a/pom.xml b/pom.xml index 3022cfba..ceb73471 100644 --- a/pom.xml +++ b/pom.xml @@ -53,6 +53,7 @@ webrtc-jni webrtc webrtc-java-media + webrtc-java-javafx webrtc-examples diff --git a/webrtc-java-javafx/pom.xml b/webrtc-java-javafx/pom.xml new file mode 100644 index 00000000..5c23291e --- /dev/null +++ b/webrtc-java-javafx/pom.xml @@ -0,0 +1,69 @@ + + + 4.0.0 + + + dev.onvoid.webrtc + webrtc-java-parent + 0.21.0-SNAPSHOT + + + webrtc-java-javafx + + webrtc-java-javafx + + JavaFX extension for webrtc-java. Shows the video of the client API in JavaFX scenes. + + + + + 17.0.20 + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + default-compile + + 11 + + + + base-compile + none + + + + 11 + + + + + + + + dev.onvoid.webrtc + webrtc-java + ${project.version} + + + + org.openjfx + javafx-graphics + ${javafx.version} + true + + + diff --git a/webrtc-java-javafx/src/main/java/dev/onvoid/webrtc/client/javafx/VideoView.java b/webrtc-java-javafx/src/main/java/dev/onvoid/webrtc/client/javafx/VideoView.java new file mode 100644 index 00000000..d3fe63c6 --- /dev/null +++ b/webrtc-java-javafx/src/main/java/dev/onvoid/webrtc/client/javafx/VideoView.java @@ -0,0 +1,277 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client.javafx; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.function.Consumer; + +import javafx.application.Platform; +import javafx.beans.property.BooleanProperty; +import javafx.beans.property.SimpleBooleanProperty; +import javafx.scene.canvas.Canvas; +import javafx.scene.canvas.GraphicsContext; +import javafx.scene.image.PixelFormat; +import javafx.scene.image.WritableImage; +import javafx.scene.layout.Region; +import javafx.scene.paint.Color; + +import dev.onvoid.webrtc.media.FourCC; +import dev.onvoid.webrtc.media.video.VideoBufferConverter; +import dev.onvoid.webrtc.media.video.VideoFrame; +import dev.onvoid.webrtc.media.video.VideoFrameBuffer; + +/** + * A JavaFX node that shows video frames, such as those of a remote track: + * 
{@code
+ * VideoView view = new VideoView();
+ * track.asVideo().onFrame(view);
+ * }
+ * Frames are converted on the thread that delivers them, so the JavaFX + * Application Thread only paints, at most once per pulse however many frames + * arrive. The picture keeps its aspect ratio and its rotation, on a black + * background, and can be mirrored for a local camera preview. + */ +public class VideoView extends Region implements Consumer { + + private final Canvas canvas = new Canvas(); + + private final BooleanProperty mirrored = new SimpleBooleanProperty(this, "mirrored"); + + /** Serializes conversions, for frames of more than one subscription. */ + private final Object convertLock = new Object(); + + /** Guards the hand-over between the converting thread and the JavaFX thread. */ + private final Object lock = new Object(); + + // Three buffers: the converting thread fills back, hands it over as + // ready, and the JavaFX thread paints from shown. None is written while + // another thread reads it. + + private Pixels back; + + private Pixels ready; + + private boolean scheduled; + + // Confined to the JavaFX Application Thread. + + private Pixels shown; + + private WritableImage image; + + private int rotation; + + + /** + * Creates a view with a preferred size of 640 by 360 pixels. + */ + public VideoView() { + getChildren().add(canvas); + setPrefSize(640, 360); + setMinSize(0, 0); + + mirrored.addListener((property, before, now) -> draw()); + } + + /** + * Shows a frame. Called with the frames of a track, from any thread; the + * frame is not released, which its subscription does. + * + * @param frame the frame to show. + */ + @Override + public void accept(VideoFrame frame) { + VideoFrameBuffer buffer = frame.buffer; + int width = buffer.getWidth(); + int height = buffer.getHeight(); + + synchronized (convertLock) { + Pixels target = back; + + if (target == null || !target.fits(width, height)) { + target = new Pixels(width, height); + } + + try { + // libyuv's ARGB is B, G, R, A in memory, which is JavaFX's BGRA. + target.data.clear(); + VideoBufferConverter.convertFromI420(buffer, target.data, FourCC.ARGB); + target.data.rewind(); + } + catch (Exception e) { + throw new IllegalStateException("The frame could not be converted", e); + } + + target.rotation = frame.rotation; + + synchronized (lock) { + back = ready; + ready = target; + + if (scheduled) { + return; + } + + scheduled = true; + } + } + + Platform.runLater(this::present); + } + + /** + * Removes the picture, leaving the background. Called on the JavaFX + * Application Thread. + */ + public void clear() { + synchronized (lock) { + ready = null; + back = null; + } + + shown = null; + image = null; + draw(); + } + + /** + * @return the property that flips the picture horizontally, as a local + * camera preview usually is. + */ + public BooleanProperty mirroredProperty() { + return mirrored; + } + + /** + * @param mirrored true to flip the picture horizontally. + */ + public void setMirrored(boolean mirrored) { + this.mirrored.set(mirrored); + } + + /** + * @return true when the picture is flipped horizontally. + */ + public boolean isMirrored() { + return mirrored.get(); + } + + /** + * Returns a copy of the picture shown, as the frame was, without rotation + * or mirroring. Called on the JavaFX Application Thread. + * + * @return the copy, or null when there is no picture. + */ + public WritableImage snapshot() { + if (image == null) { + return null; + } + + int width = (int) image.getWidth(); + int height = (int) image.getHeight(); + WritableImage copy = new WritableImage(width, height); + copy.getPixelWriter().setPixels(0, 0, width, height, image.getPixelReader(), 0, 0); + + return copy; + } + + @Override + protected void layoutChildren() { + canvas.setWidth(getWidth()); + canvas.setHeight(getHeight()); + draw(); + } + + /** Takes the newest frame over, on the JavaFX Application Thread. */ + private void present() { + Pixels next; + + synchronized (lock) { + scheduled = false; + next = ready; + + if (next == null) { + return; + } + + ready = shown; + shown = next; + } + + if (image == null || image.getWidth() != next.width || image.getHeight() != next.height) { + image = new WritableImage(next.width, next.height); + } + + image.getPixelWriter().setPixels(0, 0, next.width, next.height, + PixelFormat.getByteBgraPreInstance(), next.data, next.width * 4); + rotation = next.rotation; + + draw(); + } + + private void draw() { + GraphicsContext g = canvas.getGraphicsContext2D(); + double width = canvas.getWidth(); + double height = canvas.getHeight(); + + g.setFill(Color.BLACK); + g.fillRect(0, 0, width, height); + + if (image == null || width <= 0 || height <= 0) { + return; + } + + boolean sideways = rotation == 90 || rotation == 270; + double imageWidth = sideways ? image.getHeight() : image.getWidth(); + double imageHeight = sideways ? image.getWidth() : image.getHeight(); + double scale = Math.min(width / imageWidth, height / imageHeight); + + // Around the center: scale, rotate, mirror, then draw the image + // centered on the origin, as the Swing view does. + g.save(); + g.translate(width / 2, height / 2); + g.scale(isMirrored() ? -scale : scale, scale); + g.rotate(rotation); + g.drawImage(image, -image.getWidth() / 2, -image.getHeight() / 2); + g.restore(); + } + + + + private static final class Pixels { + + final int width; + + final int height; + + final ByteBuffer data; + + int rotation; + + + Pixels(int width, int height) { + this.width = width; + this.height = height; + this.data = ByteBuffer.allocateDirect(width * height * 4).order(ByteOrder.LITTLE_ENDIAN); + } + + boolean fits(int otherWidth, int otherHeight) { + return width == otherWidth && height == otherHeight; + } + } +} diff --git a/webrtc-java-javafx/src/main/java/module-info.java b/webrtc-java-javafx/src/main/java/module-info.java new file mode 100644 index 00000000..f79595e3 --- /dev/null +++ b/webrtc-java-javafx/src/main/java/module-info.java @@ -0,0 +1,27 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Shows the video of the webrtc-java client API in JavaFX scenes. + */ +module webrtc.java.javafx { + + requires transitive webrtc.java; + requires transitive javafx.graphics; + + exports dev.onvoid.webrtc.client.javafx; + +} diff --git a/webrtc-java-javafx/src/test/java/dev/onvoid/webrtc/client/javafx/VideoViewTest.java b/webrtc-java-javafx/src/test/java/dev/onvoid/webrtc/client/javafx/VideoViewTest.java new file mode 100644 index 00000000..1859b6ef --- /dev/null +++ b/webrtc-java-javafx/src/test/java/dev/onvoid/webrtc/client/javafx/VideoViewTest.java @@ -0,0 +1,254 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.client.javafx; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import java.nio.ByteBuffer; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import javafx.application.Platform; +import javafx.scene.Scene; +import javafx.scene.image.Image; +import javafx.scene.paint.Color; + +import dev.onvoid.webrtc.client.AudioMode; +import dev.onvoid.webrtc.client.CustomVideo; +import dev.onvoid.webrtc.client.Peer; +import dev.onvoid.webrtc.client.PeerOptions; +import dev.onvoid.webrtc.client.Signaling; +import dev.onvoid.webrtc.client.SignalingPair; +import dev.onvoid.webrtc.client.WebRtc; +import dev.onvoid.webrtc.media.video.NativeI420Buffer; +import dev.onvoid.webrtc.media.video.VideoFrame; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +/** + * Needs a display for the JavaFX toolkit, so it skips where there is none, as + * on CI runners. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@Execution(ExecutionMode.SAME_THREAD) +class VideoViewTest { + + private static final long TIMEOUT_SECONDS = 15; + + private boolean toolkit; + + private WebRtc rtc; + + + @BeforeAll + void startToolkit() { + try { + Platform.startup(() -> { }); + Platform.setImplicitExit(false); + toolkit = true; + } + catch (IllegalStateException alreadyStarted) { + toolkit = true; + } + catch (Throwable noDisplay) { + toolkit = false; + } + + // Also loads the native library the frames need. + rtc = WebRtc.builder().audio(AudioMode.HEADLESS).build(); + } + + @AfterAll + void stop() { + rtc.close(); + } + + @BeforeEach + void needsToolkit() { + assumeTrue(toolkit, "Needs a display for the JavaFX toolkit"); + } + + @Test + void showsFrame() throws Exception { + VideoView view = onFx(VideoView::new); + assertNull(onFx(view::snapshot)); + + view.accept(frame(64, 48, 128, 128)); + + Image snapshot = presented(view); + assertEquals(64, (int) snapshot.getWidth()); + assertEquals(48, (int) snapshot.getHeight()); + assertGray(snapshot.getPixelReader().getColor(32, 24)); + + onFx(() -> { + view.clear(); + return null; + }); + assertNull(onFx(view::snapshot)); + } + + @Test + void keepsAspectRatio() throws Exception { + VideoView view = sized(200, 100); + view.accept(frame(64, 48, 128, 128)); + presented(view); + + // A 4:3 picture in a 2:1 view leaves bars on the sides. + Image painted = paint(view); + assertEquals(Color.BLACK, painted.getPixelReader().getColor(5, 50)); + assertGray(painted.getPixelReader().getColor(100, 50)); + } + + @Test + void mirrors() throws Exception { + VideoView view = sized(64, 48); + view.accept(frame(64, 48, 16, 235)); + presented(view); + + assertTrue(paint(view).getPixelReader().getColor(8, 24).getBrightness() < 0.25); + + onFx(() -> { + view.setMirrored(true); + return null; + }); + + // The bright right half is now on the left. + assertTrue(paint(view).getPixelReader().getColor(8, 24).getBrightness() > 0.75); + } + + @Test + void showsRemoteVideo() throws Exception { + SignalingPair link = Signaling.loopback(); + Peer caller = rtc.createPeer(PeerOptions.builder().signaling(link.first()).build()); + Peer callee = rtc.createPeer(PeerOptions.builder().signaling(link.second()).polite(true).build()); + CustomVideo video = rtc.media().customVideo("video"); + VideoView view = onFx(VideoView::new); + + callee.onTrack(track -> track.asVideo().onFrame(view)); + caller.addTrack(video); + + ScheduledExecutorService feeder = Executors.newSingleThreadScheduledExecutor(); + + try { + feeder.scheduleAtFixedRate(() -> { + VideoFrame frame = frame(320, 240, 128, 128); + video.push(frame); + frame.release(); + }, 0, 33, TimeUnit.MILLISECONDS); + + assertEquals(320, (int) presented(view).getWidth()); + } + finally { + feeder.shutdownNow(); + caller.close(); + callee.close(); + video.close(); + } + } + + private VideoView sized(int width, int height) throws Exception { + return onFx(() -> { + VideoView view = new VideoView(); + new Scene(view, width, height); + view.resize(width, height); + view.layout(); + return view; + }); + } + + /** Waits until the view has taken a frame over and returns its picture. */ + private static Image presented(VideoView view) throws Exception { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + + while (System.nanoTime() < deadline) { + Image snapshot = onFx(view::snapshot); + + if (snapshot != null) { + return snapshot; + } + + Thread.sleep(20); + } + + throw new AssertionError("No frame reached the view"); + } + + private static Image paint(VideoView view) throws Exception { + return onFx(() -> view.snapshot(null, null)); + } + + private static T onFx(Callable action) throws Exception { + CompletableFuture result = new CompletableFuture<>(); + + Platform.runLater(() -> { + try { + result.complete(action.call()); + } + catch (Throwable e) { + result.completeExceptionally(e); + } + }); + + return result.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + + /** + * A frame whose left half has luma {@code left} and right half {@code right}, with + * neutral chroma. + */ + private static VideoFrame frame(int width, int height, int left, int right) { + NativeI420Buffer buffer = NativeI420Buffer.allocate(width, height); + ByteBuffer y = buffer.getDataY(); + + for (int row = 0; row < height; row++) { + for (int column = 0; column < width; column++) { + y.put(row * buffer.getStrideY() + column, (byte) (column < width / 2 ? left : right)); + } + } + + fill(buffer.getDataU()); + fill(buffer.getDataV()); + + return new VideoFrame(buffer, System.nanoTime()); + } + + private static void fill(ByteBuffer plane) { + for (int i = 0; i < plane.capacity(); i++) { + plane.put(i, (byte) 128); + } + } + + private static void assertGray(Color color) { + assertNotNull(color); + assertTrue(Math.abs(color.getRed() - color.getGreen()) < 0.04 && Math.abs(color.getGreen() - color.getBlue()) < 0.04 + && color.getGreen() > 0.4 && color.getGreen() < 0.63, () -> "Not mid gray: " + color); + } + +}