Skip to content

Commit abc19d5

Browse files
authored
fix: capture screens with ScreenCaptureKit on macOS (#354)
A VideoDesktopSource capturing a screen on macOS always used ScreenCapturerMac, since WebRTC tries ScreenCaptureKit only when allow_sck_capturer is set. That capturer takes its first frame from a display snapshot and afterwards only copies what its CGDisplayStream reports as changed, so when the stream stops calling back, it keeps sending the first frame, with a live cursor composited on top (#344). Screens are now captured with ScreenCaptureKit on macOS 14 and newer, and with ScreenCapturerMac as before on older releases. ScreenCaptureKit is told the source's frame rate, as it otherwise captures at the display refresh rate. Source lists stay with ScreenCapturerMac: without its system picker ScreenCaptureKit lists no screens, and both name a screen by its display ID. Window capture is unchanged. A new test switches a window of its own between black and white while the screen is captured, and checks the frames follow.
1 parent b7b7399 commit abc19d5

2 files changed

Lines changed: 181 additions & 0 deletions

File tree

‎webrtc-jni/src/main/cpp/src/media/video/VideoTrackDesktopSource.cpp‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,15 @@ namespace jni
267267

268268
#if defined(WEBRTC_MAC)
269269
options.set_allow_iosurface(true);
270+
// Screens are captured with ScreenCaptureKit on macOS 14 and newer.
271+
// The CGDisplayStream capturer otherwise used gets new pixels only
272+
// when its stream calls back, and when that stops it keeps sending the
273+
// first frame with a live cursor on top (issue #344). Older macOS
274+
// falls back to it. Only screens: window capture ignores this, and the
275+
// source list stays with the CGDisplayStream capturer in
276+
// DesktopCapturer, since ScreenCaptureKit lists no screens without its
277+
// system picker. Both name a screen by its CGDirectDisplayID.
278+
options.set_allow_sck_capturer(true);
270279
#endif
271280
#if defined(WEBRTC_WIN)
272281
options.set_allow_directx_capturer(true);
@@ -307,6 +316,12 @@ namespace jni
307316
return;
308317
}
309318

319+
#if defined(WEBRTC_MAC)
320+
// ScreenCaptureKit captures at the display refresh rate unless told
321+
// otherwise, though only one frame per interval below is taken.
322+
capturer->SetMaxFrameRate(frameRate);
323+
#endif
324+
310325
capturer->Start(this);
311326

312327
if (focusSelectedSource) {
Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
/*
2+
* Copyright 2026 Alex Andres
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package dev.onvoid.webrtc.media.video;
18+
19+
import static org.junit.jupiter.api.Assertions.assertTrue;
20+
import static org.junit.jupiter.api.Assumptions.assumeFalse;
21+
import static org.junit.jupiter.api.Assumptions.assumeTrue;
22+
23+
import dev.onvoid.webrtc.TestBase;
24+
import dev.onvoid.webrtc.media.video.desktop.DesktopSource;
25+
import dev.onvoid.webrtc.media.video.desktop.ScreenCapturer;
26+
27+
import java.awt.Color;
28+
import java.awt.EventQueue;
29+
import java.awt.Frame;
30+
import java.awt.GraphicsEnvironment;
31+
import java.awt.Rectangle;
32+
import java.nio.ByteBuffer;
33+
import java.util.ArrayList;
34+
import java.util.List;
35+
import java.util.Locale;
36+
37+
import org.junit.jupiter.api.Test;
38+
import org.junit.jupiter.api.parallel.Execution;
39+
import org.junit.jupiter.api.parallel.ExecutionMode;
40+
41+
@Execution(ExecutionMode.SAME_THREAD)
42+
class VideoDesktopSourceTests extends TestBase {
43+
44+
private static final String OS = System.getProperty("os.name").toLowerCase(Locale.ROOT);
45+
46+
@Test
47+
void screenFramesFollowScreen() throws Exception {
48+
assumeTrue(OS.contains("mac"), "covers the macOS screen capturers");
49+
assumeFalse(GraphicsEnvironment.isHeadless(), "No display");
50+
51+
DesktopSource screen = firstScreen();
52+
53+
// A window of our own, switched between black and white. A capture
54+
// that froze on its first frame keeps showing one of the two, while
55+
// still delivering frames (issue #344).
56+
Frame window = new Frame();
57+
Rectangle bounds = GraphicsEnvironment.getLocalGraphicsEnvironment()
58+
.getDefaultScreenDevice().getDefaultConfiguration().getBounds();
59+
60+
EventQueue.invokeAndWait(() -> {
61+
window.setUndecorated(true);
62+
window.setAlwaysOnTop(true);
63+
window.setBackground(Color.BLACK);
64+
window.setBounds(bounds.x + bounds.width / 10, bounds.y + bounds.height / 10,
65+
bounds.width * 8 / 10, bounds.height * 8 / 10);
66+
window.setVisible(true);
67+
window.toFront();
68+
});
69+
70+
VideoDesktopSource source = new VideoDesktopSource();
71+
source.setSourceId(screen.id, false);
72+
source.setFrameRate(10);
73+
source.setMaxFrameSize(320, 240);
74+
75+
VideoTrack track = factory.createVideoTrack("screen", source);
76+
77+
List<Double> luma = new ArrayList<>();
78+
79+
VideoTrackSink sink = frame -> {
80+
NativeI420Buffer buffer = (NativeI420Buffer) frame.buffer;
81+
double mean = meanLuma(buffer);
82+
83+
frame.release();
84+
85+
synchronized (luma) {
86+
luma.add(mean);
87+
}
88+
};
89+
90+
try {
91+
track.addSink(sink);
92+
source.start();
93+
94+
// Let the capture start on a black window, then turn it white
95+
// and back, a few times.
96+
Thread.sleep(1000);
97+
98+
for (int i = 0; i < 6; i++) {
99+
Color color = i % 2 == 0 ? Color.WHITE : Color.BLACK;
100+
101+
EventQueue.invokeAndWait(() -> {
102+
window.setBackground(color);
103+
window.repaint();
104+
});
105+
106+
Thread.sleep(500);
107+
}
108+
}
109+
finally {
110+
source.stop();
111+
track.removeSink(sink);
112+
track.dispose();
113+
source.dispose();
114+
115+
EventQueue.invokeAndWait(window::dispose);
116+
}
117+
118+
List<Double> means;
119+
120+
synchronized (luma) {
121+
means = new ArrayList<>(luma);
122+
}
123+
124+
assumeFalse(means.isEmpty(), "The screen delivered no frames");
125+
126+
double min = means.stream().mapToDouble(Double::doubleValue).min().getAsDouble();
127+
double max = means.stream().mapToDouble(Double::doubleValue).max().getAsDouble();
128+
129+
// The window covers about two thirds of the screen, so white and black
130+
// differ by more than 100 in mean luma.
131+
assertTrue(max - min > 50,
132+
"Frames did not follow the screen: " + means.size() + " frames, mean luma "
133+
+ String.format(Locale.ROOT, "%.1f to %.1f", min, max));
134+
}
135+
136+
private static double meanLuma(NativeI420Buffer buffer) {
137+
ByteBuffer data = buffer.getDataY();
138+
int stride = buffer.getStrideY();
139+
int width = buffer.getWidth();
140+
int height = buffer.getHeight();
141+
long sum = 0;
142+
143+
for (int y = 0; y < height; y++) {
144+
for (int x = 0; x < width; x++) {
145+
sum += data.get(y * stride + x) & 0xFF;
146+
}
147+
}
148+
149+
return (double) sum / (width * height);
150+
}
151+
152+
private static DesktopSource firstScreen() {
153+
ScreenCapturer capturer = new ScreenCapturer();
154+
155+
try {
156+
List<DesktopSource> screens = capturer.getDesktopSources();
157+
158+
assumeFalse(screens == null || screens.isEmpty(), "No screen to capture");
159+
160+
return screens.get(0);
161+
}
162+
finally {
163+
capturer.dispose();
164+
}
165+
}
166+
}

0 commit comments

Comments
 (0)