From cf1e09641f581be06ec2f0efb933f62d7d5176df Mon Sep 17 00:00:00 2001 From: Milosz Filimowski Date: Thu, 10 Sep 2026 12:41:12 +0200 Subject: [PATCH 1/3] feat(examples): background blur on the Fishjam camera track fishjam-chat and minimal-react apply @fishjam-cloud/video-effects through useFishjamCameraEffect, on the camera track useCamera publishes. --- .../fishjam-chat/app/_layout.tsx | 13 ++- .../fishjam-chat/app/room/[roomName].tsx | 7 ++ .../fishjam-chat/app/room/preview.tsx | 7 ++ .../fishjam-chat/babel.config.js | 10 +- .../fishjam-chat/metro.config.js | 52 +++++++++ .../mobile-client/fishjam-chat/package.json | 8 +- .../providers/BlurCameraProvider.tsx | 54 ++++++++++ .../react-client/minimal-react/package.json | 1 + .../minimal-react/src/components/App.tsx | 41 ++++++- yarn.lock | 100 ++++++++++++++++++ 10 files changed, 284 insertions(+), 9 deletions(-) create mode 100644 examples/mobile-client/fishjam-chat/metro.config.js create mode 100644 examples/mobile-client/fishjam-chat/providers/BlurCameraProvider.tsx diff --git a/examples/mobile-client/fishjam-chat/app/_layout.tsx b/examples/mobile-client/fishjam-chat/app/_layout.tsx index bade585a4..64df041a6 100644 --- a/examples/mobile-client/fishjam-chat/app/_layout.tsx +++ b/examples/mobile-client/fishjam-chat/app/_layout.tsx @@ -1,13 +1,22 @@ import { FishjamProvider, Variant } from '@fishjam-cloud/react-native-client'; -import { Stack } from 'expo-router'; +import { Stack, useRouter } from 'expo-router'; import { useEffect, useState } from 'react'; +import { BlurCameraProvider } from '../providers/BlurCameraProvider'; + import { setFishjamIdChangeCallback } from '../utils/fishjamIdStore'; const DEFAULT_FISHJAM_ID = process.env.EXPO_PUBLIC_FISHJAM_ID ?? ''; export default function RootLayout() { const [fishjamId, setFishjamId] = useState(DEFAULT_FISHJAM_ID); + const router = useRouter(); + + useEffect(() => { + if (__DEV__) { + (globalThis as { __fjRouter?: typeof router }).__fjRouter = router; + } + }, [router]); useEffect(() => { setFishjamIdChangeCallback(setFishjamId); @@ -31,6 +40,7 @@ export default function RootLayout() { Variant.VARIANT_HIGH, ], }}> + + ); } diff --git a/examples/mobile-client/fishjam-chat/app/room/[roomName].tsx b/examples/mobile-client/fishjam-chat/app/room/[roomName].tsx index 800868df9..3dfcbda1f 100644 --- a/examples/mobile-client/fishjam-chat/app/room/[roomName].tsx +++ b/examples/mobile-client/fishjam-chat/app/room/[roomName].tsx @@ -13,6 +13,7 @@ import { Platform, StyleSheet, View } from 'react-native'; import { SafeAreaView } from 'react-native-safe-area-context'; import { InCallButton, VideosGrid } from '../../components'; +import { useBlurCamera } from '../../providers/BlurCameraProvider'; export default function RoomScreen() { const { userName } = useLocalSearchParams<{ @@ -21,6 +22,7 @@ export default function RoomScreen() { }>(); const { isCameraOn, toggleCamera, stopCamera } = useCamera(); + const { isBlurEnabled, toggleBlur } = useBlurCamera(); const { isMicrophoneOn, toggleMicrophone, stopMicrophone, startMicrophone } = useMicrophone(); const { leaveRoom } = useConnection(); @@ -141,6 +143,11 @@ export default function RoomScreen() { onPress={toggleCamera} accessibilityLabel="Toggle Camera" /> + + diff --git a/examples/mobile-client/fishjam-chat/babel.config.js b/examples/mobile-client/fishjam-chat/babel.config.js index de1421ef3..5e226c3f5 100644 --- a/examples/mobile-client/fishjam-chat/babel.config.js +++ b/examples/mobile-client/fishjam-chat/babel.config.js @@ -1,9 +1,11 @@ module.exports = function (api) { api.cache(true); return { - presets: ['babel-preset-expo'], + presets: [['babel-preset-expo', { unstable_transformImportMeta: true }]], plugins: [ - 'react-native-reanimated/plugin', + // typegpu, reached through @fishjam-cloud/video-effects, ships static class blocks, which + // babel-preset-expo does not transform on its own. + '@babel/plugin-transform-class-static-block', [ 'module-resolver', { @@ -12,6 +14,10 @@ module.exports = function (api) { }, }, ], + // Compiles TypeGPU's tgpu.fn shader bodies to WGSL at build time. + 'unplugin-typegpu/babel', + // Must stay last: the worklets plugin has to see the final output of the others. + 'react-native-reanimated/plugin', ], }; }; diff --git a/examples/mobile-client/fishjam-chat/metro.config.js b/examples/mobile-client/fishjam-chat/metro.config.js new file mode 100644 index 000000000..4d7473d76 --- /dev/null +++ b/examples/mobile-client/fishjam-chat/metro.config.js @@ -0,0 +1,52 @@ +const path = require('path'); + +const { getDefaultConfig } = require('expo/metro-config'); + +const projectRoot = __dirname; +const monorepoRoot = path.resolve(projectRoot, '../../..'); + +const config = getDefaultConfig(projectRoot); + +config.watchFolders = [monorepoRoot]; + +// The segmentation weights ship as a .ssgbin file loaded through expo-asset. +config.resolver.assetExts = [...config.resolver.assetExts, 'ssgbin']; +config.resolver.nodeModulesPaths = [ + path.resolve(projectRoot, 'node_modules'), + path.resolve(monorepoRoot, 'node_modules'), +]; + +// The workspace source packages dev-pin their own copies of these, and Metro's default lookup +// finds the nested copy first. Two instances of a native module is not a duplicated-code problem, +// it is a broken app: react-native's nested 0.85.3 emits codegen this app's babel plugin cannot +// parse, and a second react-native-webgpu re-registers its native view ("Tried to register two +// views with the same name WebGPUView"). Pin these to the root copy and leave every other package +// to resolve normally, so packages that rely on their own nested dependencies still can. Every +// native module here must resolve to exactly one copy: the JS half talks to a single installed +// pod, so a second copy from a package's nested node_modules reports a version mismatch and +// refuses to run ("Worklets 0.10.2 vs 0.8.1"). +const SINGLETON_MODULES = [ + '@fishjam-cloud/react-native-webrtc', + 'react', + 'react-native', + 'react-native-nitro-modules', + 'react-native-reanimated', + 'react-native-webgpu', + 'react-native-worklets', + // Not native, but it keeps module-level registries, so it must load exactly once. + 'typegpu', +]; + +const defaultResolveRequest = config.resolver.resolveRequest; + +config.resolver.resolveRequest = (context, moduleName, platform) => { + const resolve = defaultResolveRequest ?? context.resolveRequest; + const singleton = SINGLETON_MODULES.find((name) => moduleName === name || moduleName.startsWith(`${name}/`)); + if (singleton) { + const subPath = moduleName.slice(singleton.length); + return resolve(context, path.resolve(monorepoRoot, 'node_modules', singleton) + subPath, platform); + } + return resolve(context, moduleName, platform); +}; + +module.exports = config; diff --git a/examples/mobile-client/fishjam-chat/package.json b/examples/mobile-client/fishjam-chat/package.json index 0798b0ddc..85dbce51a 100644 --- a/examples/mobile-client/fishjam-chat/package.json +++ b/examples/mobile-client/fishjam-chat/package.json @@ -14,6 +14,8 @@ "dependencies": { "@expo/vector-icons": "^15.0.3", "@fishjam-cloud/react-native-client": "workspace:*", + "@fishjam-cloud/react-native-worklets": "0.12.1", + "@fishjam-cloud/video-effects": "0.1.1", "@react-navigation/bottom-tabs": "^7.4.0", "@react-navigation/elements": "^2.6.3", "@react-navigation/native": "^7.1.8", @@ -38,16 +40,20 @@ "react-native-safe-area-context": "~5.7.0", "react-native-screens": "~4.26.0", "react-native-web": "~0.21.0", + "react-native-webgpu": "0.5.15", "react-native-worklets": "0.12.1" }, "devDependencies": { "@babel/plugin-transform-class-static-block": "^7.28.3", "@types/react": "~19.2.0", + "@webgpu/types": "0.1.65", "babel-plugin-module-resolver": "^5.0.2", "babel-preset-expo": "~57.0.10", "eslint-config-expo": "~57.0.2", "eslint-import-resolver-typescript": "^4.4.4", - "typescript": "~5.9.2" + "typegpu": "^0.12.0", + "typescript": "~5.9.2", + "unplugin-typegpu": "^0.12.0" }, "private": true } diff --git a/examples/mobile-client/fishjam-chat/providers/BlurCameraProvider.tsx b/examples/mobile-client/fishjam-chat/providers/BlurCameraProvider.tsx new file mode 100644 index 000000000..92e66a87b --- /dev/null +++ b/examples/mobile-client/fishjam-chat/providers/BlurCameraProvider.tsx @@ -0,0 +1,54 @@ +import type { VideoEffectStatus } from '@fishjam-cloud/video-effects'; +import { useBackgroundBlur } from '@fishjam-cloud/video-effects/background-blur'; +import { useFishjamCameraEffect } from '@fishjam-cloud/video-effects/fishjam-react-native'; +import { typeGpuPersonSegmentation } from '@fishjam-cloud/video-effects/segmentation/typegpu'; +import { Asset } from 'expo-asset'; +import { createContext, type ReactNode, useCallback, useContext, useEffect, useMemo, useState } from 'react'; + +const BLUR_RADIUS = 24; + +const segmentationModel = Asset.fromModule( + // eslint-disable-next-line @typescript-eslint/no-require-imports + require('@fishjam-cloud/video-effects/assets/selfie_segmenter.ssgbin'), +); +const personSegmentation = typeGpuPersonSegmentation({ modelUrl: segmentationModel.uri }); + +interface BlurCameraValue { + isBlurEnabled: boolean; + toggleBlur: () => void; + status: VideoEffectStatus; + error: Error | null; +} + +const BlurCameraContext = createContext(null); + +/** + * Owns background blur for the whole app. Blur runs on Fishjam's own camera track through the + * camera-track middleware, so the preview screen and the call screen share one camera. + */ +export function BlurCameraProvider({ children }: { children: ReactNode }) { + const [isBlurEnabled, setIsBlurEnabled] = useState(false); + const backgroundBlur = useBackgroundBlur({ segmentation: personSegmentation, radius: BLUR_RADIUS }); + const { status, error } = useFishjamCameraEffect(isBlurEnabled ? backgroundBlur : null); + const toggleBlur = useCallback(() => setIsBlurEnabled((enabled) => !enabled), []); + + useEffect(() => { + if (!__DEV__) return; + // Lets a debugger flip blur without touching the screen (see the `__fjRouter` hook in _layout). + (globalThis as { __fjToggleBlur?: () => void }).__fjToggleBlur = toggleBlur; + }, [toggleBlur]); + + const value = useMemo( + () => ({ isBlurEnabled, toggleBlur, status, error }), + [isBlurEnabled, toggleBlur, status, error], + ); + return {children}; +} + +export function useBlurCamera(): BlurCameraValue { + const value = useContext(BlurCameraContext); + if (value == null) { + throw new Error('useBlurCamera must be used inside BlurCameraProvider'); + } + return value; +} diff --git a/examples/react-client/minimal-react/package.json b/examples/react-client/minimal-react/package.json index 35374a4cd..0167a8ce0 100644 --- a/examples/react-client/minimal-react/package.json +++ b/examples/react-client/minimal-react/package.json @@ -16,6 +16,7 @@ }, "dependencies": { "@fishjam-cloud/react-client": "workspace:*", + "@fishjam-cloud/video-effects": "0.1.1", "react": "19.1.0", "react-dom": "19.1.0" }, diff --git a/examples/react-client/minimal-react/src/components/App.tsx b/examples/react-client/minimal-react/src/components/App.tsx index 38b544443..43243155e 100644 --- a/examples/react-client/minimal-react/src/components/App.tsx +++ b/examples/react-client/minimal-react/src/components/App.tsx @@ -7,11 +7,16 @@ import { Variant, } from "@fishjam-cloud/react-client"; import { useStatistics } from "@fishjam-cloud/react-client/debug"; +import { useBackgroundBlur } from "@fishjam-cloud/video-effects/background-blur"; +import { useFishjamCameraEffect } from "@fishjam-cloud/video-effects/fishjam-react"; +import { typeGpuPersonSegmentation } from "@fishjam-cloud/video-effects/segmentation/typegpu"; import { Fragment, useState } from "react"; import AudioPlayer from "./AudioPlayer"; import VideoPlayer from "./VideoPlayer"; +const personSegmentation = typeGpuPersonSegmentation(); + const variantLabel = (variant: Variant | null | undefined): string => { switch (variant) { case Variant.VARIANT_LOW: @@ -27,14 +32,24 @@ const variantLabel = (variant: Variant | null | undefined): string => { export const App = () => { const [token, setToken] = useState(""); + const [blurEnabled, setBlurEnabled] = useState(true); const { joinRoom, leaveRoom, peerStatus } = useConnection(); const { remotePeers } = usePeers(); const screenShare = useScreenShare(); - const { isCameraOn, toggleCamera } = useCamera(); + const { cameraStream, isCameraOn, toggleCamera } = useCamera(); const { isMicrophoneOn, toggleMicrophone } = useMicrophone(); const { getStatistics } = useStatistics(); + const backgroundBlur = useBackgroundBlur({ + segmentation: personSegmentation, + radius: 24, + }); + const { + status: effectStatus, + error: effectError, + retry: retryEffect, + } = useFishjamCameraEffect(blurEnabled ? backgroundBlur : null); { // for e2e test @@ -89,6 +104,14 @@ export const App = () => { Start camera + + + {effectStatus === "error" && ( + + )} + Status: {peerStatus} + Effect: {effectStatus} - {/* Render the video remote tracks from other peers*/} + {effectError && 
{effectError.message}
} + {cameraStream && ( +
+ Local camera + +
+ )} + {remotePeers.map( ({ id, cameraTrack, microphoneTrack, screenShareVideoTrack }) => { - const cameraStream = cameraTrack?.stream; + const remoteCameraStream = cameraTrack?.stream; const microphoneStream = microphoneTrack?.stream; const screenShareStream = screenShareVideoTrack?.stream; return ( - {cameraStream && ( + {remoteCameraStream && (
- +
=0.30.2" + react-native: "*" + react-native-worklets: ~0.12.0 + checksum: 10c0/a2422e87ddc9f1370fe2cc89a46d216492db965fb48707d7336d2f9bd149e7160d51c2b57b043dc6411fcd3e2e88ed65f5fcad926ac755a3eb539b2589d73a32 + languageName: node + linkType: hard + "@fishjam-cloud/ts-client@workspace:*, @fishjam-cloud/ts-client@workspace:packages/ts-client": version: 0.0.0-use.local resolution: "@fishjam-cloud/ts-client@workspace:packages/ts-client" @@ -5591,6 +5602,38 @@ __metadata: languageName: unknown linkType: soft +"@fishjam-cloud/video-effects@npm:0.1.1": + version: 0.1.1 + resolution: "@fishjam-cloud/video-effects@npm:0.1.1" + dependencies: + "@webgpu/types": "npm:0.1.65" + typed-binary: "npm:^4.3.3" + typegpu: "npm:^0.12.0" + peerDependencies: + "@fishjam-cloud/react-client": "*" + "@fishjam-cloud/react-native-client": "*" + "@fishjam-cloud/react-native-webrtc": ">=0.30.2" + react: ">=18" + react-native: "*" + react-native-webgpu: ">=0.5.15" + react-native-worklets: ">=0.12.1" + peerDependenciesMeta: + "@fishjam-cloud/react-client": + optional: true + "@fishjam-cloud/react-native-client": + optional: true + "@fishjam-cloud/react-native-webrtc": + optional: true + react-native: + optional: true + react-native-webgpu: + optional: true + react-native-worklets: + optional: true + checksum: 10c0/791d91141cc06e38752db886ce88b67dde49c5f4440c9380db669fccaf3571caf6441687c4357e9e945459a657416c12e172964f85b149732ac9a455625937d6 + languageName: node + linkType: hard + "@fishjam-cloud/webrtc-client@workspace:*, @fishjam-cloud/webrtc-client@workspace:packages/webrtc-client": version: 0.0.0-use.local resolution: "@fishjam-cloud/webrtc-client@workspace:packages/webrtc-client" @@ -5686,6 +5729,7 @@ __metadata: resolution: "@fishjam-example/minimal-react-example@workspace:examples/react-client/minimal-react" dependencies: "@fishjam-cloud/react-client": "workspace:*" + "@fishjam-cloud/video-effects": "npm:0.1.1" "@types/react": "npm:19.1.0" "@types/react-dom": "npm:19.1.0" "@vitejs/plugin-react-swc": "npm:^3.8.1" @@ -13336,10 +13380,13 @@ __metadata: "@babel/plugin-transform-class-static-block": "npm:^7.28.3" "@expo/vector-icons": "npm:^15.0.3" "@fishjam-cloud/react-native-client": "workspace:*" + "@fishjam-cloud/react-native-worklets": "npm:0.12.1" + "@fishjam-cloud/video-effects": "npm:0.1.1" "@react-navigation/bottom-tabs": "npm:^7.4.0" "@react-navigation/elements": "npm:^2.6.3" "@react-navigation/native": "npm:^7.1.8" "@types/react": "npm:~19.2.0" + "@webgpu/types": "npm:0.1.65" babel-plugin-module-resolver: "npm:^5.0.2" babel-preset-expo: "npm:~57.0.10" eslint-config-expo: "npm:~57.0.2" @@ -13365,8 +13412,11 @@ __metadata: react-native-safe-area-context: "npm:~5.7.0" react-native-screens: "npm:~4.26.0" react-native-web: "npm:~0.21.0" + react-native-webgpu: "npm:0.5.15" react-native-worklets: "npm:0.12.1" + typegpu: "npm:^0.12.0" typescript: "npm:~5.9.2" + unplugin-typegpu: "npm:^0.12.0" languageName: unknown linkType: soft @@ -20410,6 +20460,15 @@ __metadata: languageName: node linkType: hard +"tinyest-for-wgsl@npm:~0.4.1": + version: 0.4.1 + resolution: "tinyest-for-wgsl@npm:0.4.1" + dependencies: + tinyest: "npm:~0.3.3" + checksum: 10c0/66d08531d62baa0e85d205e37aceeebc3355f0b950bf12be2573524c3d9e5ef882ba59b5429c16f31db0ed3c30d9865355da593aff10a9e159e2e29a3a0004a3 + languageName: node + linkType: hard + "tinyest@npm:~0.3.1, tinyest@npm:~0.3.2": version: 0.3.2 resolution: "tinyest@npm:0.3.2" @@ -20417,6 +20476,13 @@ __metadata: languageName: node linkType: hard +"tinyest@npm:~0.3.3": + version: 0.3.3 + resolution: "tinyest@npm:0.3.3" + checksum: 10c0/4a80563d5fe4fdd7ce93dda1ae64f9f1f6822c3696e46f2abd972a610ffb1e0285ff39796778aaacc2bba148e4ec103df937fdf77f39fee3e9e006a9211c4eea + languageName: node + linkType: hard + "tinyexec@npm:^0.3.2": version: 0.3.2 resolution: "tinyexec@npm:0.3.2" @@ -20938,6 +21004,19 @@ __metadata: languageName: node linkType: hard +"typegpu@npm:^0.12.0": + version: 0.12.5 + resolution: "typegpu@npm:0.12.5" + dependencies: + tinyest: "npm:~0.3.3" + tsover-runtime: "npm:^0.0.7" + typed-binary: "npm:^4.3.3" + bin: + typegpu: ./bin.mjs + checksum: 10c0/15facb82e13aaa191b6667bf50754168cc1dcfacdd9d6302104c9d2e2f2cfc1b785589f79cfb8cac4866ce25dd09276bb9bea2a20f896e30916ca5d943950590 + languageName: node + linkType: hard + "typescript@npm:^5.8.3": version: 5.8.3 resolution: "typescript@npm:5.8.3" @@ -21134,6 +21213,27 @@ __metadata: languageName: node linkType: hard +"unplugin-typegpu@npm:^0.12.0": + version: 0.12.3 + resolution: "unplugin-typegpu@npm:0.12.3" + dependencies: + "@babel/parser": "npm:^7.29.0" + "@babel/traverse": "npm:^7.29.0" + "@babel/types": "npm:^7.29.0" + ast-kit: "npm:^2.2.0" + defu: "npm:^6.1.4" + magic-string: "npm:^0.30.21" + pathe: "npm:^2.0.3" + picomatch: "npm:^4.0.4" + tinyest: "npm:~0.3.3" + tinyest-for-wgsl: "npm:~0.4.1" + unplugin: "npm:^3.0.0" + peerDependencies: + typegpu: ^0.12.4 + checksum: 10c0/5839d57e1caddacaee7cf86a787d7eaae2b5bf230ccfd4d645201e7523cb69bfd7ae89a7500c2a81419d87333d21b7c2b7d3891bebbad46798bf721d7073623d + languageName: node + linkType: hard + "unplugin@npm:^3.0.0": version: 3.3.0 resolution: "unplugin@npm:3.3.0" From 5294b320ada06b7ed1ad95a4076ce8b5f59a279d Mon Sep 17 00:00:00 2001 From: Milosz Filimowski Date: Thu, 10 Sep 2026 12:47:31 +0200 Subject: [PATCH 2/3] chore(minimal-react): give video-effects and the app one copy of react-client --- examples/react-client/minimal-react/vite.config.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/examples/react-client/minimal-react/vite.config.ts b/examples/react-client/minimal-react/vite.config.ts index c401fdcdc..776899645 100644 --- a/examples/react-client/minimal-react/vite.config.ts +++ b/examples/react-client/minimal-react/vite.config.ts @@ -5,6 +5,15 @@ import checker from "vite-plugin-checker"; // https://vitejs.dev/config/ export default defineConfig({ + resolve: { + // @fishjam-cloud/react-client is a workspace link and @fishjam-cloud/video-effects imports it as a + // peer; without dedupe Vite hands the two a different copy and the hooks lose the provider context. + dedupe: ["react", "react-dom", "@fishjam-cloud/react-client"], + }, + optimizeDeps: { + // Pre-bundling the linked package would split its main entry from `/debug`. + exclude: ["@fishjam-cloud/react-client"], + }, server: { // https://vitejs.dev/config/server-options.html#server-host // true - listen on all addresses, including LAN and public addresses From 9bf36bdfdeeb69fba32ef12d698f73d4df1989ab Mon Sep 17 00:00:00 2001 From: Milosz Filimowski Date: Thu, 10 Sep 2026 12:56:58 +0200 Subject: [PATCH 3/3] fix(minimal-react): pass the segmentation model URL explicitly Vite pre-bundles video-effects, so the package's default model path resolves relative to the bundled chunk and misses. --- examples/react-client/minimal-react/src/components/App.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/react-client/minimal-react/src/components/App.tsx b/examples/react-client/minimal-react/src/components/App.tsx index 43243155e..514849f47 100644 --- a/examples/react-client/minimal-react/src/components/App.tsx +++ b/examples/react-client/minimal-react/src/components/App.tsx @@ -7,6 +7,7 @@ import { Variant, } from "@fishjam-cloud/react-client"; import { useStatistics } from "@fishjam-cloud/react-client/debug"; +import segmentationModelUrl from "@fishjam-cloud/video-effects/assets/selfie_segmenter.ssgbin?url"; import { useBackgroundBlur } from "@fishjam-cloud/video-effects/background-blur"; import { useFishjamCameraEffect } from "@fishjam-cloud/video-effects/fishjam-react"; import { typeGpuPersonSegmentation } from "@fishjam-cloud/video-effects/segmentation/typegpu"; @@ -15,7 +16,9 @@ import { Fragment, useState } from "react"; import AudioPlayer from "./AudioPlayer"; import VideoPlayer from "./VideoPlayer"; -const personSegmentation = typeGpuPersonSegmentation(); +const personSegmentation = typeGpuPersonSegmentation({ + modelUrl: segmentationModelUrl, +}); const variantLabel = (variant: Variant | null | undefined): string => { switch (variant) {