feat(examples): background blur on the Fishjam camera track

examples/mobile-client/fishjam-chat/app/_layout.tsx

@@ -1,13 +1,22 @@
 import { FishjamProvider, Variant } from '@fishjam-cloud/react-native-client';
-import { Stack } from 'expo-router';
+import { Stack, useRouter } from 'expo-router';
 import { useEffect, useState } from 'react';
 
+import { BlurCameraProvider } from '../providers/BlurCameraProvider';
+
 import { setFishjamIdChangeCallback } from '../utils/fishjamIdStore';
 
 const DEFAULT_FISHJAM_ID = process.env.EXPO_PUBLIC_FISHJAM_ID ?? '';
 
 export default function RootLayout() {
   const [fishjamId, setFishjamId] = useState<string>(DEFAULT_FISHJAM_ID);
+  const router = useRouter();
+
+  useEffect(() => {
+    if (__DEV__) {
+      (globalThis as { __fjRouter?: typeof router }).__fjRouter = router;
+    }
+  }, [router]);
 
   useEffect(() => {
     setFishjamIdChangeCallback(setFishjamId);
@@ -31,6 +40,7 @@ export default function RootLayout() {
           Variant.VARIANT_HIGH,
         ],
       }}>
+      <BlurCameraProvider>
       <Stack screenOptions={{ headerShown: false }}>
         <Stack.Screen name="(tabs)" options={{ title: 'Home' }} />
         <Stack.Screen
@@ -74,6 +84,7 @@ export default function RootLayout() {
           }}
         />
       </Stack>
+      </BlurCameraProvider>
     </FishjamProvider>
   );
 }

examples/mobile-client/fishjam-chat/app/room/[roomName].tsx

@@ -13,6 +13,7 @@ import { Platform, StyleSheet, View } from 'react-native';
 import { SafeAreaView } from 'react-native-safe-area-context';
 
 import { InCallButton, VideosGrid } from '../../components';
+import { useBlurCamera } from '../../providers/BlurCameraProvider';
 
 export default function RoomScreen() {
   const { userName } = useLocalSearchParams<{
@@ -21,6 +22,7 @@ export default function RoomScreen() {
   }>();
 
   const { isCameraOn, toggleCamera, stopCamera } = useCamera();
+  const { isBlurEnabled, toggleBlur } = useBlurCamera();
   const { isMicrophoneOn, toggleMicrophone, stopMicrophone, startMicrophone } =
     useMicrophone();
   const { leaveRoom } = useConnection();
@@ -141,6 +143,11 @@ export default function RoomScreen() {
           onPress={toggleCamera}
           accessibilityLabel="Toggle Camera"
         />
+        <InCallButton
+          iconName={isBlurEnabled ? 'blur' : 'blur-off'}
+          onPress={toggleBlur}
+          accessibilityLabel="Toggle Background Blur"
+        />
         <InCallButton
           iconName={screenShareStream ? 'monitor-share' : 'monitor-off'}
           onPress={handleToggleScreenShare}

examples/mobile-client/fishjam-chat/app/room/preview.tsx

@@ -13,6 +13,7 @@ import { SafeAreaView } from 'react-native-safe-area-context';
 
 import { Button, InCallButton, NoCameraView } from '../../components';
 import { useMediaPermissions } from '../../hooks/useMediaPermissions';
+import { useBlurCamera } from '../../providers/BlurCameraProvider';
 import { BrandColors } from '../../utils/Colors';
 
 export default function PreviewScreen() {
@@ -33,6 +34,7 @@ export default function PreviewScreen() {
   const { joinRoom, leaveRoom } = useConnection();
 
   const { permissionsGranted, openSettings } = useMediaPermissions();
+  const { isBlurEnabled, toggleBlur } = useBlurCamera();
 
   const [isInitialized, setIsInitialized] = useState(false);
   const [isJoining, setIsJoining] = useState(false);
@@ -150,6 +152,11 @@ export default function PreviewScreen() {
           onPress={toggleCamera}
           accessibilityLabel="Toggle Camera"
         />
+        <InCallButton
+          iconName={isBlurEnabled ? 'blur' : 'blur-off'}
+          onPress={toggleBlur}
+          accessibilityLabel="Toggle Background Blur"
+        />
       </View>
 
       <View style={styles.joinButton}>

examples/mobile-client/fishjam-chat/babel.config.js

@@ -1,9 +1,11 @@
 module.exports = function (api) {
   api.cache(true);
   return {
-    presets: ['babel-preset-expo'],
+    presets: [['babel-preset-expo', { unstable_transformImportMeta: true }]],
     plugins: [
-      'react-native-reanimated/plugin',
+      // typegpu, reached through @fishjam-cloud/video-effects, ships static class blocks, which
+      // babel-preset-expo does not transform on its own.
+      '@babel/plugin-transform-class-static-block',
       [
         'module-resolver',
         {
@@ -12,6 +14,10 @@ module.exports = function (api) {
           },
         },
       ],
+      // Compiles TypeGPU's tgpu.fn shader bodies to WGSL at build time.
+      'unplugin-typegpu/babel',
+      // Must stay last: the worklets plugin has to see the final output of the others.
+      'react-native-reanimated/plugin',
     ],
   };
 };

examples/mobile-client/fishjam-chat/metro.config.js

@@ -0,0 +1,52 @@
+const path = require('path');
+
+const { getDefaultConfig } = require('expo/metro-config');
+
+const projectRoot = __dirname;
+const monorepoRoot = path.resolve(projectRoot, '../../..');
+
+const config = getDefaultConfig(projectRoot);
+
+config.watchFolders = [monorepoRoot];
+
+// The segmentation weights ship as a .ssgbin file loaded through expo-asset.
+config.resolver.assetExts = [...config.resolver.assetExts, 'ssgbin'];
+config.resolver.nodeModulesPaths = [
+  path.resolve(projectRoot, 'node_modules'),
+  path.resolve(monorepoRoot, 'node_modules'),
+];
+
+// The workspace source packages dev-pin their own copies of these, and Metro's default lookup
+// finds the nested copy first. Two instances of a native module is not a duplicated-code problem,
+// it is a broken app: react-native's nested 0.85.3 emits codegen this app's babel plugin cannot
+// parse, and a second react-native-webgpu re-registers its native view ("Tried to register two
+// views with the same name WebGPUView"). Pin these to the root copy and leave every other package
+// to resolve normally, so packages that rely on their own nested dependencies still can. Every
+// native module here must resolve to exactly one copy: the JS half talks to a single installed
+// pod, so a second copy from a package's nested node_modules reports a version mismatch and
+// refuses to run ("Worklets 0.10.2 vs 0.8.1").
+const SINGLETON_MODULES = [
+  '@fishjam-cloud/react-native-webrtc',
+  'react',
+  'react-native',
+  'react-native-nitro-modules',
+  'react-native-reanimated',
+  'react-native-webgpu',
+  'react-native-worklets',
+  // Not native, but it keeps module-level registries, so it must load exactly once.
+  'typegpu',
+];
+
+const defaultResolveRequest = config.resolver.resolveRequest;
+
+config.resolver.resolveRequest = (context, moduleName, platform) => {
+  const resolve = defaultResolveRequest ?? context.resolveRequest;
+  const singleton = SINGLETON_MODULES.find((name) => moduleName === name || moduleName.startsWith(`${name}/`));
+  if (singleton) {
+    const subPath = moduleName.slice(singleton.length);
+    return resolve(context, path.resolve(monorepoRoot, 'node_modules', singleton) + subPath, platform);
+  }
+  return resolve(context, moduleName, platform);
+};
+
+module.exports = config;

examples/mobile-client/fishjam-chat/package.json

@@ -14,6 +14,8 @@
   "dependencies": {
     "@expo/vector-icons": "^15.0.3",
     "@fishjam-cloud/react-native-client": "workspace:*",
+    "@fishjam-cloud/react-native-worklets": "0.12.1",
+    "@fishjam-cloud/video-effects": "0.1.1",
     "@react-navigation/bottom-tabs": "^7.4.0",
     "@react-navigation/elements": "^2.6.3",
     "@react-navigation/native": "^7.1.8",
@@ -38,16 +40,20 @@
     "react-native-safe-area-context": "~5.7.0",
     "react-native-screens": "~4.26.0",
     "react-native-web": "~0.21.0",
+    "react-native-webgpu": "0.5.15",
     "react-native-worklets": "0.12.1"
   },
   "devDependencies": {
     "@babel/plugin-transform-class-static-block": "^7.28.3",
     "@types/react": "~19.2.0",
+    "@webgpu/types": "0.1.65",
     "babel-plugin-module-resolver": "^5.0.2",
     "babel-preset-expo": "~57.0.10",
     "eslint-config-expo": "~57.0.2",
     "eslint-import-resolver-typescript": "^4.4.4",
-    "typescript": "~5.9.2"
+    "typegpu": "^0.12.0",
+    "typescript": "~5.9.2",
+    "unplugin-typegpu": "^0.12.0"
   },
   "private": true
 }

examples/mobile-client/fishjam-chat/providers/BlurCameraProvider.tsx

@@ -0,0 +1,54 @@
+import type { VideoEffectStatus } from '@fishjam-cloud/video-effects';
+import { useBackgroundBlur } from '@fishjam-cloud/video-effects/background-blur';
+import { useFishjamCameraEffect } from '@fishjam-cloud/video-effects/fishjam-react-native';
+import { typeGpuPersonSegmentation } from '@fishjam-cloud/video-effects/segmentation/typegpu';
+import { Asset } from 'expo-asset';
+import { createContext, type ReactNode, useCallback, useContext, useEffect, useMemo, useState } from 'react';
+
+const BLUR_RADIUS = 24;
+
+const segmentationModel = Asset.fromModule(
+  // eslint-disable-next-line @typescript-eslint/no-require-imports
+  require('@fishjam-cloud/video-effects/assets/selfie_segmenter.ssgbin'),
+);
+const personSegmentation = typeGpuPersonSegmentation({ modelUrl: segmentationModel.uri });
+
+interface BlurCameraValue {
+  isBlurEnabled: boolean;
+  toggleBlur: () => void;
+  status: VideoEffectStatus;
+  error: Error | null;
+}
+
+const BlurCameraContext = createContext<BlurCameraValue | null>(null);
+
+/**
+ * Owns background blur for the whole app. Blur runs on Fishjam's own camera track through the
+ * camera-track middleware, so the preview screen and the call screen share one camera.
+ */
+export function BlurCameraProvider({ children }: { children: ReactNode }) {
+  const [isBlurEnabled, setIsBlurEnabled] = useState(false);
+  const backgroundBlur = useBackgroundBlur({ segmentation: personSegmentation, radius: BLUR_RADIUS });
+  const { status, error } = useFishjamCameraEffect(isBlurEnabled ? backgroundBlur : null);
+  const toggleBlur = useCallback(() => setIsBlurEnabled((enabled) => !enabled), []);
+
+  useEffect(() => {
+    if (!__DEV__) return;
+    // Lets a debugger flip blur without touching the screen (see the `__fjRouter` hook in _layout).
+    (globalThis as { __fjToggleBlur?: () => void }).__fjToggleBlur = toggleBlur;
+  }, [toggleBlur]);
+
+  const value = useMemo(
+    () => ({ isBlurEnabled, toggleBlur, status, error }),
+    [isBlurEnabled, toggleBlur, status, error],
+  );
+  return <BlurCameraContext.Provider value={value}>{children}</BlurCameraContext.Provider>;
+}
+
+export function useBlurCamera(): BlurCameraValue {
+  const value = useContext(BlurCameraContext);
+  if (value == null) {
+    throw new Error('useBlurCamera must be used inside BlurCameraProvider');
+  }
+  return value;
+}

examples/react-client/minimal-react/package.json

@@ -16,6 +16,7 @@
   },
   "dependencies": {
     "@fishjam-cloud/react-client": "workspace:*",
+    "@fishjam-cloud/video-effects": "0.1.1",
     "react": "19.1.0",
     "react-dom": "19.1.0"
   },

examples/react-client/minimal-react/src/components/App.tsx

@@ -7,11 +7,19 @@ import {
   Variant,
 } from "@fishjam-cloud/react-client";
 import { useStatistics } from "@fishjam-cloud/react-client/debug";
+import segmentationModelUrl from "@fishjam-cloud/video-effects/assets/selfie_segmenter.ssgbin?url";
+import { useBackgroundBlur } from "@fishjam-cloud/video-effects/background-blur";
+import { useFishjamCameraEffect } from "@fishjam-cloud/video-effects/fishjam-react";
+import { typeGpuPersonSegmentation } from "@fishjam-cloud/video-effects/segmentation/typegpu";
 import { Fragment, useState } from "react";
 
 import AudioPlayer from "./AudioPlayer";
 import VideoPlayer from "./VideoPlayer";
 
+const personSegmentation = typeGpuPersonSegmentation({
+  modelUrl: segmentationModelUrl,
+});
+
 const variantLabel = (variant: Variant | null | undefined): string => {
   switch (variant) {
     case Variant.VARIANT_LOW:
@@ -27,14 +35,24 @@ const variantLabel = (variant: Variant | null | undefined): string => {
 
 export const App = () => {
   const [token, setToken] = useState("");
+  const [blurEnabled, setBlurEnabled] = useState(true);
 
   const { joinRoom, leaveRoom, peerStatus } = useConnection();
 
   const { remotePeers } = usePeers();
   const screenShare = useScreenShare();
-  const { isCameraOn, toggleCamera } = useCamera();
+  const { cameraStream, isCameraOn, toggleCamera } = useCamera();
   const { isMicrophoneOn, toggleMicrophone } = useMicrophone();
   const { getStatistics } = useStatistics();
+  const backgroundBlur = useBackgroundBlur({
+    segmentation: personSegmentation,
+    radius: 24,
+  });
+  const {
+    status: effectStatus,
+    error: effectError,
+    retry: retryEffect,
+  } = useFishjamCameraEffect(blurEnabled ? backgroundBlur : null);
 
   {
     // for e2e test
@@ -89,6 +107,14 @@ export const App = () => {
           Start camera
         </button>
 
+        <button onClick={() => setBlurEnabled((enabled) => !enabled)}>
+          {blurEnabled ? "Disable blur" : "Enable blur"}
+        </button>
+
+        {effectStatus === "error" && (
+          <button onClick={retryEffect}>Retry effect</button>
+        )}
+
         <button
           disabled={isMicrophoneOn || peerStatus !== "connected"}
           onClick={toggleMicrophone}
@@ -97,20 +123,28 @@ export const App = () => {
         </button>
 
         <span>Status: {peerStatus}</span>
+        <span>Effect: {effectStatus}</span>
       </div>
 
-      {/* Render the video remote tracks from other peers*/}
+      {effectError && <pre>{effectError.message}</pre>}
+      {cameraStream && (
+        <div>
+          <strong>Local camera</strong>
+          <VideoPlayer stream={cameraStream} peerId="local" />
+        </div>
+      )}
+
       {remotePeers.map(
         ({ id, cameraTrack, microphoneTrack, screenShareVideoTrack }) => {
-          const cameraStream = cameraTrack?.stream;
+          const remoteCameraStream = cameraTrack?.stream;
           const microphoneStream = microphoneTrack?.stream;
           const screenShareStream = screenShareVideoTrack?.stream;
 
           return (
             <Fragment key={id}>
-              {cameraStream && (
+              {remoteCameraStream && (
                 <div>
-                  <VideoPlayer stream={cameraStream} peerId={id} />
+                  <VideoPlayer stream={remoteCameraStream} peerId={id} />
                   <div
                     style={{
                       display: "flex",

examples/react-client/minimal-react/vite.config.ts

@@ -5,6 +5,15 @@ import checker from "vite-plugin-checker";
 
 // https://vitejs.dev/config/
 export default defineConfig({
+  resolve: {
+    // @fishjam-cloud/react-client is a workspace link and @fishjam-cloud/video-effects imports it as a
+    // peer; without dedupe Vite hands the two a different copy and the hooks lose the provider context.
+    dedupe: ["react", "react-dom", "@fishjam-cloud/react-client"],
+  },
+  optimizeDeps: {
+    // Pre-bundling the linked package would split its main entry from `/debug`.
+    exclude: ["@fishjam-cloud/react-client"],
+  },
   server: {
     // https://vitejs.dev/config/server-options.html#server-host
     // true - listen on all addresses, including LAN and public addresses