diff --git a/AGENTS.md b/AGENTS.md index 807c4734b..df44d1b20 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,3 +8,7 @@ Non-obvious setup caveat: - `packages/protobufs/protos` and `packages/react-native-webrtc` are git submodules. They must be initialized before `yarn install`, otherwise install fails during resolution with `Workspace not found (@fishjam-cloud/react-native-webrtc@workspace:*)`. Run `git submodule update --init --recursive` first (the startup script does this). - Validated commands: `yarn build`, `yarn test:unit`, and `yarn lint:check` all pass. `yarn test:e2e` needs Playwright browsers and a running backend. + +## Conventions + +- Log through the SDK logger (`getLogger` in `packages/webrtc-client/src/logger.ts`, threaded as `logger`), never `console.*` directly. It respects the `debug` flag and adds the `[FISHJAM]` prefix. diff --git a/examples/mobile-client/fishjam-chat/app/room/[roomName].tsx b/examples/mobile-client/fishjam-chat/app/room/[roomName].tsx index f81ad8a1f..10778e839 100644 --- a/examples/mobile-client/fishjam-chat/app/room/[roomName].tsx +++ b/examples/mobile-client/fishjam-chat/app/room/[roomName].tsx @@ -155,7 +155,9 @@ export default function RoomScreen() { - setCameraTrackMiddleware(isBlurOn ? null : backgroundBlur).catch(reportBackgroundBlurFailure) + setCameraTrackMiddleware(isBlurOn ? null : backgroundBlur).catch( + reportBackgroundBlurFailure, + ) } /> - setCameraTrackMiddleware(isBlurOn ? null : backgroundBlur).catch(reportBackgroundBlurFailure) + setCameraTrackMiddleware(isBlurOn ? null : backgroundBlur).catch( + reportBackgroundBlurFailure, + ) } /> diff --git a/packages/mobile-client/src/index.ts b/packages/mobile-client/src/index.ts index 88a1005b6..e04e06051 100644 --- a/packages/mobile-client/src/index.ts +++ b/packages/mobile-client/src/index.ts @@ -124,6 +124,7 @@ export type { RoomType, UseSandboxProps, BandwidthLimits, + BandwidthLimitsInput, Brand, DeviceError, DeviceItem, diff --git a/packages/react-client/src/FishjamProvider.tsx b/packages/react-client/src/FishjamProvider.tsx index f5559afe9..11d4466fb 100644 --- a/packages/react-client/src/FishjamProvider.tsx +++ b/packages/react-client/src/FishjamProvider.tsx @@ -17,8 +17,8 @@ import { useFishjamClientState } from "./hooks/internal/useFishjamClientState"; import { usePeerStatus } from "./hooks/internal/usePeerStatus"; import { useScreenShareManager } from "./hooks/internal/useScreenshareManager"; import { useTrackManager } from "./hooks/internal/useTrackManager"; -import type { BandwidthLimits, PersistLastDeviceHandlers, StreamConfig } from "./types/public"; -import { mergeWithDefaultBandwitdthLimits } from "./utils/bandwidth"; +import type { BandwidthLimitsInput, PersistLastDeviceHandlers, StreamConfig } from "./types/public"; +import { mergeWithDefaultBandwidthLimits } from "./utils/bandwidth"; import { getLastDevice, saveLastDevice } from "./utils/localStorage"; /** @@ -42,9 +42,10 @@ export interface FishjamProviderProps extends PropsWithChildren { */ persistLastDevice?: boolean | PersistLastDeviceHandlers; /** - * Adjust max bandwidth limit for a single stream and simulcast. + * Adjust max bandwidth limit (in kbps) for a single stream and each simulcast layer. + * Missing values default to `MAX_BANDWIDTH_LIMITS`; values above the cap are clamped to it. */ - bandwidthLimits?: Partial; + bandwidthLimits?: BandwidthLimitsInput; /** * Configure whether to use video simulcast and which quality layers to send if so. */ @@ -96,7 +97,7 @@ export function FishjamProvider(props: FishjamProviderProps) { const peerStatus = usePeerStatus(fishjamClientRef.current); const mergedBandwidthLimits = useMemo( - () => mergeWithDefaultBandwitdthLimits(props.bandwidthLimits), + () => mergeWithDefaultBandwidthLimits(props.bandwidthLimits), [props.bandwidthLimits], ); diff --git a/packages/react-client/src/index.ts b/packages/react-client/src/index.ts index 5f404ec44..5b3845891 100644 --- a/packages/react-client/src/index.ts +++ b/packages/react-client/src/index.ts @@ -29,6 +29,7 @@ export { useUpdatePeerMetadata } from "./hooks/useUpdatePeerMetadata"; export { useVAD } from "./hooks/useVAD"; export type { BandwidthLimits, + BandwidthLimitsInput, Brand, CustomSource, DeviceError, diff --git a/packages/react-client/src/types/public.ts b/packages/react-client/src/types/public.ts index 576558475..97fc29098 100644 --- a/packages/react-client/src/types/public.ts +++ b/packages/react-client/src/types/public.ts @@ -72,6 +72,11 @@ export type StreamConfig = { sentQualities?: Variant[] | false }; export type BandwidthLimits = { singleStream: number; simulcast: SimulcastBandwidthLimits }; +/** + * Bandwidth limits accepted by `FishjamProvider`. Every value is optional; missing ones fall back to `MAX_BANDWIDTH_LIMITS`. + */ +export type BandwidthLimitsInput = { singleStream?: number; simulcast?: Partial }; + export type DeviceType = "audio" | "video"; export type DeviceError = diff --git a/packages/react-client/src/utils/bandwidth.test.ts b/packages/react-client/src/utils/bandwidth.test.ts new file mode 100644 index 000000000..1058f98d4 --- /dev/null +++ b/packages/react-client/src/utils/bandwidth.test.ts @@ -0,0 +1,20 @@ +import { MAX_BANDWIDTH_LIMITS, Variant } from "@fishjam-cloud/ts-client"; +import { expect, it } from "vitest"; + +import { mergeWithDefaultBandwidthLimits } from "./bandwidth"; + +it("defaults every limit to the cap", () => { + expect(mergeWithDefaultBandwidthLimits()).toEqual({ + singleStream: MAX_BANDWIDTH_LIMITS.singleStream, + simulcast: MAX_BANDWIDTH_LIMITS.simulcast, + }); +}); + +it("keeps user values and fills the missing simulcast layers with the cap", () => { + const merged = mergeWithDefaultBandwidthLimits({ singleStream: 800, simulcast: { [Variant.VARIANT_HIGH]: 1000 } }); + + expect(merged.singleStream).toBe(800); + expect(merged.simulcast[Variant.VARIANT_HIGH]).toBe(1000); + expect(merged.simulcast[Variant.VARIANT_MEDIUM]).toBe(MAX_BANDWIDTH_LIMITS.simulcast[Variant.VARIANT_MEDIUM]); + expect(merged.simulcast[Variant.VARIANT_LOW]).toBe(MAX_BANDWIDTH_LIMITS.simulcast[Variant.VARIANT_LOW]); +}); diff --git a/packages/react-client/src/utils/bandwidth.ts b/packages/react-client/src/utils/bandwidth.ts index 6877946e0..6610d80ca 100644 --- a/packages/react-client/src/utils/bandwidth.ts +++ b/packages/react-client/src/utils/bandwidth.ts @@ -1,10 +1,8 @@ -import { Variant } from "@fishjam-cloud/ts-client"; +import { MAX_BANDWIDTH_LIMITS } from "@fishjam-cloud/ts-client"; -import type { BandwidthLimits } from "../types/public"; +import type { BandwidthLimits, BandwidthLimitsInput } from "../types/public"; -export const ALL_VARIANTS_SIMULCAST = [Variant.VARIANT_LOW, Variant.VARIANT_MEDIUM, Variant.VARIANT_HIGH] as const; - -export const mergeWithDefaultBandwitdthLimits = (limits?: Partial): BandwidthLimits => ({ - singleStream: limits?.singleStream ?? 0, - simulcast: limits?.simulcast ?? { [Variant.VARIANT_LOW]: 0, [Variant.VARIANT_MEDIUM]: 0, [Variant.VARIANT_HIGH]: 0 }, +export const mergeWithDefaultBandwidthLimits = (limits?: BandwidthLimitsInput): BandwidthLimits => ({ + singleStream: limits?.singleStream ?? MAX_BANDWIDTH_LIMITS.singleStream, + simulcast: { ...MAX_BANDWIDTH_LIMITS.simulcast, ...limits?.simulcast }, }); diff --git a/packages/react-client/src/utils/track.ts b/packages/react-client/src/utils/track.ts index aaac0485a..1cf76cfe4 100644 --- a/packages/react-client/src/utils/track.ts +++ b/packages/react-client/src/utils/track.ts @@ -1,5 +1,5 @@ -import type { FishjamClient, SimulcastConfig, TrackContext, TrackMetadata } from "@fishjam-cloud/ts-client"; -import { Variant } from "@fishjam-cloud/ts-client"; +import type { FishjamClient, SimulcastConfig, TrackContext, TrackMetadata, Variant } from "@fishjam-cloud/ts-client"; +import { SIMULCAST_VARIANTS } from "@fishjam-cloud/ts-client"; import type { BandwidthLimits, Track, TrackId } from "../types/public"; @@ -53,8 +53,7 @@ export function setupOnEndedCallback( } const getDisabledEncodings = (activeEncodings: Variant[] = []) => { - const allEncodings: Variant[] = [Variant.VARIANT_LOW, Variant.VARIANT_MEDIUM, Variant.VARIANT_HIGH]; - return allEncodings.filter((encoding) => !activeEncodings.includes(encoding)); + return SIMULCAST_VARIANTS.filter((encoding) => !activeEncodings.includes(encoding)); }; export const getConfigAndBandwidthFromProps = ( diff --git a/packages/ts-client/src/FishjamClient.ts b/packages/ts-client/src/FishjamClient.ts index 9674d14ab..6c328deaa 100644 --- a/packages/ts-client/src/FishjamClient.ts +++ b/packages/ts-client/src/FishjamClient.ts @@ -588,7 +588,7 @@ export class FishjamClient { if (!this.webrtc) throw this.handleWebRTCNotInitialized(); if (this.isAudioOnlyConnection && track.kind !== 'audio') throw new TrackTypeError(); @@ -663,14 +663,16 @@ export class FishjamClient} Success */ - public async setTrackBandwidth(trackId: string, bandwidth: BandwidthLimit): Promise { + public async setTrackBandwidth(trackId: string, bandwidth: TrackBandwidthLimit): Promise { if (!this.webrtc) throw this.handleWebRTCNotInitialized(); await this.webrtc.setTrackBandwidth(trackId, bandwidth); return true; @@ -681,7 +683,8 @@ export class FishjamClient { expect(joined).toHaveBeenCalledOnce(); }); + it("forwards per-variant simulcast limits to setTrackBandwidth", async () => { + const setTrackBandwidth = vi.spyOn(TsClient.prototype, "setTrackBandwidth").mockResolvedValue(true); + vi.spyOn(TsClient.prototype, "connect").mockResolvedValue(); + const client = new FishjamClient(); + await client.connect(connectConfig); + const limits = new Map([[Variant.VARIANT_HIGH, 900]]); + + await expect(client.setTrackBandwidth("track-id", limits)).resolves.toBe(true); + + expect(setTrackBandwidth).toHaveBeenCalledWith("track-id", limits); + }); + it("does not take ownership of tracks passed through the signalling seam", async () => { const publication = deferred(); vi.spyOn(TsClient.prototype, "addTrack").mockReturnValue(publication.promise); diff --git a/packages/tsunami/src/FishjamClient.ts b/packages/tsunami/src/FishjamClient.ts index 7729c2e0a..5a7ce3187 100644 --- a/packages/tsunami/src/FishjamClient.ts +++ b/packages/tsunami/src/FishjamClient.ts @@ -119,7 +119,7 @@ export class FishjamClient this.getTsClient().replaceTrack(trackId, newTrack)); } - public async setTrackBandwidth(trackId: string, bandwidth: BandwidthLimit): Promise { + public async setTrackBandwidth(trackId: string, bandwidth: TrackBandwidthLimit): Promise { return this.resources.run(() => this.getTsClient().setTrackBandwidth(trackId, bandwidth)); } diff --git a/packages/webrtc-client/src/bandwidth.ts b/packages/webrtc-client/src/bandwidth.ts new file mode 100644 index 000000000..ed861f068 --- /dev/null +++ b/packages/webrtc-client/src/bandwidth.ts @@ -0,0 +1,181 @@ +import type { MediaEvent_VariantBitrate } from '@fishjam-cloud/protobufs/peer'; +import { Variant } from '@fishjam-cloud/protobufs/shared'; + +import { getTrackKind, type TrackContextImpl } from './internal'; +import type { BandwidthLimit, Logger, SimulcastBandwidthLimit, TrackBandwidthLimit } from './types'; + +export type Bitrate = number; + +// Throughout the SDK "kbps" means 1024 bps, so that the values match the ones reported by the browser. +export const kbpsToBps = (kbps: BandwidthLimit): Bitrate => kbps * 1024; +export const bpsToKbps = (bps: Bitrate): BandwidthLimit => Math.round(bps / 1024); + +/** + * Hard caps (in kbps) for the bitrate of an outgoing video track. + * A limit passed to the SDK is never allowed to exceed these values, and an unset (0) limit resolves to them. + */ +export const MAX_BANDWIDTH_LIMITS = { + singleStream: 1500, + simulcast: { + [Variant.VARIANT_LOW]: 150, + [Variant.VARIANT_MEDIUM]: 500, + [Variant.VARIANT_HIGH]: 1500, + }, +} as const satisfies { singleStream: BandwidthLimit; simulcast: Record }; + +/** The variants that have a simulcast layer, ordered from the lowest to the highest resolution. */ +export const SIMULCAST_VARIANTS = [Variant.VARIANT_LOW, Variant.VARIANT_MEDIUM, Variant.VARIANT_HIGH] as const; + +export type SimulcastVariant = (typeof SIMULCAST_VARIANTS)[number]; + +const isSimulcastVariant = (variant: Variant): variant is SimulcastVariant => + (SIMULCAST_VARIANTS as readonly Variant[]).includes(variant); + +const resolveAgainstCap = ( + limit: BandwidthLimit | undefined, + cap: BandwidthLimit, + label: string, + logger: Logger, +): BandwidthLimit => { + if (!limit || limit <= 0) return cap; + if (limit > cap) { + logger.warn(`Desired ${label} bandwidth of ${limit} kbps exceeds the cap of ${cap} kbps`); + return cap; + } + return limit; +}; + +/** + * Resolves a single-stream video limit against {@link MAX_BANDWIDTH_LIMITS.singleStream}. + * 0 or a negative value means "use the cap"; higher values are clamped to it. + */ +export const resolveSingleStreamLimit = (limit: BandwidthLimit, logger: Logger): BandwidthLimit => + resolveAgainstCap(limit, MAX_BANDWIDTH_LIMITS.singleStream, 'single stream', logger); + +/** + * Resolves per-variant simulcast limits against {@link MAX_BANDWIDTH_LIMITS.simulcast}. + * Every simulcast variant is present in the result: a missing, 0 or negative entry resolves to that variant's cap, + * and higher values are clamped to it. + */ +export const resolveSimulcastLimits = (limits: SimulcastBandwidthLimit, logger: Logger): SimulcastBandwidthLimit => { + const resolved: SimulcastBandwidthLimit = new Map(); + for (const variant of SIMULCAST_VARIANTS) { + const cap = MAX_BANDWIDTH_LIMITS.simulcast[variant]; + resolved.set(variant, resolveAgainstCap(limits.get(variant), cap, Variant[variant], logger)); + } + return resolved; +}; + +/** + * Resolves the limit of one simulcast layer against its cap. Throws for a variant that has no layer. + */ +export const resolveVariantBandwidthLimit = ( + variant: Variant, + limit: BandwidthLimit, + logger: Logger, +): BandwidthLimit => { + if (!isSimulcastVariant(variant)) throw new Error(`${Variant[variant]} is not a simulcast variant`); + + return resolveAgainstCap(limit, MAX_BANDWIDTH_LIMITS.simulcast[variant], Variant[variant], logger); +}; + +// The suggested bitrate values are based on our internal tests. +export const defaultBitrates = { + audio: 50_000 as Bitrate, + video: kbpsToBps(MAX_BANDWIDTH_LIMITS.singleStream), +}; + +export const SIMULCAST_LAYER_SCALE: Record = { + [Variant.VARIANT_LOW]: 4, + [Variant.VARIANT_MEDIUM]: 2, + [Variant.VARIANT_HIGH]: 1, +}; + +export const splitBandwidth = ( + rtcRtpEncodingParameters: RTCRtpEncodingParameters[], + maxBandwidth: number, + logger: Logger, +): RTCRtpEncodingParameters[] => { + const bandwidth = kbpsToBps(maxBandwidth); + + if (bandwidth === 0) { + return rtcRtpEncodingParameters.map((encoding) => ({ + ...encoding, + maxBitrate: undefined, + })); + } + + if (rtcRtpEncodingParameters.length === 0) { + // This most likely is a race condition. Log an error and prevent catastrophic failure + logger.error("Attempted to limit bandwidth of the track that doesn't have any encodings"); + return rtcRtpEncodingParameters.map((encoding) => ({ ...encoding })); + } + if (!rtcRtpEncodingParameters[0]) throw new Error('RTCRtpEncodingParameters is in invalid state'); + + // We are solving the following equation: + // x + (k0/k1)^2 * x + (k0/k2)^2 * x + ... + (k0/kn)^2 * x = bandwidth + // where x is the bitrate for the first encoding, kn are scaleResolutionDownBy factors + // square is dictated by the fact that k0/kn is a scale factor, but we are interested in the total number of pixels in the image + const firstScaleDownBy = rtcRtpEncodingParameters[0].scaleResolutionDownBy || 1; + const bitrate_parts = rtcRtpEncodingParameters.reduce( + (acc, value) => acc + (firstScaleDownBy / (value.scaleResolutionDownBy || 1)) ** 2, + 0, + ); + const x = bandwidth / bitrate_parts; + + return rtcRtpEncodingParameters.map((encoding) => ({ + ...encoding, + maxBitrate: x * (firstScaleDownBy / (encoding.scaleResolutionDownBy || 1)) ** 2, + })); +}; + +/** Splits a total budget (in kbps) across the simulcast layers proportionally to their pixel count. */ +export const splitSimulcastBudget = (budget: number, logger: Logger): SimulcastBandwidthLimit => { + const layers = SIMULCAST_VARIANTS.map((variant) => ({ scaleResolutionDownBy: SIMULCAST_LAYER_SCALE[variant] })); + const split = splitBandwidth(layers, budget, logger); + + return new Map( + SIMULCAST_VARIANTS.map((variant, index) => { + const maxBitrate = split[index]?.maxBitrate; + return [variant, maxBitrate ? bpsToKbps(maxBitrate) : 0]; + }), + ); +}; + +/** + * Resolves the limit requested for a video track against {@link MAX_BANDWIDTH_LIMITS}. + * A number is a budget for the whole track, a Map holds per-layer limits; 0 means "use the cap(s)". + * - single stream: the number is clamped to the single-stream cap; a Map is rejected + * - simulcast: a number is split across the layers, then every layer is clamped to its own cap + */ +export const resolveTrackBandwidthLimit = ( + requested: TrackBandwidthLimit, + simulcast: boolean, + logger: Logger, +): TrackBandwidthLimit => { + if (!simulcast) { + if (typeof requested !== 'number') throw new Error('A non-simulcast track expects a single bandwidth limit'); + return resolveSingleStreamLimit(requested, logger); + } + + if (typeof requested !== 'number') return resolveSimulcastLimits(requested, logger); + + const layers = requested > 0 ? splitSimulcastBudget(requested, logger) : new Map(); + return resolveSimulcastLimits(layers, logger); +}; + +/** + * Builds the per-variant bitrates (in bps) reported to the server for a local track, + * derived from the limits the encoder was configured with. + */ +export const getVariantBitrates = (trackContext: TrackContextImpl): MediaEvent_VariantBitrate[] => { + const { maxBandwidth } = trackContext; + + if (typeof maxBandwidth === 'number') { + // 0 means the limit was never set, so report the default for the track kind. + const fallback = getTrackKind(trackContext) === 'audio' ? defaultBitrates.audio : defaultBitrates.video; + return [{ variant: Variant.VARIANT_UNSPECIFIED, bitrate: kbpsToBps(maxBandwidth) || fallback }]; + } + + return [...maxBandwidth.entries()].map(([variant, limit]) => ({ variant, bitrate: kbpsToBps(limit) })); +}; diff --git a/packages/webrtc-client/src/bitrate.ts b/packages/webrtc-client/src/bitrate.ts deleted file mode 100644 index 732b031a8..000000000 --- a/packages/webrtc-client/src/bitrate.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { Variant } from '@fishjam-cloud/protobufs/shared'; - -import { type BandwidthLimit } from './types'; - -export type Bitrate = number; -export type Bitrates = Record | Bitrate; - -// The suggested bitrate values are based on our internal tests. -export const defaultBitrates = { - audio: 50_000 as Bitrate, - video: 2_500_000 as Bitrate, -} as const; - -export const UNLIMITED_BANDWIDTH: Bitrate = 0 as Bitrate; - -// The suggested bitrate values are based on our internal tests. -export const defaultSimulcastBitrates: { - [key in Variant]: BandwidthLimit; -} = { - [Variant.VARIANT_HIGH]: 2_500_000, - [Variant.VARIANT_MEDIUM]: 500_000, - [Variant.VARIANT_LOW]: 150_000, - [Variant.VARIANT_UNSPECIFIED]: 0, - [Variant.UNRECOGNIZED]: 0, -}; diff --git a/packages/webrtc-client/src/index.ts b/packages/webrtc-client/src/index.ts index 5cb9016ff..77e8706aa 100644 --- a/packages/webrtc-client/src/index.ts +++ b/packages/webrtc-client/src/index.ts @@ -1,3 +1,5 @@ +export type { SimulcastVariant } from './bandwidth'; +export { MAX_BANDWIDTH_LIMITS, resolveTrackBandwidthLimit, SIMULCAST_VARIANTS } from './bandwidth'; export { getLogger } from './logger'; export type { MediaEvent, SerializedMediaEvent } from './mediaEvent'; export type { diff --git a/packages/webrtc-client/src/internal.ts b/packages/webrtc-client/src/internal.ts index 2a6de7dab..0f51318c4 100644 --- a/packages/webrtc-client/src/internal.ts +++ b/packages/webrtc-client/src/internal.ts @@ -61,3 +61,6 @@ export class TrackContextImpl export type EndpointWithTrackContext = Omit & { tracks: Map; }; + +export const getTrackKind = (trackContext: TrackContextImpl): TrackKind | null => + (trackContext.track?.kind as TrackKind | undefined) ?? trackContext.trackKind; diff --git a/packages/webrtc-client/src/tracks/Local.ts b/packages/webrtc-client/src/tracks/Local.ts index 83a4b488c..279f64ef3 100644 --- a/packages/webrtc-client/src/tracks/Local.ts +++ b/packages/webrtc-client/src/tracks/Local.ts @@ -1,4 +1,3 @@ -import type { MediaEvent_VariantBitrate } from '@fishjam-cloud/protobufs/peer'; import { MediaEvent as PeerMediaEvent, MediaEvent_DisableTrackVariant, @@ -10,14 +9,16 @@ import { MediaEvent_UpdateTrackMetadata, } from '@fishjam-cloud/protobufs/peer'; import { type MediaEvent_Track_SimulcastConfig } from '@fishjam-cloud/protobufs/server'; -import { Variant } from '@fishjam-cloud/protobufs/shared'; +import type { Variant } from '@fishjam-cloud/protobufs/shared'; +import { getVariantBitrates, resolveTrackBandwidthLimit, resolveVariantBandwidthLimit } from '../bandwidth'; import type { ConnectionManager } from '../ConnectionManager'; import type { EndpointWithTrackContext } from '../internal'; -import { isTrackKind, TrackContextImpl } from '../internal'; +import { getTrackKind, isTrackKind, TrackContextImpl } from '../internal'; import type { BandwidthLimit, LocalTrackId, + Logger, MetadataJson, MLineId, RemoteTrackId, @@ -28,6 +29,8 @@ import type { WebRTCEndpoint } from '../webRTCEndpoint'; import { LocalTrack } from './LocalTrack'; import type { EndpointId, TrackId } from './TrackCommon'; +const isVideoTrack = (trackManager: LocalTrack) => getTrackKind(trackManager.trackContext) === 'video'; + /** * This class encapsulates methods related to handling the list of local tracks and local endpoint. * It stores and mutates part of the client state for this local peer. @@ -51,6 +54,7 @@ export class Local { private readonly sendMediaEvent: (mediaEvent: PeerMediaEvent) => void; private connection: ConnectionManager | null = null; + private readonly logger: Logger; constructor( emit: >( @@ -58,9 +62,11 @@ export class Local { ...args: Parameters[E]> ) => void, sendMediaEvent: (mediaEvent: PeerMediaEvent) => void, + logger: Logger, ) { this.emit = emit; this.sendMediaEvent = sendMediaEvent; + this.logger = logger; } public updateSenders = () => { @@ -116,7 +122,7 @@ export class Local { this.localEndpoint.tracks.set(trackId, trackContext); - const trackManager = new LocalTrack(connection, trackId, trackContext); + const trackManager = new LocalTrack(connection, trackId, trackContext, this.logger); this.localTracks[trackId] = trackManager; return trackManager; }; @@ -155,23 +161,21 @@ export class Local { return this.localEndpoint; }; - public setTrackBandwidth = async (trackId: string, bandwidth: BandwidthLimit): Promise => { - // FIXME: maxBandwidth in TrackContext is not updated - + public setTrackBandwidth = async (trackId: string, bandwidth: TrackBandwidthLimit): Promise => { const trackManager = this.localTracks[trackId]; if (!trackManager) throw new Error(`Cannot find ${trackId}`); - await trackManager.setTrackBandwidth(bandwidth); + const resolvedBandwidth = this.resolveTrackBandwidth(trackManager, bandwidth); + + await trackManager.setTrackBandwidth(resolvedBandwidth); + trackManager.trackContext.maxBandwidth = resolvedBandwidth; const trackBitrates = MediaEvent_TrackBitrates.create({ trackId, - variantBitrates: [{ variant: Variant.VARIANT_UNSPECIFIED, bitrate: bandwidth }], + variantBitrates: getVariantBitrates(trackManager.trackContext), }); this.sendMediaEvent(PeerMediaEvent.create({ trackBitrates })); - this.emit('localTrackBandwidthSet', { - trackId, - bandwidth, - }); + this.emit('localTrackBandwidthSet', { trackId, bandwidth: resolvedBandwidth }); }; public getTrackIdToTrack = (): Map => { @@ -185,35 +189,43 @@ export class Local { this.localEndpoint.id = endpointId; }; + /** Audio limits are applied as given; video limits go through {@link resolveTrackBandwidthLimit}. */ + private resolveTrackBandwidth = (trackManager: LocalTrack, requested: TrackBandwidthLimit): TrackBandwidthLimit => { + if (!isVideoTrack(trackManager)) { + if (typeof requested !== 'number') throw new Error('An audio track expects a single bandwidth limit'); + return requested; + } + + return resolveTrackBandwidthLimit( + requested, + trackManager.trackContext.simulcastConfig?.enabled ?? false, + this.logger, + ); + }; + public setEncodingBandwidth = async (trackId: TrackId, rid: Variant, bandwidth: BandwidthLimit): Promise => { const trackManager = this.localTracks[trackId]; if (!trackManager) throw new Error(`Cannot find ${trackId}`); - await trackManager.setEncodingBandwidth(rid, bandwidth); + const resolvedBandwidth = isVideoTrack(trackManager) + ? resolveVariantBandwidthLimit(rid, bandwidth, this.logger) + : bandwidth; - const bitrates = trackManager.getTrackBitrates(); + await trackManager.setEncodingBandwidth(rid, resolvedBandwidth); - let variantBitrates: MediaEvent_VariantBitrate[] = []; - - if (typeof bitrates === 'number') { - variantBitrates = [{ variant: Variant.VARIANT_UNSPECIFIED, bitrate: bitrates }]; - } else if (bitrates) { - variantBitrates = Object.entries(bitrates).map(([variant, bitrate]) => ({ - variant: Number(variant) as Variant, - bitrate, - })); - } + const { maxBandwidth } = trackManager.trackContext; + if (maxBandwidth instanceof Map) maxBandwidth.set(rid, resolvedBandwidth); const trackBitrates = MediaEvent_TrackBitrates.create({ trackId, - variantBitrates, + variantBitrates: getVariantBitrates(trackManager.trackContext), }); this.sendMediaEvent(PeerMediaEvent.create({ trackBitrates })); this.emit('localTrackEncodingBandwidthSet', { trackId, rid, - bandwidth, + bandwidth: resolvedBandwidth, }); }; @@ -298,9 +310,14 @@ export class Local { {}, ); - // TODO add bitrates private getTrackIdToTrackBitrates = (): Record => - Object.values(this.localTracks).reduce((acc, { id }) => ({ ...acc, [id]: { bitrate: 1_500_000 } }), {}); + Object.values(this.localTracks).reduce( + (acc, { id, trackContext }) => ({ + ...acc, + [id]: MediaEvent_TrackBitrates.create({ trackId: id, variantBitrates: getVariantBitrates(trackContext) }), + }), + {}, + ); private getMidToTrackId = (): Record => { if (!this.connection) return {}; diff --git a/packages/webrtc-client/src/tracks/LocalTrack.ts b/packages/webrtc-client/src/tracks/LocalTrack.ts index 0e918bf2e..16548c175 100644 --- a/packages/webrtc-client/src/tracks/LocalTrack.ts +++ b/packages/webrtc-client/src/tracks/LocalTrack.ts @@ -1,16 +1,14 @@ import { Variant } from '@fishjam-cloud/protobufs/shared'; -import type { Bitrate, Bitrates } from '../bitrate'; -import { defaultBitrates, defaultSimulcastBitrates, UNLIMITED_BANDWIDTH } from '../bitrate'; +import { kbpsToBps } from '../bandwidth'; import type { ConnectionManager } from '../ConnectionManager'; import type { TrackContextImpl } from '../internal'; -import type { BandwidthLimit, LocalTrackId, MediaStreamTrackId, MLineId, TrackKind } from '../types'; -// import { generateCustomEvent } from '../mediaEvent'; +import type { BandwidthLimit, LocalTrackId, Logger, MediaStreamTrackId, MLineId, TrackBandwidthLimit } from '../types'; import type { WebRTCEndpoint } from '../webRTCEndpoint'; import { encodingToVariantMap, getEncodingParameters } from './encodings'; import { emitMutableEvents, getActionType } from './muteTrackUtils'; import type { TrackCommon, TrackEncodings, TrackId } from './TrackCommon'; -import { createTransceiverConfig } from './transceivers'; +import { calculateSimulcastEncodings, createTransceiverConfig } from './transceivers'; /** * This is a wrapper over `TrackContext` that adds additional properties such as: @@ -56,11 +54,18 @@ export class LocalTrack implements TrackCommon { }; public connection: ConnectionManager | undefined; - - constructor(connection: ConnectionManager | undefined, id: LocalTrackId, trackContext: TrackContextImpl) { + private readonly logger: Logger; + + constructor( + connection: ConnectionManager | undefined, + id: LocalTrackId, + trackContext: TrackContextImpl, + logger: Logger, + ) { this.connection = connection; this.id = id; this.trackContext = trackContext; + this.logger = logger; // todo maybe we could remove this object and use sender.getParameters().encodings.encodingParameter.active instead if (trackContext.track?.id) { @@ -99,7 +104,7 @@ export class LocalTrack implements TrackCommon { if (!this.connection) throw new Error(`There is no active RTCPeerConnection`); - const transceiverConfig = createTransceiverConfig(this.trackContext); + const transceiverConfig = createTransceiverConfig(this.trackContext, this.logger); this.updateEncodings(); @@ -161,12 +166,16 @@ export class LocalTrack implements TrackCommon { } }; - public setTrackBandwidth = (bandwidth: BandwidthLimit): Promise => { + /** Writes an already resolved limit to the sender: a number for a single stream, a Map for simulcast layers. */ + public setTrackBandwidth = (bandwidth: TrackBandwidthLimit): Promise => { if (!this.sender) throw new Error(`RTCRtpSender for track ${this.id} not found`); const parameters = this.sender.getParameters(); - parameters.encodings = getEncodingParameters(parameters, bandwidth); + parameters.encodings = + typeof bandwidth === 'number' + ? getEncodingParameters(parameters, bandwidth, this.logger) + : calculateSimulcastEncodings(parameters.encodings, bandwidth); return this.sender.setParameters(parameters); }; @@ -182,7 +191,7 @@ export class LocalTrack implements TrackCommon { } else if (bandwidth === 0) { delete encoding.maxBitrate; } else { - encoding.maxBitrate = bandwidth * 1024; + encoding.maxBitrate = kbpsToBps(bandwidth); } return this.sender.setParameters(parameters); @@ -210,44 +219,6 @@ export class LocalTrack implements TrackCommon { this.mLineId = mLineId; }; - private isNotSimulcastTrack = (encodings: RTCRtpEncodingParameters[]): boolean => - encodings.length === 1 && !encodings[0]!.rid; - - public getTrackBitrates = (): Bitrates | undefined => { - const trackContext = this.trackContext; - const kind = this.trackContext.track?.kind as TrackKind | undefined; - - if (!trackContext.track) { - if (!trackContext.trackKind) { - throw new Error('trackContext.trackKind is empty'); - } - - return defaultBitrates[trackContext.trackKind]; - } - - if (!this.sender) return undefined; - - const encodings = this.sender.getParameters().encodings; - - if (this.isNotSimulcastTrack(encodings)) { - return encodings[0]!.maxBitrate || (kind ? defaultBitrates[kind] : UNLIMITED_BANDWIDTH); - } else if (kind === 'audio') { - throw 'Audio track cannot have multiple encodings'; - } - - return encodings - .filter((encoding) => encoding.rid) - .reduce( - (acc, encoding) => { - const variant = encodingToVariantMap[encoding.rid!] ?? Variant.VARIANT_UNSPECIFIED; - - acc[variant] = encoding.maxBitrate || defaultSimulcastBitrates[variant]; - return acc; - }, - {} as Record, - ); - }; - public getAudioLevel = async (): Promise<{ level: number } | null> => { if (!this.sender) return null; @@ -261,15 +232,4 @@ export class LocalTrack implements TrackCommon { return null; } }; - - public createTrackVariantBitratesEvent = () => { - // TODO implement this when simulcast is supported - // return generateCustomEvent({ - // type: 'trackVariantBitrates', - // data: { - // trackId: this.id, - // variantBitrates: this.getTrackBitrates(), - // }, - // }); - }; } diff --git a/packages/webrtc-client/src/tracks/LocalTrackManager.ts b/packages/webrtc-client/src/tracks/LocalTrackManager.ts index 74e247380..1a2a47aff 100644 --- a/packages/webrtc-client/src/tracks/LocalTrackManager.ts +++ b/packages/webrtc-client/src/tracks/LocalTrackManager.ts @@ -48,19 +48,11 @@ export class LocalTrackManager { this.ongoingRenegotiation = false; }; - public parseAddTrack = ( - track: MediaStreamTrack, - simulcastConfig: SimulcastConfig, - maxBandwidth: TrackBandwidthLimit, - ) => { + public parseAddTrack = (track: MediaStreamTrack) => { if (this.getEndpointId() === '') { throw new Error('Cannot add tracks before being accepted by the server'); } - if (!simulcastConfig.enabled && !(typeof maxBandwidth === 'number')) { - throw new Error('Invalid type of `maxBandwidth` argument for a non-simulcast track, expected: number'); - } - if (this.connection?.isTrackInUse(track)) { throw new Error("This track was already added to peerConnection, it can't be added again!"); } diff --git a/packages/webrtc-client/src/tracks/Remote.ts b/packages/webrtc-client/src/tracks/Remote.ts index 6a67490ad..9b71bac27 100644 --- a/packages/webrtc-client/src/tracks/Remote.ts +++ b/packages/webrtc-client/src/tracks/Remote.ts @@ -8,6 +8,7 @@ import type { EndpointWithTrackContext } from '../internal'; import { TrackContextImpl } from '../internal'; import type { EncodingReason, + Logger, MetadataJson, RemoteTrackId, TrackContext, @@ -26,6 +27,7 @@ export class Remote { ...args: Parameters[E]> ) => void; private readonly sendMediaEvent: (mediaEvent: PeerMediaEvent) => void; + private readonly logger: Logger; constructor( emit: >( @@ -33,9 +35,11 @@ export class Remote { ...args: Parameters[E]> ) => void, sendMediaEvent: (mediaEvent: PeerMediaEvent) => void, + logger: Logger, ) { this.emit = emit; this.sendMediaEvent = sendMediaEvent; + this.logger = logger; } public getTrackByMid = (mid: string): RemoteTrack => { @@ -185,7 +189,7 @@ export class Remote { remoteTrack.trackContext.vadStatus = nextStatus; remoteTrack.trackContext.emit('voiceActivityChanged', remoteTrack.trackContext); } else { - console.warn('Received unknown vad status: ', vadStatus); + this.logger.warn('Received unknown vad status: ', vadStatus); } }; @@ -220,7 +224,7 @@ export class Remote { variant, }); } catch (e) { - console.warn(e); + this.logger.warn(e); } }; diff --git a/packages/webrtc-client/src/tracks/bandwidth.ts b/packages/webrtc-client/src/tracks/bandwidth.ts deleted file mode 100644 index fd476ba03..000000000 --- a/packages/webrtc-client/src/tracks/bandwidth.ts +++ /dev/null @@ -1,36 +0,0 @@ -export const splitBandwidth = ( - rtcRtpEncodingParameters: RTCRtpEncodingParameters[], - maxBandwidth: number, -): RTCRtpEncodingParameters[] => { - const bandwidth = maxBandwidth * 1024; - - if (bandwidth === 0) { - return rtcRtpEncodingParameters.map((encoding) => ({ - ...encoding, - maxBitrate: undefined, - })); - } - - if (rtcRtpEncodingParameters.length === 0) { - // This most likely is a race condition. Log an error and prevent catastrophic failure - console.error("Attempted to limit bandwidth of the track that doesn't have any encodings"); - return rtcRtpEncodingParameters.map((encoding) => ({ ...encoding })); - } - if (!rtcRtpEncodingParameters[0]) throw new Error('RTCRtpEncodingParameters is in invalid state'); - - // We are solving the following equation: - // x + (k0/k1)^2 * x + (k0/k2)^2 * x + ... + (k0/kn)^2 * x = bandwidth - // where x is the bitrate for the first encoding, kn are scaleResolutionDownBy factors - // square is dictated by the fact that k0/kn is a scale factor, but we are interested in the total number of pixels in the image - const firstScaleDownBy = rtcRtpEncodingParameters[0].scaleResolutionDownBy || 1; - const bitrate_parts = rtcRtpEncodingParameters.reduce( - (acc, value) => acc + (firstScaleDownBy / (value.scaleResolutionDownBy || 1)) ** 2, - 0, - ); - const x = bandwidth / bitrate_parts; - - return rtcRtpEncodingParameters.map((encoding) => ({ - ...encoding, - maxBitrate: x * (firstScaleDownBy / (encoding.scaleResolutionDownBy || 1)) ** 2, - })); -}; diff --git a/packages/webrtc-client/src/tracks/encodings.ts b/packages/webrtc-client/src/tracks/encodings.ts index d69f31582..2f4a4914d 100644 --- a/packages/webrtc-client/src/tracks/encodings.ts +++ b/packages/webrtc-client/src/tracks/encodings.ts @@ -1,14 +1,18 @@ import { Variant } from '@fishjam-cloud/protobufs/shared'; -import { splitBandwidth } from './bandwidth'; +import { splitBandwidth } from '../bandwidth'; +import type { Logger } from '../types'; export const getEncodingParameters = ( parameters: RTCRtpSendParameters, bandwidth: number, + logger: Logger, ): RTCRtpEncodingParameters[] => { const unlimitedEncodings = [{}]; - return parameters.encodings.length === 0 ? unlimitedEncodings : splitBandwidth(parameters.encodings, bandwidth); + return parameters.encodings.length === 0 + ? unlimitedEncodings + : splitBandwidth(parameters.encodings, bandwidth, logger); }; export const encodingToVariantMap: Record = { diff --git a/packages/webrtc-client/src/tracks/transceivers.ts b/packages/webrtc-client/src/tracks/transceivers.ts index d8267bc8c..11d3cfd55 100644 --- a/packages/webrtc-client/src/tracks/transceivers.ts +++ b/packages/webrtc-client/src/tracks/transceivers.ts @@ -1,18 +1,18 @@ import { Variant } from '@fishjam-cloud/protobufs/shared'; +import { kbpsToBps, SIMULCAST_LAYER_SCALE, splitBandwidth } from '../bandwidth'; import type { TrackContextImpl } from '../internal'; -import type { SimulcastBandwidthLimit, TrackBandwidthLimit } from '../types'; -import { splitBandwidth } from './bandwidth'; +import type { Logger, SimulcastBandwidthLimit, TrackBandwidthLimit } from '../types'; import { encodingToVariantMap } from './encodings'; -export const createTransceiverConfig = (trackContext: TrackContextImpl): RTCRtpTransceiverInit => { +export const createTransceiverConfig = (trackContext: TrackContextImpl, logger: Logger): RTCRtpTransceiverInit => { if (!trackContext.track) throw new Error(`Cannot create transceiver config for `); if (trackContext.track.kind === 'audio') { return createAudioTransceiverConfig(trackContext.stream); } - return createVideoTransceiverConfig(trackContext, trackContext.maxBandwidth); + return createVideoTransceiverConfig(trackContext, trackContext.maxBandwidth, logger); }; const createAudioTransceiverConfig = (stream: MediaStream | null): RTCRtpTransceiverInit => { @@ -25,29 +25,18 @@ const createAudioTransceiverConfig = (stream: MediaStream | null): RTCRtpTransce const createVideoTransceiverConfig = ( trackContext: TrackContextImpl, maxBandwidth: TrackBandwidthLimit, + logger: Logger, ): RTCRtpTransceiverInit => { if (!trackContext.simulcastConfig) throw new Error(`Simulcast config for track ${trackContext.trackId} not found.`); if (trackContext.simulcastConfig.enabled) { - let simulcastConfig: Map; - - if (maxBandwidth === 0) { - simulcastConfig = new Map([ - [Variant.VARIANT_LOW, 0], - [Variant.VARIANT_MEDIUM, 0], - [Variant.VARIANT_HIGH, 0], - ]); - } else if (typeof maxBandwidth === 'number') { - throw new Error('Invalid bandwidth limit for simulcast track.'); - } else { - simulcastConfig = maxBandwidth; - } - - return createSimulcastTransceiverConfig(trackContext, simulcastConfig); + if (typeof maxBandwidth === 'number') throw new Error('Invalid bandwidth limit for simulcast track.'); + + return createSimulcastTransceiverConfig(trackContext, maxBandwidth); } if (typeof maxBandwidth === 'number') { - return createNonSimulcastTransceiverConfig(trackContext, maxBandwidth); + return createNonSimulcastTransceiverConfig(trackContext, maxBandwidth, logger); } throw new Error('LocalTrack is in invalid state!'); @@ -56,10 +45,11 @@ const createVideoTransceiverConfig = ( const createNonSimulcastTransceiverConfig = ( trackContext: TrackContextImpl, maxBandwidth: number, + logger: Logger, ): RTCRtpTransceiverInit => { return { direction: 'sendonly', - sendEncodings: splitBandwidth([{ active: true }], maxBandwidth), + sendEncodings: splitBandwidth([{ active: true }], maxBandwidth, logger), streams: trackContext.stream ? [trackContext.stream] : [], }; }; @@ -77,13 +67,13 @@ const createSimulcastTransceiverConfig = ( rid: 'l', active: activeEncodings.includes(Variant.VARIANT_LOW), // maxBitrate: 4_000_000, - scaleResolutionDownBy: 4.0, + scaleResolutionDownBy: SIMULCAST_LAYER_SCALE[Variant.VARIANT_LOW], // scalabilityMode: "L1T" + TEMPORAL_LAYERS_COUNT, }, { rid: 'm', active: activeEncodings.includes(Variant.VARIANT_MEDIUM), - scaleResolutionDownBy: 2.0, + scaleResolutionDownBy: SIMULCAST_LAYER_SCALE[Variant.VARIANT_MEDIUM], }, { rid: 'h', @@ -103,7 +93,10 @@ const createSimulcastTransceiverConfig = ( }; }; -const calculateSimulcastEncodings = (encodings: RTCRtpEncodingParameters[], maxBandwidth: SimulcastBandwidthLimit) => { +export const calculateSimulcastEncodings = ( + encodings: RTCRtpEncodingParameters[], + maxBandwidth: SimulcastBandwidthLimit, +) => { return encodings .filter((encoding) => encoding.rid) .map((encoding) => { @@ -113,7 +106,7 @@ const calculateSimulcastEncodings = (encodings: RTCRtpEncodingParameters[], maxB return { ...encoding, - maxBitrate: limit > 0 ? limit * 1024 : undefined, + maxBitrate: limit > 0 ? kbpsToBps(limit) : undefined, } satisfies RTCRtpEncodingParameters; }); }; diff --git a/packages/webrtc-client/src/types.ts b/packages/webrtc-client/src/types.ts index f2ddec4dc..065be8617 100644 --- a/packages/webrtc-client/src/types.ts +++ b/packages/webrtc-client/src/types.ts @@ -22,14 +22,15 @@ export type TrackKind = 'audio' | 'video'; export type VadStatus = 'speech' | 'silence'; /** - * Type describing maximal bandwidth that can be used, in kbps. 0 is interpreted as unlimited bandwidth. + * Type describing maximal bandwidth that can be used, in kbps. + * Video limits are clamped to `MAX_BANDWIDTH_LIMITS`; 0 is interpreted as "use the cap". */ export type BandwidthLimit = number; /** * Type describing bandwidth limit for simulcast track. * It is a mapping (encoding => BandwidthLimit). - * If encoding isn't present in this mapping, it will be assumed that this particular encoding shouldn't have any bandwidth limit + * If encoding isn't present in this mapping, the cap from `MAX_BANDWIDTH_LIMITS.simulcast` is used for it. */ export type SimulcastBandwidthLimit = Map; @@ -239,7 +240,7 @@ export interface WebRTCEndpointEvents { localTrackUnmuted: (event: { trackId: string }) => void; - localTrackBandwidthSet: (event: { trackId: string; bandwidth: BandwidthLimit }) => void; + localTrackBandwidthSet: (event: { trackId: string; bandwidth: TrackBandwidthLimit }) => void; localTrackEncodingBandwidthSet: (event: { trackId: string; rid: Variant; bandwidth: BandwidthLimit }) => void; diff --git a/packages/webrtc-client/src/webRTCEndpoint.ts b/packages/webrtc-client/src/webRTCEndpoint.ts index 3b889f526..07144a6f6 100644 --- a/packages/webrtc-client/src/webRTCEndpoint.ts +++ b/packages/webrtc-client/src/webRTCEndpoint.ts @@ -18,6 +18,7 @@ import EventEmitter from 'events'; import type TypedEmitter from 'typed-emitter'; import { v4 as uuidv4 } from 'uuid'; +import { resolveTrackBandwidthLimit } from './bandwidth'; import { CommandsQueue } from './CommandsQueue'; import { ConnectionManager } from './ConnectionManager'; import { DataChannelManager } from './dataChannels/DataChannelManager'; @@ -70,8 +71,8 @@ export class WebRTCEndpoint extends (EventEmitter as new () => TypedEmitter TypedEmitter 0 ? maxBandwidth : 0; + const resolvedMaxBandwidth = this.resolveInitialBandwidth(track, simulcastConfig, maxBandwidth); try { if (!stream) trackStream.addTrack(track); @@ -443,7 +443,7 @@ export class WebRTCEndpoint extends (EventEmitter as new () => TypedEmitter this.localTrackManager.parseAddTrack(track, simulcastConfig, resolvedMaxBandwidth), + parse: () => this.localTrackManager.parseAddTrack(track), resolve: 'after-renegotiation', resolutionNotifier, batchable: true, @@ -464,6 +464,16 @@ export class WebRTCEndpoint extends (EventEmitter as new () => TypedEmitter TypedEmitter} success */ - public setTrackBandwidth(trackId: string, bandwidth: BandwidthLimit): Promise { + public setTrackBandwidth(trackId: string, bandwidth: TrackBandwidthLimit): Promise { if (!this.connectionManager) throw new Error(`There is no active RTCPeerConnection`); return this.local.setTrackBandwidth(trackId, bandwidth); @@ -543,7 +554,7 @@ export class WebRTCEndpoint extends (EventEmitter as new () => TypedEmitter { diff --git a/packages/webrtc-client/tests/methods/addTrackMethod.test.ts b/packages/webrtc-client/tests/methods/addTrackMethod.test.ts index e134aa822..ab64211d6 100644 --- a/packages/webrtc-client/tests/methods/addTrackMethod.test.ts +++ b/packages/webrtc-client/tests/methods/addTrackMethod.test.ts @@ -8,6 +8,8 @@ import { createTransceiverConfig } from '../../src/tracks/transceivers'; import { createConnectedEventWithOneEndpoint, mockTrack } from '../fixtures'; import { mockMediaStream, mockRTCPeerConnection } from '../mocks'; +const logger = { debug: () => {}, warn: () => {}, error: () => {} }; + it('Adding track invokes renegotiation', async () => { const webRTCEndpoint = new WebRTCEndpoint(); mockMediaStream(); @@ -61,13 +63,46 @@ it('Simulcast transceiver config includes the stream', () => { ); trackContext.track = videoTrack; trackContext.stream = stream; - trackContext.maxBandwidth = 0; + trackContext.maxBandwidth = new Map(); - const config = createTransceiverConfig(trackContext); + const config = createTransceiverConfig(trackContext, logger); expect(config.streams).toEqual([stream]); }); +it('Adding a simulcast track keeps the per-variant bandwidth limits', async () => { + mockRTCPeerConnection(); + mockMediaStream(); + + const webRTCEndpoint = new WebRTCEndpoint(); + const serializedEvent = serializeServerMediaEvent({ connected: createConnectedEventWithOneEndpoint() }); + await webRTCEndpoint.receiveMediaEvent(serializedEvent); + + const limits = new Map([ + [Variant.VARIANT_LOW, 150], + [Variant.VARIANT_MEDIUM, 500], + [Variant.VARIANT_HIGH, 1500], + ]); + + webRTCEndpoint.addTrack( + mockTrack, + undefined, + { + enabled: true, + enabledVariants: [Variant.VARIANT_LOW, Variant.VARIANT_MEDIUM, Variant.VARIANT_HIGH], + disabledVariants: [], + }, + limits, + ); + + const [trackContext] = [...webRTCEndpoint['local'].getTrackIdToTrack().values()]; + expect(trackContext!.maxBandwidth).toEqual(limits); + + const config = createTransceiverConfig(trackContext!, logger); + const high = config.sendEncodings!.find((encoding) => encoding.rid === 'h'); + expect(high?.maxBitrate).toBe(1500 * 1024); +}); + it('Adding track before being accepted by the server throws error', async () => { // Given mockRTCPeerConnection(); diff --git a/packages/webrtc-client/tests/methods/bandwidthLimits.test.ts b/packages/webrtc-client/tests/methods/bandwidthLimits.test.ts new file mode 100644 index 000000000..495547da1 --- /dev/null +++ b/packages/webrtc-client/tests/methods/bandwidthLimits.test.ts @@ -0,0 +1,352 @@ +import { FakeMediaStreamTrack } from 'fake-mediastreamtrack'; +import { afterEach, expect, it, vi } from 'vitest'; + +import { MAX_BANDWIDTH_LIMITS, Variant, WebRTCEndpoint } from '../../src'; +import { resolveTrackBandwidthLimit, resolveVariantBandwidthLimit, splitSimulcastBudget } from '../../src/bandwidth'; +import { deserializePeerMediaEvent, serializeServerMediaEvent } from '../../src/mediaEvent'; +import { createTransceiverConfig } from '../../src/tracks/transceivers'; +import { createAddLocalTrackSDPOffer, createConnectedEventWithOneEndpoint } from '../fixtures'; +import { mockMediaStream, mockRTCPeerConnection } from '../mocks'; + +const KBPS = 1024; + +const logger = { debug: vi.fn(), warn: vi.fn(), error: vi.fn() }; + +afterEach(() => { + vi.restoreAllMocks(); + logger.warn.mockClear(); +}); + +it('resolveTrackBandwidthLimit resolves 0 to the single-stream cap', () => { + expect(resolveTrackBandwidthLimit(0, false, logger)).toBe(MAX_BANDWIDTH_LIMITS.singleStream); + expect(resolveTrackBandwidthLimit(-5, false, logger)).toBe(MAX_BANDWIDTH_LIMITS.singleStream); +}); + +it('resolveTrackBandwidthLimit keeps a single-stream value below the cap', () => { + expect(resolveTrackBandwidthLimit(800, false, logger)).toBe(800); +}); + +it('resolveTrackBandwidthLimit lowers a single-stream value above the cap and warns', () => { + expect(resolveTrackBandwidthLimit(8000, false, logger)).toBe(MAX_BANDWIDTH_LIMITS.singleStream); + expect(logger.warn).toHaveBeenCalledOnce(); +}); + +it('resolveTrackBandwidthLimit fills missing simulcast variants with caps and clamps the rest', () => { + const limits = new Map([ + [Variant.VARIANT_MEDIUM, 300], + [Variant.VARIANT_HIGH, 9000], + ]); + + const resolved = resolveTrackBandwidthLimit(limits, true, logger) as Map; + + expect(resolved.get(Variant.VARIANT_LOW)).toBe(MAX_BANDWIDTH_LIMITS.simulcast[Variant.VARIANT_LOW]); + expect(resolved.get(Variant.VARIANT_MEDIUM)).toBe(300); + expect(resolved.get(Variant.VARIANT_HIGH)).toBe(MAX_BANDWIDTH_LIMITS.simulcast[Variant.VARIANT_HIGH]); +}); + +it('resolveVariantBandwidthLimit uses the cap of the given variant', () => { + expect(resolveVariantBandwidthLimit(Variant.VARIANT_LOW, 1000, logger)).toBe( + MAX_BANDWIDTH_LIMITS.simulcast[Variant.VARIANT_LOW], + ); + expect(resolveVariantBandwidthLimit(Variant.VARIANT_HIGH, 1000, logger)).toBe(1000); +}); + +it('resolveVariantBandwidthLimit rejects a variant that has no simulcast layer', () => { + expect(() => resolveVariantBandwidthLimit(Variant.VARIANT_UNSPECIFIED, 1000, logger)).toThrow(); +}); + +const connect = async () => { + const webRTCEndpoint = new WebRTCEndpoint(); + await webRTCEndpoint.receiveMediaEvent( + serializeServerMediaEvent({ connected: createConnectedEventWithOneEndpoint() }), + ); + + const offers: ReturnType['sdpOffer'][] = []; + webRTCEndpoint.on('sendMediaEvent', (mediaEvent) => { + const event = deserializePeerMediaEvent(mediaEvent); + if (event.sdpOffer) offers.push(event.sdpOffer); + }); + + return { webRTCEndpoint, offers }; +}; + +const trackContexts = (webRTCEndpoint: WebRTCEndpoint) => [...webRTCEndpoint['local'].getTrackIdToTrack().values()]; + +it('addTrack without a limit uses the single-stream cap for video and no limit for audio', async () => { + mockRTCPeerConnection(); + mockMediaStream(); + const { webRTCEndpoint } = await connect(); + + webRTCEndpoint.addTrack(new FakeMediaStreamTrack({ kind: 'video' })); + webRTCEndpoint.addTrack(new FakeMediaStreamTrack({ kind: 'audio' })); + await webRTCEndpoint.receiveMediaEvent(serializeServerMediaEvent({ offerData: createAddLocalTrackSDPOffer() })); + + const [video, audio] = trackContexts(webRTCEndpoint); + expect(video!.maxBandwidth).toBe(MAX_BANDWIDTH_LIMITS.singleStream); + expect(audio!.maxBandwidth).toBe(0); +}); + +it('addTrack on a simulcast track without a limit resolves to a Map of the per-variant caps', async () => { + mockRTCPeerConnection(); + mockMediaStream(); + const { webRTCEndpoint } = await connect(); + + webRTCEndpoint.addTrack(new FakeMediaStreamTrack({ kind: 'video' }), undefined, { + enabled: true, + enabledVariants: [Variant.VARIANT_LOW, Variant.VARIANT_MEDIUM, Variant.VARIANT_HIGH], + disabledVariants: [], + }); + + const [video] = trackContexts(webRTCEndpoint); + const caps = new Map( + Object.entries(MAX_BANDWIDTH_LIMITS.simulcast).map(([variant, limit]) => [Number(variant), limit]), + ); + expect(video!.maxBandwidth).toEqual(caps); + expect(() => createTransceiverConfig(video!, logger)).not.toThrow(); +}); + +it('addTrack clamps a simulcast limit above the cap', async () => { + mockRTCPeerConnection(); + mockMediaStream(); + const { webRTCEndpoint } = await connect(); + + webRTCEndpoint.addTrack( + new FakeMediaStreamTrack({ kind: 'video' }), + undefined, + { enabled: true, enabledVariants: [Variant.VARIANT_LOW, Variant.VARIANT_HIGH], disabledVariants: [] }, + new Map([[Variant.VARIANT_HIGH, 6000]]), + ); + + const [video] = trackContexts(webRTCEndpoint); + const limits = video!.maxBandwidth as Map; + expect(limits.get(Variant.VARIANT_HIGH)).toBe(MAX_BANDWIDTH_LIMITS.simulcast[Variant.VARIANT_HIGH]); + expect(limits.get(Variant.VARIANT_LOW)).toBe(MAX_BANDWIDTH_LIMITS.simulcast[Variant.VARIANT_LOW]); +}); + +it('SDP offer reports the configured bitrates per variant in bps', async () => { + mockRTCPeerConnection(); + mockMediaStream(); + const { webRTCEndpoint, offers } = await connect(); + + webRTCEndpoint.addTrack( + new FakeMediaStreamTrack({ kind: 'video' }), + { type: 'camera' }, + { + enabled: true, + enabledVariants: [Variant.VARIANT_LOW, Variant.VARIANT_MEDIUM, Variant.VARIANT_HIGH], + disabledVariants: [], + }, + new Map([ + [Variant.VARIANT_LOW, 100], + [Variant.VARIANT_MEDIUM, 400], + [Variant.VARIANT_HIGH, 1200], + ]), + ); + webRTCEndpoint.addTrack(new FakeMediaStreamTrack({ kind: 'video' }), { type: 'screen' }, undefined, 900); + webRTCEndpoint.addTrack(new FakeMediaStreamTrack({ kind: 'audio' }), { type: 'audio' }); + await webRTCEndpoint.receiveMediaEvent(serializeServerMediaEvent({ offerData: createAddLocalTrackSDPOffer() })); + + expect(offers).toHaveLength(1); + const bitrates = offers[0]!.trackIdToBitrates; + const [simulcastId, singleId, audioId] = trackContexts(webRTCEndpoint).map((t) => t.trackId); + + expect(bitrates[simulcastId!]!.variantBitrates).toEqual([ + { variant: Variant.VARIANT_LOW, bitrate: 100 * KBPS }, + { variant: Variant.VARIANT_MEDIUM, bitrate: 400 * KBPS }, + { variant: Variant.VARIANT_HIGH, bitrate: 1200 * KBPS }, + ]); + expect(bitrates[singleId!]!.variantBitrates).toEqual([{ variant: Variant.VARIANT_UNSPECIFIED, bitrate: 900 * KBPS }]); + expect(bitrates[audioId!]!.variantBitrates).toEqual([{ variant: Variant.VARIANT_UNSPECIFIED, bitrate: 50_000 }]); +}); + +it('splitSimulcastBudget divides a budget by pixel count', () => { + // 1 : 4 : 16 → 71 / 286 / 1143 kbps + const limits = splitSimulcastBudget(1500, logger); + + expect([...limits.keys()]).toEqual([Variant.VARIANT_LOW, Variant.VARIANT_MEDIUM, Variant.VARIANT_HIGH]); + expect(limits.get(Variant.VARIANT_LOW)).toBe(71); + expect(limits.get(Variant.VARIANT_MEDIUM)).toBe(286); + expect(limits.get(Variant.VARIANT_HIGH)).toBe(1143); +}); + +it('setTrackBandwidth passes audio values through and caps video values', async () => { + mockRTCPeerConnection(); + mockMediaStream(); + const { webRTCEndpoint } = await connect(); + + webRTCEndpoint.addTrack(new FakeMediaStreamTrack({ kind: 'audio' })); + webRTCEndpoint.addTrack(new FakeMediaStreamTrack({ kind: 'video' })); + await webRTCEndpoint.receiveMediaEvent(serializeServerMediaEvent({ offerData: createAddLocalTrackSDPOffer() })); + + const [audio, video] = trackContexts(webRTCEndpoint); + await webRTCEndpoint.setTrackBandwidth(audio!.trackId, 5000); + await webRTCEndpoint.setTrackBandwidth(video!.trackId, 5000); + + expect(audio!.maxBandwidth).toBe(5000); + expect(video!.maxBandwidth).toBe(MAX_BANDWIDTH_LIMITS.singleStream); +}); + +it('setTrackBandwidth on an audio track reports the configured bitrate to the server', async () => { + mockRTCPeerConnection(); + mockMediaStream(); + const { webRTCEndpoint } = await connect(); + + const reported: number[] = []; + webRTCEndpoint.on('sendMediaEvent', (mediaEvent) => { + const bitrates = deserializePeerMediaEvent(mediaEvent).trackBitrates?.variantBitrates; + if (bitrates) reported.push(...bitrates.map((entry) => entry.bitrate)); + }); + + webRTCEndpoint.addTrack(new FakeMediaStreamTrack({ kind: 'audio' })); + await webRTCEndpoint.receiveMediaEvent(serializeServerMediaEvent({ offerData: createAddLocalTrackSDPOffer() })); + + const [audio] = trackContexts(webRTCEndpoint); + await webRTCEndpoint.setTrackBandwidth(audio!.trackId, 64); + + expect(reported).toEqual([64 * KBPS]); +}); + +const ALL_VARIANTS = [Variant.VARIANT_LOW, Variant.VARIANT_MEDIUM, Variant.VARIANT_HIGH]; + +const connectWithSimulcastTrack = async () => { + mockRTCPeerConnection(); + mockMediaStream(); + const { webRTCEndpoint } = await connect(); + + webRTCEndpoint.addTrack(new FakeMediaStreamTrack({ kind: 'video' }), undefined, { + enabled: true, + enabledVariants: ALL_VARIANTS, + disabledVariants: [], + }); + await webRTCEndpoint.receiveMediaEvent(serializeServerMediaEvent({ offerData: createAddLocalTrackSDPOffer() })); + + const [video] = trackContexts(webRTCEndpoint); + return { webRTCEndpoint, trackId: video!.trackId, limits: () => video!.maxBandwidth as Map }; +}; + +it('setTrackBandwidth(0) on a simulcast track sets every layer to its cap', async () => { + const { webRTCEndpoint, trackId, limits } = await connectWithSimulcastTrack(); + + await webRTCEndpoint.setTrackBandwidth(trackId, 0); + + expect(limits().get(Variant.VARIANT_LOW)).toBe(150); + expect(limits().get(Variant.VARIANT_MEDIUM)).toBe(500); + expect(limits().get(Variant.VARIANT_HIGH)).toBe(1500); +}); + +it('setTrackBandwidth on a simulcast track lets the high layer reach its full cap', async () => { + const { webRTCEndpoint, trackId, limits } = await connectWithSimulcastTrack(); + + // 2100 split 1 : 4 : 16 → 100 / 400 / 1600, then the high layer is clamped to 1500 + await webRTCEndpoint.setTrackBandwidth(trackId, 2100); + + expect(limits().get(Variant.VARIANT_LOW)).toBe(100); + expect(limits().get(Variant.VARIANT_MEDIUM)).toBe(400); + expect(limits().get(Variant.VARIANT_HIGH)).toBe(1500); +}); + +it('setTrackBandwidth on a simulcast track clamps every layer when the budget is huge', async () => { + const { webRTCEndpoint, trackId, limits } = await connectWithSimulcastTrack(); + + await webRTCEndpoint.setTrackBandwidth(trackId, 100_000); + + expect(limits().get(Variant.VARIANT_LOW)).toBe(150); + expect(limits().get(Variant.VARIANT_MEDIUM)).toBe(500); + expect(limits().get(Variant.VARIANT_HIGH)).toBe(1500); +}); + +it('addTrack splits a number on a simulcast track and clamps each layer', async () => { + mockRTCPeerConnection(); + mockMediaStream(); + const { webRTCEndpoint } = await connect(); + + webRTCEndpoint.addTrack( + new FakeMediaStreamTrack({ kind: 'video' }), + undefined, + { enabled: true, enabledVariants: ALL_VARIANTS, disabledVariants: [] }, + 2100, + ); + + const [video] = trackContexts(webRTCEndpoint); + const limits = video!.maxBandwidth as Map; + expect(limits.get(Variant.VARIANT_LOW)).toBe(100); + expect(limits.get(Variant.VARIANT_MEDIUM)).toBe(400); + expect(limits.get(Variant.VARIANT_HIGH)).toBe(1500); +}); + +it('setTrackBandwidth accepts a Map on a simulcast track', async () => { + const { webRTCEndpoint, trackId, limits } = await connectWithSimulcastTrack(); + + await webRTCEndpoint.setTrackBandwidth(trackId, new Map([[Variant.VARIANT_HIGH, 900]])); + + expect(limits().get(Variant.VARIANT_LOW)).toBe(150); + expect(limits().get(Variant.VARIANT_MEDIUM)).toBe(500); + expect(limits().get(Variant.VARIANT_HIGH)).toBe(900); +}); + +it('addTrack rejects a Map on a non-simulcast track before registering it', async () => { + mockRTCPeerConnection(); + mockMediaStream(); + const { webRTCEndpoint } = await connect(); + + await expect( + webRTCEndpoint.addTrack(new FakeMediaStreamTrack({ kind: 'video' }), undefined, undefined, new Map()), + ).rejects.toThrow('non-simulcast track'); + expect(trackContexts(webRTCEndpoint)).toHaveLength(0); +}); + +const reportedBitrates = (webRTCEndpoint: WebRTCEndpoint) => { + const reported: { variant: Variant; bitrate: number }[][] = []; + webRTCEndpoint.on('sendMediaEvent', (mediaEvent) => { + const bitrates = deserializePeerMediaEvent(mediaEvent).trackBitrates?.variantBitrates; + if (bitrates) reported.push(bitrates.map(({ variant, bitrate }) => ({ variant, bitrate }))); + }); + return reported; +}; + +it('setEncodingBandwidth clamps an over-cap layer, stores it and reports every layer in bps', async () => { + const { webRTCEndpoint, trackId, limits } = await connectWithSimulcastTrack(); + const reported = reportedBitrates(webRTCEndpoint); + + await webRTCEndpoint.setEncodingBandwidth(trackId, Variant.VARIANT_MEDIUM, 9000); + + expect(limits().get(Variant.VARIANT_MEDIUM)).toBe(MAX_BANDWIDTH_LIMITS.simulcast[Variant.VARIANT_MEDIUM]); + expect(limits().get(Variant.VARIANT_LOW)).toBe(MAX_BANDWIDTH_LIMITS.simulcast[Variant.VARIANT_LOW]); + expect(limits().get(Variant.VARIANT_HIGH)).toBe(MAX_BANDWIDTH_LIMITS.simulcast[Variant.VARIANT_HIGH]); + expect(reported).toEqual([ + [ + { variant: Variant.VARIANT_LOW, bitrate: 150 * KBPS }, + { variant: Variant.VARIANT_MEDIUM, bitrate: 500 * KBPS }, + { variant: Variant.VARIANT_HIGH, bitrate: 1500 * KBPS }, + ], + ]); +}); + +it('setEncodingBandwidth(0) resolves a layer to its cap and reports it in bps', async () => { + const { webRTCEndpoint, trackId, limits } = await connectWithSimulcastTrack(); + await webRTCEndpoint.setEncodingBandwidth(trackId, Variant.VARIANT_HIGH, 700); + const reported = reportedBitrates(webRTCEndpoint); + + await webRTCEndpoint.setEncodingBandwidth(trackId, Variant.VARIANT_HIGH, 0); + + expect(limits().get(Variant.VARIANT_HIGH)).toBe(MAX_BANDWIDTH_LIMITS.simulcast[Variant.VARIANT_HIGH]); + expect(reported).toHaveLength(1); + expect(reported[0]).toContainEqual({ variant: Variant.VARIANT_HIGH, bitrate: 1500 * KBPS }); +}); + +it('setEncodingBandwidth keeps a below-cap layer and reports only that layer changed', async () => { + const { webRTCEndpoint, trackId, limits } = await connectWithSimulcastTrack(); + const reported = reportedBitrates(webRTCEndpoint); + + await webRTCEndpoint.setEncodingBandwidth(trackId, Variant.VARIANT_HIGH, 700); + + expect(limits().get(Variant.VARIANT_HIGH)).toBe(700); + expect(limits().get(Variant.VARIANT_MEDIUM)).toBe(MAX_BANDWIDTH_LIMITS.simulcast[Variant.VARIANT_MEDIUM]); + expect(reported).toEqual([ + [ + { variant: Variant.VARIANT_LOW, bitrate: 150 * KBPS }, + { variant: Variant.VARIANT_MEDIUM, bitrate: 500 * KBPS }, + { variant: Variant.VARIANT_HIGH, bitrate: 700 * KBPS }, + ], + ]); +}); diff --git a/packages/webrtc-client/tests/mocks.ts b/packages/webrtc-client/tests/mocks.ts index 9496b0320..53f7a9b0f 100644 --- a/packages/webrtc-client/tests/mocks.ts +++ b/packages/webrtc-client/tests/mocks.ts @@ -30,10 +30,12 @@ export const mockRTCPeerConnection = (): { addTransceiverCallback(trackOrKind, init); const sender: any = {}; - sender.getParameters = () => { - // @ts-ignore - const encodings: RTCRtpEncodingParameters[] = [{}]; - return { encodings: encodings } as RTCRtpSendParameters; + let encodings: RTCRtpEncodingParameters[] = init?.sendEncodings?.map((encoding) => ({ ...encoding })) ?? [{}]; + sender.getParameters = () => + ({ encodings: encodings.map((encoding) => ({ ...encoding })) }) as RTCRtpSendParameters; + sender.setParameters = (parameters: RTCRtpSendParameters) => { + encodings = parameters.encodings; + return Promise.resolve(); }; if (init?.direction === 'sendonly') {