From 1a36ff764027a957049cb33817101391e490bc1f Mon Sep 17 00:00:00 2001 From: Shijing Xian Date: Fri, 23 Oct 2026 15:20:32 -0700 Subject: [PATCH] Add proportional jitter to the default reconnect delays DefaultReconnectPolicy returned a fixed delay per retry count, so every client disconnected by the same event waited exactly 100ms, then 300ms, then 300ms, in lockstep. They retried as one herd, and because the schedule is identical they could stay synchronised across attempts rather than drifting apart. Scale each delay by a random factor in [0.5, 1.5) so clients spread further apart with every retry. A first retry configured as zero draws from a fixed 0-500ms window instead, since there is no base delay to scale; a zero at any later index still means retry immediately. Port of livekit/client-sdk-js#2124. Co-Authored-By: Claude Opus 5 (1M context) --- .changeset/reconnect-delay-jitter.md | 9 ++ .../room/network/DefaultReconnectPolicy.kt | 19 ++- .../network/DefaultReconnectPolicyTest.kt | 126 ++++++++++++++++++ 3 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 .changeset/reconnect-delay-jitter.md create mode 100644 livekit-android-test/src/test/java/io/livekit/android/room/network/DefaultReconnectPolicyTest.kt diff --git a/.changeset/reconnect-delay-jitter.md b/.changeset/reconnect-delay-jitter.md new file mode 100644 index 000000000..cbc1ccf7d --- /dev/null +++ b/.changeset/reconnect-delay-jitter.md @@ -0,0 +1,9 @@ +--- +"client-sdk-android": patch +--- + +Add proportional jitter to the default reconnect delays. `DefaultReconnectPolicy` returned a fixed delay for each retry count, so every client disconnected by the same event — a server restart, a region failover, a dropped uplink — waited exactly 100ms, then 300ms, then 300ms, and so on, in lockstep. They retried as one herd and could keep re-synchronising across attempts. + +Each delay is now scaled by a random factor between 0.5 and 1.5, spreading clients further apart with every retry. A first retry whose configured delay is zero instead draws from a fixed 0–500ms window, since there is no base delay to scale; a zero at any later index still means retry immediately. + +This matches the JavaScript SDK (livekit/client-sdk-js#2124). diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/network/DefaultReconnectPolicy.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/network/DefaultReconnectPolicy.kt index 5593d3134..99a98d4d1 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/network/DefaultReconnectPolicy.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/network/DefaultReconnectPolicy.kt @@ -16,6 +16,7 @@ package io.livekit.android.room.network +import kotlin.random.Random import kotlin.time.Duration import kotlin.time.Duration.Companion.milliseconds import kotlin.time.Duration.Companion.seconds @@ -45,13 +46,29 @@ class DefaultReconnectPolicy( return null } - return retryDelays[context.retryCount] + val retryDelay = retryDelays[context.retryCount] + + // The first retry has no base delay to scale, so it gets a fixed window instead. A zero at + // a later index is an explicit request for an immediate retry and passes through as zero. + if (context.retryCount == 0 && retryDelay == Duration.ZERO) { + return Random.nextDouble(MAX_FIRST_RETRY_JITTER_IN_MS).milliseconds + } + + // Jitter scales with the delay (+/-50%) so clients spread out further on later retries. + return retryDelay * (0.5 + Random.nextDouble()) } companion object { val DEFAULT_MAX_RECONNECTION_TIMEOUT = 60.seconds + /** + * Upper bound of the random delay used when the first retry's base delay is zero. It keeps + * a brief network blip recovering quickly while spreading out clients that disconnect + * together. + */ + private const val MAX_FIRST_RETRY_JITTER_IN_MS = 500.0 + val DEFAULT_MAX_RETRY_DELAY = 5.seconds val DEFAULT_RETRY_DELAYS = listOf( diff --git a/livekit-android-test/src/test/java/io/livekit/android/room/network/DefaultReconnectPolicyTest.kt b/livekit-android-test/src/test/java/io/livekit/android/room/network/DefaultReconnectPolicyTest.kt new file mode 100644 index 000000000..46c11c6de --- /dev/null +++ b/livekit-android-test/src/test/java/io/livekit/android/room/network/DefaultReconnectPolicyTest.kt @@ -0,0 +1,126 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.android.room.network + +import io.livekit.android.room.network.DefaultReconnectPolicy.Companion.DEFAULT_RETRY_DELAYS +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import kotlin.time.Duration +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds + +class DefaultReconnectPolicyTest { + + private fun delayAt(policy: DefaultReconnectPolicy, retryCount: Int): Duration? = + policy.getNextRetryDelay(ReconnectContext(retryCount = retryCount, elapsedTime = Duration.ZERO)) + + @Test + fun `stops retrying once the delays are exhausted`() { + val policy = DefaultReconnectPolicy() + + assertNull(delayAt(policy, DEFAULT_RETRY_DELAYS.size)) + } + + @Test + fun `stops retrying once the reconnection timeout has elapsed`() { + val policy = DefaultReconnectPolicy() + + assertNull( + policy.getNextRetryDelay( + ReconnectContext(retryCount = 0, elapsedTime = 61.seconds), + ), + ) + } + + @Test + fun `spreads out clients that disconnect at the same moment`() { + val policy = DefaultReconnectPolicy() + + // Every client retrying on the same schedule reconnects in lockstep, so the + // delay must vary between callers rather than being a fixed constant. + val observed = (1..100).map { delayAt(policy, 0) }.toSet() + + assertTrue( + "every caller got the same first retry delay: $observed", + observed.size > 1, + ) + } + + @Test + fun `keeps jitter proportional to the base delay`() { + val policy = DefaultReconnectPolicy() + + for (retryCount in DEFAULT_RETRY_DELAYS.indices) { + val base = DEFAULT_RETRY_DELAYS[retryCount] + + repeat(100) { + val delay = delayAt(policy, retryCount) + + assertNotNull("no delay for retry $retryCount", delay) + assertTrue( + "retry $retryCount delay $delay below half of $base", + delay!! >= base * 0.5, + ) + assertTrue( + "retry $retryCount delay $delay above 1.5x of $base", + delay <= base * 1.5, + ) + } + } + } + + @Test + fun `gives a zero first delay a bounded random window`() { + // A caller asking for an immediate first retry still needs the herd spread out, + // but there is no base delay to scale, so a fixed window is used instead. + val policy = DefaultReconnectPolicy( + retryDelays = listOf(Duration.ZERO, 300.milliseconds), + ) + + val observed = (1..100).map { + val delay = delayAt(policy, 0) + assertNotNull(delay) + assertTrue("zero-base delay $delay was negative", delay!! >= Duration.ZERO) + assertTrue("zero-base delay $delay exceeded the window", delay < 500.milliseconds) + delay + }.toSet() + + assertTrue("zero-base first retry did not vary: $observed", observed.size > 1) + } + + @Test + fun `honours a custom delay list length`() { + val policy = DefaultReconnectPolicy(retryDelays = listOf(100.milliseconds)) + + assertNotNull(delayAt(policy, 0)) + assertNull(delayAt(policy, 1)) + } + + @Test + fun `passes through an explicit zero at a later index`() { + // Only the first retry gets the fixed window; a later zero is an explicit + // request to retry immediately. + val policy = DefaultReconnectPolicy( + retryDelays = listOf(100.milliseconds, Duration.ZERO), + ) + + assertEquals(Duration.ZERO, delayAt(policy, 1)) + } +}