diff --git a/.changeset/add-livekit-telemetry-crate.md b/.changeset/add-livekit-telemetry-crate.md new file mode 100644 index 000000000..4e6eb7d95 --- /dev/null +++ b/.changeset/add-livekit-telemetry-crate.md @@ -0,0 +1,5 @@ +--- +livekit-telemetry: minor +--- + +Add `livekit-telemetry`, the shared client telemetry core: records (events, warn/error logs, spans, RTC stats windows) are buffered on the device, committed to a write-ahead cache (memory, or fsynced files with `storage_dir`) and exported as OTLP/HTTP logs and traces to the LiveKit Cloud project each Room belongs to. A Room hands over only its server URL and token (`Scope::set_server`, again on every refresh); the core validates the URL, reads the token's grant and expiry, and uploads each record with the token of the Room and project that captured it. Every collector answer is classified (partial success, 4xx, 413 split, 401/403 disabled vs credential, 404, 429/503 `Retry-After`/`RetryInfo`, 5xx, no answer) with jittered backoff per destination, soft and hard device holds, loss counters by reason (`lk.telemetry.report`, `Telemetry::stats`), Room-scoped custom events and correlation attributes, and an opt-out that purges everything unsent. Defaults export and window RTC stats once a minute; `LK_TELEMETRY_ENDPOINT` points uploads at a local collector for tests. diff --git a/Cargo.lock b/Cargo.lock index afcc5a91c..811a82f7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3812,6 +3812,26 @@ dependencies = [ "url", ] +[[package]] +name = "livekit-telemetry" +version = "0.1.0" +dependencies = [ + "async-trait", + "base64 0.22.1", + "flate2", + "livekit-net", + "log", + "opentelemetry-proto", + "prost 0.14.4", + "prost-types 0.14.4", + "rand 0.9.5", + "serde_json", + "thiserror 2.0.19", + "tokio", + "uniffi", + "url", +] + [[package]] name = "livekit-token" version = "0.2.1" @@ -5037,6 +5057,46 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror 2.0.19", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost 0.14.4", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry", + "percent-encoding", + "portable-atomic", + "rand 0.9.5", + "thiserror 2.0.19", +] + [[package]] name = "orbclient" version = "0.3.55" diff --git a/Cargo.toml b/Cargo.toml index 2bb66801a..265d7a29d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "livekit-ffi", "livekit-uniffi", "livekit-datatrack", + "livekit-telemetry", "livekit-token", "livekit-token-source", "livekit-ffi-node-bindings", @@ -60,6 +61,7 @@ livekit-api = { version = "0.8.1", path = "livekit-api" } livekit-capture = { version = "0.1.3", path = "livekit-capture" } livekit-ffi = { version = "0.12.81", path = "livekit-ffi" } livekit-datatrack = { version = "0.2.1", path = "livekit-datatrack" } +livekit-telemetry = { version = "0.1.0", path = "livekit-telemetry" } livekit-signaling = { version = "0.1.4", path = "livekit-signaling" } livekit-token = { version = "0.2.1", path = "livekit-token" } livekit-token-source = { version = "0.1.3", path = "livekit-token-source" } diff --git a/knope.toml b/knope.toml index bf337b338..039ed4fed 100644 --- a/knope.toml +++ b/knope.toml @@ -178,6 +178,14 @@ versioned_files = [ ] changelog = "livekit-datatrack/CHANGELOG.md" +[packages.livekit-telemetry] +versioned_files = [ + "livekit-telemetry/Cargo.toml", + "Cargo.lock", + { path = "Cargo.toml", dependency = "livekit-telemetry" }, +] +changelog = "livekit-telemetry/CHANGELOG.md" + [packages.livekit-common] versioned_files = [ "livekit-common/Cargo.toml", diff --git a/livekit-telemetry/CHANGELOG.md b/livekit-telemetry/CHANGELOG.md new file mode 100644 index 000000000..825c32f0d --- /dev/null +++ b/livekit-telemetry/CHANGELOG.md @@ -0,0 +1 @@ +# Changelog diff --git a/livekit-telemetry/Cargo.toml b/livekit-telemetry/Cargo.toml new file mode 100644 index 000000000..0943e1689 --- /dev/null +++ b/livekit-telemetry/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "livekit-telemetry" +description = "Client telemetry core for LiveKit: buffers events on-device and exports them as OTLP" +version = "0.1.0" +readme = "README.md" +license.workspace = true +edition.workspace = true +repository.workspace = true + +[dependencies] +tokio = { workspace = true, default-features = false, features = ["macros", "sync", "time"] } +log = { workspace = true } +thiserror = { workspace = true } +async-trait = "0.1" +prost = { workspace = true } +# `google.protobuf.Any`/`Duration` for the `google.rpc.Status` error body (RetryInfo). +prost-types = { workspace = true } +rand = { workspace = true } +# Unverified JWT claims (observability grant, expiry); both already linked by livekit-api. +base64 = "0.22" +serde_json = { workspace = true } +# Server URLs are parsed, never string-matched, before a token may follow them. +url = "2.3" +# gzip for request bodies; the deflate backend (miniz_oxide) is already linked by data streams. +flate2 = "1" +# OTLP message types only (`gen-tonic-messages` = prost structs, no tonic). Same prost as +# livekit-protocol so a single prost is linked. +opentelemetry-proto = { version = "0.32", default-features = false, features = ["logs", "trace", "gen-tonic-messages"] } +livekit-net = { workspace = true, optional = true } +uniffi = { workspace = true, features = ["scaffolding-ffi-buffer-fns"], optional = true } + +[features] +# Default HTTP transport over the pluggable `livekit-net` client (native backend, or one +# the host registered with `livekit_net::set_http_client`). +net = ["dep:livekit-net"] +uniffi = ["dep:uniffi"] + +[dev-dependencies] +tokio = { workspace = true, default-features = false, features = ["rt", "rt-multi-thread", "macros", "sync", "time", "test-util", "net", "io-util"] } + +# How CI checks this crate's features, read by +# `.github/workflows/feature-combinations-curated.yml` via `cargo metadata`. +[package.metadata.feature-combinations] +mode = "powerset" diff --git a/livekit-telemetry/README.md b/livekit-telemetry/README.md new file mode 100644 index 000000000..1809183d9 --- /dev/null +++ b/livekit-telemetry/README.md @@ -0,0 +1,81 @@ +# LiveKit Telemetry + +**Important**: +This is an internal crate that powers client telemetry in LiveKit client SDKs (through +`livekit-uniffi`: Swift, Kotlin, Dart) and is not meant to be used directly. + +The core buffers records on the device and ships them out-of-band as standard +[OTLP/HTTP](https://opentelemetry.io/docs/specs/otlp/) logs and traces to the LiveKit Cloud +project a room belongs to. Everything hard lives here once — destination and credentials, +batching, encoding, retry, persistence, holds, loss accounting — while platforms provide only +what they are uniquely placed to: OS signals (instruments) and the byte-moving transport. + +```text + SDK / instruments ──emit()──▶ Telemetry ──▶ Store ──drain──▶ Exporter ──push──▶ BatchCache ──upload oldest-first──▶ TelemetryTransport + (source) (records) (tick · encode) (MemoryCache | FileCache) (NetTransport | host HTTP) +``` + +| Layer | OTel equivalent | Type | +|---|---|---| +| source | `Logger.emit`, `Tracer.start` | [`Telemetry`], [`Scope`] (one per room), [`Span`] | +| store | `BatchLogRecordProcessor` queue | `Store` (in-memory, drop-oldest, bounded) | +| exporter | batch processor timer + exporter | [`Exporter`] (actor, spawn `run()`) | +| cache | disk buffering | [`BatchCache`]: [`MemoryCache`], [`FileCache`] (`storage_dir`) | +| transport | exporter's HTTP client | [`TelemetryTransport`]; `NetTransport` over `livekit-net` (feature `net`) | + +## Usage + +```rust +# use std::sync::Arc; +# use livekit_telemetry::*; +# struct Discard; +# #[async_trait::async_trait] +# impl TelemetryTransport for Discard { +# async fn send(&self, _: ExportRequest) -> Result { +# Ok(ExportResponse::accepted()) +# } +# } +# #[tokio::main(flavor = "current_thread")] async fn main() { +let config = TelemetryConfig { storage_dir: None, ..Default::default() }; +let (telemetry, exporter) = Telemetry::new(config, Arc::new(Discard)); +tokio::spawn(exporter.run()); + +// One scope per room. Its server URL and token decide where its records go; hand the token +// over again on every refresh. +let room = telemetry.begin_scope(); +room.set_server("wss://my-project.livekit.cloud", ""); +room.set_attribute("app.call_id", Some("c-42".into())); +room.emit_custom("checkout", vec![Attribute::new("plan", "pro")]); + +let connect = room.start(SpanName::Connect, None); +connect.step(SpanStep::WsOpen); +connect.end(SpanOutcome::Ok, None); + +telemetry.set_device_state(DeviceState { thermal: ThermalState::Serious, ..Default::default() }); +telemetry.shutdown().await; // cache, then upload what the network allows +println!("{}", telemetry.stats()); // drops by reason, uploads, cached batches +# } +``` + +Event names, attributes, cadences and the upload policy are defined in [`SPEC.md`](SPEC.md). + +## Design notes + +- **The room decides the destination.** A platform passes only the server URL and the token + ([`Scope::set_server`]). The core derives the ingest URL (LiveKit Cloud hosts only), reads + the token's unverified claims (observability grant, expiry) and routes each batch by its + owner — project and session — so two rooms on two projects never share a token. +- **Write-ahead cache.** Every batch is encoded, gzipped and cached *before* the network is + tried, then uploaded oldest-first and removed on an answer. Tokens are attached at upload + time and never persisted. +- **The core reads every answer.** Transports return status, headers and body; the core alone + classifies OTLP partial success, 4xx, 413 (split), 429/503 `Retry-After` and `RetryInfo`, + and Cloud's disabled and credential answers. +- **Loss is counted, never silent.** Every way a record can be lost has a counter; the deltas + ride along with the next batch as `lk.telemetry.report` and are readable locally through + [`Telemetry::stats`]. +- **Device state comes from the host; the policy lives here.** Thermal, power, memory, + network and lifecycle are OS APIs the host watches and pushes through + [`Telemetry::set_device_state`]; the core stretches its cadence and holds uploads. +- **Size.** OTLP types come from `opentelemetry-proto` (`gen-tonic-messages`, no tonic); JWT + claims are read with `base64` and `serde_json`, both already linked by the UniFFI library. diff --git a/livekit-telemetry/SPEC.md b/livekit-telemetry/SPEC.md new file mode 100644 index 000000000..0da641e92 --- /dev/null +++ b/livekit-telemetry/SPEC.md @@ -0,0 +1,569 @@ +# Client telemetry spec + +Source of truth for event names, attributes and cadences emitted by LiveKit client SDKs. +Additive-only by convention; LiveKit-defined names carry the `lk.` prefix, everything else +follows [OpenTelemetry semantic conventions](https://github.com/open-telemetry/semantic-conventions). + +## Resource attributes + +Set once per pipeline (`TelemetryConfig.resource`): + +| Key | Who sets it | Example | +|---|---|---| +| `service.name` | platform SDK | `livekit-client-swift` | +| `service.version` | platform SDK | `2.9.0` | +| `os.name`, `os.version` | platform SDK | `iOS`, `18.5` | +| `device.model.identifier` | platform SDK | `iPhone16,1` | +| `telemetry.sdk.name/language/version` | core | `livekit-telemetry`, `rust`, `0.1.0` | + +## Pipeline, scopes and destination + +One pipeline per process — started at SDK init, so audio pre-initialization, permission failures +and connect attempts that never reach a server are captured — and one **scope** per room (one +call). A scope is a trace id plus the attributes attached to its records (`lk.room.sid`, +`lk.participant.identity`, …); spans, RTC windows and events are filed under the scope that +produced them, and `session.id` (OTel semconv) is written on every record as an attribute. A log +record emitted inside a room's span is filed under that room's scope; anything emitted outside +a scope — device state, pre-room errors, self-telemetry — belongs to the pipeline's own process +scope. Scopes are not ended: a room's last record is simply its last. + +### Destination and credentials + +The platform passes exactly two things per room, through `Scope::set_server(url, token)`: the +LiveKit server URL the room connects to and the participant token it connects with — at connect, +and again with **every** refreshed token (the SFU sends one right after join, then every few +minutes). The call is cheap and idempotent. There is no client-side endpoint, header or sink. + +- **Ingest URL:** derived from the server URL's host, `https:///observability/client/{logs,traces}/otlp/v0`, + for LiveKit Cloud hosts only (`*.livekit.cloud`, including `*.staging.livekit.cloud`). Any other + host (self-hosted, OSS) has no ingest: one local warning, and that room's records are dropped + at the door instead of cached. +- **Token:** `Authorization: Bearer `. The core reads the token's *unverified* claims: the + observability grant (`observability.write` or `observability.clientWrite`) and `exp`. It never + sends a token known to be expired or refused; batches wait for the next token instead + (a hard hold). A project whose first token has no grant never opted in: nothing is collected + for it. A refreshed token without the grant (today's SFU drops it) does not replace a granted + one that is still valid. Tokens live in memory only — never in a batch, never on disk. +- **Server URL validation:** parsed with WHATWG URL rules, never string-matched. A token is only + sent to `https://.livekit.cloud/…` built from the parsed host alone: TLS scheme + (`wss`/`https`), a domain under the Cloud suffix with a label of its own, the default port, no + userinfo. +- **Ownership:** every record captures its owner when it is captured — its session and the + project that session is routed to at that moment — and keeps it: a Room that reconnects to + another project takes nothing queued or cached along. Credentials are keyed by (project, + session): a live Room uploads with its **own** latest token for that project, never another + Room's. A Room's records captured before it had a server go to its own first project, never to + another Room's. Process-level records (device state, pre-room errors, self-telemetry) go to + the project most recently handed a token, with that project's latest token; so do batches from + a previous launch, which wait — up to the 24 h age limit — for a token of the same project. + The answer to a request is attributed to the project it was sent to — its 404, disable or + pause never lands on another project. A session's credentials stay while the session is alive + (its Room, or records, windows or spans still referencing it) — for every project it was routed + to, so records captured for an earlier project can still be sent; that is the residual: a live + Room keeps one credential per project it has used — and while cached batches need them; + project-level copies only while a live session is routed there or the backlog has batches for + it. A token the collector refused is recorded by identity (a hash) and never sent again from + any slot, until it expires (a token without `exp` stays refused for the process; like the + per-host project table, which keeps one small entry per host ever seen, that grows only with + what a process meets — bounded in practice, not by a cap). A past, + negative or non-numeric `exp` counts as expired. +- **Waiting** for a first destination or a usable token is uncapped, bounded only by the cache. + +Local end-to-end tests point everything at an OpenTelemetry collector of their own with the +`LK_TELEMETRY_ENDPOINT` environment variable, read by the core at start (a base URL gets +`/v1/logs` and `/v1/traces`; a URL ending in `logs` is used as is). It is not part of any platform +API. With it, every batch goes there without Cloud rules or tokens. + +## Events + +An event with no `body` is exported with its name as the body as well as in `event_name`: log +viewers key their line on the body, and not every backend surfaces `event_name` yet. + +```yaml +event: lk.ping +area: sdk +severity: info +attributes: + lk.ping.seq: int # optional, monotonically increasing per pipeline +cadence: on demand — pipeline smoke test, never emitted in production paths +platforms: all +``` + +```yaml +event: lk.telemetry.report +area: sdk (self-telemetry) +severity: info +attributes: # counts since the previous report + lk.telemetry.uploads.sent: int # batches accepted + lk.telemetry.uploads.bytes: int # compressed bytes accepted — what telemetry cost the uplink + lk.telemetry.uploads.failed: int # attempts that failed transiently (no answer, 429, 5xx) + lk.telemetry.cache.batches: int # batches waiting in the cache right now (a gauge) + # the rest only when non-zero: + lk.telemetry.uploads.timeouts: int # attempts that hit export_timeout_ms + lk.telemetry.uploads.unauthorized: int # 401/403 answers: tokens the collector refused + lk.telemetry.holds.capped: int # soft holds that reached the 60 s cap + lk.telemetry.cache.write_errors: int # batches the disk refused (full, gone), kept in memory + lk.telemetry.dropped.queue_full: int # records evicted from the in-memory queue + lk.telemetry.dropped.cache_error: int # records no cache, not even memory, could take + lk.telemetry.dropped.cache_full: int # records evicted by the cache's size / file-count bound + lk.telemetry.dropped.expired: int # records in batches past the 24 h age limit + lk.telemetry.dropped.corrupt: int # records in cached batches that failed their CRC + lk.telemetry.dropped.invalid: int # custom events / attributes over the limits (rejected) + lk.telemetry.dropped.rejected: int # records the collector rejected (final 4xx/5xx, partial success) + lk.telemetry.dropped.oversized: int # single records larger than the collector accepts (413) + lk.telemetry.dropped.throttled: int # records evicted from the cache during a server-directed pause + lk.telemetry.dropped.rate_limited: int # discrete events dropped by the flood guard +cadence: appended to the next upload whenever a loss, a failure, a refusal, a capped hold or a + disk error happened since the previous report — never its own request, never persisted + on its own, so a broken uploader never reports through itself — and once at shutdown as + the session summary, so fleet-wide success rates have denominators. Losses by policy + (a project that receives nothing, the opt-out) are local only (`Telemetry::stats`). +platforms: all +``` + +```yaml +event: lk.device.thermal.changed +area: device +attributes: + lk.device.thermal.state: enum(nominal | fair | serious | critical) +cadence: on change (+ initial value on the first `set_device_state`); `unknown` (no thermal + source on the platform, the default) is no reading: no event, no stretch +platforms: ios, macos, android — optional elsewhere +``` + +```yaml +event: lk.device.low_power.changed +area: device +attributes: + lk.device.low_power.enabled: bool +cadence: on change (+ initial value); `None` (no source on the platform, the default) is no + reading: no event, no stretch +platforms: ios, macos, android — optional elsewhere +``` + +```yaml +event: lk.device.app_state.changed +area: device +attributes: + lk.device.app_state: enum(foreground | background) +cadence: on change (+ initial value); entering background also forces a flush +platforms: all +``` + +```yaml +event: lk.device.memory.changed +area: device +attributes: + lk.device.memory.pressure: enum(normal | warning | critical) + # Apple: DispatchSource memory-pressure levels; Android onTrimMemory: RUNNING_LOW / + # BACKGROUND → warning, RUNNING_CRITICAL / COMPLETE → critical +cadence: on change (+ initial value) +platforms: ios, macos, android — optional elsewhere +``` + +```yaml +event: lk.device.network.changed +area: device +attributes: + network.connection.type: enum(wifi | cell | wired | vpn | bluetooth | other | unavailable | unknown) # OTel semconv + lk.device.network.expensive: bool # cellular / hotspot (NWPath.isExpensive, metered) + lk.device.network.constrained: bool # Low Data Mode / Data Saver / navigator.connection.saveData +cadence: on change of any attribute (+ initial value) +platforms: ios, macos, android — web: Chromium only +``` + +```yaml +event: lk.device.battery.changed +area: device +attributes: + hw.battery.charge: double # 0.0–1.0 (OTel hardware semconv) + hw.battery.state: enum(charging | discharging) # OTel hardware semconv +cadence: on charging change and when the level crosses 20 % or 10 % unplugged — never per + percent; silent where the level is unknown (desktops, tvOS) +platforms: ios, android — optional elsewhere +``` + +```yaml +event: lk.device.audio_route.changed +area: device +attributes: + lk.device.audio_route.reason: enum(new_device | old_device_unavailable | category_change | override | wake_from_sleep | no_suitable_route | route_configuration_change | unknown) # AVAudioSession names; `unknown` where the platform gives none + lk.device.audio_route.outputs: string # comma-separated enum(speaker | receiver | wired_headset | bluetooth | car_audio | air_play | hdmi | usb | other) +cadence: on change +platforms: ios — android: audio device callbacks; optional elsewhere +``` + +```yaml +event: lk.device.audio.interruption +area: device +attributes: + lk.device.audio.interruption: enum(began | ended) +cadence: on change +platforms: ios — android: audio focus loss/gain; optional elsewhere +``` + +```yaml +event: lk.device.capture.failed +area: device +severity: warn +attributes: + lk.device.capture.device: enum(camera | microphone | screen_share) + lk.device.capture.reason: enum(permission_denied | not_found | in_use | disconnected | other) # the getUserMedia failure taxonomy +cadence: on failure +platforms: all — ios: authorization status, capture interruptions; android: permission checks, camera callbacks; web: DOMException names +``` + +## Cadence policy + +`flush_interval × factor` and `stats_window × factor`, capped at 4× (60 s → 4 min at the default +cadence: one export and one RTC window per minute, conservative for the collector's shared +per-project quota). Factors multiply; a change applies at the next tick, and a *shorter* period +applies at once (pressure relieved → no waiting out a stretched period). + +| Condition | factor | source | +|---|---|---| +| thermal `serious` | 2 | host, `DeviceState.thermal` | +| thermal `critical` | 4 | host | +| memory pressure `warning` | 2 | host, `DeviceState.memory` | +| memory pressure `critical` | 4 | host | +| low-power mode | 2 | host | +| background | 2 | host | +| battery ≤ 20 % and unplugged | 2 | host, `DeviceState.battery_*` | +| constrained network (Low Data Mode / Data Saver) | 2 | host, `DeviceState.network_constrained` | +| encoder CPU-limited: an outbound track's `qualityLimitationDurations.cpu` grew within the last 60 s | 2 | core, from `record_stats` | + +CPU is never measured by the pipeline itself (measuring CPU costs CPU): thermal state is the OS's +judgement and `qualityLimitationReason` is WebRTC's. The core also paces the platform's +`getStats()` polling (`Scope::stats_poll_interval_ms`): every second while a subscribe waits for +its first media, else twice per (stretched) window — 30 s by default. + +## Upload policy — telemetry never wins over media + +Uploads are shaped, not just batched: + +- **One request in flight**, oldest batch first, polled by the exporter's loop: commands, ticks, + subscribe deadlines and wake-ups are served while it is out. Each answer is classified by the + core (see *Collector answers*); a transport returns status, headers and body and fails only + without a response. A pause stops uploads **to that destination** — other projects carry on — + never collection: records keep landing in the cache and ship when it lifts. +- **Retry:** local backoff starts at 1 s and doubles per consecutive failure up to 60 s, with + full jitter (a uniform wait in `[0, backoff]`). A delay the server names — `Retry-After` + (delay-seconds or HTTP-date, RFC 9110 §10.2.3) or `RetryInfo.retry_delay` — is honored in + full even when longer, validated (garbage ignored, negative → now, overflow ignored) and + clamped only to the 24 h age limit; neither shutdown nor a hold's escape cuts it short. + Running out of patience never deletes: only the cache's age and size bound what is kept. +- **Budget:** while a Room is in a call (it has a server and has not disconnected), at most + `max_batches_per_upload` (default 4) requests per flush interval — an allowance refilled at + each tick, 413 halves included, each request at most `max_batch_bytes` of protobuf before gzip + — however many wake-ups happen; a backlog (offline period, previous launch) waits its turn + oldest first. With no Room in a call nothing is metered: each pass sends the whole cache. A new + batch is encoded only at the tick, when the queue crosses `flush_threshold_bytes`, or when the + app enters the background; a lifted route, token or hold only re-runs a pass over what is + cached (within the allowance); a subscribe or a cadence change only re-reads deadlines. + Entering the background and an explicit `flush()` drain the whole cache without the allowance, + within every hold and pause. `shutdown` drains without the budget within + `export_timeout_ms`, then cancels the request on the wire and stops the exporter; what did not + go out stays cached. +- **Holds** — nothing is sent, everything keeps flowing into the write-ahead cache. The policy is + evaluated before every request, not once per pass, and a hold changing (or the app entering the + background, or the cadence changing) wakes the exporter: + - *hard* (no escape, for as long as they last): the device is offline; no usable token + (missing, expired, grant-less, refused); the project disabled data recording; the opt-out. + - *soft* (at most 60 s, on its own clock rather than the next tick, then one batch goes out and + the hold starts over — the cap that bounds the policy when its signals lie): an `lk.connect` or `lk.reconnect` span is open (signaling + and ICE/DTLS own the uplink); Low Data Mode / Data Saver; battery ≤ 10 % unplugged. The cap is + not scheduled while a hard hold is on; it resumes when the hard hold lifts. + `qualityLimitationDurations.bandwidth` is deliberately *not* a hold: WebRTC reports it for + minutes during a normal ramp-up and for as long as an encoder stalls. +- **Bytes:** bodies are gzipped (level 1, `Content-Encoding: gzip`) when cached, so a batch is + 5–10× smaller on disk and on the wire and a replay costs no CPU; a cached batch is CRC-checked + before it is sent. A request never carries more than `max_batch_size` (512) records, nor more + than `max_batch_bytes` (1 MiB) of encoded protobuf — checked on the encoded body, session + attributes and the self-report included (the report takes one of the `max_batch_size` + places); a batch over it is halved until it fits, and a single record over it is dropped and + counted as `oversized`. Caller strings a span or scope retains are bounded: names, keys and + error types ≤ 128 bytes, values and Room identities ≤ 1 KiB (anything longer is not kept and + is counted as `invalid`). When the queue reaches + `flush_threshold_bytes` (256 KiB) it is exported at once instead of at the next tick. +- **Backlog:** nested bounds, every eviction counted — the queue (`max_queue_size`, 2048 + records), the cache's size (`max_cache_bytes`, 4 MiB compressed) and file count (512 + batches), and age (24 h, enforced at start and while running; the monotonic clock must agree + for this launch's own batches, so a clock set forward cannot expire them). Oldest goes first. +- **Durability:** a batch is committed when the cache's `push` returns: `FileCache` writes a + `.tmp`, `fsync`s it, renames it into place and `fsync`s the directory (on Unix; elsewhere a + directory cannot be synced and that step is skipped). If any step fails the file is removed + and the batch is kept in memory instead (counted as a write error) — never claimed committed. + A 413 split is journaled (halves written, synced and renamed to `@.pend`, + directory synced, parent deleted = commit point, halves published); an interrupted split is + finished or rolled back when the cache opens, before anything is evicted, so every record is + there exactly once. A split that fails after its commit point has succeeded: its halves wait in + the journal and the next listing of the cache publishes them (so they upload, and an opt-out + purges and counts them). Splits and recovery share a process-wide lock, so a reconfigure's + cache on the same directory never sees the draining pipeline's split half-done; stray files + are swept only when a cache opens. A crash loses what was not committed yet — records since the last tick + (≤ one flush interval, ≤ 2048 queued), open spans, open RTC windows. Duplicates: a crash + between an answer and its delete sends that one batch again at the next launch; a delete that + fails without a crash leaves the batch pending deletion (never sent again in this launch, but + sent once more after a restart if it is still there). A disk that refuses writes gets batches + kept in memory (bounded by `max_cache_bytes`, oldest evicted and counted), which survive + failures but not the process. A batch that exists but cannot be read right now (file + protection, permissions) is kept, not counted corrupt. +- **Priority:** every request carries `Priority: u=7` (RFC 9218, lowest urgency) for HTTP/2+ + hops that implement it. +- **Redirects:** a 3xx the transport returns drops the batch. Transports must not forward + `Authorization` across origins; the `livekit-net` native client strips it when a redirect + changes the host or the port (tested). A scheme-only change on the same explicit port is not + covered by those tests. +- **Threads:** pipeline work runs on the SDK's runtime; none of it is on a media or UI thread and + `emit`/`record_stats` never block on it. + +### Collector answers + +| Answer | Batch | Pipeline | +|---|---|---| +| 2xx | removed | — | +| 2xx with OTLP `partial_success` | removed; rejected records counted, never retried | — | +| 400, 3xx, other 4xx | dropped, counted `rejected` | — | +| 413 | replaced by two halves in one cache transaction (both committed where the batch was, on disk stays on disk, nothing evicted in between) and retried at once, down to one record; if the cache cannot take the halves the batch stays whole; a lone oversized record is dropped, counted `oversized` | — | +| 401/403 "data recording is disabled by owner" | purged with the project's whole cache | project silent for the process | +| other 401/403 | kept | that token is never sent again; the project's batches wait for the next token | +| 404 | purged with the project's cache | project silent until its next token (a new connect or refresh) | +| 429 | kept | paused for `Retry-After`, else `RetryInfo`, else 60 s | +| 503 with `Retry-After`/`RetryInfo` | kept | paused for that long | +| 502, 503, 504; any error with `RetryInfo` (Cloud's retryable 500) | kept | paused for the named delay, else backoff | +| other 5xx | dropped, counted `rejected` | — | +| no answer: timeout, connection, DNS, TLS | kept | backoff; never a capability decision | +| invalid request (transport-side) | dropped, counted `rejected` | — | + +## Log records + +A `TelemetryEvent` with an empty `name` is a plain log record (OTLP log without `event_name`): +`severity` + `body` (the message) + `code.function.name`, `code.file.path`, `code.line.number` +(semconv), `lk.log.source` (`sdk` | `ffi` | `webrtc`) and `lk.log.logger` (type, module or file). The +platform hands the core a typed `LogRecord` via `log(record)`; the core applies the floor: WebRTC only +at `error`, the SDK and the core at the configured `log_severity`, the core's own telemetry module +never. Only `warn` and `error` records +leave the device; `trace`/`debug`/`info` are dropped in `emit`. + +The core's own warnings and errors (Rust `log` records from `livekit*` targets, never +`livekit_telemetry*`) reach the pipeline by themselves through `livekit-uniffi`'s log forwarder: +it copies them as `ffi` records — every JWT-shaped substring (`eyJ…` `.` … `.` …) masked as +``, so a token quoted in an error never leaves the device — and forwards the console entry +unchanged. The copy is active wherever the platform calls `log_forward_bootstrap` (Swift's +default `OSLogger` with `ffi: true` does), whatever level it passes: that level filters only what +is forwarded to the platform's console (the global `log` level is kept at `warn` or looser) — Rust allows one logger per process, and without the +forwarder the core's records go nowhere. Platforms must not feed forwarded Rust log entries +(what `log_forward_receive` returns) to `telemetry_log` / `log(record)`: the core already copied +them, so they would be counted twice. `telemetry_log` is for the platform's own and WebRTC's +lines. + +## App API: custom events, correlation attributes, opt-out + +Three things an app can do; everything else is the SDK's. + +- **Custom events** are room-scoped: `Scope::emit_custom(name, attributes)` (Swift: + `room.emitTelemetryEvent(_:attributes:)`). The core prefixes the name with `custom.` + (`checkout` → `custom.checkout`), so a custom event can never collide with, or spoof, an + `lk.*` event and the backend can filter or quota the namespace as a whole. Severity is `info`; + custom events count against the flood guard like any discrete event. +- **Correlation attributes** are room-scoped too: `Scope::set_attribute(key, value | nil)` — + `app.call_id`, `enduser.id` — inherited by the room's subsequent logs, spans, events and RTC + windows. Values are snapshotted when a record is captured: a later change never rewrites what + is queued, and a change closes the Room's open RTC windows first, so a window never mixes + readings taken under two values. A custom event's own attributes override the room's. There is no public + process-wide attribute (a mutable global would leak across simultaneous rooms); the + process-level set is internal, for SDK metadata. +- **Limits:** names and keys ≤ 128 UTF-8 bytes, string values ≤ 1024 bytes, ≤ 64 attributes per + room and per event. SDK-owned keys are reserved: `lk.*` (room, participant, track, outcome …) + and `session.id`. Anything else is rejected — never truncated, a truncated id collides — and + counted as `lk.telemetry.dropped.invalid`. +- **Opt-out** is process-wide and synchronous: when `telemetry_disable()` returns, no scope is + handed out, every existing scope and instrument captures nothing and later configures are + refused; the purge — cancel scheduled work, delete everything not yet sent (queue, open spans + and windows, every cached batch on disk) — then runs on the core's runtime + (`telemetry_shutdown` / `telemetry_flush` await it; `global::disable()` returns it as a + future). + Install, instrument start/stop and the opt-out run under one lifecycle lock, so an instrument + never starts after the opt-out stopped it. Every pipeline generation still alive (one replaced + but still draining included) is revoked — the flag is checked under the lock of every structure + that commits data (queue, spans, RTC windows, the cache's write gate), which the purge also + takes, so nothing lands after it — then its exporter's request on the wire is cancelled and + the exporter awaited, and later configures in the process are refused. The pull queue never + hands out a request its exporter gave up on, whether the host awaits `next()` or polls + `try_next()` (Dart, from a timer: no Rust future ever holds a continuation of an isolate that + may die; such hosts pass no instruments either, so Rust never calls into them). Data already sent cannot be recalled: at most the + one batch per generation already on the wire, or already handed to a pull-queue host. If the + storage refuses a delete the purge is reported incomplete (logged; `global::disable` and + `Telemetry::purge` return `false`; the exported `telemetry_disable` returns nothing). Platforms + mark the call TODO pending the token discussion. + +## Flood guard + +Discrete events (`emit`) are capped at `max_events_per_10min` (default 300, design doc); what +exceeds it is dropped and reported as `lk.telemetry.dropped.rate_limited`. `lk.rtc.stats.sample` +windows and `lk.telemetry.report` are exempt. + +```yaml +event: lk.rtc.stats.sample +area: rtc +severity: info +cadence: one per track and direction per stats window (default 60 s, stretched by the cadence + factor); closed early on background, when the track leaves (`track_ended`), at + disconnect and at shutdown. Produced by the core from raw readings — one getStats() + report per peer connection (`record_peer_stats`), polled when the core says + (`stats_poll_interval_ms`). Keyed by session and track: two rooms receiving the same + track keep two windows. +attributes: + lk.track.sid: string + lk.track.kind: enum(audio | video) + lk.track.direction: enum(inbound | outbound) + lk.rtc.codec: string # mimeType, when known + lk.rtc.window_ms: int # actual window length + lk.rtc.samples: int # readings in the window + # cumulative counters — the last reading's value, monotonic (W3C webrtc-stats model) + lk.rtc.bytes: int + lk.rtc.packets: int + lk.rtc.packets_lost: int # inbound + lk.rtc.freeze_count: int # inbound video + lk.rtc.freezes_duration_ms: int # inbound video + lk.rtc.concealed_samples: int # inbound audio + lk.rtc.concealment_events: int # inbound audio + lk.rtc.jitter_buffer_delay_ms: int # inbound + lk.rtc.jitter_buffer_emitted_count: int # inbound + lk.rtc.quality_limitation.bandwidth_ms: int # outbound video + lk.rtc.quality_limitation.cpu_ms: int # outbound video + lk.rtc.quality_limitation.other_ms: int # outbound video + lk.rtc.pause_count: int # inbound video + lk.rtc.pauses_duration_ms: int # inbound video + lk.rtc.silent_concealed_samples: int # inbound audio + lk.rtc.interruption_count: int # inbound audio + lk.rtc.interruptions_duration_ms: int # inbound audio + # gauges — min / max / avg over the window + lk.rtc.jitter_ms.{min,max,avg}: double + lk.rtc.rtt_ms.{min,max,avg}: double # remote-inbound RTT for outbound, candidate-pair for inbound + lk.rtc.fps.{min,max,avg}: double # video + lk.rtc.audio_level.{min,max,avg}: double # audio +platforms: all +``` + +```yaml +event: lk.room.disconnected +area: session +severity: info (client_initiated) | warn (anything else) +cadence: once, when the Room leaves connected for good — never on a reconnect +attributes: + lk.disconnect.reason: enum(client_initiated | duplicate_identity | server_shutdown | participant_removed | room_deleted | state_mismatch | join_failure | migration | signal_close | room_closed | user_unavailable | user_rejected | sip_trunk_failure | connection_timeout | media_failure | agent_error | reconnect_failed | unknown) # the protocol's DisconnectReason, plus the client giving up +platforms: all +``` + +## Spans + +A span is **one attempt** at an operation. The scope (one Room connection lifetime, across +reconnects) is the trace; its id is generated by the core when the pipeline starts and rides on +every span and log record. Spans are exported when they end — never a long-lived scope span. + +| Rule | Value | +|---|---| +| Names | `lk.connect`, `lk.reconnect`, `lk.publish`, `lk.subscribe` — verbs, never ids | +| Kind | `CLIENT` for connect/reconnect (a call to the SFU), `INTERNAL` otherwise | +| Status | OTel `Unset` on success **and** cancellation, `Error` (+ `error.type`, message) on failure | +| `lk.outcome` | always present: `ok` \| `error` \| `cancelled` — rollups read this, never the status | +| `error.type` | platform-defined, a type name (≤ 128 bytes), never a message: e.g. Swift sends `LiveKitError.`, `CancellationError` or the Swift error type; dashboards group by it per `service.name` | +| Checkpoints | span events in the span's envelope (`ws_open`, `join_recv`, `pc_connected`, `attempt 2 full`, …); real events stay log records pointing at the span via `span_id` | +| Limits | 128 events and 128 attributes per span (OTel defaults); 256 open spans per pipeline | + +```yaml +span: lk.connect +kind: client +attributes: + lk.connect.attempt: int # 1 for the user-initiated connect +checkpoints: + required: ws_open, signal, join_recv, pc_created # every platform, in this order + best-effort: engine, pc_connected, offer_sent, answer_sent, room_connected # where the SDK has the moment +outcome: ok | error (error.type) | cancelled +``` + +Dashboards compute connect phases only from the required checkpoints; best-effort ones refine +a platform's own view and may be missing or ordered differently (Android reports seven of the +nine today). + +```yaml +span: lk.reconnect +kind: client +attributes: + lk.reconnect.reason: enum(signal_disconnected | publisher_failed | subscriber_failed | transport_failed | switch_candidate | network_changed | debug | unknown) + lk.reconnect.mode: enum(quick | full) # mode of the last attempt + lk.reconnect.attempts: int +checkpoints: "attempt " per attempt +outcome: ok | error | cancelled # cancelled when disconnect() or a newer reconnect wins +``` + +```yaml +span: lk.publish +kind: internal +parent: the ambient span, when any; a pre-connect publish (a microphone published before the + connect completes) is its own span in the session's trace — `lk.connect` has ended by then +attributes: + lk.track.kind: enum(audio | video) + lk.track.source: enum(camera | microphone | screen_share | screen_share_audio | unknown) + lk.track.sid: string # on success +outcome: ok | error (error.type) | cancelled +``` + +```yaml +span: lk.subscribe +kind: internal +starts: when the intent to subscribe exists — a remote publish under autoSubscribe, or the + manual subscribe call. Tracks already in the room at join: platforms call + `subscribe_started` for each at connect (the join response lists them), so `lk.subscribe` + measures from join on every platform; a `subscribed` with no intent before it still opens + the span (a fallback, measured from the confirmation) +ends: at first media (the first inbound stats reading with bytes; the core sees it) → ok; + unsubscribe / unpublish before media → cancelled; + subscription failure → error; no media within 30 s → error (error.type = timed_out), + enforced on the core's own clock (no reading needed) and kept through a later disconnect +owner: the core (`Scope::subscribe_started / subscribed / track_ended / subscribe_failed`); + an SDK only reports the remote track's lifecycle +attributes: + lk.track.sid: string + lk.track.kind: enum(audio | video) + lk.track.source: enum(camera | microphone | screen_share | screen_share_audio | unknown) + lk.participant.remote_identity: string +checkpoints: subscribed, first_media +``` + +## Typed surface + +Everything the SDKs have in common enters the core typed; the core owns the keys, the bodies and +the policy. `Attribute { key, value }` survives only as the open bag: `emit_custom`, +`set_attribute`, and `Span::set_attribute` for app-defined spans. + +| Platform calls | The core produces | +|---|---| +| `Scope::set_server(url, token)` — at connect and on every token refresh | the room's destination: `https:///observability/client/{logs,traces}/otlp/v0` for Cloud hosts, the token's grant and expiry read, its batches routed to its project with its own token | +| `Scope::emit_custom(name, attributes)`, `Scope::set_attribute(key, value)` | `custom.` events and correlation attributes, validated and snapshotted (see *App API*) | +| `telemetry_disable()` | the opt-out, in effect when it returns: capture stops; everything unsent is then deleted | +| `TelemetryConfig.sdk: TelemetryResource { sdk: Sdk, sdk_version, os_name, os_version, device_model }` | `service.name = livekit-client-`, `service.version`, `os.*`, `device.model.identifier`, plus `telemetry.sdk.*` | +| `log(LogRecord { severity, source: LogSource, body, logger, function, file, line, timestamp_ns, span_id })` | a record with `code.function.name`, `code.file.path`, `code.line.number`, `lk.log.source`, `lk.log.logger`; the per-source floor (WebRTC at `error`, own module never) | +| `Scope::set_room(RoomIdentity { sid, name, participant_sid, participant_identity })` | `lk.room.*`, `lk.participant.*` on every record of the scope | +| `Scope::start(SpanName, parent) -> Span`; `Span::detached(name)` | an OTLP span (`lk.connect` / `lk.reconnect` are `client`, the rest `internal`); `Reconnect { reason }` sets `lk.reconnect.reason` | +| `Span::step(SpanStep)` | a span event named `ws_open` … `room_connected`, `subscribed`, `first_media`, `attempt N quick|full` (which also sets `lk.reconnect.attempts` / `.mode`) | +| `Span::set_track(SpanTrack { sid, kind, source, remote_identity })` | `lk.track.sid`, `lk.track.kind`, `lk.track.source`, `lk.participant.remote_identity` | +| `Span::end(outcome, error)` / `fail(error)` / `cancel()` | status, `error.type`, `lk.outcome`; ending twice is a no-op | +| `Span::describe()` | `lk.connect: ws_open +1.49s, signal +0.03s, total 1.83s, ok` — the console line, identical on every platform | +| `Span::context()` | `TraceContext { trace_id, span_id }` for log correlation; `None` when detached | +| `device_event(DeviceEvent::{AudioRouteChanged, AudioInterruption, CaptureFailed})` | `lk.device.audio_route.changed`, `lk.device.audio.interruption`, `lk.device.capture.failed` with display bodies; every value is a shared enum (`AudioOutput`, `CaptureDevice`, `CaptureFailure`) | +| `Scope::subscribe_started(SpanTrack)`, `subscribed(SpanTrack)`, `subscribe_failed(sid, error_type)` | the `lk.subscribe` span, ended by the core at the first inbound reading with bytes, or `timed_out` after 30 s; tracks already in the room at join: `subscribe_started` at connect (a lone `subscribed` still opens the span, as a fallback) | +| `Scope::track_ended(sid)` | a pending subscribe ends (`cancelled`, or `timed_out` past its deadline); the track's last RTC window ships; its per-track state is retired | +| `Scope::record_peer_stats(Vec, tracks: {MediaStreamTrack id → sid}, ts)` | one peer connection's raw `getStats()` report → the core finds each track's RTP streams (`trackIdentifier`, or the `media-source` an `outbound-rtp` names), resolves codec / RTT, converts seconds to ms and records one sample per stream | +| `Scope::stats_poll_interval_ms()` | when to poll `getStats()` next: 1 s while a subscribe awaits first media, else half the stretched window | +| `Scope::record_stats_report(sid, kind, direction, Vec, ts)` | the same mapping for one track's report (platforms that poll per track) | +| `DisconnectReason::from_proto(i32)`, `ReconnectReason::from_proto(i32)` | the protocol numbers → the shared enums | +| `Scope::log(LogRecord)` | a record filed under the session without an ambient span (Dart has no task-local outside a zone); same floor and filters as `Telemetry::log` | +| `Scope::disconnected(DisconnectReason)` | `lk.room.disconnected` with `lk.disconnect.reason` — info when the client hung up, warn otherwise; pending subscribes end, the session's RTC windows ship and its state is retired | +| `RtcStatsSample.layer` (rid, ssrc or stats id) | simulcast layers folded into one monotonic series per track before windowing | + +Timing rule: span calls are synchronous and stamp the clock inside the core, so the only skew is +the FFI call. Anything that may cross an executor hop before reaching the core (a log record) carries +its own `timestamp_ns` from capture. Context propagation — the "current" span — stays with the +platform runtime (task-local, coroutine context, zone); that is the one piece a core cannot own. diff --git a/livekit-telemetry/src/cache.rs b/livekit-telemetry/src/cache.rs new file mode 100644 index 000000000..b1ac99da3 --- /dev/null +++ b/livekit-telemetry/src/cache.rs @@ -0,0 +1,993 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{ + fs, io, + path::{Path, PathBuf}, + sync::{atomic::Ordering, Mutex}, +}; + +/// Queue of encoded OTLP batches between the [`Exporter`](crate::Exporter) and the transport. +/// +/// The exporter writes every batch here *before* trying the network (write-ahead), uploads +/// oldest-first, and removes what the collector accepted or rejected. Two implementations ship: +/// [`MemoryCache`] (default) and [`FileCache`] (survives crashes and restarts); anything else — +/// a database, a platform store — plugs in through +/// [`Telemetry::with_cache`](crate::Telemetry::with_cache). +/// +/// Ids are chosen by the exporter as `---[-]`, made of +/// filename-safe characters: sortable, so [`pending`](Self::pending) is a plain sort, and +/// prefixed with the creation time so an implementation can expire old batches without touching +/// file timestamps. Implementations bound their own footprint by evicting the oldest batches and +/// report what they evicted, so every loss is counted. Storage failures are reported, never +/// swallowed: a delete that did not happen must not look like one that did. +pub trait BatchCache: Send + Sync { + /// Store one encoded batch under `id`: once this returns `Ok` the batch is committed (for + /// [`FileCache`], on disk). Returns the ids of older batches evicted to make room (the + /// exporter counts their events as dropped). + fn push(&self, id: &str, body: &[u8]) -> io::Result>; + /// Replace the batch `old` with `new` (a split): all of `new` is committed before `old` + /// goes, and nothing is evicted in between, so a crash can duplicate records but never lose + /// them. On `Err`, `old` is still there. Returns the ids evicted afterwards. + fn replace(&self, old: &str, new: &[(String, Vec)]) -> io::Result>; + /// Ids of stored batches, oldest first. + fn pending(&self) -> Vec; + /// The body stored under `id`: `ErrorKind::NotFound` when it is gone, another error when it + /// exists but cannot be read right now (e.g. a locked device's file protection). + fn read(&self, id: &str) -> io::Result>; + /// Delete one batch; a batch already gone is not an error. + fn remove(&self, id: &str) -> io::Result<()>; + /// Discard everything (telemetry disabled: nothing may be replayed later); `Err` when + /// something could not be deleted. + fn clear(&self) -> io::Result<()>; + /// Ids evicted before the exporter could see them (over the bounds when the cache + /// was opened), handed over once so they are counted as lost. + fn take_evicted(&self) -> Vec { + Vec::new() + } +} + +/// Batches a cache will hold regardless of their size. `max_cache_bytes` is the size policy; +/// this only stops a long offline stretch at a 1 s cadence from leaving thousands of tiny files +/// in the directory. +// ponytail: a flat cap rather than another config knob; make it one if a host ever needs +// buffering at a scale where the file count matters. +pub(crate) const MAX_BATCHES: usize = 512; + +/// In-memory [`BatchCache`]: batches that could not be uploaded wait for the next attempt, +/// bounded by `max_bytes` and [`MAX_BATCHES`] (oldest evicted). Lost with the process. +// ponytail: a Vec with remove(0) — a handful of small batches, and it shares Vec code the +// binary already has instead of pulling in VecDeque's ring-buffer instantiations. +pub struct MemoryCache { + batches: Mutex)>>, + max_bytes: usize, +} + +impl MemoryCache { + /// An empty cache holding at most `max_bytes` of batches. + pub fn new(max_bytes: u64) -> Self { + Self { + batches: Mutex::new(Vec::new()), + max_bytes: usize::try_from(max_bytes).unwrap_or(usize::MAX), + } + } + + fn lock(&self) -> std::sync::MutexGuard<'_, Vec<(String, Vec)>> { + self.batches.lock().unwrap_or_else(|e| e.into_inner()) + } +} + +impl BatchCache for MemoryCache { + fn push(&self, id: &str, body: &[u8]) -> io::Result> { + let mut batches = self.lock(); + batches.push((id.to_owned(), body.to_vec())); + batches.sort_by(|a, b| a.0.cmp(&b.0)); + Ok(self.evict(&mut batches)) + } + + fn replace(&self, old: &str, new: &[(String, Vec)]) -> io::Result> { + let mut batches = self.lock(); + batches.retain(|(id, _)| id != old); + batches.extend(new.iter().cloned()); + batches.sort_by(|a, b| a.0.cmp(&b.0)); + Ok(self.evict(&mut batches)) + } + + fn pending(&self) -> Vec { + self.lock().iter().map(|(id, _)| id.clone()).collect() + } + + fn read(&self, id: &str) -> io::Result> { + self.lock() + .iter() + .find(|(i, _)| i == id) + .map(|(_, body)| body.clone()) + .ok_or_else(|| io::ErrorKind::NotFound.into()) + } + + fn remove(&self, id: &str) -> io::Result<()> { + self.lock().retain(|(i, _)| i != id); + Ok(()) + } + + fn clear(&self) -> io::Result<()> { + self.lock().clear(); + Ok(()) + } +} + +impl MemoryCache { + /// Drop the oldest batches until the rest fit; returns their ids. + fn evict(&self, batches: &mut Vec<(String, Vec)>) -> Vec { + let mut total: usize = batches.iter().map(|(_, b)| b.len()).sum(); + let mut evicted = Vec::new(); + while (total > self.max_bytes || batches.len() > MAX_BATCHES) && batches.len() > 1 { + let (id, body) = batches.remove(0); + total -= body.len(); + evicted.push(id); + } + evicted + } +} + +const EXT: &str = "otlp"; + +/// On-disk [`BatchCache`]: one file per encoded batch, so a crash or an offline shutdown loses +/// nothing and the next launch replays what is left. +/// +/// Files are written as `.tmp` and renamed into place, so a crash never leaves a half batch +/// readable; eviction is drop-oldest above `max_bytes` or [`MAX_BATCHES`] files. Age is the +/// exporter's call (24 h, judged by the timestamp in the id rather than file metadata — an +/// Apple "required reason" API). +pub struct FileCache { + dir: PathBuf, + max_bytes: u64, + /// File-count bound: [`MAX_BATCHES`] (lowered only by tests). + max_batches: usize, + /// What opening pruned, until [`BatchCache::take_evicted`] collects it. + evicted: Mutex>, + /// Publishing a journaled split failed: listings stop retrying (and logging) until the cache + /// writes again. + recovery_failed: std::sync::atomic::AtomicBool, + /// `recover` runs so far (tests only). + #[cfg(test)] + recoveries: std::sync::atomic::AtomicUsize, + /// Test-only fault points around every storage step (see [`Step`]). + #[cfg(test)] + fault: Mutex>, +} + +/// A storage step: the points a test can fail (return an error) or crash at (panic), +/// deterministically. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Step { + /// Creating and writing a `.tmp` (the path is passed). + Write, + /// `fsync` of a `.tmp`. + Sync, + /// Renaming a `.tmp` into place, or into a replacement's journal (`.pend`). + Rename, + /// `fsync` of the directory. + SyncDir, + /// Deleting a batch (including a replaced one — a split's commit point). + Delete, + /// Publishing a committed replacement half (`.pend` → batch). + Publish, +} + +#[cfg(test)] +type Fault = Box io::Result<()> + Send + Sync>; + +const PEND: &str = "pend"; +/// Separates parent and half in a journal file name. Batch ids never contain it (ids are digits, +/// hex, `l`/`t`, `-` and a URL-parsed host, which cannot hold `@`). +const JOURNAL: char = '@'; + +/// Held across a split's journaled section and across recovery, by every [`FileCache`] in the +/// process: a reconfigure opens a second cache on the directory while the replaced pipeline still +/// drains, and recovery must never see that pipeline's split half-done — it would delete the +/// journal of a split about to commit, losing the halves. +// ponytail: process-wide rather than per directory; splits are rare and short (a few fsyncs). +static SPLITS: Mutex<()> = Mutex::new(()); + +fn splits() -> std::sync::MutexGuard<'static, ()> { + SPLITS.lock().unwrap_or_else(|e| e.into_inner()) +} + +impl FileCache { + /// Open the cache directory (created if missing; its parent must exist), finish or roll back + /// a split interrupted by a crash — before anything is evicted — and discard stray or + /// half-written files (only here: a replaced pipeline may still be writing its own). + pub fn open(dir: impl Into, max_bytes: u64) -> io::Result { + let dir = dir.into(); + // ponytail: `create_dir`, not `create_dir_all` — the recursive variant drags in + // `Path::components` machinery (~3 KiB) for a parent the host always provides. + match fs::create_dir(&dir) { + Err(err) if err.kind() != io::ErrorKind::AlreadyExists => return Err(err), + _ => {} + } + let cache = Self { + dir, + max_bytes, + max_batches: MAX_BATCHES, + evicted: Mutex::new(Vec::new()), + recovery_failed: Default::default(), + #[cfg(test)] + recoveries: Default::default(), + #[cfg(test)] + fault: Mutex::new(None), + }; + cache.recover()?; + { + // Under `SPLITS` too: another cache's split in progress has `.tmp` files here. + let _splits = splits(); + for entry in fs::read_dir(&cache.dir)?.flatten() { + let path = entry.path(); + if !is_batch(&path) && path.extension().is_none_or(|e| e != PEND) { + let _ = fs::remove_file(&path); + } + } + } + let evicted = cache.prune()?; + *cache.evicted.lock().unwrap_or_else(|e| e.into_inner()) = evicted; + Ok(cache) + } + + /// A cache with a lower file-count bound (tests only). + #[cfg(test)] + pub(crate) fn open_with_max_batches( + dir: impl Into, + max_bytes: u64, + max_batches: usize, + ) -> io::Result { + let mut cache = Self::open(dir, max_bytes)?; + cache.max_batches = max_batches; + Ok(cache) + } + + /// Fail or crash at storage steps from now on (tests only). + #[cfg(test)] + pub(crate) fn inject( + &self, + fault: impl Fn(Step, &Path) -> io::Result<()> + Send + Sync + 'static, + ) { + *self.fault.lock().unwrap_or_else(|e| e.into_inner()) = Some(Box::new(fault)); + } + + #[allow(unused_variables)] + fn step(&self, step: Step, path: &Path) -> io::Result<()> { + #[cfg(test)] + if let Some(fault) = self.fault.lock().unwrap_or_else(|e| e.into_inner()).as_ref() { + return fault(step, path); + } + Ok(()) + } + + fn path(&self, id: &str, ext: &str) -> PathBuf { + self.dir.join(format!("{id}.{ext}")) + } + + /// A replacement half journaled under its parent: `@.pend`. + fn pend(&self, parent: &str, half: &str) -> PathBuf { + self.dir.join(format!("{parent}{JOURNAL}{half}.{PEND}")) + } + + /// The journal files in the directory. + fn journal(&self) -> io::Result> { + Ok(fs::read_dir(&self.dir)? + .flatten() + .map(|e| e.path()) + .filter(|p| p.extension().is_some_and(|e| e == PEND)) + .collect()) + } + + /// Write `body` to `tmp` and flush it to the device (`fsync`), so the rename that publishes + /// it can never expose a file whose contents are still only in the page cache. + fn write_synced(&self, tmp: &Path, body: &[u8]) -> io::Result<()> { + let write = |path: &Path| -> io::Result<()> { + self.step(Step::Write, path)?; + let mut file = fs::File::create(path)?; + io::Write::write_all(&mut file, body)?; + self.step(Step::Sync, path)?; + file.sync_all() + }; + match write(tmp) { + // iOS may purge the whole Caches subdirectory while the app runs: recreate it once. + Err(err) if err.kind() == io::ErrorKind::NotFound => { + fs::create_dir(&self.dir)?; + write(tmp) + } + other => other, + } + } + + fn rename(&self, from: &Path, to: &Path, step: Step) -> io::Result<()> { + self.step(step, to)?; + fs::rename(from, to) + } + + /// Make the directory's entries (renames, deletes) durable. Failures are errors; only a + /// platform that cannot sync a directory at all (not Unix) skips it. + fn sync_dir(&self) -> io::Result<()> { + self.step(Step::SyncDir, &self.dir)?; + #[cfg(unix)] + fs::File::open(&self.dir)?.sync_all()?; + Ok(()) + } + + /// The batch ids in the directory, oldest first; an unreadable directory is an error. A + /// split committed but not yet published (its publish failed) is published first, so its + /// halves are pending — uploaded, purged and counted — like any other batch. + fn ids(&self) -> io::Result> { + let list = || -> io::Result<(Vec, bool)> { + let (mut ids, mut journaled) = (Vec::new(), false); + for path in fs::read_dir(&self.dir)?.flatten().map(|e| e.path()) { + journaled |= path.extension().is_some_and(|e| e == PEND); + if let Some(id) = + path.file_stem().and_then(|s| s.to_str()).filter(|_| is_batch(&path)) + { + ids.push(id.to_owned()); + } + } + Ok((ids, journaled)) + }; + let (mut ids, journaled) = list()?; + if journaled && !self.recovery_failed.load(Ordering::Relaxed) { + match self.recover() { + Ok(()) => ids = list()?.0, + Err(err) => { + self.recovery_failed.store(true, Ordering::Relaxed); + log::debug!("journaled split not published yet: {err}; retried after a write"); + } + } + } + ids.sort_unstable(); + Ok(ids) + } + + /// Finish or roll back a split a crash interrupted. Its halves are journaled as + /// `@.pend` before the parent is deleted (the commit point): with the parent + /// still there the journal is discarded (the parent is whole); without it the halves are + /// published. A rebinding — one half whose id extends its host-less parent's with the project + /// the parent lacks — is rolled forward instead: a journal file is complete once it exists (its + /// contents are `fsync`ed before the rename), and only it records the project. Either way + /// every record is there exactly once. Under [`SPLITS`], so no split is in progress: a + /// journal next to its parent is a crash's or a failed rollback's. + fn recover(&self) -> io::Result<()> { + let _splits = splits(); + #[cfg(test)] + self.recoveries.fetch_add(1, Ordering::Relaxed); + let journal = self.journal()?; + if journal.is_empty() { + return Ok(()); + } + for pend in journal { + let Some((parent, half)) = + pend.file_stem().and_then(|s| s.to_str()).and_then(|s| s.split_once(JOURNAL)) + else { + let _ = fs::remove_file(&pend); + continue; + }; + // A host-less id ends with its empty host; a split's halves rename the sequence. + let rebinding = + parent.ends_with('-') && half.len() > parent.len() && half.starts_with(parent); + if self.path(parent, EXT).exists() { + if !rebinding { + fs::remove_file(&pend)?; + continue; + } + // Parent first: a crash in between leaves a committed journal, published below. + fs::remove_file(self.path(parent, EXT))?; + } + fs::rename(&pend, self.path(half, EXT))?; + } + self.sync_dir() + } + + /// Delete the oldest batches until the rest fit `max_bytes` and the file-count bound (a + /// committed split is published by the listing first). Returns the ids actually removed. + fn prune(&self) -> io::Result> { + let mut removed = Vec::new(); + let kept = self.ids()?; + let sizes: Vec = kept + .iter() + .map(|id| fs::metadata(self.path(id, EXT)).map(|m| m.len()).unwrap_or(0)) + .collect(); + let mut total: u64 = sizes.iter().sum(); + let mut count = kept.len(); + for (id, len) in kept.iter().zip(sizes) { + if total <= self.max_bytes && count <= self.max_batches { + break; + } + // A batch that could not be deleted still takes its room: keep evicting, report only + // what really went. + if self.remove(id).is_ok() { + total -= len; + count -= 1; + removed.push(id.clone()); + } + } + Ok(removed) + } + + /// The journaled section of [`BatchCache::replace`], under [`SPLITS`]. `Err` only before the + /// commit point, with `old` whole. + fn split(&self, old: &str, new: &[(String, Vec)]) -> io::Result<()> { + let _splits = splits(); + let pends: Vec = new.iter().map(|(half, _)| self.pend(old, half)).collect(); + let roll_back = |err: io::Error| { + for ((half, _), pend) in new.iter().zip(&pends) { + let _ = fs::remove_file(self.path(half, "tmp")); + let _ = fs::remove_file(pend); + } + Err(err) + }; + for ((half, body), pend) in new.iter().zip(&pends) { + let tmp = self.path(half, "tmp"); + if let Err(err) = + self.write_synced(&tmp, body).and_then(|()| self.rename(&tmp, pend, Step::Rename)) + { + return roll_back(err); + } + } + if let Err(err) = self.sync_dir() { + return roll_back(err); + } + if let Err(err) = self + .step(Step::Delete, &self.path(old, EXT)) + .and_then(|()| fs::remove_file(self.path(old, EXT))) + { + return roll_back(err); + } + // Committed: the records are the halves now, whatever follows. A failure from here leaves + // them journaled — published by the next listing (`ids`), purged by an opt-out — so the + // split succeeded and must not be read as "the parent is still there". + let published = self + .sync_dir() + .and_then(|()| { + new.iter().zip(&pends).try_for_each(|((half, _), pend)| { + self.rename(pend, &self.path(half, EXT), Step::Publish) + }) + }) + .and_then(|()| self.sync_dir()); + if let Err(err) = published { + log::warn!("split committed but not published yet ({err}); it will be"); + } + Ok(()) + } +} + +/// The durability boundary: a batch is committed once `push` returns `Ok` — its file was written +/// to a `.tmp`, `fsync`ed, renamed into place and the directory `fsync`ed; if any of that fails +/// the file is removed and `push` errs (the caller keeps the batch in memory instead). A crash +/// before that point loses only the batch being written (its `.tmp` is removed at the next +/// open). Deletes are `fsync`ed the same way. A split (`replace`) is journaled: see +/// [`recover`](FileCache::recover). +impl BatchCache for FileCache { + fn push(&self, id: &str, body: &[u8]) -> io::Result> { + let (tmp, dest) = (self.path(id, "tmp"), self.path(id, EXT)); + let written = self + .write_synced(&tmp, body) + .and_then(|()| self.rename(&tmp, &dest, Step::Rename)) + .and_then(|()| self.sync_dir()); + if let Err(err) = written { + // Not committed: nothing of it may stay (ENOSPC leaves a truncated `.tmp`, a failed + // directory sync a published file of unknown durability). + let _ = fs::remove_file(&tmp); + let _ = fs::remove_file(&dest); + return Err(err); + } + // Committed. Eviction trouble must not turn that into an error (the caller would keep a + // second copy in memory): it is logged, and the bound is enforced at the next push. + self.recovery_failed.store(false, Ordering::Relaxed); + Ok(self.prune().unwrap_or_else(|err| { + log::debug!("could not prune the cache: {err}"); + Vec::new() + })) + } + + /// Journaled: both halves are written, `fsync`ed and renamed to `@.pend`, the + /// directory is `fsync`ed, then `old` is deleted — the commit point — and the halves are + /// published. Before the commit point any failure rolls back and `old` stays whole; after it + /// the halves are safe in the journal and published by the next listing. Nothing is evicted + /// until the replacement is complete. + fn replace(&self, old: &str, new: &[(String, Vec)]) -> io::Result> { + self.split(old, new)?; + self.recovery_failed.store(false, Ordering::Relaxed); + Ok(self.prune().unwrap_or_default()) + } + + fn pending(&self) -> Vec { + self.ids().unwrap_or_default() + } + + fn read(&self, id: &str) -> io::Result> { + fs::read(self.path(id, EXT)) + } + + fn remove(&self, id: &str) -> io::Result<()> { + let path = self.path(id, EXT); + self.step(Step::Delete, &path)?; + match fs::remove_file(&path) { + Err(err) if err.kind() != io::ErrorKind::NotFound => Err(err), + _ => self.sync_dir(), + } + } + + fn clear(&self) -> io::Result<()> { + // Try every file — batches and journaled split halves — then report the first failure, + // an unreadable directory included. + let mut first = Ok(()); + for id in self.ids()? { + if let Err(err) = self.remove(&id) { + first = first.and(Err(err)); + } + } + for pend in self.journal()? { + if let Err(err) = fs::remove_file(&pend) { + first = first.and(Err(err)); + } + } + first.and_then(|()| self.sync_dir()) + } + + fn take_evicted(&self) -> Vec { + std::mem::take(&mut *self.evicted.lock().unwrap_or_else(|e| e.into_inner())) + } +} + +/// A cache that never refuses a batch: what the primary cannot store (disk full, directory +/// gone) goes to memory instead, so it still uploads — it just no longer survives the process. +/// Every spill is counted as `cache_write_errors`. +pub(crate) struct SpillCache { + primary: std::sync::Arc, + spill: MemoryCache, + counters: std::sync::Arc, + /// The consent barrier at the commit point: writes check the opt-out under `gate`, and + /// `clear` takes `gate` too, so no write can land after a purge has cleared. + revoked: std::sync::Arc, + gate: Mutex<()>, +} + +impl SpillCache { + pub fn new( + primary: std::sync::Arc, + max_bytes: u64, + counters: std::sync::Arc, + revoked: std::sync::Arc, + ) -> Self { + Self { + primary, + spill: MemoryCache::new(max_bytes), + counters, + revoked, + gate: Mutex::new(()), + } + } + + /// Hold the write gate; `Err` once the app opted out. + fn open_gate(&self) -> io::Result> { + let gate = self.gate.lock().unwrap_or_else(|e| e.into_inner()); + if self.revoked.load(std::sync::atomic::Ordering::SeqCst) { + return Err(io::Error::other("telemetry was disabled")); + } + Ok(gate) + } +} + +impl BatchCache for SpillCache { + fn push(&self, id: &str, body: &[u8]) -> io::Result> { + let _gate = self.open_gate()?; + match self.primary.push(id, body) { + Ok(evicted) => Ok(evicted), + Err(err) => { + let first = self.counters.snapshot().cache_write_errors == 0; + crate::stats::Counters::add(&self.counters.cache_write_errors, 1); + if first { + log::warn!( + "cannot write the telemetry cache ({err}); keeping batches in memory" + ); + } + self.spill.push(id, body) + } + } + } + + /// A batch spilled to memory is split in memory; one on disk is split on disk or not at all + /// — never moved from disk to memory, where a crash would lose it. + fn replace(&self, old: &str, new: &[(String, Vec)]) -> io::Result> { + let _gate = self.open_gate()?; + if self.spill.pending().iter().any(|id| id == old) { + self.spill.replace(old, new) + } else { + self.primary.replace(old, new) + } + } + + fn pending(&self) -> Vec { + let mut ids = self.primary.pending(); + ids.extend(self.spill.pending()); + ids.sort_unstable(); + ids + } + + fn read(&self, id: &str) -> io::Result> { + self.spill.read(id).or_else(|_| self.primary.read(id)) + } + + fn remove(&self, id: &str) -> io::Result<()> { + self.spill.remove(id)?; + self.primary.remove(id) + } + + fn clear(&self) -> io::Result<()> { + let _gate = self.gate.lock().unwrap_or_else(|e| e.into_inner()); + self.spill.clear()?; + self.primary.clear() + } + + fn take_evicted(&self) -> Vec { + self.primary.take_evicted() + } +} + +/// A batch file: our extension and a creation stamp in its name. Anything else in the directory +/// (a half-written `.tmp`, a stray or mangled file) is removed when the cache is pruned. +fn is_batch(path: &Path) -> bool { + path.extension().is_some_and(|e| e == EXT) && stamp(path).is_some() +} + +/// The creation time encoded in a batch file name. +fn stamp(path: &Path) -> Option { + path.file_stem()?.to_str()?.split('-').next()?.parse().ok() +} + +#[cfg(test)] +pub(crate) fn temp_dir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir() + .join(format!("livekit-telemetry-{tag}-{}", crate::event::now_unix_nanos())); + let _ = fs::remove_dir_all(&dir); + dir +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::*; + + /// Ids the exporter would mint: a real timestamp (fixed for the test run, so ids stay + /// stable however slow the disk) + seq. + fn id(n: u64) -> String { + static BASE: std::sync::OnceLock = std::sync::OnceLock::new(); + let base = + *BASE.get_or_init(|| crate::event::now_unix_nanos() / 1_000_000_000 * 1_000_000_000); + format!("{:020}-{n:06}-1", base + n) + } + + #[test] + fn memory_cache_evicts_oldest_beyond_max_bytes() { + let cache = MemoryCache::new(25); + for (n, body) in [b"aaaaaaaaaa", b"bbbbbbbbbb", b"cccccccccc"].into_iter().enumerate() { + cache.push(&id(n as u64), body).expect("push"); + } + assert_eq!(cache.pending(), [id(1), id(2)]); + assert_eq!(cache.read(&id(1)).expect("read"), b"bbbbbbbbbb"); + cache.remove(&id(1)).expect("remove"); + assert_eq!(cache.pending(), [id(2)]); + cache.clear().expect("clear"); + assert!(cache.pending().is_empty()); + } + + #[test] + fn file_cache_evicts_oldest_beyond_max_bytes_and_drops_stray_tmp() { + let dir = temp_dir("cache"); + let cache = FileCache::open(&dir, 25).expect("open"); + fs::write(dir.join("crashed.tmp"), b"half").expect("write"); + for (n, body) in [b"aaaaaaaaaa", b"bbbbbbbbbb", b"cccccccccc"].into_iter().enumerate() { + cache.push(&id(n as u64), body).expect("push"); + } + assert_eq!(cache.pending(), [id(1), id(2)], "10-byte batches under a 25-byte cap"); + assert_eq!(cache.read(&id(1)).expect("read"), b"bbbbbbbbbb"); + // A stray `.tmp` goes at the next open, never mid-run: it may be a replaced pipeline's + // write in progress. + assert!(dir.join("crashed.tmp").exists()); + let cache = FileCache::open(&dir, 25).expect("reopen"); + assert!(!dir.join("crashed.tmp").exists()); + cache.clear().expect("clear"); + assert!(cache.pending().is_empty()); + let _ = fs::remove_dir_all(&dir); + } + + /// Days offline at a one-second cadence are tiny batches, not big ones: the byte cap alone + /// would leave thousands of files in the directory. + #[test] + fn file_cache_caps_the_number_of_batches() { + let dir = temp_dir("count"); + let cache = FileCache::open(&dir, 1 << 20).expect("open"); + // `id` stamps the current second, so keep the ids rather than recomputing them. + let ids: Vec = (0..(MAX_BATCHES as u64 + 8)).map(id).collect(); + for batch in &ids { + cache.push(batch, b"x").expect("push"); + } + let kept = cache.pending(); + assert_eq!(kept.len(), MAX_BATCHES, "far under the byte cap, still bounded"); + assert_eq!(kept.first(), Some(&ids[8]), "the oldest went first"); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn file_cache_recreates_a_purged_directory() { + let dir = temp_dir("purged"); + let cache = FileCache::open(&dir, 1 << 20).expect("open"); + fs::remove_dir_all(&dir).expect("purge like iOS does"); + cache.push(&id(1), b"after purge").expect("push recreates the dir"); + assert_eq!(cache.pending(), [id(1)]); + let _ = fs::remove_dir_all(&dir); + } + + #[cfg(unix)] + #[test] + fn file_cache_failed_write_leaves_no_partial_file() { + use std::os::unix::fs::PermissionsExt; + let dir = temp_dir("readonly"); + let cache = FileCache::open(&dir, 1 << 20).expect("open"); + // Stand-in for ENOSPC: any write into the directory fails. + fs::set_permissions(&dir, fs::Permissions::from_mode(0o555)).expect("chmod"); + let result = cache.push(&id(1), b"no room"); + fs::set_permissions(&dir, fs::Permissions::from_mode(0o755)).expect("chmod back"); + assert!(result.is_err()); + assert_eq!(fs::read_dir(&dir).expect("dir").count(), 0, "no stray .tmp"); + let _ = fs::remove_dir_all(&dir); + } + + /// Findings r1-7 / r2-7 / r2-12: a real split, crashed at every one of its storage steps and + /// reopened under byte and file-count pressure, leaves every record exactly once — the + /// parent whole, or both halves — and recovery runs before anything is evicted. + #[test] + fn a_split_crashed_at_every_step_loses_and_duplicates_nothing() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let (parent, a, b) = (id(1), format!("{}a", id(1)), format!("{}b", id(1))); + let halves = [(a.clone(), vec![b'a'; 60]), (b.clone(), vec![b'b'; 60])]; + // Room for the parent and not a byte more / not a file more: a duplicate would force an + // eviction. + // (The split's two halves take two files: the file bound is two — a duplicate would + // need three.) + for (max_bytes, max_batches) in [(120, MAX_BATCHES), (1 << 20, 2)] { + let mut step = 0; + loop { + let dir = temp_dir(&format!("split-{step}-{max_batches}")); + let cache = + FileCache::open_with_max_batches(&dir, max_bytes, max_batches).expect("open"); + cache.push(&parent, &[b'p'; 120]).expect("push"); + let calls = Arc::new(AtomicUsize::new(0)); + let (counter, crash_at) = (calls.clone(), step); + cache.inject(move |_, _| { + if counter.fetch_add(1, Ordering::SeqCst) == crash_at { + panic!("killed"); + } + Ok(()) + }); + let crashed = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _ = cache.replace(&parent, &halves); + })) + .is_err(); + drop(cache); + let cache = + FileCache::open_with_max_batches(&dir, max_bytes, max_batches).expect("reopen"); + assert!(cache.take_evicted().is_empty(), "step {step}: nothing evicted"); + let pending = cache.pending(); + let whole = pending == [parent.clone()] + && cache.read(&parent).expect("parent").len() == 120; + let split = pending == [a.clone(), b.clone()]; + assert!(whole || split, "step {step}: {pending:?}"); + let _ = fs::remove_dir_all(&dir); + if !crashed { + assert!(split, "a replace that ran to the end split the batch"); + break; + } + step += 1; + } + assert!( + step >= 10, + "every write, sync, rename, delete and publish was crashed at: {step}" + ); + } + } + + /// Finding r3-2: a split that fails after its commit point (a failed publish, a failed + /// directory sync) has succeeded — its halves are safe in the journal and published by the + /// next listing — and an opt-out deletes the journal too, leaving the directory empty. + #[test] + fn a_split_failing_after_its_commit_point_is_kept_and_purgeable() { + use std::sync::atomic::{AtomicBool, Ordering}; + let (parent, a, b) = (id(1), format!("{}a", id(1)), format!("{}b", id(1))); + let halves = [(a.clone(), b"half a".to_vec()), (b.clone(), b"half b".to_vec())]; + for failing in [Step::Publish, Step::SyncDir] { + let dir = temp_dir(&format!("post-commit-{failing:?}")); + let cache = FileCache::open(&dir, 1 << 20).expect("open"); + cache.push(&parent, b"parent").expect("push"); + let committed = Arc::new(AtomicBool::new(false)); + let seen = committed.clone(); + cache.inject(move |step, _| { + if step == Step::Delete { + seen.store(true, Ordering::SeqCst); + } + if step == failing && seen.load(Ordering::SeqCst) { + return Err(io::Error::other("injected")); + } + Ok(()) + }); + let ids = cache.replace(&parent, &halves).expect("committed: not an error"); + assert!(!cache.pending().contains(&parent), "{failing:?}: the parent is gone"); + cache.inject(|_, _| Ok(())); // the storage recovers; then the app opts out + assert!(cache.clear().is_ok()); + assert_eq!( + fs::read_dir(&dir).expect("dir").count(), + 0, + "{failing:?}: nothing survives the opt-out" + ); + let _ = ids; + + // Without the opt-out, the next listing publishes the journaled halves. + let dir2 = temp_dir(&format!("post-commit-publish-{failing:?}")); + let cache = FileCache::open(&dir2, 1 << 20).expect("open"); + cache.push(&parent, b"parent").expect("push"); + let flag = Arc::new(AtomicBool::new(true)); + let fail = flag.clone(); + let committed = Arc::new(AtomicBool::new(false)); + let seen = committed.clone(); + cache.inject(move |step, _| { + if step == Step::Delete { + seen.store(true, Ordering::SeqCst); + } + if step == failing && seen.load(Ordering::SeqCst) && fail.load(Ordering::SeqCst) { + return Err(io::Error::other("injected")); + } + Ok(()) + }); + cache.replace(&parent, &halves).expect("committed"); + flag.store(false, Ordering::SeqCst); + cache.push(&id(9), b"next").expect("push"); + assert_eq!(cache.pending(), [a.clone(), b.clone(), id(9)], "{failing:?}: published"); + let _ = fs::remove_dir_all(&dir); + let _ = fs::remove_dir_all(&dir2); + } + } + + /// Finding r4-NB: a reconfigure opens a second cache on the directory while the replaced + /// pipeline still drains. Its open, pushes and listings must never see that pipeline's split + /// half-done — deleting the journal of a split about to commit would lose both halves. + #[test] + fn a_second_cache_on_the_directory_never_breaks_a_split_in_progress() { + let dir = temp_dir("two-generations"); + let (parent, a, b) = (id(1), format!("{}a", id(1)), format!("{}b", id(1))); + let halves = [(a.clone(), b"half a".to_vec()), (b.clone(), b"half b".to_vec())]; + let old = FileCache::open(&dir, 1 << 20).expect("open"); + old.push(&parent, b"parent").expect("push"); + let other = Arc::new(Mutex::new(None)); + let (handle, path) = (other.clone(), dir.clone()); + old.inject(move |step, _| { + // Journaled, parent still there, commit next: the new generation starts now. + let mut handle = handle.lock().expect("lock"); + if step == Step::Delete && handle.is_none() { + let path = path.clone(); + *handle = Some(std::thread::spawn(move || { + let new = FileCache::open(&path, 1 << 20).expect("open"); + new.push(&id(2), b"new generation").expect("push"); + })); + // Give it every chance to reach the directory mid-split. + std::thread::sleep(std::time::Duration::from_millis(100)); + } + Ok(()) + }); + old.replace(&parent, &halves).expect("split"); + let new = other.lock().expect("lock").take().expect("the new generation ran"); + new.join().expect("the new generation succeeded"); + let reopened = FileCache::open(&dir, 1 << 20).expect("reopen"); + assert_eq!(reopened.pending(), [a, b, id(2)], "every record exactly once"); + let _ = fs::remove_dir_all(&dir); + } + + /// Round-5 review: a journal that cannot be published is not retried (and logged) by every + /// listing — only again after the cache has written something. + #[cfg(unix)] + #[test] + fn a_stuck_journal_is_retried_after_a_write_not_every_listing() { + let dir = temp_dir("stuck-journal"); + let cache = FileCache::open(&dir, 1 << 20).expect("open"); + let (parent, half) = (id(1), format!("{}a", id(1))); + fs::write(cache.pend(&parent, &half), b"half").expect("journal"); + // The half's name is taken by a non-empty directory: the publishing rename fails. + fs::create_dir(cache.path(&half, EXT)).expect("blocker"); + fs::write(cache.path(&half, EXT).join("x"), b"").expect("blocker content"); + let before = cache.recoveries.load(Ordering::Relaxed); + for _ in 0..5 { + cache.pending(); + } + assert_eq!(cache.recoveries.load(Ordering::Relaxed), before + 1, "tried once"); + cache.push(&id(2), b"next").expect("push"); + assert_eq!(cache.recoveries.load(Ordering::Relaxed), before + 2, "a write retries"); + fs::remove_dir_all(cache.path(&half, EXT)).expect("unblock"); + cache.push(&id(3), b"next").expect("push"); + assert_eq!(cache.pending(), [half, id(2), id(3)], "published once unblocked"); + let _ = fs::remove_dir_all(&dir); + } + + /// Finding r2-12: a write killed half-way (bytes on disk, no rename) leaves no partial batch + /// and costs no committed one. + #[test] + fn a_write_killed_half_way_leaves_no_partial_batch() { + let dir = temp_dir("torn"); + let cache = FileCache::open(&dir, 1 << 20).expect("open"); + cache.push(&id(1), b"committed").expect("push"); + cache.inject(|step, path| { + if step == Step::Write { + fs::write(path, b"half of a batc").expect("partial"); + panic!("killed mid-write"); + } + Ok(()) + }); + let killed = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _ = cache.push(&id(2), b"half of a batch that never lands"); + })); + assert!(killed.is_err()); + drop(cache); + let cache = FileCache::open(&dir, 1 << 20).expect("reopen"); + assert_eq!(cache.pending(), [id(1)]); + assert_eq!(cache.read(&id(1)).expect("read"), b"committed"); + assert_eq!(fs::read_dir(&dir).expect("dir").count(), 1, "the torn .tmp is gone"); + let _ = fs::remove_dir_all(&dir); + } + + /// Finding r2-8: a push whose file or directory cannot be made durable is not committed — + /// it errs and leaves nothing behind — and a split that cannot be made durable before its + /// commit point keeps the parent. + #[test] + fn durability_failures_are_errors_not_silent_successes() { + for failing in [Step::Sync, Step::SyncDir] { + let dir = temp_dir("sync"); + let cache = FileCache::open(&dir, 1 << 20).expect("open"); + cache.push(&id(1), b"parent").expect("push"); + cache.inject(move |step, _| { + if step == failing { + return Err(io::Error::other("fsync failed")); + } + Ok(()) + }); + assert!(cache.push(&id(2), b"new").is_err(), "{failing:?}: not committed"); + assert_eq!(cache.pending(), [id(1)], "{failing:?}: nothing left of it"); + let halves = + [(format!("{}a", id(1)), b"a".to_vec()), (format!("{}b", id(1)), b"b".to_vec())]; + assert!(cache.replace(&id(1), &halves).is_err()); + assert_eq!(cache.pending(), [id(1)], "{failing:?}: the parent stays whole"); + let _ = fs::remove_dir_all(&dir); + } + } + + /// Finding r2-8: an unreadable directory is an error for `clear`, not an empty success. + #[cfg(unix)] + #[test] + fn clearing_an_unreadable_directory_fails() { + use std::os::unix::fs::PermissionsExt; + let dir = temp_dir("unreadable"); + let cache = FileCache::open(&dir, 1 << 20).expect("open"); + cache.push(&id(1), b"batch").expect("push"); + fs::set_permissions(&dir, fs::Permissions::from_mode(0o300)).expect("chmod"); + assert!(cache.clear().is_err(), "cannot list, cannot claim it is empty"); + fs::set_permissions(&dir, fs::Permissions::from_mode(0o755)).expect("chmod"); + assert_eq!(cache.pending(), [id(1)]); + let _ = fs::remove_dir_all(&dir); + } +} diff --git a/livekit-telemetry/src/destination.rs b/livekit-telemetry/src/destination.rs new file mode 100644 index 000000000..b8045ab60 --- /dev/null +++ b/livekit-telemetry/src/destination.rs @@ -0,0 +1,768 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Where batches go and with which credential — decided here, once, for every platform. +//! +//! A platform hands over exactly two things: the LiveKit server URL a room connects to and the +//! participant token it connects with (again on every token refresh). The core derives the +//! ingest URL, checks that the host is LiveKit Cloud, reads the token's unverified claims (the +//! observability grant, the expiry), and routes every batch to the project its session belongs +//! to, so two rooms talking to two projects never share a token or a destination. + +use std::{ + collections::HashMap, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use base64::{ + alphabet, + engine::{DecodePaddingMode, GeneralPurpose, GeneralPurposeConfig}, + Engine, +}; +use tokio::time::Instant; + +/// The environment variable that points every upload at a collector of your own — a local +/// OpenTelemetry collector for end-to-end tests. Not part of any platform API: set it in the +/// test process's environment before the pipeline starts. Either a base URL +/// (`http://localhost:4318`, OTLP paths `/v1/logs` and `/v1/traces` are appended) or a full logs +/// URL ending in `logs` (its traces URL is derived by replacing that segment). +pub const ENDPOINT_OVERRIDE_ENV: &str = "LK_TELEMETRY_ENDPOINT"; + +/// The longest a token is believed to stay valid, whatever its `exp` says. +const MAX_TOKEN_LIFETIME: Duration = Duration::from_secs(366 * 24 * 60 * 60); + +/// LiveKit Cloud project hosts: `.livekit.cloud` in production, +/// `.staging.livekit.cloud` in staging. Self-hosted servers have no ingest. +const CLOUD_SUFFIX: &str = ".livekit.cloud"; + +/// Which OTLP signal a batch carries; picks the ingest path. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Signal { + Logs, + Traces, +} + +/// A request target: URL per signal and the bearer token, when one is due. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct Target { + /// The project the batch is sent to (`None` for the test override): what its answer is + /// attributed to — a 404, a disable, a pause. + pub project: Option, + pub logs: String, + pub traces: String, + pub token: Option, +} + +impl Target { + pub fn url(&self, signal: Signal) -> &str { + match signal { + Signal::Logs => &self.logs, + Signal::Traces => &self.traces, + } + } +} + +/// What the unverified claims of a participant token say — enough to not send with a token the +/// collector is known to refuse. The signature is the collector's business. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct Token { + raw: String, + /// `observability.write` or `observability.clientWrite`. + grant: bool, + /// From `exp`, on the monotonic clock (so tests can move it). + expires: Option, + /// The credential's identity (a hash of the raw token): what a refusal is recorded against. + id: u64, +} + +impl Token { + pub fn parse(raw: &str) -> Self { + let claims = raw + .split('.') + .nth(1) + .and_then(|payload| { + const URL_SAFE: GeneralPurpose = GeneralPurpose::new( + &alphabet::URL_SAFE, + GeneralPurposeConfig::new() + .with_decode_padding_mode(DecodePaddingMode::Indifferent), + ); + URL_SAFE.decode(payload).ok() + }) + .and_then(|json| serde_json::from_slice::(&json).ok()); + let grant = claims.as_ref().is_some_and(|c| { + let observability = &c["observability"]; + observability["write"].as_bool() == Some(true) + || observability["clientWrite"].as_bool() == Some(true) + }); + // Untrusted input, read conservatively: a past or negative `exp` is expired, and so is + // one that is not a number at all; a fractional one counts in whole seconds; one beyond + // any real token lifetime is clamped (an `Instant` that far out would overflow). Only a + // token without `exp` has no known expiry. + let expires = claims.as_ref().and_then(|c| c.get("exp")).map(|exp| { + let now = + SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs_f64(); + let left = match exp.as_f64() { + Some(exp) if exp.is_finite() && exp > now => { + Duration::from_secs_f64((exp - now).min(MAX_TOKEN_LIFETIME.as_secs_f64())) + } + _ => Duration::ZERO, + }; + Instant::now().checked_add(left).unwrap_or_else(Instant::now) + }); + let id = { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + raw.hash(&mut hasher); + hasher.finish() + }; + Self { raw: raw.to_owned(), grant, expires, id } + } + + fn expired(&self) -> bool { + self.expires.is_some_and(|at| Instant::now() >= at) + } + + /// May be sent: carries the grant, not expired, not refused. + fn usable(&self, refused: &Refusals) -> bool { + self.grant && !self.expired() && !refused.contains(self) + } +} + +/// Why a project receives nothing (anymore). Each is logged once. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Dead { + /// Not a LiveKit Cloud host (self-hosted, a typo): there is no ingest to talk to. + NotCloud, + /// The project's own token carries no observability grant: the customer did not opt in. + NoGrant, + /// The collector answered 404: no client ingest on this host. Revived by the next token + /// (a new connect or refresh), so a misrouted answer cannot silence a project for good. + NotFound, + /// The collector said the project's data recording is disabled by its owner. + Disabled, +} + +/// One LiveKit Cloud project, keyed by its host. +#[derive(Debug, Default)] +struct Project { + /// The latest token any room handed over for the project: what process-level batches and + /// batches from a previous launch are sent with. + latest: Option, + /// A token with the grant was seen: the customer opted in. Refreshed tokens may lack the + /// grant (today's server drops it on refresh) without withdrawing that. + consented: bool, + dead: Option, +} + +/// Offer `new` for a token slot: a refused token is never taken, into any slot; a token with the +/// grant wins; one without it does not replace a usable granted token (today's refreshes drop +/// the grant). +fn offer(slot: &mut Option, new: &Token, refused: &Refusals) { + if refused.contains(new) { + return; + } + let keep = slot.as_ref().is_some_and(|t| t.raw == new.raw || (!new.grant && t.usable(refused))); + if !keep { + *slot = Some(new.clone()); + } +} + +/// Credentials the collector refused, by identity, shared by every slot: a refused token is never +/// sent again, whichever Room or project hands it over. An entry lives until its token expires; +/// tokens without `exp` stay refused for the life of the process. +// ponytail: grows with refusals of tokens without `exp` (LiveKit tokens always carry one); a cap +// if a server ever mints tokens without expiry. +#[derive(Debug, Default)] +struct Refusals(HashMap>); + +impl Refusals { + fn contains(&self, token: &Token) -> bool { + self.0.contains_key(&token.id) + } + + /// Returns whether the refusal is new. + fn add(&mut self, token: &Token) -> bool { + let now = Instant::now(); + self.0.retain(|_, expires| expires.is_none_or(|at| at > now)); + self.0.insert(token.id, token.expires).is_none() + } +} + +/// The owner of process-level records (device state, pre-room errors, self-telemetry): no Room, +/// so they go to the latest project with its latest token. +pub(crate) const PROCESS_OWNER: &str = "process"; + +/// Where a batch is headed. +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum Route { + /// Ready: send now. + Send(Target), + /// Keep it cached: no destination yet, or no usable token (missing, expired, grant-less, + /// refused) until the platform hands over a fresh one. A hard hold: no escape hatch sends + /// it anyway. + Wait, + /// Drop it: its project receives nothing. + Drop, +} + +/// Every project and session this process talks to, plus the test override. Tokens live here, +/// in memory only: never persisted, never in a batch. +#[derive(Debug, Default)] +pub(crate) struct Destinations { + projects: HashMap, + /// Each Room session's own latest token per project, by (host, session id): its batches for + /// that project are sent with it and never with another Room's, nor with the token it later + /// got for another project. + sessions: HashMap<(String, String), Token>, + /// Each Room session's first project: where the records it captured before it had a server + /// go. A Room's records never fall back to another Room's project. + rooms: HashMap, + refused: Refusals, + /// The host process-level batches (device state, pre-room logs, self-telemetry) go to: the + /// project most recently handed a token, while it is alive. + latest: Option, + /// [`ENDPOINT_OVERRIDE_ENV`]: every batch goes there, no Cloud rules, no token. + endpoint_override: Option, +} + +impl Destinations { + pub fn new(endpoint_override: Option<&str>) -> Self { + Self { + endpoint_override: endpoint_override.filter(|e| !e.is_empty()).map(override_target), + ..Self::default() + } + } + + /// The first project a Room session was routed to: where what it captured before it had a + /// server belongs. + pub fn first_project(&self, owner: &str) -> Option { + self.rooms.get(owner).cloned() + } + + pub fn has_override(&self) -> bool { + self.endpoint_override.is_some() + } + + /// Session `owner`'s server URL and token (at connect, and again on every refresh). Returns + /// the project host its batches carry, or `None` when the URL has no host. + pub fn set(&mut self, url: &str, token: &str, owner: &str) -> Option { + let Server { key: host, cloud } = Server::parse(url)?; + if self.endpoint_override.is_some() { + return Some(host); + } + let project = self.projects.entry(host.clone()).or_default(); + if project.dead.is_none() && !cloud { + project.dead = Some(Dead::NotCloud); + log::warn!("{host} is not LiveKit Cloud: client telemetry stays on this device"); + } + if matches!(project.dead, Some(Dead::NotCloud | Dead::Disabled)) { + return Some(host); + } + let token = Token::parse(token); + if project.dead == Some(Dead::NotFound) { + log::debug!("new token for {host}: trying its ingest again"); + project.dead = None; + } + if token.grant { + project.consented = true; + if project.dead == Some(Dead::NoGrant) { + project.dead = None; + } + } else if !project.consented && project.dead.is_none() { + project.dead = Some(Dead::NoGrant); + log::info!("the token for {host} has no observability grant: telemetry stays local"); + } + offer(&mut project.latest, &token, &self.refused); + let key = (host.clone(), owner.to_owned()); + let mut own = self.sessions.remove(&key); + offer(&mut own, &token, &self.refused); + self.sessions.extend(own.map(|own| (key, own))); + self.rooms.entry(owner.to_owned()).or_insert_with(|| host.clone()); + self.latest = Some(host.clone()); + Some(host) + } + + /// Where a batch goes: `host` is its project as captured with its records, `owner` the + /// session that produced them. A Room known to this process sends with its own latest token + /// for that project and waits when it has none; process-level batches and a previous + /// launch's use the project's latest. A Room's records captured before it had a server go + /// to its first project, never to another Room's. + pub fn route(&self, host: Option<&str>, owner: Option<&str>) -> Route { + if let Some(target) = &self.endpoint_override { + return Route::Send(target.clone()); + } + let room = owner.filter(|o| *o != PROCESS_OWNER); + let host = match (host, room) { + (Some(host), _) => host, + (None, Some(room)) => match self.rooms.get(room) { + Some(first) => first.as_str(), + None => return Route::Wait, + }, + (None, None) => match self.fallback() { + Some(host) => host, + // Nothing to talk to yet: wait. Only dead projects so far: nobody will ever + // take process-level data, so it is not kept either. + None if self.all_dead() => return Route::Drop, + None => return Route::Wait, + }, + }; + let Some(project) = self.projects.get(host) else { return Route::Wait }; + if project.dead.is_some() { + return Route::Drop; + } + let token = match room { + Some(room) if self.rooms.contains_key(room) => { + self.sessions.get(&(host.to_owned(), room.to_owned())) + } + _ => project.latest.as_ref(), + }; + match token { + Some(token) if token.usable(&self.refused) => Route::Send(Target { + project: Some(host.to_owned()), + logs: ingest_url(host, Signal::Logs), + traces: ingest_url(host, Signal::Traces), + token: Some(token.raw.clone()), + }), + _ => Route::Wait, + } + } + + /// The host of the project process-level batches go to: the latest, else any alive one. + fn fallback(&self) -> Option<&str> { + let alive = |host: &&String| self.projects.get(*host).is_some_and(|p| p.dead.is_none()); + self.latest + .as_ref() + .filter(alive) + .or_else(|| self.projects.keys().find(alive)) + .map(String::as_str) + } + + fn all_dead(&self) -> bool { + !self.projects.is_empty() && self.projects.values().all(|p| p.dead.is_some()) + } + + /// Whether a session routed to `host` should still collect: `false` once its project is + /// known to receive nothing, so its records are dropped at the door instead of cached. + pub fn alive(&self, host: Option<&str>) -> bool { + self.endpoint_override.is_some() || !matches!(self.route(host, None), Route::Drop) + } + + /// The collector refused `token` (401/403): it is not sent again, from any slot; batches that + /// would use it wait for the next one. + pub fn refuse(&mut self, token: &str) { + if self.refused.add(&Token::parse(token)) { + log::warn!("the collector refused a token; waiting for a new one"); + } + } + + /// Keep credentials only where something still needs them. `live` maps every session still + /// alive — held by its Room, or by records, windows or spans not yet cached — to its current + /// project; its credentials stay (for each project it was routed to: records it captured for + /// an earlier project may still be queued). `backlog` is (project or none, owner) of every + /// cached batch: their credentials stay too. Project-level copies stay while a live session + /// is routed there or the backlog has batches for the project (process-level batches: the + /// latest project). + pub fn retain_owners( + &mut self, + live: &HashMap>, + backlog: &std::collections::HashSet<(Option, String)>, + ) { + // A Room's batch captured before it had a server is cached without a project: it goes + // to that Room's first project, so it backs that project's credential. + let backlog: std::collections::HashSet<(Option, String)> = backlog + .iter() + .map(|(host, owner)| match (host, self.rooms.get(owner)) { + (None, Some(first)) if owner != PROCESS_OWNER => { + (Some(first.clone()), owner.clone()) + } + _ => (host.clone(), owner.clone()), + }) + .collect(); + let backed = + |host: &str, owner: &str| backlog.contains(&(Some(host.to_owned()), owner.to_owned())); + self.sessions.retain(|(host, owner), _| backed(host, owner) || live.contains_key(owner)); + self.rooms + .retain(|owner, _| live.contains_key(owner) || backlog.iter().any(|(_, o)| o == owner)); + let unrouted = backlog.iter().any(|(host, _)| host.is_none()); + let latest = self.latest.clone(); + for (host, project) in &mut self.projects { + let needed = live.values().any(|route| route.as_deref() == Some(host)) + || backlog.iter().any(|(h, _)| h.as_deref() == Some(host)) + || (unrouted && latest.as_deref() == Some(host)); + if !needed { + project.latest = None; + } + } + } + + /// How many credentials are held, Room and project copies together. + #[cfg(test)] + pub fn credentials(&self) -> usize { + self.sessions.len() + self.projects.values().filter(|p| p.latest.is_some()).count() + } + + /// Try `host`'s ingest again if it answered 404 (a reconnect). Returns whether it did. + pub fn revive(&mut self, host: &str) -> bool { + match self.projects.get_mut(host) { + Some(project) if project.dead == Some(Dead::NotFound) => { + project.dead = None; + true + } + _ => false, + } + } + + /// The project on `host` receives nothing (see [`Dead`] for how long). + pub fn kill(&mut self, host: &str, why: Dead) { + let project = self.projects.entry(host.to_owned()).or_default(); + if project.dead.is_none() { + match why { + Dead::NotFound => log::warn!("{host} has no client telemetry ingest; going silent"), + Dead::Disabled => log::warn!("{host} disabled data recording; going silent"), + Dead::NotCloud | Dead::NoGrant => {} + } + project.dead = Some(why); + } + } +} + +/// A server URL, parsed (WHATWG URL rules, the same a browser or `URLSession` applies): its +/// routing key — the canonical host, with the port when one is given — and whether it is a +/// LiveKit Cloud project a token may be sent to. +struct Server { + key: String, + cloud: bool, +} + +impl Server { + fn parse(url: &str) -> Option { + let url = url::Url::parse(url).ok()?; + let host = url.host_str().filter(|h| !h.is_empty())?.to_ascii_lowercase(); + let key = match url.port() { + Some(port) => format!("{host}:{port}"), + None => host.clone(), + }; + // Credentials only ever go to `https://.livekit.cloud/…`, built from the + // validated host alone: a TLS scheme, a domain under the Cloud suffix with a label of its + // own, the default port, no userinfo. Anything else — a look-alike that only a string + // check would accept (`evil.example\.livekit.cloud` is host `evil.example`), a + // plaintext scheme, an odd port — is not Cloud. + let cloud = matches!(url.scheme(), "wss" | "https") + && matches!(url.host(), Some(url::Host::Domain(_))) + && host.strip_suffix(CLOUD_SUFFIX).is_some_and(|label| !label.is_empty()) + && url.port().is_none() + && url.username().is_empty() + && url.password().is_none(); + Some(Self { key, cloud }) + } +} + +/// `https:///observability/client/{logs,traces}/otlp/v0`: the client table set. +fn ingest_url(host: &str, signal: Signal) -> String { + let signal = match signal { + Signal::Logs => "logs", + Signal::Traces => "traces", + }; + format!("https://{host}/observability/client/{signal}/otlp/v0") +} + +fn override_target(endpoint: &str) -> Target { + let endpoint = endpoint.trim_end_matches('/'); + let (logs, traces) = match endpoint.rsplit_once("logs") { + Some((before, after)) => (endpoint.to_owned(), format!("{before}traces{after}")), + None => (format!("{endpoint}/v1/logs"), format!("{endpoint}/v1/traces")), + }; + Target { project: None, logs, traces, token: None } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + /// An unsigned JWT with these claims — enough for the core, which never verifies. + pub(crate) fn jwt(claims: serde_json::Value) -> String { + let engine = base64::engine::general_purpose::URL_SAFE_NO_PAD; + format!( + "{}.{}.sig", + engine.encode(br#"{"alg":"HS256"}"#), + engine.encode(claims.to_string()) + ) + } + + pub(crate) fn granted(ttl_secs: u64) -> String { + let exp = SystemTime::now().duration_since(UNIX_EPOCH).expect("clock").as_secs() + ttl_secs; + jwt(serde_json::json!({ "exp": exp, "observability": { "write": true } })) + } + + pub(crate) fn grantless(ttl_secs: u64) -> String { + let exp = SystemTime::now().duration_since(UNIX_EPOCH).expect("clock").as_secs() + ttl_secs; + jwt(serde_json::json!({ "exp": exp, "video": { "roomJoin": true } })) + } + + fn send(route: Route) -> Target { + match route { + Route::Send(target) => target, + other => panic!("expected a target, got {other:?}"), + } + } + + #[test] + fn only_livekit_cloud_hosts_get_an_ingest_url() { + let mut d = Destinations::default(); + let token = granted(600); + let host = d.set("wss://Proj.LiveKit.Cloud/rtc?access_token=x#f", &token, "s"); + assert_eq!(host.as_deref(), Some("proj.livekit.cloud")); + let target = send(d.route(host.as_deref(), Some("s"))); + assert_eq!(target.logs, "https://proj.livekit.cloud/observability/client/logs/otlp/v0"); + assert_eq!(target.traces, "https://proj.livekit.cloud/observability/client/traces/otlp/v0"); + assert_eq!(target.token.as_deref(), Some(token.as_str())); + + let staging = d.set("https://p.staging.livekit.cloud", &token, "s"); + assert!(matches!(d.route(staging.as_deref(), None), Route::Send(_)), "staging is Cloud"); + + for url in + ["ws://localhost:7880", "wss://livekit.example.com", "wss://livekit.cloud.evil.io"] + { + let host = d.set(url, &token, "s"); + assert_eq!(d.route(host.as_deref(), None), Route::Drop, "{url}: no ingest"); + } + assert_eq!(d.set("nonsense", &token, "s"), None); + let empty = d.set("wss:///rtc", &token, "s"); // the parser reads host `rtc` + assert_eq!(d.route(empty.as_deref(), None), Route::Drop); + } + + /// Finding r1-1: whatever a string check would accept, the token only goes to a host the URL + /// parser agrees is `.livekit.cloud`, over TLS, on the default port, without + /// userinfo — and the endpoint is built from that host alone. + #[test] + fn look_alike_server_urls_never_get_the_token() { + let token = granted(600); + for url in [ + "wss://evil.example\\.livekit.cloud", + "wss://evil.example\\x.livekit.cloud/rtc", + "wss://user:pass@p.livekit.cloud", + "wss://evil.example@p.livekit.cloud", + "wss://p.livekit.cloud:8443", + "ws://p.livekit.cloud", + "http://p.livekit.cloud", + "wss://livekit.cloud", + "wss://.livekit.cloud", + "wss://p.livekit.cloud.evil.io", + "wss://p.livekit.cloud%2eevil.io", + "wss://evil.io#.livekit.cloud", + "wss://evil.io?.livekit.cloud", + "wss://[::1]", + "wss://1.2.3.4", + "wss://p.livekit.cloud\u{0}.evil", + ] { + let mut d = Destinations::default(); + let host = d.set(url, &token, "s"); + let route = d.route(host.as_deref(), Some("s")); + assert!(!matches!(route, Route::Send(_)), "{url} → {host:?} must not get the token"); + } + let mut d = Destinations::default(); + let host = d.set("WSS://P.LiveKit.Cloud:443/rtc?x=1#y", &token, "s"); + let target = send(d.route(host.as_deref(), Some("s"))); + assert_eq!(target.logs, "https://p.livekit.cloud/observability/client/logs/otlp/v0"); + } + + #[test] + fn a_token_without_the_grant_is_not_consent() { + let mut d = Destinations::default(); + let host = d.set("wss://p.livekit.cloud", &grantless(600), "s"); + assert_eq!(d.route(host.as_deref(), Some("s")), Route::Drop, "never opted in"); + assert_eq!(d.route(None, None), Route::Drop, "and no project takes process data"); + let host = d.set("wss://p.livekit.cloud", &granted(600), "s"); + assert!(matches!(d.route(host.as_deref(), Some("s")), Route::Send(_)), "a grant opts in"); + } + + #[tokio::test(start_paused = true)] + async fn a_refresh_that_drops_the_grant_keeps_the_granted_token_until_it_expires() { + let mut d = Destinations::default(); + let join = granted(60); + let host = d.set("wss://p.livekit.cloud", &join, "s"); + d.set("wss://p.livekit.cloud", &grantless(600), "s"); + assert_eq!(send(d.route(host.as_deref(), Some("s"))).token, Some(join), "still granted"); + tokio::time::advance(Duration::from_secs(61)).await; + assert_eq!(d.route(host.as_deref(), Some("s")), Route::Wait, "expired: hold, never drop"); + let refreshed = granted(600); + d.set("wss://p.livekit.cloud", &refreshed, "s"); + assert_eq!(send(d.route(host.as_deref(), Some("s"))).token, Some(refreshed)); + } + + #[test] + fn a_refused_token_is_never_sent_again() { + let mut d = Destinations::default(); + let first = granted(600); + let host = d.set("wss://p.livekit.cloud", &first, "s").expect("host"); + d.refuse(&first); + assert_eq!(d.route(Some(&host), Some("s")), Route::Wait); + d.set("wss://p.livekit.cloud", &first, "s"); + assert_eq!(d.route(Some(&host), Some("s")), Route::Wait, "handing it over again: still no"); + d.set("wss://p.livekit.cloud", &granted(900), "s"); + assert!(matches!(d.route(Some(&host), Some("s")), Route::Send(_))); + } + + #[test] + fn rooms_never_borrow_each_others_tokens() { + let mut d = Destinations::default(); + let (a, b, c) = (granted(600), granted(700), granted(800)); + let host_a = d.set("wss://a.livekit.cloud", &a, "room-a"); + let host_b = d.set("wss://b.livekit.cloud", &b, "room-b"); + d.set("wss://a.livekit.cloud", &c, "room-c"); // a second room on project a + let target = send(d.route(host_a.as_deref(), Some("room-a"))); + assert!(target.logs.starts_with("https://a.") && target.token == Some(a.clone())); + let target = send(d.route(host_a.as_deref(), Some("room-c"))); + assert_eq!(target.token, Some(c.clone()), "same project, its own token"); + let target = send(d.route(host_b.as_deref(), Some("room-b"))); + assert!(target.logs.starts_with("https://b.") && target.token == Some(b)); + d.refuse(&a); + assert_eq!(d.route(host_a.as_deref(), Some("room-a")), Route::Wait, "no borrowing c"); + let previous_launch = send(d.route(host_a.as_deref(), Some("gone"))); + assert_eq!(previous_launch.token, Some(c), "a previous launch's batch: project's latest"); + d.kill("b.livekit.cloud", Dead::Disabled); + assert!(send(d.route(None, None)).logs.starts_with("https://a."), "process: a live one"); + } + + /// Finding r1-2: a Room's batches keep the project they were captured for — after it + /// reconnects to another project they still go to the first with the first project's token — + /// and records it captured before it had a server never go to another Room's project. + #[test] + fn ownership_survives_a_room_changing_projects() { + let mut d = Destinations::default(); + let (a, b) = (granted(600), granted(700)); + let host_a = d.set("wss://a.livekit.cloud", &a, "room").expect("a"); + d.set("wss://b.livekit.cloud", &b, "room"); + let target = send(d.route(Some(&host_a), Some("room"))); + assert!(target.logs.starts_with("https://a.") && target.token == Some(a)); + + d.set("wss://c.livekit.cloud", &granted(800), "other"); + assert_eq!(d.route(None, Some("unconnected")), Route::Wait, "no borrowing c"); + let early = send(d.route(None, Some("room"))); + assert!(early.logs.starts_with("https://a."), "a Room's first project"); + assert!(matches!(d.route(None, Some(PROCESS_OWNER)), Route::Send(_))); + } + + /// Findings r1-13 / r2-4: a refusal belongs to the credential — no slot, Room, project or + /// process route can make it usable again — and credentials follow the live Rooms and their + /// backlog, project-level copies included. + #[test] + fn refusals_stick_and_credentials_follow_live_rooms() { + let mut d = Destinations::default(); + let tokens: Vec = (0..40).map(|n| granted(600 + n)).collect(); + for (n, token) in tokens.iter().enumerate() { + d.set("wss://p.livekit.cloud", token, &format!("room-{n}")); + d.refuse(token); + } + for (n, token) in tokens.iter().enumerate() { + let room = format!("room-{n}"); + d.set("wss://p.livekit.cloud", token, &room); // handed over again + assert_eq!(d.route(Some("p.livekit.cloud"), Some(&room)), Route::Wait, "{n}"); + d.set("wss://p.livekit.cloud", token, &format!("new-{n}")); // by a new Room + assert_eq!(d.route(Some("p.livekit.cloud"), Some(&format!("new-{n}"))), Route::Wait); + assert_eq!(d.route(None, Some(PROCESS_OWNER)), Route::Wait, "{n}: process route"); + assert_eq!(d.route(Some("p.livekit.cloud"), Some("gone")), Route::Wait, "{n}: restart"); + } + // Refuse A, let B become the latest, hand A over again: A stays refused everywhere. + let (a, b) = (granted(900), granted(901)); + d.set("wss://q.livekit.cloud", &a, "ra"); + d.refuse(&a); + d.set("wss://q.livekit.cloud", &b, "rb"); + d.set("wss://q.livekit.cloud", &a, "ra"); + let process = send(d.route(Some("q.livekit.cloud"), Some("gone"))); + assert_eq!(process.token, Some(b), "the project copy is b, never the refused a"); + + // Churn: 50 projects, their Rooms gone, nothing cached: nothing is retained. + for n in 0..50 { + d.set(&format!("wss://c{n}.livekit.cloud"), &granted(1000 + n), &format!("churn-{n}")); + } + let live: HashMap> = + [("rb".to_owned(), Some("q.livekit.cloud".to_owned()))].into(); + d.retain_owners(&live, &Default::default()); + assert_eq!(d.credentials(), 2, "rb's token and q's project copy, nothing else"); + let backlog = [(Some("q.livekit.cloud".to_owned()), "ra".to_owned())].into(); + d.set("wss://q.livekit.cloud", &granted(902), "ra"); + d.retain_owners(&Default::default(), &backlog); + assert_eq!(d.credentials(), 2, "a gone Room's backlog keeps its credential and q's copy"); + } + + #[test] + fn not_found_recovers_with_the_next_token_disabled_does_not() { + let mut d = Destinations::default(); + let host = d.set("wss://p.livekit.cloud", &granted(600), "s").expect("host"); + d.kill(&host, Dead::NotFound); + assert_eq!(d.route(Some(&host), Some("s")), Route::Drop); + d.set("wss://p.livekit.cloud", &granted(700), "s"); + assert!(matches!(d.route(Some(&host), Some("s")), Route::Send(_)), "a new connect retries"); + d.kill(&host, Dead::Disabled); + d.set("wss://p.livekit.cloud", &granted(800), "s"); + assert_eq!(d.route(Some(&host), Some("s")), Route::Drop, "the owner's switch stands"); + } + + #[test] + fn the_override_takes_everything_without_a_token() { + let d = Destinations::new(Some("http://localhost:4318/")); + let target = send(d.route(None, None)); + assert_eq!(target.logs, "http://localhost:4318/v1/logs"); + assert_eq!(target.traces, "http://localhost:4318/v1/traces"); + assert_eq!(target.token, None); + let full = Destinations::new(Some("https://h/observability/client/logs/otlp/v0")); + assert_eq!( + send(full.route(Some("ignored"), None)).traces, + "https://h/observability/client/traces/otlp/v0" + ); + let mut local = Destinations::new(Some("http://localhost:4318")); + let host = local.set("ws://localhost:7880", "not-a-jwt", "s"); + assert!(matches!(local.route(host.as_deref(), Some("s")), Route::Send(_))); + } + + /// Finding r2-11: a past, negative, fractional-past or malformed `exp` is expired (a hard + /// hold); only a missing one means no known expiry. + #[tokio::test(start_paused = true)] + async fn untrusted_expiry_claims_fail_closed() { + let now = SystemTime::now().duration_since(UNIX_EPOCH).expect("clock").as_secs_f64(); + let token = |exp: serde_json::Value| { + jwt(serde_json::json!({ "exp": exp, "observability": { "write": true } })) + }; + for exp in [ + serde_json::json!(-1), + serde_json::json!(0), + serde_json::json!(now - 0.5), + serde_json::json!(-1e308), + serde_json::json!("tomorrow"), + serde_json::json!(true), + serde_json::json!(null), + serde_json::json!({}), + ] { + let mut d = Destinations::default(); + let host = d.set("wss://p.livekit.cloud", &token(exp.clone()), "s"); + assert_eq!(d.route(host.as_deref(), Some("s")), Route::Wait, "exp {exp}: expired"); + } + let mut d = Destinations::default(); + let host = d.set("wss://p.livekit.cloud", &token(serde_json::json!(now + 60.9)), "s"); + assert!(matches!(d.route(host.as_deref(), Some("s")), Route::Send(_)), "fractional future"); + let host = d.set("wss://p.livekit.cloud", &token(serde_json::json!(1e300)), "s"); + assert!(matches!(d.route(host.as_deref(), Some("s")), Route::Send(_)), "extreme: clamped"); + } + + #[test] + fn tokens_are_read_not_verified() { + assert!(Token::parse(&granted(60)).grant); + let client_write = jwt(serde_json::json!({ "observability": { "clientWrite": true } })); + let parsed = Token::parse(&client_write); + assert!(parsed.grant && parsed.expires.is_none(), "no exp: no known expiry"); + assert!(!Token::parse("garbage").grant); + assert!(!Token::parse("a.!!!.c").grant); + } +} diff --git a/livekit-telemetry/src/device.rs b/livekit-telemetry/src/device.rs new file mode 100644 index 000000000..714623aa0 --- /dev/null +++ b/livekit-telemetry/src/device.rs @@ -0,0 +1,520 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::TelemetryEvent; + +/// Thermal pressure as the OS reports it (`ProcessInfo.thermalState`, `PowerManager` thermal +/// status, the web Compute Pressure API — all four share this vocabulary). +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum ThermalState { + /// The platform has no thermal source: no pressure assumed, no change event. + #[default] + Unknown, + Nominal, + Fair, + Serious, + Critical, +} + +/// Whether the app is in front of the user. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum AppState { + #[default] + Foreground, + Background, +} + +/// Memory pressure as the OS reports it (`DISPATCH_MEMORYPRESSURE_*`, `onTrimMemory` levels: +/// `RUNNING_LOW`/`BACKGROUND` → warning, `RUNNING_CRITICAL`/`COMPLETE` → critical). +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum MemoryPressure { + #[default] + Normal, + Warning, + Critical, +} + +/// The active network path, in OTel `network.connection.type` terms. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum NetworkType { + #[default] + Unknown, + Wifi, + Cell, + Wired, + /// A VPN tunnel is the active interface (Android `TRANSPORT_VPN`, WebRTC candidate `vpn`). + Vpn, + /// Bluetooth tethering (Android `TRANSPORT_BLUETOOTH`, WebRTC candidate `bluetooth`). + Bluetooth, + /// Connected over something the platform does not name (`NWInterface.InterfaceType.other`, + /// Android `TRANSPORT_USB` / `LOWPAN`, …). + Other, + Unavailable, +} + +/// What the host observes about the device; pushed with +/// [`Telemetry::set_device_state`](crate::Telemetry::set_device_state) whenever anything changes. +/// +/// The host owns the OS APIs (thermal, power, memory, battery, network path, lifecycle) — they +/// are not reachable from Rust without a JVM/ObjC bridge — and the core owns what to do with +/// them: emit the change events from `SPEC.md`, stretch its own cadence and hold uploads so +/// telemetry never competes with the call. CPU is deliberately not measured here (measuring CPU +/// costs CPU): thermal state is the OS's judgement, and WebRTC's own `qualityLimitationReason` +/// (see [`RtcStatsSample`](crate::RtcStatsSample)) says when the encoder is CPU-starved. +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, Copy, PartialEq, Default)] +pub struct DeviceState { + pub thermal: ThermalState, + /// Low Power Mode / Battery Saver; `None` when the platform has no source (no change event, + /// no stretch). + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub low_power_mode: Option, + pub app_state: AppState, + // No `uniffi(default)` on enum fields: uniffi 0.31's Swift generator rejects `Default` there. + pub memory: MemoryPressure, + pub network: NetworkType, + /// Cellular / personal hotspot (`NWPath.isExpensive`, metered on Android). + #[cfg_attr(feature = "uniffi", uniffi(default = false))] + pub network_expensive: bool, + /// The user asked for less traffic: Low Data Mode (`NWPath.isConstrained`), Data Saver, + /// `navigator.connection.saveData`. + #[cfg_attr(feature = "uniffi", uniffi(default = false))] + pub network_constrained: bool, + /// Battery charge in percent; `None` when unknown (desktops, tvOS, monitoring disabled). + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub battery_level: Option, + #[cfg_attr(feature = "uniffi", uniffi(default = false))] + pub battery_charging: bool, +} + +impl DeviceState { + /// Below this, unplugged, the pipeline still records but stops uploading (a soft hold). + const BATTERY_HOLD_PERCENT: u32 = 10; + /// Below this, unplugged, the cadence doubles. + const BATTERY_LOW_PERCENT: u32 = 20; + + /// Multiplier on the flush interval and stats window: 1× at rest, up to 4× under pressure + /// (60 s → 4 min at the default cadence). Serious thermal, memory + /// warning, low-power mode, background, low battery and a constrained network each double + /// it; critical thermal or memory quadruple it. + pub fn cadence_factor(&self) -> u32 { + let thermal = match self.thermal { + ThermalState::Unknown | ThermalState::Nominal | ThermalState::Fair => 1, + ThermalState::Serious => 2, + ThermalState::Critical => 4, + }; + let memory = match self.memory { + MemoryPressure::Normal => 1, + MemoryPressure::Warning => 2, + MemoryPressure::Critical => 4, + }; + let doubled = [ + self.low_power_mode == Some(true), + self.app_state == AppState::Background, + self.battery_bucket() >= 1, + self.network_constrained, + ] + .iter() + .filter(|&&on| on) + .count() as u32; + ((thermal * memory) << doubled).min(4) + } + + /// A soft hold: the user asked for less traffic (Low Data Mode / Data Saver) or the battery + /// is nearly empty and unplugged. Uploads wait (at most a minute at a time); data keeps + /// flowing into the cache meanwhile. Being offline (`network: Unavailable`) is a hard hold: + /// nothing is attempted until the network is back. + pub fn holds_uploads(&self) -> bool { + self.network_constrained || self.battery_bucket() == 2 + } + + /// 0 = fine or unknown, 1 = low (≤ 20 %), 2 = nearly empty (≤ 10 %); always 0 on charge. + /// Per-percent battery updates only matter when they cross one of these. + fn battery_bucket(&self) -> u8 { + match self.battery_level { + Some(level) if !self.battery_charging && level <= Self::BATTERY_HOLD_PERCENT => 2, + Some(level) if !self.battery_charging && level <= Self::BATTERY_LOW_PERCENT => 1, + _ => 0, + } + } + + /// The `SPEC.md` events describing what changed between `previous` and `self` + /// (everything, when there is no previous state: "initial value at session start"). + pub(crate) fn change_events(&self, previous: Option<&DeviceState>) -> Vec { + let mut events = Vec::new(); + // An unknown reading is no reading: it never makes a change event. + if self.thermal != ThermalState::Unknown + && previous.is_none_or(|p| p.thermal != self.thermal) + { + events.push( + TelemetryEvent::new("lk.device.thermal.changed") + .with_body(format!("thermal: {}", self.thermal.as_str())) + .with_attribute("lk.device.thermal.state", self.thermal.as_str()), + ); + } + if let Some(low_power) = self + .low_power_mode + .filter(|_| previous.is_none_or(|p| p.low_power_mode != self.low_power_mode)) + { + events.push( + TelemetryEvent::new("lk.device.low_power.changed") + .with_body(format!("low power mode: {}", if low_power { "on" } else { "off" })) + .with_attribute("lk.device.low_power.enabled", low_power), + ); + } + if previous.is_none_or(|p| p.app_state != self.app_state) { + events.push( + TelemetryEvent::new("lk.device.app_state.changed") + .with_body(format!("app: {}", self.app_state.as_str())) + .with_attribute("lk.device.app_state", self.app_state.as_str()), + ); + } + if previous.is_none_or(|p| p.memory != self.memory) { + events.push( + TelemetryEvent::new("lk.device.memory.changed") + .with_body(format!("memory pressure: {}", self.memory.as_str())) + .with_attribute("lk.device.memory.pressure", self.memory.as_str()), + ); + } + if previous.is_none_or(|p| { + (p.network, p.network_expensive, p.network_constrained) + != (self.network, self.network_expensive, self.network_constrained) + }) { + events.push( + TelemetryEvent::new("lk.device.network.changed") + .with_body(format!( + "network: {}{}{}", + self.network.as_str(), + if self.network_expensive { ", expensive" } else { "" }, + if self.network_constrained { ", constrained" } else { "" } + )) + .with_attribute("network.connection.type", self.network.as_str()) + .with_attribute("lk.device.network.expensive", self.network_expensive) + .with_attribute("lk.device.network.constrained", self.network_constrained), + ); + } + // Per-percent updates are noise; report charging flips and bucket crossings. + if self.battery_level.is_some() + && previous.is_none_or(|p| { + p.battery_charging != self.battery_charging + || p.battery_bucket() != self.battery_bucket() + || p.battery_level.is_none() + }) + { + events.push( + TelemetryEvent::new("lk.device.battery.changed") + .with_body(format!( + "battery: {}%{}", + self.battery_level.unwrap_or(0), + if self.battery_charging { ", charging" } else { "" } + )) + .with_attribute( + "lk.device.battery.level", + self.battery_level.unwrap_or(0) as i64, + ) + .with_attribute("lk.device.battery.charging", self.battery_charging), + ); + } + events + } +} + +impl ThermalState { + fn as_str(self) -> &'static str { + match self { + ThermalState::Unknown => "unknown", + ThermalState::Nominal => "nominal", + ThermalState::Fair => "fair", + ThermalState::Serious => "serious", + ThermalState::Critical => "critical", + } + } +} + +impl AppState { + fn as_str(self) -> &'static str { + match self { + AppState::Foreground => "foreground", + AppState::Background => "background", + } + } +} + +impl MemoryPressure { + fn as_str(self) -> &'static str { + match self { + MemoryPressure::Normal => "normal", + MemoryPressure::Warning => "warning", + MemoryPressure::Critical => "critical", + } + } +} + +impl NetworkType { + fn as_str(self) -> &'static str { + match self { + NetworkType::Unknown => "unknown", + NetworkType::Wifi => "wifi", + NetworkType::Cell => "cell", + NetworkType::Wired => "wired", + NetworkType::Vpn => "vpn", + NetworkType::Bluetooth => "bluetooth", + NetworkType::Other => "other", + NetworkType::Unavailable => "unavailable", + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cadence_stretches_under_pressure_and_caps_at_four() { + assert_eq!(DeviceState::default().cadence_factor(), 1); + let serious = DeviceState { thermal: ThermalState::Serious, ..Default::default() }; + assert_eq!(serious.cadence_factor(), 2); + let low_power = DeviceState { low_power_mode: Some(true), ..Default::default() }; + assert_eq!(low_power.cadence_factor(), 2); + let tight = DeviceState { memory: MemoryPressure::Warning, ..Default::default() }; + assert_eq!(tight.cadence_factor(), 2); + let low_battery = DeviceState { battery_level: Some(15), ..Default::default() }; + assert_eq!(low_battery.cadence_factor(), 2); + let charging = DeviceState { battery_charging: true, ..low_battery }; + assert_eq!(charging.cadence_factor(), 1, "a charging battery is not low"); + let everything = DeviceState { + thermal: ThermalState::Critical, + low_power_mode: Some(true), + app_state: AppState::Background, + memory: MemoryPressure::Critical, + network_constrained: true, + ..Default::default() + }; + assert_eq!(everything.cadence_factor(), 4); + } + + #[test] + fn uploads_hold_on_constrained_network_or_empty_battery() { + assert!(!DeviceState::default().holds_uploads()); + let saver = DeviceState { network_constrained: true, ..Default::default() }; + assert!(saver.holds_uploads()); + let empty = DeviceState { battery_level: Some(8), ..Default::default() }; + assert!(empty.holds_uploads()); + let plugged = DeviceState { battery_charging: true, ..empty }; + assert!(!plugged.holds_uploads()); + let low = DeviceState { battery_level: Some(15), ..Default::default() }; + assert!(!low.holds_uploads(), "low only stretches the cadence"); + } + + #[test] + fn only_changed_fields_produce_events() { + let initial = DeviceState { + thermal: ThermalState::Nominal, + low_power_mode: Some(false), + ..Default::default() + }; + let names: Vec<_> = initial.change_events(None).into_iter().map(|e| e.name).collect(); + assert_eq!( + names, + [ + "lk.device.thermal.changed", + "lk.device.low_power.changed", + "lk.device.app_state.changed", + "lk.device.memory.changed", + "lk.device.network.changed", + ], + "initial values at session start; battery is unknown so it stays silent" + ); + let hot = DeviceState { thermal: ThermalState::Critical, ..initial }; + let events = hot.change_events(Some(&initial)); + assert_eq!(events.len(), 1); + assert_eq!(events[0].name, "lk.device.thermal.changed"); + assert!(hot.change_events(Some(&hot)).is_empty()); + } + + /// Flutter review: a platform without a thermal or low-power source reports them unknown — + /// never a made-up nominal/off: no change event, no stretch, no hold. + #[test] + fn unknown_thermal_and_low_power_are_silent_and_neutral() { + let unknown = DeviceState::default(); + assert_eq!((unknown.thermal, unknown.low_power_mode), (ThermalState::Unknown, None)); + let names: Vec<_> = unknown.change_events(None).into_iter().map(|e| e.name).collect(); + assert_eq!( + names, + [ + "lk.device.app_state.changed", + "lk.device.memory.changed", + "lk.device.network.changed" + ] + ); + assert_eq!(unknown.cadence_factor(), 1); + assert!(!unknown.holds_uploads()); + let known = DeviceState { + thermal: ThermalState::Serious, + low_power_mode: Some(true), + ..Default::default() + }; + assert!(unknown.change_events(Some(&known)).is_empty(), "losing the source is silent"); + assert_eq!(known.change_events(Some(&unknown)).len(), 2, "a first real reading reports"); + } + + #[test] + fn battery_reports_bucket_crossings_not_every_percent() { + let at = |level| DeviceState { battery_level: Some(level), ..Default::default() }; + assert_eq!(at(80).change_events(Some(&DeviceState::default())).len(), 1, "first reading"); + assert!(at(79).change_events(Some(&at(80))).is_empty()); + assert_eq!(at(20).change_events(Some(&at(21))).len(), 1, "crossed into low"); + assert_eq!(at(10).change_events(Some(&at(11))).len(), 1, "crossed into hold"); + let charging = DeviceState { battery_charging: true, ..at(10) }; + assert_eq!(charging.change_events(Some(&at(10))).len(), 1, "plugged in"); + } +} + +/// Why the audio route changed, in platform-neutral terms (AVAudioSession reasons; Android's +/// device added/removed callbacks map to `NewDevice` / `OldDeviceUnavailable`). +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AudioRouteReason { + NewDevice, + OldDeviceUnavailable, + CategoryChange, + Override, + WakeFromSleep, + NoSuitableRoute, + RouteConfigurationChange, + Unknown, +} + +/// Where the audio goes, in platform-neutral terms (AVAudioSession port types, Android's +/// `AudioDevice`; the web's `MediaDeviceInfo.kind` only ever gives `other`). +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AudioOutput { + Speaker, + /// The phone earpiece (`builtInReceiver`, Android `Earpiece`). + Receiver, + WiredHeadset, + Bluetooth, + CarAudio, + AirPlay, + Hdmi, + Usb, + Other, +} + +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CaptureDevice { + Camera, + Microphone, + ScreenShare, +} + +/// Why a capture device could not be used: the `getUserMedia` failure taxonomy, which every +/// platform maps onto (authorization status, `onCameraDisconnected`, `NotReadableError`, …). +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CaptureFailure { + /// The user (or MDM) said no: the most common pre-connect failure on mobile. + PermissionDenied, + NotFound, + /// Another app holds the device (`NotReadableError`, `TrackStartError`). + InUse, + /// The device went away mid-capture (unplugged, `onCameraDisconnected`). + Disconnected, + Other, +} + +/// Things that happen to the device mid-call and explain what the media did next. Not state +/// (see [`DeviceState`]): each one is a record with a display body. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DeviceEvent { + /// The outputs after the change, in the order the platform lists them. + AudioRouteChanged { outputs: Vec, reason: AudioRouteReason }, + /// A phone call, Siri, another app taking audio focus: `began`, then the matching `ended`. + AudioInterruption { began: bool }, + /// A capture device could not be, or stay, used. + CaptureFailed { device: CaptureDevice, reason: CaptureFailure }, +} + +pub(crate) fn snake(debug: impl std::fmt::Debug) -> String { + let mut out = String::new(); + for (i, c) in format!("{debug:?}").chars().enumerate() { + if c.is_uppercase() && i > 0 { + out.push('_'); + } + out.push(c.to_ascii_lowercase()); + } + out +} + +impl DeviceEvent { + pub(crate) fn into_event(self) -> crate::TelemetryEvent { + use crate::{Severity, TelemetryEvent}; + match self { + Self::AudioRouteChanged { outputs, reason } => { + let outputs = outputs.into_iter().map(snake).collect::>().join(","); + let reason = snake(reason); + TelemetryEvent::new("lk.device.audio_route.changed") + .with_body(format!("audio route: {outputs} ({reason})")) + .with_attribute("lk.device.audio_route.outputs", outputs) + .with_attribute("lk.device.audio_route.reason", reason) + } + Self::AudioInterruption { began } => { + let phase = if began { "began" } else { "ended" }; + TelemetryEvent::new("lk.device.audio.interruption") + .with_body(format!("audio interruption {phase}")) + .with_attribute("lk.device.audio.interruption", phase) + } + Self::CaptureFailed { device, reason } => { + let (device, reason) = (snake(device), snake(reason)); + TelemetryEvent::new("lk.device.capture.failed") + .with_severity(Severity::Warn) + .with_body(format!("capture failed: {device} ({reason})")) + .with_attribute("lk.device.capture.device", device) + .with_attribute("lk.device.capture.reason", reason) + } + } + } +} + +#[cfg(test)] +mod event_tests { + use super::*; + + #[test] + fn device_events_have_bodies_and_snake_case_values() { + let event = DeviceEvent::AudioRouteChanged { + outputs: vec![AudioOutput::Speaker], + reason: AudioRouteReason::OldDeviceUnavailable, + } + .into_event(); + assert_eq!(event.body.as_deref(), Some("audio route: speaker (old_device_unavailable)")); + let denied = DeviceEvent::CaptureFailed { + device: CaptureDevice::ScreenShare, + reason: CaptureFailure::PermissionDenied, + } + .into_event(); + assert_eq!( + denied.body.as_deref(), + Some("capture failed: screen_share (permission_denied)") + ); + assert_eq!(denied.severity, crate::Severity::Warn); + } +} diff --git a/livekit-telemetry/src/event.rs b/livekit-telemetry/src/event.rs new file mode 100644 index 000000000..957de0027 --- /dev/null +++ b/livekit-telemetry/src/event.rs @@ -0,0 +1,282 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::time::{SystemTime, UNIX_EPOCH}; + +/// A discrete telemetry event. +/// +/// Exported as one OTLP log record whose `event_name` is [`name`](Self::name), following the +/// OTel logs data model (events are log records with a top-level event name). +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, PartialEq)] +pub struct TelemetryEvent { + /// Event name. LiveKit-defined events use the `lk.` prefix (e.g. `lk.ping`); see `SPEC.md`. + pub name: String, + pub severity: Severity, + /// Optional human-readable message (the OTLP log record body). + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub body: Option, + pub attributes: Vec, + /// Wall-clock time in nanoseconds since the Unix epoch. `None` stamps the event at emit time. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub timestamp_ns: Option, + /// The in-flight span this record belongs to (a handle from `begin_span`), if any. The trace + /// id is always the session's and is attached by the core. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub span_id: Option, +} + +impl TelemetryEvent { + /// An `Info` event without attributes, stamped when emitted. + pub fn new(name: impl Into) -> Self { + Self { + name: name.into(), + severity: Severity::Info, + body: None, + attributes: Vec::new(), + timestamp_ns: None, + span_id: None, + } + } + + /// Link this record to an in-flight span. + pub fn in_span(mut self, span: u64) -> Self { + self.span_id = Some(span); + self + } + + /// The same event with `severity`. + pub fn with_severity(mut self, severity: Severity) -> Self { + self.severity = severity; + self + } + + /// The same event with a display body. + pub fn with_body(mut self, body: impl Into) -> Self { + self.body = Some(body.into()); + self + } + + /// The same event with one more attribute. + pub fn with_attribute( + mut self, + key: impl Into, + value: impl Into, + ) -> Self { + self.attributes.push(Attribute::new(key, value)); + self + } + + /// A consumer's own event. Always namespaced under `custom.` so it can never be mistaken for + /// a LiveKit-defined `lk.*` event, and the backend can filter or quota it separately; + /// attributes keep the caller's namespace (`acme.checkout.step`). + pub fn custom(name: &str, attributes: Vec) -> Self { + let name = format!("custom.{}", name.trim_start_matches("custom.")); + Self { attributes, body: Some(name.clone()), ..Self::new(name) } + } + + /// Rough encoded size — strings plus a fixed overhead per field. Drives the byte bounds on + /// queue flushing and request size; cheaper than encoding and close enough for both. + pub fn size_hint(&self) -> usize { + 32 + self.name.len() + + self.body.as_ref().map_or(0, String::len) + + self.attributes.iter().map(|a| 4 + a.key.len() + a.value.size_hint()).sum::() + } +} + +/// Event severity, mapped onto the OTel severity numbers (`TRACE`=1 … `ERROR`=17). +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Severity { + Trace, + Debug, + Info, + Warn, + Error, +} + +/// Where a log line came from. WebRTC is chatty at warn, so only its errors become records; +/// the SDK and the core use the configured floor. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LogSource { + Sdk, + Ffi, + WebRtc, +} + +impl LogSource { + fn as_str(self) -> &'static str { + match self { + Self::Sdk => "sdk", + Self::Ffi => "ffi", + Self::WebRtc => "webrtc", + } + } +} + +/// A log line as the platform captured it, where it happened. The core turns it into a record: +/// semconv `code.*` attributes, `lk.log.source`, `lk.log.logger`, filed under the span's session. +/// Stamp `timestamp_ns` at capture; the record may cross an executor hop before it gets here. +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, PartialEq)] +pub struct LogRecord { + pub severity: Severity, + pub source: LogSource, + /// The line itself (the OTLP log body). Not `message`: AGENTS.md keeps that name off every + /// exported record. + pub body: String, + /// The logger: a type, module or file name (`Room`, `livekit::rtc_engine`, `sctp.cc`). + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub logger: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub function: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub file: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub line: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub timestamp_ns: Option, + /// The in-flight span this line was logged under, if any. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub span_id: Option, +} + +impl From for TelemetryEvent { + fn from(record: LogRecord) -> Self { + let mut event = TelemetryEvent::new("") + .with_severity(record.severity) + .with_body(record.body) + .with_attribute("lk.log.source", record.source.as_str()); + if let Some(logger) = record.logger.filter(|s| !s.is_empty()) { + event = event.with_attribute("lk.log.logger", logger); + } + if let Some(function) = record.function.filter(|s| !s.is_empty()) { + event = event.with_attribute("code.function.name", function); + } + if let Some(file) = record.file.filter(|s| !s.is_empty()) { + event = event.with_attribute("code.file.path", file); + } + if let Some(line) = record.line.filter(|l| *l > 0) { + event = event.with_attribute("code.line.number", line as i64); + } + event.timestamp_ns = record.timestamp_ns; + event.span_id = record.span_id; + event + } +} + +/// A key/value attribute on an event or on the resource. +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, PartialEq)] +pub struct Attribute { + pub key: String, + pub value: AttributeValue, +} + +impl Attribute { + /// A key/value pair. + pub fn new(key: impl Into, value: impl Into) -> Self { + Self { key: key.into(), value: value.into() } + } +} + +/// Attribute value: the scalar subset of OTLP `AnyValue`. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, PartialEq)] +pub enum AttributeValue { + Str(String), + Int(i64), + Double(f64), + Bool(bool), +} + +impl From<&str> for AttributeValue { + fn from(value: &str) -> Self { + Self::Str(value.to_owned()) + } +} + +impl From for AttributeValue { + fn from(value: String) -> Self { + Self::Str(value) + } +} + +impl From for AttributeValue { + fn from(value: i64) -> Self { + Self::Int(value) + } +} + +impl From for AttributeValue { + fn from(value: f64) -> Self { + Self::Double(value) + } +} + +impl From for AttributeValue { + fn from(value: bool) -> Self { + Self::Bool(value) + } +} + +/// Current wall-clock time in nanoseconds since the Unix epoch (0 if the clock is before 1970). +pub(crate) fn now_unix_nanos() -> u64 { + let now = + SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_nanos() as u64).unwrap_or(0); + #[cfg(test)] + let now = now.saturating_add_signed(CLOCK_JUMP_NS.with(|jump| jump.get())); + now +} + +#[cfg(test)] +thread_local! { + /// How far a test moved the wall clock, on this (current-thread runtime) thread. + pub(crate) static CLOCK_JUMP_NS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Limits on what an app hands over: long enough for any real identifier, short enough that one +/// app cannot bloat every record. Over-long input is rejected and counted, never truncated — a +/// truncated id silently collides with another. +pub(crate) const MAX_NAME_BYTES: usize = 128; +pub(crate) const MAX_KEY_BYTES: usize = 128; +pub(crate) const MAX_VALUE_BYTES: usize = 1024; +/// Custom attributes per room, and per custom event. +pub(crate) const MAX_CUSTOM_ATTRIBUTES: usize = 64; + +/// Keys the SDK owns: an app can neither set nor override them (`lk.*` — room, participant, +/// track, outcome — and the session id). +pub(crate) fn reserved(key: &str) -> bool { + key.starts_with("lk.") || key == "session.id" +} + +/// Whether an app-provided attribute is within the limits and outside the SDK's namespace. +pub(crate) fn valid_custom(key: &str, value: Option<&AttributeValue>) -> bool { + let value_ok = match value { + Some(AttributeValue::Str(s)) => s.len() <= MAX_VALUE_BYTES, + _ => true, + }; + !key.is_empty() && key.len() <= MAX_KEY_BYTES && !reserved(key) && value_ok +} + +impl AttributeValue { + /// Rough encoded size of the value. + pub(crate) fn size_hint(&self) -> usize { + match self { + AttributeValue::Str(s) => s.len(), + _ => 8, + } + } +} diff --git a/livekit-telemetry/src/exporter.rs b/livekit-telemetry/src/exporter.rs new file mode 100644 index 000000000..b7f6992b4 --- /dev/null +++ b/livekit-telemetry/src/exporter.rs @@ -0,0 +1,1368 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{ + collections::{HashMap, VecDeque}, + future::Future, + io::Write, + pin::Pin, + sync::{atomic::AtomicU64, Arc}, + time::Duration, +}; + +use flate2::{write::GzEncoder, Compression}; +use tokio::sync::{mpsc, oneshot}; +use tokio::time::{sleep_until, timeout, Instant}; + +use crate::{ + destination::{Dead, Route, Signal, Target, PROCESS_OWNER}, + event::now_unix_nanos, + otlp, + scope::ScopeState, + stats::{Counters, Snapshot, TelemetryStatus}, + store::Queued, + telemetry::Shared, + transport::Verdict, + AppState, DeviceState, ExportError, ExportRequest, MemoryPressure, NetworkType, + TelemetryTransport, ThermalState, +}; + +/// Who a batch belongs to: its session's project (`None` until the session has a server) and +/// the session itself. +#[derive(Debug, PartialEq)] +struct Owner { + host: Option, + session: String, +} + +/// Local retry backoff: 1 s doubling per consecutive failure up to 60 s, with full jitter +/// (a uniform wait in `[0, backoff]`) so a fleet that failed together does not retry together. +/// Retrying never gives up on a batch: the cache's age and size bound what is kept. +const RETRY_BASE: Duration = Duration::from_secs(1); +const RETRY_CAP: Duration = Duration::from_secs(60); +/// Delays the server asks for (`Retry-After`, `RetryInfo`) are honored even beyond +/// [`RETRY_CAP`]; clamped only to the cache's age limit, past which there is nothing to wait for. +const MAX_SERVER_DELAY: Duration = Duration::from_secs(24 * 60 * 60); +/// A cached batch older than this is dropped (`expired`) — at start and while running. +const MAX_AGE: Duration = Duration::from_secs(24 * 60 * 60); +/// Soft holds (see [`Exporter::hold_reason`]) last at most this long before one batch goes out +/// anyway: the cap that bounds the policy when its signals lie. Hard holds have no such escape. +const MAX_HOLD: Duration = Duration::from_secs(60); +/// While one of these is open the uplink belongs to signaling and ICE/DTLS. +const SENSITIVE_SPANS: &[&str] = &["lk.connect", "lk.reconnect"]; +/// RFC 9218 request priority: lowest urgency, not incremental. A hint for HTTP/2+ hops that +/// implement it; the host transport marks the local traffic class (see `TelemetryTransport`). +const PRIORITY: &str = "u=7"; + +impl Signal { + fn tag(self) -> char { + match self { + Signal::Logs => 'l', + Signal::Traces => 't', + } + } +} + +/// A cached batch's id, which is also its ownership record: +/// `-----`. Sorting ids sorts batches oldest +/// first; the host and the session that produced it pick destination and token at upload time +/// (no host: process-level, sent to the latest project). No token is ever written to disk. +struct BatchId<'a> { + /// Creation time, unix ns. + stamp: u64, + count: u64, + signal: Signal, + owner: Option<&'a str>, + host: Option<&'a str>, +} + +impl<'a> BatchId<'a> { + fn format(signal: Signal, seq: &str, count: u64, owner: &str, host: Option<&str>) -> String { + let (stamp, tag, host) = (now_unix_nanos(), signal.tag(), host.unwrap_or_default()); + format!("{stamp:020}-{seq}-{count}-{tag}-{owner}-{host}") + } + + fn parse(id: &'a str) -> Self { + let mut parts = id.splitn(6, '-'); + let stamp = parts.next().and_then(|n| n.parse().ok()).unwrap_or(0); + let count = parts.nth(1).and_then(|n| n.parse().ok()).unwrap_or(0); + let signal = match parts.next() { + Some("t") => Signal::Traces, + _ => Signal::Logs, + }; + let owner = parts.next().filter(|o| !o.is_empty()); + Self { stamp, count, signal, owner, host: parts.next().filter(|h| !h.is_empty()) } + } + + /// The id of one half of this batch after a 413: same stamp and owner, so it keeps the + /// original's place in line, `seq` extended with `a`/`b`. + fn half(id: &str, which: char, count: u64) -> String { + let mut parts: Vec<&str> = id.splitn(6, '-').collect(); + let seq = format!("{}{which}", parts.get(1).copied().unwrap_or_default()); + let count = count.to_string(); + if parts.len() >= 3 { + parts[1] = &seq; + parts[2] = &count; + } + parts.join("-") + } +} + +/// The records a cached batch id says it holds. +pub(crate) fn events_in(id: &str) -> u64 { + BatchId::parse(id).count +} + +pub(crate) enum Command { + Flush(oneshot::Sender<()>), + Shutdown(oneshot::Sender<()>), + /// Opt-out: stop without uploading and delete everything held. + Purge(oneshot::Sender<()>), +} + +/// What one request achieved. +enum Outcome { + Answered(Verdict), + /// No answer: offline, connection or TLS failure, timeout. + NoAnswer { + timed_out: bool, + reason: String, + }, +} + +/// The request on the wire, polled by the actor's loop so commands and deadlines keep being +/// served while it is out. +struct InFlight { + id: String, + signal: Signal, + /// Which destination's pause the answer drives. + key: String, + target: Target, + body: Vec, + request: Pin + Send>>, +} + +/// One pass over the cache: the batches still to look at and the requests it may make. +#[derive(Default)] +struct Round { + queue: VecDeque, + budget: usize, + attempts: usize, + /// A batch waited for a token or a destination. + waiting: bool, + /// A batch waited for its destination's pause. + paused: bool, + /// The one-batch escape of a soft hold: the hold does not stop it. + escape: bool, + /// Its requests are charged to the interval's allowance: next to a call, and not a drain, + /// flush, background pass or escape. + metered: bool, + /// The hold policy stopped the pass. + held: bool, +} + +/// A destination's pause after a failure: uploads to it wait, others carry on. +#[derive(Debug, Default)] +struct Pause { + /// Retry after this (backoff or server delay); `None` once it has passed. + until: Option, + /// The part the server asked for: honored in full, even while draining. + server_until: Option, + /// Consecutive failures: the backoff exponent. + failing: u32, +} + +/// Background actor that turns stored events into OTLP requests. +/// +/// The role of OTel's `BatchLogRecordProcessor` + OTLP exporter in one place. Every tick it +/// [`enqueue`](Self::enqueue)s: drains the queue and the finished spans, appends an +/// `lk.telemetry.report` when something was dropped or failed since the last one, encodes and +/// gzips one batch per destination project and writes it to the [`BatchCache`](crate::BatchCache) +/// *before* any network is involved; then it [`begin_round`](Self::begin_round)s the cache oldest-first +/// through the [`TelemetryTransport`], one request at a time. What the collector answers decides +/// what happens to the batch ([`Verdict`]); a failure pauses uploads with jittered exponential +/// backoff, a throttle for as long as the collector asked, and collection never stops meanwhile. +/// +/// Telemetry must never win over media, so uploads are shaped as well as batched: at most +/// `max_batches_per_upload` per tick while a session may be live, and none at all while the room +/// is connecting or reconnecting or while the device asks for quiet ([`DeviceState::holds_uploads`]) +/// — bounded by [`MAX_HOLD`]. Yielding to media on the wire is the transport's job: every request +/// carries `Priority: u=7` (RFC 9218) and a gzipped body. +/// +/// The tick period is `flush_interval_ms × cadence factor`: device pressure and a CPU-limited +/// encoder stretch it up to 4×, and entering the background flushes once immediately (the app +/// may be suspended any moment). +/// +/// Drive it with `spawn(exporter.run())` on the consumer's runtime. It stops after +/// [`Telemetry::shutdown`](crate::Telemetry::shutdown) or when the last +/// [`Telemetry`](crate::Telemetry) handle is dropped. +pub struct Exporter { + shared: Arc, + transport: Arc, + commands: mpsc::UnboundedReceiver, + /// Per-destination pauses after failures, by destination key (the project host). + pauses: HashMap, + /// The upload pass in progress, and its request on the wire. + round: Option, + inflight: Option, + /// Another pass is due when this one ends (new batches arrived meanwhile). + rerun: bool, + /// `flush()` callers, answered when the pass ends. + flush_waiters: Vec>, + /// The next pass comes from an explicit `flush()` or entering the background: it has no + /// per-pass budget. + flushing: bool, + /// Requests left in this flush interval (`max_batches_per_upload`, refilled every tick), + /// charged while a Room is in a call: wake-ups between ticks never add requests beyond it. + allowance: usize, + /// Batches done with whose delete failed: never sent again, deletion retried each pass. + undeletable: std::collections::HashSet, + /// When this launch wrote each cached batch, on the monotonic clock (see + /// [`Exporter::expired`]). + born: HashMap, + /// Policy transitions are logged once, not per tick: the hold reason last logged and the + /// cadence factor last logged. + hold_logged: Option<&'static str>, + cadence_logged: u32, + seq: u64, + /// Counter values at the last `lk.telemetry.report`. + last_report: Snapshot, + /// When the current upload hold began, for the [`MAX_HOLD`] cap. + held_since: Option, + /// Shutting down: the call is over, so only the device's own holds still apply. + draining: bool, + /// Append an `lk.telemetry.report` to the next batch even if nothing went wrong (the + /// shutdown summary). + force_report: bool, +} + +impl Exporter { + pub(crate) fn new( + shared: Arc, + transport: Arc, + commands: mpsc::UnboundedReceiver, + ) -> Self { + Self { + shared, + transport, + commands, + pauses: HashMap::new(), + round: None, + inflight: None, + rerun: false, + flush_waiters: Vec::new(), + undeletable: std::collections::HashSet::new(), + flushing: false, + allowance: 0, + born: HashMap::new(), + hold_logged: None, + cadence_logged: 1, + seq: 0, + last_report: Snapshot::default(), + held_since: None, + draining: false, + force_report: false, + } + } + + /// Run until shut down: replay whatever the cache holds, then export on every tick and on + /// demand. A request on the wire never blocks the loop: commands, ticks and deadlines are + /// served while it is out. + pub async fn run(mut self) { + self.begin_round(); + // Deadlines rather than tickers: `next = last + period`, re-derived every loop, so a + // cadence change applies to the pending tick in both directions (pressure postpones it, + // relief brings it forward) and a missed tick never bursts. The first flush is immediate, + // the first window closes a full period after it opens. + let mut last_flush = Instant::now() - self.period(self.shared.config.flush_interval_ms); + let mut last_window = Instant::now(); + loop { + self.log_cadence(); + let next_flush = last_flush + self.period(self.shared.config.flush_interval_ms); + let next_window = last_window + self.period(self.shared.config.stats_window_ms); + let retry = self.retry_deadline(); + let subscribe = self.subscribe_deadline(); + tokio::select! { + outcome = answer(&mut self.inflight) => self.on_answer(outcome), + _ = sleep_until(next_flush) => { + // A window due at the same moment closes first, so it rides this batch. + if Instant::now() >= next_window { + self.close_windows(); + last_window = Instant::now(); + } + // A new interval: a new allowance of requests. + self.allowance = self.shared.config.max_batches_per_upload.max(1) as usize; + self.export_pending(); + last_flush = Instant::now(); + } + _ = sleep_until(next_window) => { + self.close_windows(); + last_window = Instant::now(); + } + // A pause ran out or a soft hold reached its cap: go without waiting for a tick. + _ = at(retry) => { + self.clear_expired_pauses(); + self.begin_round(); + } + _ = at(subscribe) => self.expire_subscribes(), + // Coalesced wake-ups. Only the queue crossing its threshold or the app going to + // the background encode a new batch; a lifted route, token or hold re-runs a pass + // over what is cached (within the interval's allowance); a subscribe or a cadence + // change only makes the loop re-read its deadlines. So the cadence stays one + // export per interval. + _ = self.shared.wake.notified() => { + let taken = |flag: &std::sync::atomic::AtomicBool| { + flag.swap(false, std::sync::atomic::Ordering::SeqCst) + }; + let background = taken(&self.shared.backgrounded); + let overflow = taken(&self.shared.overflowed); + let release = taken(&self.shared.released); + if background { + // The app may be suspended any moment: close the RTC windows and get + // everything into the cache and out — the whole cache, like a flush, + // whatever is left of the interval's allowance. + self.close_windows(); + self.flushing = true; + } + if background || overflow { + self.export_pending(); + } else if release { + self.begin_round(); + } + } + command = self.commands.recv() => match command { + Some(Command::Flush(done)) => { + // An explicit flush drains the cache: no per-pass budget, but every hold + // and pause still applies. + self.flushing = true; + self.export_pending(); + self.answer_when_done(done); + } + + Some(Command::Shutdown(done)) => { + self.drain().await; + let _ = done.send(()); + return; + } + Some(Command::Purge(done)) => { + // Cancel the request on the wire: its answer must not bring anything back. + self.inflight = None; + self.round = None; + self.shared.clear(); + self.answer_flushes(); + let _ = done.send(()); + return; + } + // Every `Telemetry` handle is gone. + None => { + self.drain().await; + return; + } + }, + } + } + } + + /// `base_ms × cadence factor`. + fn period(&self, base_ms: u64) -> Duration { + Duration::from_millis(base_ms.max(1)) * self.cadence_factor() + } + + fn device(&self) -> DeviceState { + self.shared.device.lock().unwrap_or_else(|e| e.into_inner()).unwrap_or_default() + } + + fn cadence_factor(&self) -> u32 { + self.shared.cadence_factor() + } + + fn cpu_limited(&self) -> bool { + self.shared.windows.lock().unwrap_or_else(|e| e.into_inner()).cpu_limited() + } + + /// Whether a Room is in a call: it has a server and has not disconnected. + fn in_call(&self) -> bool { + let scopes = self.shared.scopes.lock().unwrap_or_else(|e| e.into_inner()); + scopes.iter().any(|s| s.upgrade().is_some_and(|s| s.in_call())) + } + + /// Every live session's earliest subscribe deadline. + fn subscribe_deadline(&self) -> Option { + let scopes = self.shared.scopes.lock().unwrap_or_else(|e| e.into_inner()); + scopes.iter().filter_map(|s| s.upgrade()?.subscribe_deadline()).min() + } + + fn expire_subscribes(&self) { + let scopes: Vec> = { + let scopes = self.shared.scopes.lock().unwrap_or_else(|e| e.into_inner()); + scopes.iter().filter_map(|s| s.upgrade()).collect() + }; + for scope in scopes { + scope.expire_subscribes(); + } + } + + /// One debug line per cadence change, naming what stretched it. + fn log_cadence(&mut self) { + let factor = self.cadence_factor(); + if factor == self.cadence_logged { + return; + } + self.cadence_logged = factor; + let device = self.device(); + let mut why = Vec::new(); + if !matches!(device.thermal, ThermalState::Unknown | ThermalState::Nominal) { + why.push(format!("thermal {:?}", device.thermal).to_lowercase()); + } + if device.memory != MemoryPressure::Normal { + why.push(format!("memory {:?}", device.memory).to_lowercase()); + } + if device.low_power_mode == Some(true) { + why.push("low power mode".to_string()); + } + if device.app_state == AppState::Background { + why.push("background".to_string()); + } + if self.cpu_limited() { + why.push("encoder cpu-limited".to_string()); + } + log::debug!( + "cadence ×{factor}, flush every {}s ({})", + self.period(self.shared.config.flush_interval_ms).as_secs(), + if why.is_empty() { "pressure over".to_string() } else { why.join(", ") } + ); + } + + /// Last chance: everything queued into the cache and out, within `export_timeout_ms`. + /// Ignores the failure backoff, the batch budget and the session holds (the call is over); + /// respects server-directed delays and the device's own holds. At the deadline the request + /// on the wire is cancelled and the actor stops: its batch stays cached for the next launch. + async fn drain(&mut self) { + let deadline = + Instant::now() + Duration::from_millis(self.shared.config.export_timeout_ms.max(1)); + self.draining = true; + self.force_report = true; + // Pending subscribes end now and ride the last batch; nothing keeps a span afterwards. + self.shared.end_pending_subscribes(true); + for pause in self.pauses.values_mut() { + pause.until = pause.server_until.filter(|t| Instant::now() < *t); + } + self.close_windows(); + self.export_pending(); + let mut commands_open = true; + while self.round.is_some() || self.inflight.is_some() { + tokio::select! { + outcome = answer(&mut self.inflight) => self.on_answer(outcome), + _ = sleep_until(deadline) => { + log::debug!("shutdown deadline: cancelling the upload in flight"); + self.inflight = None; + self.round = None; + break; + } + // The opt-out reaches a draining generation too: cancel, clear, confirm. + command = self.commands.recv(), if commands_open => match command { + Some(Command::Purge(done)) => { + self.inflight = None; + self.round = None; + self.shared.clear(); + let _ = done.send(()); + break; + } + Some(Command::Flush(done)) => self.answer_when_done(done), + Some(Command::Shutdown(done)) => { + let _ = done.send(()); + } + None => commands_open = false, + }, + } + // A pass that ended with a request left to make (a split, the rerun) goes on. + if self.inflight.is_none() && self.round.is_some() { + self.advance(); + } + } + self.answer_flushes(); + let left = self.shared.cache.pending().len(); + if left > 0 { + log::debug!("{left} batches still cached at shutdown (replayed next start)"); + } + } + + /// Turn every open RTC stats window into its `lk.rtc.stats.sample` event. Windows bypass the + /// flood guard: they are the pipeline's own, bounded output. + fn close_windows(&mut self) { + let events = self.shared.windows.lock().unwrap_or_else(|e| e.into_inner()).close(); + for event in events { + self.shared.store.push(event); + } + } + + fn export_pending(&mut self) { + self.enqueue(); + self.begin_round(); + } + + /// Split records by owner — the project their session routes to, and the session — so each + /// batch has exactly one destination and one token. Records whose project receives nothing + /// are dropped here, counted as `disabled`. + fn by_owner( + &self, + items: Vec, + owner_of: impl Fn(&T) -> (&ScopeState, Option<&String>), + ) -> Vec<(Owner, Vec)> { + let destinations = self.destinations(); + let mut groups: Vec<(Owner, Vec)> = Vec::new(); + for item in items { + let (session, route) = owner_of(&item); + let process = session.trace_id == self.shared.process.trace_id; + let session = if process { PROCESS_OWNER.to_owned() } else { session.hex() }; + // The project captured with the record, never the session's current one — or, for a + // Room record captured before its first server, that Room's first project, so the + // cached batch names its project and replays after a restart. + let host = match route { + Some(route) => Some(route.clone()), + None if !process => destinations.first_project(&session), + None => None, + }; + let owner = Owner { host, session }; + if !destinations.alive(owner.host.as_deref()) { + Counters::add(&self.shared.counters.disabled, 1); + continue; + } + match groups.iter_mut().find(|(o, _)| *o == owner) { + Some((_, group)) => group.push(item), + None => groups.push((owner, vec![item])), + } + } + groups + } + + /// Encode everything queued — log records and finished spans — into the cache. No network. + fn enqueue(&mut self) { + self.enqueue_spans(); + let config = &self.shared.config; + let (max, max_bytes) = ( + config.max_batch_size.max(1) as usize, + usize::try_from(config.max_batch_bytes.max(1)).unwrap_or(usize::MAX), + ); + // One report per pass at most: a report that is itself dropped (oversized) or evicts + // another batch must not make the next report due in the same pass, or with + // `max_batch_size` 1 it would take the only place forever and no record would move. + let mut reported = false; + loop { + // Self-telemetry rides along with real data: never its own request, never its own + // cadence, and only when there is something to report — plus once at shutdown. It + // takes one of the batch's `max_batch_size` places. + let now = self.shared.counters.snapshot(); + let delta = now.since(&self.last_report); + let report_due = !reported && (delta.has_problems() || self.force_report); + let room = if report_due { max - 1 } else { max }; + let mut batch = self.shared.store.drain(room, max_bytes); + // Real data to ride along with (with `max_batch_size` 1 the report goes first, alone); + // the shutdown summary goes out even with nothing else queued. + let data = !batch.is_empty() || (room == 0 && !self.shared.store.is_empty()); + if !data && !self.force_report { + return; + } + if report_due { + let cached = self.shared.cache.pending().len() as u64; + // The host's console gets the cumulative line through the FFI log path. + log::debug!("{}", crate::stats::TelemetryStats::new(now, cached, self.status())); + let report = delta.report(cached); + batch.push(Queued::new(report, self.shared.process.clone())); + self.last_report = now; + self.force_report = false; + reported = true; + } + let global = self.shared.global.lock().unwrap_or_else(|e| e.into_inner()).clone(); + for (owner, records) in self.by_owner(batch, |q| (&q.session, q.route.as_ref())) { + let count = records.len() as u64; + let body = otlp::encode_logs(&self.shared.config.resource, &global, records); + self.push_batch(Signal::Logs, &owner, count, &body); + } + } + } + + /// Finished spans travel as their own batches on the traces signal: every one of them, so a + /// burst larger than a batch (or a shutdown) leaves nothing behind in memory. + fn enqueue_spans(&mut self) { + loop { + let (spans, dropped) = { + let mut registry = self.shared.spans.lock().unwrap_or_else(|e| e.into_inner()); + ( + registry.drain( + self.shared.config.max_batch_size.max(1) as usize, + usize::try_from(self.shared.config.max_batch_bytes.max(1)) + .unwrap_or(usize::MAX), + ), + registry.take_dropped(), + ) + }; + Counters::add(&self.shared.counters.queue_full, dropped); + if spans.is_empty() { + return; + } + let global = self.shared.global.lock().unwrap_or_else(|e| e.into_inner()).clone(); + for (owner, spans) in self.by_owner(spans, |s| (&s.session, s.route.as_ref())) { + let count = spans.len() as u64; + let body = otlp::encode_spans(&self.shared.config.resource, &global, spans); + self.push_batch(Signal::Traces, &owner, count, &body); + } + } + } + + /// Gzip the encoded batch and cache it. Compressed at rest as well as on the wire: the cache + /// holds 5–10× more, the disk write shrinks, and a replay costs no CPU. + /// + /// The record-size estimates that cut batches cannot see what encoding adds (session and + /// pipeline attributes, the self-report), so the real encoded size is checked here: a batch + /// over `max_batch_bytes` is halved until it fits, or is a single record. + fn push_batch(&mut self, signal: Signal, owner: &Owner, count: u64, body: &[u8]) { + let limit = + usize::try_from(self.shared.config.max_batch_bytes.max(1)).unwrap_or(usize::MAX); + if body.len() > limit { + let halves = match signal { + Signal::Logs => otlp::split_logs(body), + Signal::Traces => otlp::split_spans(body), + }; + if let Some([(first, first_count), (second, second_count)]) = halves { + self.push_batch(signal, owner, first_count, &first); + self.push_batch(signal, owner, second_count, &second); + } else { + // One record larger than a request may be: never cached, never sent. + log::warn!("a record of {} bytes exceeds max_batch_bytes; dropped", body.len()); + Counters::add(&self.shared.counters.oversized, count); + } + return; + } + let body = gzip(body); + self.seq += 1; + let seq = format!("{:06}", self.seq); + let id = BatchId::format(signal, &seq, count, &owner.session, owner.host.as_deref()); + self.store_batch(id, &body); + } + + /// Write one gzipped batch to the cache — unless the app opted out meanwhile. + fn store_batch(&mut self, id: String, body: &[u8]) { + if self.shared.revoked() { + return; + } + self.born.insert(id.clone(), Instant::now()); + match self.shared.cache.push(&id, body) { + Ok(evicted) => self.count_evicted(&evicted), + Err(err) => { + log::debug!("could not cache {id}: {err}"); + Counters::add(&self.shared.counters.cache_error, events_in(&id)); + self.born.remove(&id); + } + } + } + + /// Older batches pushed out by the cache's bounds: lost, but counted. Losing them mid-hold is + /// its own answer — the collector held us off longer than the cache could carry — so it is + /// counted apart from an ordinary overflow. + fn count_evicted(&mut self, evicted: &[String]) { + let now = Instant::now(); + let counters = self.shared.counters.clone(); + for id in evicted { + // `throttled` only when the evicted batch's own destination is under a server pause — + // for a host-less batch (process-level, pre-connect), the project it routes to. + let batch = BatchId::parse(id); + let host = match self.destinations().route(batch.host, batch.owner) { + Route::Send(target) => target.project.unwrap_or_default(), + _ => batch.host.unwrap_or_default().to_owned(), + }; + let held = + self.pauses.get(&host).is_some_and(|p| p.server_until.is_some_and(|t| now < t)); + let counter = if held { &counters.throttled } else { &counters.cache_full }; + Counters::add(counter, events_in(id)); + self.born.remove(id); + } + } + + /// Why uploads should wait right now, if they should. Data keeps flowing into the cache + /// meanwhile — write-ahead caching is what makes holding free. + fn hold_reason(&self) -> Option<&'static str> { + if self.device().holds_uploads() { + return Some("device asks for quiet"); + } + if self.draining { + return None; + } + if self.shared.spans.lock().unwrap_or_else(|e| e.into_inner()).any_open(SENSITIVE_SPANS) { + return Some("connecting"); + } + None + } + + fn status(&self) -> TelemetryStatus { + *self.shared.status.lock().unwrap_or_else(|e| e.into_inner()) + } + + /// Record the policy state; true when it changed (log once, not per tick). + fn set_status(&self, status: TelemetryStatus) -> bool { + let mut current = self.shared.status.lock().unwrap_or_else(|e| e.into_inner()); + let changed = *current != status; + *current = status; + changed + } + + /// Log a hold transition once. + fn log_hold(&mut self, reason: Option<&'static str>, backlog: usize) { + match (self.hold_logged, reason) { + (None, Some(now)) => log::debug!("held ({now}), backlog {backlog}"), + (Some(was), Some(now)) if was != now => { + log::debug!("held ({now}, was {was}), backlog {backlog}") + } + (Some(was), None) => match self.held_since { + Some(since) => log::debug!( + "resumed after {}s ({was}), backlog {backlog}", + since.elapsed().as_secs() + ), + None => log::debug!("resumed ({was}), backlog {backlog}"), + }, + _ => {} + } + self.hold_logged = reason; + } + + /// How many batches this call may send: none while held — a hard hold (offline) for as long + /// as it lasts, a soft one up to [`MAX_HOLD`], then one — otherwise what is left of the + /// interval's allowance when `metered`, everything when not. + fn budget(&mut self, backlog: usize, metered: bool) -> (usize, bool) { + if self.offline() { + self.log_hold(Some("offline"), backlog); + return (0, false); + } + let reason = self.hold_reason(); + self.log_hold(reason, backlog); + let Some(reason) = reason else { + self.held_since = None; + return (if metered { self.allowance } else { usize::MAX }, false); + }; + let since = *self.held_since.get_or_insert_with(Instant::now); + if since.elapsed() < MAX_HOLD { + log::trace!("holding {backlog} batches: {reason}"); + return (0, false); + } + log::warn!("held {}s ({reason}): sending one batch anyway", MAX_HOLD.as_secs()); + // Held long enough: one batch goes out, then the hold starts over. + self.held_since = Some(Instant::now()); + Counters::add(&self.shared.counters.hold_cap_hits, 1); + (1, true) + } + + /// A hard hold: nothing is attempted while the device is offline. + fn offline(&self) -> bool { + self.device().network == NetworkType::Unavailable + } + + /// Whether the pass may make its next request: the hold policy is evaluated before every + /// request, not once per pass. A hard hold stops the pass; a soft hold that began meanwhile + /// stops it too — unless this pass is the hold's one-batch escape (or a shutdown drain). + fn may_send(&mut self) -> bool { + // The opt-out stops a pass mid-way (a second guard: `advance` checks it as well). + if self.offline() || self.shared.revoked() { + return false; + } + if self.draining || self.round.as_ref().is_some_and(|r| r.escape) { + return true; + } + if self.hold_reason().is_some() { + self.held_since.get_or_insert_with(Instant::now); + return false; + } + true + } + + /// Whether a cached batch is past [`MAX_AGE`]. Wall-clock age decides for batches from a + /// previous launch; for this launch's own the monotonic clock must agree, so a clock set + /// forward cannot expire a fresh backlog (and one set back only delays expiry). + fn expired(&self, id: &str) -> bool { + let wall = Duration::from_nanos(now_unix_nanos().saturating_sub(BatchId::parse(id).stamp)); + wall > MAX_AGE && self.born.get(id).is_none_or(|born| born.elapsed() > MAX_AGE) + } + + /// When the loop must wake to upload without a tick: the earliest pause to run out, or a + /// soft hold reaching its cap while batches wait. + fn retry_deadline(&self) -> Option { + let pause = self.pauses.values().filter_map(|p| p.until).min(); + // A hard hold suppresses the soft hold's cap: waking for it could send nothing. It comes + // back when the hard hold lifts (a device change wakes the loop). + let hold = self + .held_since + .filter(|_| self.round.is_none() && !self.offline()) + .filter(|_| !self.shared.cache.pending().is_empty()) + .map(|since| since + MAX_HOLD); + pause.into_iter().chain(hold).min() + } + + fn clear_expired_pauses(&mut self) { + let now = Instant::now(); + for pause in self.pauses.values_mut() { + pause.until = pause.until.filter(|t| now < *t); + pause.server_until = pause.server_until.filter(|t| now < *t); + } + } + + /// Answer a `flush()` once the pass it started is over. + fn answer_when_done(&mut self, done: oneshot::Sender<()>) { + if self.round.is_some() || self.inflight.is_some() { + // Callers that stopped waiting are not kept. + self.flush_waiters.retain(|waiter| !waiter.is_closed()); + self.flush_waiters.push(done); + } else { + let _ = done.send(()); + } + } + + fn answer_flushes(&mut self) { + for done in self.flush_waiters.drain(..) { + let _ = done.send(()); + } + } + + /// Start a pass over the cache: oldest first, within this tick's budget, one request at a + /// time. A pass already running picks the new batches up in a second pass. + fn begin_round(&mut self) { + #[cfg(test)] + self.shared.passes.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if self.shared.revoked() { + return; + } + if self.round.is_some() || self.inflight.is_some() { + self.rerun = true; + return; + } + // An explicit flush applies to this pass only, whatever it finds. + let flushing = std::mem::take(&mut self.flushing); + let counters = self.shared.counters.clone(); + self.retry_deletes(); + self.bind_pre_connect(); + for id in self.shared.cache.pending() { + if self.expired(&id) { + self.forget(&id, Some(&counters.expired)); + } + } + self.retire_credentials(); + if self.nobody_listens() { + self.purge_dead(); + self.set_status(TelemetryStatus::Off); + return self.answer_flushes(); + } + let pending = self.shared.cache.pending(); + if pending.is_empty() { + self.held_since = None; + self.set_status(TelemetryStatus::Ok); + return self.answer_flushes(); + } + // Only next to a call is a pass charged to the interval's allowance. + let metered = !(self.draining || flushing) && self.in_call(); + let (budget, escape) = self.budget(pending.len(), metered); + if budget == 0 { + if self.held_since.is_some() || self.offline() { + self.set_status(TelemetryStatus::Held); + } + return self.answer_flushes(); + } + let metered = metered && !escape; + self.round = + Some(Round { queue: pending.into(), budget, escape, metered, ..Round::default() }); + self.advance(); + } + + /// Take the pass to its next request, or to its end. + fn advance(&mut self) { + if self.inflight.is_some() || self.shared.revoked() { + return; + } + let counters = self.shared.counters.clone(); + loop { + let next = match self.round.as_mut() { + None => return, + Some(round) if round.attempts >= round.budget => None, + Some(round) => round.queue.pop_front(), + }; + let Some(id) = next else { return self.finish_round() }; + if self.undeletable.contains(&id) { + continue; + } + if !self.may_send() { + self.mark(|round| round.held = true); + return self.finish_round(); + } + let (route, signal) = { + let batch = BatchId::parse(&id); + (self.destinations().route(batch.host, batch.owner), batch.signal) + }; + let target = match route { + Route::Send(target) => target, + // A hard hold: no token that may be used (missing, expired, grant-less, refused). + Route::Wait => { + self.mark(|round| round.waiting = true); + continue; + } + Route::Drop => { + self.forget(&id, Some(&counters.disabled)); + continue; + } + }; + // The project the request goes to — and nothing else — is what its answer is about. + let key = target.project.clone().unwrap_or_default(); + if self.pauses.get(&key).is_some_and(|p| p.until.is_some_and(|t| Instant::now() < t)) { + self.mark(|round| round.paused = true); + continue; + } + let body = match self.shared.cache.read(&id) { + Ok(body) if intact(&body) => body, + Ok(_) => { + log::debug!("cached batch {id} is corrupt; dropped"); + self.forget(&id, Some(&counters.corrupt)); + continue; + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + self.born.remove(&id); + continue; + } + // There but unreadable right now (file protection on a locked device, a + // permission): not corrupt — it is left alone and tried again later. + Err(err) => { + log::debug!("cached batch {id} is inaccessible ({err}); trying later"); + self.mark(|round| round.waiting = true); + continue; + } + }; + self.mark(|round| round.attempts += 1); + if self.round.as_ref().is_some_and(|r| r.metered) { + self.allowance = self.allowance.saturating_sub(1); + } + let request = self.request(&body, signal, &target); + self.inflight = Some(InFlight { id, signal, key, target, body, request }); + return; + } + } + + fn mark(&mut self, f: impl FnOnce(&mut Round)) { + if let Some(round) = self.round.as_mut() { + f(round); + } + } + + /// The pass is over: report where uploads stand, then start the next pass if one is due. + fn finish_round(&mut self) { + let Some(round) = self.round.take() else { return }; + let now = Instant::now(); + let paused = |server: bool| { + self.pauses.values().any(|p| { + let until = if server { p.server_until } else { p.until }; + until.is_some_and(|t| now < t) + }) + }; + let status = if paused(true) { + TelemetryStatus::Throttled + } else if paused(false) { + TelemetryStatus::Paused + } else if round.held { + TelemetryStatus::Held + } else if round.waiting && round.attempts == 0 { + TelemetryStatus::Waiting + } else if self.held_since.is_some() { + TelemetryStatus::Held + } else { + TelemetryStatus::Ok + }; + if self.set_status(status) && status == TelemetryStatus::Waiting { + log::debug!("waiting for a destination or a usable token"); + } + if std::mem::take(&mut self.rerun) { + self.begin_round(); + } + if self.round.is_none() && self.inflight.is_none() { + self.answer_flushes(); + } + } + + /// What the collector answered to the request on the wire, acted on; then the pass goes on. + fn on_answer(&mut self, outcome: Outcome) { + let Some(InFlight { id, signal, key, target, body, .. }) = self.inflight.take() else { + return; + }; + // The app opted out while the request was out: whatever came back, nothing is written, + // retried or rescheduled. + if self.shared.revoked() { + self.round = None; + return; + } + let counters = self.shared.counters.clone(); + match outcome { + Outcome::Answered(Verdict::Accepted { rejected, reason }) => { + self.forget(&id, None); + Counters::add(&counters.uploads_sent, 1); + Counters::add(&counters.upload_bytes, body.len() as u64); + // The batch's other records were accepted; the refused ones are counted, never + // retried (OTLP partial success). + if rejected > 0 { + log::warn!("collector refused {rejected} records: {reason}"); + Counters::add(&counters.rejected, rejected); + } + self.recovered(&key); + } + Outcome::Answered(Verdict::TooLarge) => { + // Split in two; the halves go at once, down to single records — each one a + // request of its own against this pass's budget. + match self.split(&id, &body, signal) { + Ok(halves) if halves.is_empty() => { + log::warn!("a single record is larger than the collector accepts; dropped"); + self.forget(&id, Some(&counters.oversized)); + } + Ok(halves) => { + if let Some(round) = self.round.as_mut() { + for half in halves.into_iter().rev() { + round.queue.push_front(half); + } + } + } + Err(err) => { + // The batch stays whole where it was; try the split again later. + self.back_off(&key, None, &format!("cannot store the split batch: {err}")); + } + } + } + Outcome::Answered(Verdict::Rejected(reason)) => { + log::warn!("batch rejected by the collector, data lost: {reason}"); + self.forget(&id, Some(&counters.rejected)); + self.recovered(&key); + } + Outcome::Answered( + verdict @ (Verdict::Unauthorized(_) | Verdict::NotFound | Verdict::Disabled), + ) if self.has_override() => { + log::warn!("the collector at {} refused a batch: {verdict:?}", target.logs); + self.forget(&id, Some(&counters.rejected)); + } + Outcome::Answered(Verdict::Unauthorized(reason)) => { + // A credential problem, not a data problem: this token is not used again and the + // batch waits for the next one. + log::debug!("upload unauthorized: {reason}"); + Counters::add(&counters.auth_denied, 1); + if let Some(token) = &target.token { + self.destinations().refuse(token); + } + if let Some(round) = self.round.as_mut() { + round.waiting = true; + } + } + Outcome::Answered(Verdict::NotFound) => { + self.kill(target.project.as_deref(), Dead::NotFound) + } + Outcome::Answered(Verdict::Disabled) => { + self.kill(target.project.as_deref(), Dead::Disabled) + } + Outcome::Answered(Verdict::Throttled { delay_ms, reason }) => { + Counters::add(&counters.upload_failures, 1); + self.back_off(&key, Some(Duration::from_millis(delay_ms)), &reason); + } + Outcome::Answered(Verdict::Retry { delay_ms, reason }) => { + Counters::add(&counters.upload_failures, 1); + self.back_off(&key, delay_ms.map(Duration::from_millis), &reason); + } + Outcome::NoAnswer { timed_out, reason } => { + let counter = + if timed_out { &counters.upload_timeouts } else { &counters.upload_failures }; + Counters::add(counter, 1); + self.back_off(&key, None, &reason); + } + } + self.advance(); + } + + /// 413: re-encode the batch as two halves and replace it in the cache transactionally — + /// both halves committed where the batch was (on disk stays on disk) before it goes, nothing + /// evicted in between. `Ok` holds the halves' ids, oldest first, or none for a single record + /// (it cannot shrink); `Err` means the cache could not take the halves and the batch is + /// still there, whole. + fn split(&mut self, id: &str, body: &[u8], signal: Signal) -> std::io::Result> { + let Some(encoded) = gunzip(body) else { return Ok(Vec::new()) }; + let halves = match signal { + Signal::Logs => otlp::split_logs(&encoded), + Signal::Traces => otlp::split_spans(&encoded), + }; + let Some([(first, first_count), (second, second_count)]) = halves else { + return Ok(Vec::new()); + }; + let new = [ + (BatchId::half(id, 'a', first_count), gzip(&first)), + (BatchId::half(id, 'b', second_count), gzip(&second)), + ]; + let evicted = self.shared.cache.replace(id, &new)?; + self.count_evicted(&evicted); + self.born.remove(id); + let now = Instant::now(); + for (half, _) in &new { + self.born.insert(half.clone(), now); + } + log::debug!( + "413: split a batch of {} into {first_count} + {second_count}", + first_count + second_count + ); + Ok(new.into_iter().map(|(half, _)| half).collect()) + } + + /// A Room batch cached before the Room had a server names no project; once the Room's first + /// project is known it is rewritten under an id that names it (a journaled replace, as + /// crash-safe as a push), so it replays after a restart, when this launch's Room → project + /// map is gone. Recovery rolls an interrupted rewrite forward once its journal is durable + /// ([`FileCache`](crate::cache::FileCache)); a crash before that leaves the batch unbound. + fn bind_pre_connect(&mut self) { + for id in self.shared.cache.pending() { + let batch = BatchId::parse(&id); + let (None, Some(owner)) = (batch.host, batch.owner) else { continue }; + // Accepted already, its delete pending: a new id would be sent again. + if owner == PROCESS_OWNER || self.undeletable.contains(&id) { + continue; + } + let Some(project) = self.destinations().first_project(owner) else { continue }; + let Ok(body) = self.shared.cache.read(&id) else { continue }; + // A host-less id ends with the empty host: appending one is the bound id. + let bound = format!("{id}{project}"); + if let Ok(evicted) = self.shared.cache.replace(&id, &[(bound.clone(), body)]) { + if let Some(born) = self.born.remove(&id) { + self.born.insert(bound, born); + } + self.count_evicted(&evicted); + } + } + } + + /// Keep credentials only for Rooms still alive or with batches still cached. + fn retire_credentials(&self) { + let live: HashMap> = { + let scopes = self.shared.scopes.lock().unwrap_or_else(|e| e.into_inner()); + scopes.iter().filter_map(|s| s.upgrade()).map(|s| (s.hex(), s.route())).collect() + }; + let backlog = self + .shared + .cache + .pending() + .iter() + .filter_map(|id| { + let batch = BatchId::parse(id); + Some((batch.host.map(str::to_owned), batch.owner?.to_owned())) + }) + .collect(); + self.destinations().retain_owners(&live, &backlog); + } + + fn destinations(&self) -> std::sync::MutexGuard<'_, crate::destination::Destinations> { + self.shared.destinations.lock().unwrap_or_else(|e| e.into_inner()) + } + + fn has_override(&self) -> bool { + self.destinations().has_override() + } + + /// Every project the process talked to receives nothing: telemetry is off. + fn nobody_listens(&self) -> bool { + matches!(self.destinations().route(None, None), Route::Drop) + } + + /// The project on `host` receives nothing more: its cached batches go now. + fn kill(&mut self, host: Option<&str>, why: Dead) { + if let Some(host) = host { + self.destinations().kill(host, why); + } + self.purge_dead(); + } + + /// Delete every cached batch whose project receives nothing, counted as `disabled`. + fn purge_dead(&mut self) { + let counters = self.shared.counters.clone(); + for id in self.shared.cache.pending() { + let batch = BatchId::parse(&id); + if self.destinations().route(batch.host, batch.owner) == Route::Drop { + self.forget(&id, Some(&counters.disabled)); + } + } + } + + /// Remove a cached batch; when it was lost, count its records against that reason. A + /// delete that fails leaves the batch pending deletion: never uploaded again, deleted as soon + /// as the storage lets us (a restart before then may send it once more). + fn forget(&mut self, id: &str, lost: Option<&AtomicU64>) { + if let Err(err) = self.shared.cache.remove(id) { + log::warn!("cannot delete cached batch {id} ({err}); it will not be sent again"); + self.undeletable.insert(id.to_owned()); + } + self.born.remove(id); + if let Some(counter) = lost { + Counters::add(counter, events_in(id)); + } + } + + /// Try the deletes that failed before. + fn retry_deletes(&mut self) { + let cache = self.shared.cache.clone(); + self.undeletable.retain(|id| cache.remove(id).is_err()); + } + + fn recovered(&mut self, key: &str) { + if let Some(pause) = self.pauses.remove(key) { + log::info!( + "uploads recovered after {} failures, backlog {}", + pause.failing, + self.shared.cache.pending().len() + ); + } + } + + /// Pause uploads to one destination after a failure — the others carry on: for the delay the + /// server named, honored in full (neither shutdown nor a hold's escape hatch cuts it short), + /// else for a jittered exponential backoff. + fn back_off(&mut self, key: &str, server_delay: Option, reason: &str) { + let pause = self.pauses.entry(key.to_owned()).or_default(); + pause.failing += 1; + let wait = match server_delay { + Some(delay) => { + let delay = delay.min(MAX_SERVER_DELAY); + pause.server_until = Some(Instant::now() + delay); + delay + } + None => backoff(pause.failing), + }; + pause.until = Some(Instant::now() + wait); + let failing = pause.failing; + let backlog = self.shared.cache.pending().len(); + let wait = wait.as_secs_f64(); + if failing == 1 { + log::warn!("upload failed: {reason}; retrying in {wait:.1}s, backlog {backlog}"); + } else { + log::debug!( + "upload failed ({failing} in a row): {reason}; retrying in {wait:.1}s, backlog {backlog}" + ); + } + } + + /// One request, bounded by `export_timeout_ms`, as a future the loop polls. Never retried + /// here: the answer decides. + fn request( + &self, + body: &[u8], + signal: Signal, + target: &Target, + ) -> Pin + Send>> { + let mut headers = HashMap::from([ + ("Content-Type".to_owned(), otlp::CONTENT_TYPE.to_owned()), + ("Content-Encoding".to_owned(), "gzip".to_owned()), + ("Priority".to_owned(), PRIORITY.to_owned()), + ]); + if let Some(token) = &target.token { + headers.insert("Authorization".to_owned(), format!("Bearer {token}")); + } + let request = + ExportRequest { url: target.url(signal).to_owned(), headers, body: body.to_vec() }; + let bound = Duration::from_millis(self.shared.config.export_timeout_ms.max(1)); + let transport = self.transport.clone(); + Box::pin(async move { + match timeout(bound, transport.send(request)).await { + Ok(Ok(response)) => Outcome::Answered(Verdict::of(&response)), + Ok(Err(ExportError::Retryable { reason, retry_after_ms: Some(ms) })) => { + Outcome::Answered(Verdict::Throttled { delay_ms: ms, reason }) + } + Ok(Err(ExportError::Retryable { reason, retry_after_ms: None })) => { + Outcome::NoAnswer { timed_out: false, reason } + } + Ok(Err(ExportError::Rejected { reason })) => { + Outcome::Answered(Verdict::Rejected(reason)) + } + Ok(Err(ExportError::Disabled)) => Outcome::Answered(Verdict::Disabled), + Err(_) => Outcome::NoAnswer { timed_out: true, reason: "timed out".to_owned() }, + } + }) + } +} + +/// The answer to the request on the wire, or never while none is out. +async fn answer(inflight: &mut Option) -> Outcome { + match inflight { + Some(inflight) => inflight.request.as_mut().await, + None => std::future::pending().await, + } +} + +/// Sleep until `deadline`, or forever without one. +async fn at(deadline: Option) { + match deadline { + Some(deadline) => sleep_until(deadline).await, + None => std::future::pending().await, + } +} + +/// The `failures`-th consecutive failure's wait: [`RETRY_BASE`] doubling up to [`RETRY_CAP`], +/// fully jittered: uniform in `[0, backoff]`. +fn backoff(failures: u32) -> Duration { + let full = RETRY_BASE.saturating_mul(1 << failures.saturating_sub(1).min(16)).min(RETRY_CAP); + full.mul_f64(rand::random::()) +} + +/// A cached batch is intact when it is one complete gzip member (RFC 1952) whose CRC checks +/// out: a truncated or bit-flipped file is caught here, never sent. +fn intact(body: &[u8]) -> bool { + body.starts_with(&[0x1f, 0x8b]) && gunzip(body).is_some() +} + +fn gunzip(body: &[u8]) -> Option> { + use std::io::Read; + let mut out = Vec::with_capacity(body.len() * 4); + flate2::read::GzDecoder::new(body).read_to_end(&mut out).ok()?; + Some(out) +} + +/// Level 1: protobuf with repeated attribute keys shrinks 5–10× already; higher levels buy little +/// for more CPU. +fn gzip(body: &[u8]) -> Vec { + let mut encoder = GzEncoder::new(Vec::with_capacity(body.len() / 4), Compression::fast()); + // Writing into a Vec cannot fail. + let _ = encoder.write_all(body); + encoder.finish().unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn batch_ids_carry_their_owner() { + let id = + BatchId::format(Signal::Traces, "000007", 12, "abc", Some("my-proj.livekit.cloud")); + let parsed = BatchId::parse(&id); + assert_eq!((parsed.count, parsed.signal), (12, Signal::Traces)); + assert_eq!(parsed.owner, Some("abc")); + assert_eq!(parsed.host, Some("my-proj.livekit.cloud"), "hosts may contain dashes"); + let process = BatchId::format(Signal::Logs, "000008", 3, "def", None); + assert_eq!(BatchId::parse(&process).host, None); + assert_eq!(events_in(&process), 3); + assert!(id < process, "ids sort oldest first"); + let (a, b) = (BatchId::half(&id, 'a', 6), BatchId::half(&id, 'b', 6)); + assert!(id < a && a < b && b < process, "halves keep their place in line"); + assert_eq!((events_in(&a), BatchId::parse(&b).host), (6, parsed.host)); + } + + #[test] + fn backoff_doubles_with_full_jitter_up_to_a_minute() { + for (failures, full) in [(1, 1), (2, 2), (3, 4), (6, 32), (7, 60), (40, 60)] { + let full = Duration::from_secs(full); + let waits: Vec = (0..64).map(|_| backoff(failures)).collect(); + assert!(waits.iter().all(|w| *w <= full), "{failures}: {waits:?} vs {full:?}"); + assert!(waits.iter().any(|w| *w < full / 2), "{failures}: jitter spans the range"); + } + } +} diff --git a/livekit-telemetry/src/lib.rs b/livekit-telemetry/src/lib.rs new file mode 100644 index 000000000..c4045e13e --- /dev/null +++ b/livekit-telemetry/src/lib.rs @@ -0,0 +1,74 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![doc = include_str!("../README.md")] + +/// Event data model: what SDKs push in. +mod event; + +/// Bounded in-memory queue between `emit` and the exporter. +mod store; + +/// Pipeline health counters and the `lk.telemetry.report` event. +mod stats; + +/// Host-reported device state and the cadence policy derived from it. +mod device; + +/// RTC stats samples and their on-device windowing. +mod rtc; + +/// Spans: one attempt at an operation, with explicit handles across the FFI. +mod scope; +#[cfg_attr(test, allow(dead_code))] // `Spans::open_count` serves the device contract tests +mod span; + +/// Batch exporter actor: timer, OTLP encoding, retry policy. +mod exporter; + +/// OTLP/HTTP protobuf encoding of a batch. +mod otlp; + +/// Queue of encoded batches between the exporter and the transport (memory or disk). +mod cache; + +/// OTLP protobuf types (re-exported from `opentelemetry-proto`). +mod proto; + +/// Transport seam: how encoded batches leave the device. +mod transport; + +/// Where batches go: server URL + token → ingest URL, grant, expiry, per-project routing. +mod destination; + +/// Entry point and configuration. +#[allow(dead_code)] // `weak_commands` serves `global`; test hooks serve the pipeline tests +mod telemetry; +mod trace; + +pub use cache::{BatchCache, FileCache, MemoryCache}; +pub use destination::ENDPOINT_OVERRIDE_ENV; +pub use device::*; +pub use event::*; +pub use exporter::Exporter; +pub use rtc::{RtcStat, RtcStatsSample, StreamDirection, TrackKind}; +pub use scope::{DisconnectReason, RoomIdentity, Scope}; +pub use span::SpanOutcome; +pub use stats::{TelemetryStats, TelemetryStatus}; +pub use telemetry::*; +pub use trace::*; +pub use transport::*; + +#[cfg(feature = "uniffi")] +uniffi::setup_scaffolding!(); diff --git a/livekit-telemetry/src/otlp.rs b/livekit-telemetry/src/otlp.rs new file mode 100644 index 000000000..1276000d0 --- /dev/null +++ b/livekit-telemetry/src/otlp.rs @@ -0,0 +1,273 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use prost::Message; + +use crate::span::SpanKind; +use crate::{ + event::now_unix_nanos, + proto::opentelemetry::proto::{ + collector::{logs::v1::ExportLogsServiceRequest, trace::v1::ExportTraceServiceRequest}, + common::v1::{any_value, AnyValue, InstrumentationScope, KeyValue}, + logs::v1::{LogRecord, ResourceLogs, ScopeLogs, SeverityNumber}, + resource::v1::Resource, + trace::v1::{span, status, ResourceSpans, ScopeSpans, Span, Status}, + }, + span::SpanRecord, + store::Queued, + Attribute, AttributeValue, Severity, SpanOutcome, +}; + +pub(crate) const CONTENT_TYPE: &str = "application/x-protobuf"; + +fn resource(attributes: &[Attribute]) -> Option { + Some(Resource { + attributes: attributes.iter().map(KeyValue::from).collect(), + ..Default::default() + }) +} + +fn scope() -> Option { + Some(InstrumentationScope { + name: env!("CARGO_PKG_NAME").to_owned(), + version: env!("CARGO_PKG_VERSION").to_owned(), + ..Default::default() + }) +} + +/// Encode one batch as an OTLP `ExportLogsServiceRequest`: one resource, one instrumentation +/// scope (this crate), one log record per event. Every record carries its session's trace id and +/// attributes; records emitted inside a span carry its span id too. +pub(crate) fn encode_logs( + resource_attributes: &[Attribute], + global: &[Attribute], + events: Vec, +) -> Vec { + ExportLogsServiceRequest { + resource_logs: vec![ResourceLogs { + resource: resource(resource_attributes), + scope_logs: vec![ScopeLogs { + scope: scope(), + log_records: events.into_iter().map(|e| log_record(e, global)).collect(), + ..Default::default() + }], + ..Default::default() + }], + } + .encode_to_vec() +} + +/// Encode finished spans as an OTLP `ExportTraceServiceRequest`, each under its session's trace id. +pub(crate) fn encode_spans( + resource_attributes: &[Attribute], + global: &[Attribute], + spans: Vec, +) -> Vec { + ExportTraceServiceRequest { + resource_spans: vec![ResourceSpans { + resource: resource(resource_attributes), + scope_spans: vec![ScopeSpans { + scope: scope(), + spans: spans.into_iter().map(|s| otlp_span(s, global)).collect(), + ..Default::default() + }], + ..Default::default() + }], + } + .encode_to_vec() +} + +/// An encoded batch cut in two halves (a 413 answer), each with its record count; `None` when it +/// holds a single record or is not the one-resource, one-scope shape this crate encodes. +pub(crate) type Halves = Option<[(Vec, u64); 2]>; + +pub(crate) fn split_logs(encoded: &[u8]) -> Halves { + let mut request = ExportLogsServiceRequest::decode(encoded).ok()?; + let [resource] = &mut request.resource_logs[..] else { return None }; + let [scope] = &mut resource.scope_logs[..] else { return None }; + let n = scope.log_records.len(); + if n < 2 { + return None; + } + let second = scope.log_records.split_off(n / 2); + let first = request.encode_to_vec(); + request.resource_logs[0].scope_logs[0].log_records = second; + Some([(first, (n / 2) as u64), (request.encode_to_vec(), (n - n / 2) as u64)]) +} + +pub(crate) fn split_spans(encoded: &[u8]) -> Halves { + let mut request = ExportTraceServiceRequest::decode(encoded).ok()?; + let [resource] = &mut request.resource_spans[..] else { return None }; + let [scope] = &mut resource.scope_spans[..] else { return None }; + let n = scope.spans.len(); + if n < 2 { + return None; + } + let second = scope.spans.split_off(n / 2); + let first = request.encode_to_vec(); + request.resource_spans[0].scope_spans[0].spans = second; + Some([(first, (n / 2) as u64), (request.encode_to_vec(), (n - n / 2) as u64)]) +} + +fn log_record(Queued { mut event, session, .. }: Queued, global: &[Attribute]) -> LogRecord { + session.decorate(&mut event.attributes, global); + let time_unix_nano = event.timestamp_ns.unwrap_or_else(now_unix_nanos); + // Events carry a display body (OTel: "a string display message of the event"); the name is + // the last resort so no event ever renders as an empty line. `otel.event.name` (semconv 1.39) + // duplicates `EventName` for backends that do not surface the field yet. + let body = event.body.or_else(|| (!event.name.is_empty()).then(|| event.name.clone())); + if !event.name.is_empty() { + event.attributes.push(Attribute::new("otel.event.name", event.name.clone())); + } + LogRecord { + time_unix_nano, + observed_time_unix_nano: time_unix_nano, + severity_number: SeverityNumber::from(event.severity) as i32, + severity_text: severity_text(event.severity).to_owned(), + body: body.map(|text| AnyValue { value: Some(any_value::Value::StringValue(text)) }), + attributes: event.attributes.iter().map(KeyValue::from).collect(), + event_name: event.name, + trace_id: session.trace_id.to_vec(), + span_id: event.span_id.map(|id| id.to_be_bytes().to_vec()).unwrap_or_default(), + ..Default::default() + } +} + +fn otlp_span(mut record: SpanRecord, global: &[Attribute]) -> Span { + let session = record.session.clone(); + session.decorate(&mut record.attributes, global); + let mut attributes: Vec = record.attributes.iter().map(KeyValue::from).collect(); + attributes.extend(record.outcome_attributes().iter().map(KeyValue::from)); + Span { + trace_id: session.trace_id.to_vec(), + span_id: record.span_id.to_be_bytes().to_vec(), + parent_span_id: record.parent_span_id.map(|p| p.to_be_bytes().to_vec()).unwrap_or_default(), + name: record.name, + kind: match record.kind { + SpanKind::Internal => span::SpanKind::Internal, + SpanKind::Client => span::SpanKind::Client, + } as i32, + start_time_unix_nano: record.start_ns, + end_time_unix_nano: record.end_ns, + attributes, + events: record + .events + .into_iter() + .map(|e| span::Event { + time_unix_nano: e.time_ns, + name: e.name, + attributes: e.attributes.iter().map(KeyValue::from).collect(), + ..Default::default() + }) + .collect(), + // OTel: instrumentation should not set `Ok`; success and cancellation stay `Unset` and + // are told apart by `lk.outcome`. + status: Some(Status { + code: match record.outcome { + SpanOutcome::Error => status::StatusCode::Error, + SpanOutcome::Ok | SpanOutcome::Cancelled => status::StatusCode::Unset, + } as i32, + message: record.error_type.unwrap_or_default(), + }), + ..Default::default() + } +} + +impl From for SeverityNumber { + fn from(severity: Severity) -> Self { + match severity { + Severity::Trace => SeverityNumber::Trace, + Severity::Debug => SeverityNumber::Debug, + Severity::Info => SeverityNumber::Info, + Severity::Warn => SeverityNumber::Warn, + Severity::Error => SeverityNumber::Error, + } + } +} + +fn severity_text(severity: Severity) -> &'static str { + match severity { + Severity::Trace => "TRACE", + Severity::Debug => "DEBUG", + Severity::Info => "INFO", + Severity::Warn => "WARN", + Severity::Error => "ERROR", + } +} + +impl From<&Attribute> for KeyValue { + fn from(attribute: &Attribute) -> Self { + let value = match &attribute.value { + AttributeValue::Str(s) => any_value::Value::StringValue(s.clone()), + AttributeValue::Int(i) => any_value::Value::IntValue(*i), + AttributeValue::Double(d) => any_value::Value::DoubleValue(*d), + AttributeValue::Bool(b) => any_value::Value::BoolValue(*b), + }; + KeyValue { + key: attribute.key.clone(), + value: Some(AnyValue { value: Some(value) }), + ..Default::default() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TelemetryEvent; + + #[test] + fn encodes_events_as_otlp_log_records() { + let resource = [Attribute::new("service.name", "test")]; + let event = TelemetryEvent::new("lk.ping") + .with_severity(Severity::Warn) + .with_body("hi") + .with_attribute("lk.ping.seq", 7i64); + let session = crate::scope::ScopeState::with_trace_id([7u8; 16]); + let bytes = encode_logs(&resource, &[], vec![Queued::new(event, session)]); + + let decoded = ExportLogsServiceRequest::decode(&bytes[..]).expect("valid OTLP"); + let resource_logs = &decoded.resource_logs[0]; + let res_attr = &resource_logs.resource.as_ref().expect("resource").attributes[0]; + assert_eq!(res_attr.key, "service.name"); + let scope_logs = &resource_logs.scope_logs[0]; + assert_eq!(scope_logs.scope.as_ref().expect("scope").name, "livekit-telemetry"); + let record = &scope_logs.log_records[0]; + assert_eq!(record.event_name, "lk.ping"); + assert_eq!(record.trace_id, vec![7u8; 16]); + assert!(record.span_id.is_empty()); + assert_eq!(record.severity_number, SeverityNumber::Warn as i32); + assert_eq!(record.severity_text, "WARN"); + assert!(record.time_unix_nano > 0); + assert_eq!(record.attributes[0].key, "lk.ping.seq"); + assert_eq!( + record.attributes[0].value.as_ref().and_then(|v| v.value.clone()), + Some(any_value::Value::IntValue(7)) + ); + } + + #[test] + fn events_without_a_body_carry_their_name_as_body() { + let session = crate::scope::ScopeState::with_trace_id([7u8; 16]); + let event = TelemetryEvent::new("lk.rtc.stats.sample"); + let bytes = encode_logs(&[], &[], vec![Queued::new(event, session)]); + let decoded = ExportLogsServiceRequest::decode(&bytes[..]).expect("valid OTLP"); + let record = &decoded.resource_logs[0].scope_logs[0].log_records[0]; + assert_eq!(record.event_name, "lk.rtc.stats.sample"); + assert_eq!( + record.body.as_ref().and_then(|b| b.value.clone()), + Some(any_value::Value::StringValue("lk.rtc.stats.sample".into())) + ); + } +} diff --git a/livekit-telemetry/src/proto/mod.rs b/livekit-telemetry/src/proto/mod.rs new file mode 100644 index 000000000..195d5e2bb --- /dev/null +++ b/livekit-telemetry/src/proto/mod.rs @@ -0,0 +1,26 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! OTLP protobuf types from the upstream `opentelemetry-proto` crate (prost-generated, +//! `gen-tonic-messages` only — no tonic/gRPC). The crate tracks a newer proto revision than +//! the OTLP 1.x stable surface, so construct its messages with `..Default::default()`. +//! +//! Only the types are used; the `opentelemetry`/`opentelemetry_sdk` crates it depends on are +//! dead code here and LTO removes them from release binaries (measured: +8 bytes on iOS). + +pub mod opentelemetry { + pub mod proto { + pub use opentelemetry_proto::tonic::{collector, common, logs, resource, trace}; + } +} diff --git a/livekit-telemetry/src/rtc.rs b/livekit-telemetry/src/rtc.rs new file mode 100644 index 000000000..cd29faeef --- /dev/null +++ b/livekit-telemetry/src/rtc.rs @@ -0,0 +1,1002 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{collections::HashMap, sync::Arc, time::Duration}; + +use tokio::time::Instant; + +use crate::{ + event::now_unix_nanos, scope::ScopeState, store::Queued, Attribute, AttributeValue, + TelemetryEvent, +}; + +/// Audio or video. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum TrackKind { + Audio, + Video, +} + +/// Received (inbound) or sent (outbound) media. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum StreamDirection { + Inbound, + Outbound, +} + +/// One reading of a track's RTP statistics, as `getStats()` reports them. +/// +/// Cumulative counters (`bytes`, `packets`, `freeze_count`, …) are passed through as reported — +/// monotonic, paired with their denominators (the W3C webrtc-stats model) so any layer can +/// recompute rates and a dropped window never corrupts the next. Gauges (`jitter_ms`, `rtt_ms`, +/// `frames_per_second`, `audio_level`) are summarised per window as min/max/avg. Fields a +/// platform or direction does not have stay `None` and are omitted from the wire. +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, PartialEq)] +pub struct RtcStatsSample { + pub track_sid: String, + pub kind: TrackKind, + pub direction: StreamDirection, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub codec: Option, + // Cumulative counters. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub bytes: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub packets: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub packets_lost: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub freeze_count: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub freezes_duration_ms: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub concealed_samples: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub concealment_events: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub jitter_buffer_delay_ms: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub jitter_buffer_emitted_count: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub quality_limitation_bandwidth_ms: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub quality_limitation_cpu_ms: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub quality_limitation_other_ms: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub pause_count: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub pauses_duration_ms: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub silent_concealed_samples: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub interruption_count: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub interruptions_duration_ms: Option, + // Gauges. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub jitter_ms: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub rtt_ms: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub frames_per_second: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub audio_level: Option, + /// When the reading was taken; `None` = now. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub timestamp_ns: Option, + /// The RTP stream this sample describes when a track has several (simulcast layers): any + /// stable id (`rid`, ssrc, the stats id). The core folds layers into the track; a platform + /// never sums them itself. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub layer: Option, +} + +/// One entry of a WebRTC `RTCStatsReport`, as the platform got it: the entry's `type`, `id` and +/// its standard members (W3C webrtc-stats names; nested maps flattened with a dot, e.g. +/// `qualityLimitationDurations.cpu`). Numbers may arrive as `Int`, `Double` or numeric `Str`. +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, PartialEq)] +pub struct RtcStat { + pub kind: String, + pub id: String, + pub members: HashMap, +} + +fn num(stat: &RtcStat, key: &str) -> Option { + match stat.members.get(key)? { + AttributeValue::Int(i) => Some(*i as f64), + AttributeValue::Double(d) => Some(*d), + AttributeValue::Str(s) => s.parse().ok(), + AttributeValue::Bool(_) => None, + } +} + +fn count(stat: &RtcStat, key: &str) -> Option { + num(stat, key).map(|v| v.max(0.0) as u64) +} + +/// A duration member (seconds, per webrtc-stats) as whole milliseconds. +fn ms(stat: &RtcStat, key: &str) -> Option { + num(stat, key).map(|s| (s.max(0.0) * 1000.0) as u64) +} + +fn text(stat: &RtcStat, key: &str) -> Option { + match stat.members.get(key)? { + AttributeValue::Str(s) => Some(s.clone()), + _ => None, + } +} + +/// The samples in one track's `getStats()` report: one per `outbound-rtp` (tagged with its +/// layer) or `inbound-rtp` entry of `direction`, codec resolved through `codecId`, RTT from the +/// `remote-inbound-rtp` that reports on the stream (outbound) or the nominated candidate pair +/// (inbound), durations converted from seconds to milliseconds. One mapping for every SDK. +pub(crate) fn samples_from_report( + track_sid: &str, + kind: TrackKind, + direction: StreamDirection, + report: &[RtcStat], + timestamp_ns: Option, +) -> Vec { + let index = Index::new(report); + index + .rtp + .iter() + .filter(|(_, d)| *d == direction) + .map(|(stat, direction)| index.sample(stat, track_sid, kind, *direction, timestamp_ns)) + .collect() +} + +/// The samples in a whole peer connection's `getStats()` report: every RTP stream whose track +/// is in `tracks` (MediaStreamTrack id → track sid; `trackIdentifier` of an `inbound-rtp`, or of +/// the `media-source` an `outbound-rtp` names), kind from the stream's `kind`. Streams of +/// unknown tracks are skipped. One call per peer connection per poll: no platform walks its +/// participants and tracks. +pub(crate) fn samples_from_peer_report( + report: &[RtcStat], + tracks: &HashMap, + timestamp_ns: Option, +) -> Vec { + let index = Index::new(report); + index + .rtp + .iter() + .filter_map(|(stat, direction)| { + let track_id = match direction { + StreamDirection::Inbound => text(stat, "trackIdentifier"), + StreamDirection::Outbound => text(stat, "mediaSourceId") + .and_then(|id| index.by_id.get(id.as_str()).copied()) + .filter(|source| source.kind == "media-source") + .and_then(|source| text(source, "trackIdentifier")), + }?; + let sid = tracks.get(&track_id)?; + let kind = match text(stat, "kind").or_else(|| text(stat, "mediaType"))?.as_str() { + "audio" => TrackKind::Audio, + "video" => TrackKind::Video, + _ => return None, + }; + Some(index.sample(stat, sid, kind, *direction, timestamp_ns)) + }) + .collect() +} + +/// A report indexed once: entries by id, the `remote-inbound-rtp` reporting on each outbound +/// stream, the nominated candidate pair's RTT, the RTP streams with their direction. Resolving a +/// stream's codec, RTT or media source is then a lookup, not a scan of the report. +struct Index<'a> { + by_id: HashMap<&'a str, &'a RtcStat>, + remote_by_local: HashMap, + pair_rtt_ms: Option, + rtp: Vec<(&'a RtcStat, StreamDirection)>, +} + +impl<'a> Index<'a> { + fn new(report: &'a [RtcStat]) -> Self { + let mut index = Self { + by_id: HashMap::with_capacity(report.len()), + remote_by_local: HashMap::new(), + pair_rtt_ms: None, + rtp: Vec::new(), + }; + for stat in report { + index.by_id.insert(stat.id.as_str(), stat); + match stat.kind.as_str() { + "outbound-rtp" => index.rtp.push((stat, StreamDirection::Outbound)), + "inbound-rtp" => index.rtp.push((stat, StreamDirection::Inbound)), + "remote-inbound-rtp" => { + if let Some(local) = text(stat, "localId") { + index.remote_by_local.insert(local, stat); + } + } + "candidate-pair" if index.pair_rtt_ms.is_none() => { + let nominated = + matches!(stat.members.get("nominated"), Some(AttributeValue::Bool(true))) + || text(stat, "state").as_deref() == Some("succeeded"); + if nominated { + index.pair_rtt_ms = num(stat, "currentRoundTripTime").map(|s| s * 1000.0); + } + } + _ => {} + } + } + index + } + + /// One RTP stream entry as a sample. + fn sample( + &self, + stat: &RtcStat, + track_sid: &str, + kind: TrackKind, + direction: StreamDirection, + timestamp_ns: Option, + ) -> RtcStatsSample { + let mut sample = RtcStatsSample::new(track_sid, kind, direction); + sample.codec = text(stat, "codecId") + .and_then(|id| self.by_id.get(id.as_str()).copied()) + .filter(|codec| codec.kind == "codec") + .and_then(|codec| text(codec, "mimeType")); + sample.timestamp_ns = timestamp_ns; + match direction { + StreamDirection::Outbound => { + sample.layer = Some(text(stat, "rid").unwrap_or_else(|| stat.id.clone())); + sample.bytes = count(stat, "bytesSent"); + sample.packets = count(stat, "packetsSent"); + sample.frames_per_second = num(stat, "framesPerSecond"); + sample.quality_limitation_bandwidth_ms = + ms(stat, "qualityLimitationDurations.bandwidth"); + sample.quality_limitation_cpu_ms = ms(stat, "qualityLimitationDurations.cpu"); + sample.quality_limitation_other_ms = ms(stat, "qualityLimitationDurations.other"); + sample.rtt_ms = self + .remote_by_local + .get(&stat.id) + .and_then(|remote| num(remote, "roundTripTime")) + .map(|s| s * 1000.0); + } + StreamDirection::Inbound => { + sample.bytes = count(stat, "bytesReceived"); + sample.packets = count(stat, "packetsReceived"); + sample.packets_lost = count(stat, "packetsLost"); + sample.freeze_count = count(stat, "freezeCount"); + sample.freezes_duration_ms = ms(stat, "totalFreezesDuration"); + sample.pause_count = count(stat, "pauseCount"); + sample.pauses_duration_ms = ms(stat, "totalPausesDuration"); + sample.concealed_samples = count(stat, "concealedSamples"); + sample.silent_concealed_samples = count(stat, "silentConcealedSamples"); + sample.concealment_events = count(stat, "concealmentEvents"); + sample.interruption_count = count(stat, "interruptionCount"); + sample.interruptions_duration_ms = ms(stat, "totalInterruptionDuration"); + sample.jitter_buffer_delay_ms = ms(stat, "jitterBufferDelay"); + sample.jitter_buffer_emitted_count = count(stat, "jitterBufferEmittedCount"); + sample.jitter_ms = num(stat, "jitter").map(|s| s * 1000.0); + sample.frames_per_second = num(stat, "framesPerSecond"); + sample.audio_level = num(stat, "audioLevel"); + sample.rtt_ms = self.pair_rtt_ms; + } + } + sample + } +} + +impl RtcStatsSample { + /// A sample with every optional field unset. + pub fn new(track_sid: impl Into, kind: TrackKind, direction: StreamDirection) -> Self { + Self { + track_sid: track_sid.into(), + kind, + direction, + codec: None, + bytes: None, + packets: None, + packets_lost: None, + freeze_count: None, + freezes_duration_ms: None, + concealed_samples: None, + concealment_events: None, + jitter_buffer_delay_ms: None, + jitter_buffer_emitted_count: None, + quality_limitation_bandwidth_ms: None, + quality_limitation_cpu_ms: None, + quality_limitation_other_ms: None, + pause_count: None, + pauses_duration_ms: None, + silent_concealed_samples: None, + interruption_count: None, + interruptions_duration_ms: None, + jitter_ms: None, + rtt_ms: None, + frames_per_second: None, + audio_level: None, + timestamp_ns: None, + layer: None, + } + } +} + +/// min / max / avg of a gauge over a window. +#[derive(Debug, Default, Clone, Copy)] +struct Gauge { + min: f64, + max: f64, + sum: f64, + n: u32, +} + +impl Gauge { + fn add(&mut self, value: Option) { + let Some(value) = value else { return }; + if self.n == 0 { + self.min = value; + self.max = value; + } else { + self.min = self.min.min(value); + self.max = self.max.max(value); + } + self.sum += value; + self.n += 1; + } + + fn attach(&self, event: TelemetryEvent, key: &str) -> TelemetryEvent { + if self.n == 0 { + return event; + } + event + .with_attribute(format!("{key}.min"), self.min) + .with_attribute(format!("{key}.max"), self.max) + .with_attribute(format!("{key}.avg"), self.sum / self.n as f64) + } +} + +/// Samples of one track in one direction accumulated since the window opened. +struct Window { + session: Arc, + /// The window's owner, captured when it opened: the session's project and its correlation + /// attributes then. A change to either closes the window first (see `StatsWindows::split`). + route: Option, + custom: Vec, + start_ns: u64, + samples: u32, + /// Cumulative counters at the window's first reading, for the display body's deltas. + first: RtcStatsSample, + last: RtcStatsSample, + jitter: Gauge, + rtt: Gauge, + fps: Gauge, + audio_level: Gauge, +} + +impl Window { + fn open(start_ns: u64, first: RtcStatsSample, session: Arc) -> Self { + let mut window = Self { + route: session.route(), + custom: session.custom_snapshot(), + session, + start_ns, + samples: 0, + first: first.clone(), + last: first.clone(), + jitter: Gauge::default(), + rtt: Gauge::default(), + fps: Gauge::default(), + audio_level: Gauge::default(), + }; + window.add(first); + window + } + + /// `video outbound: 1204 kbps, loss 0.4%, rtt 48 ms, 30 fps` — deltas over the window. + fn summary(&self, window_ms: u64) -> String { + let mut parts = Vec::new(); + let delta = |a: Option, b: Option| Some(a?.saturating_sub(b?)); + if let (Some(bytes), true) = (delta(self.last.bytes, self.first.bytes), window_ms > 0) { + parts.push(format!("{} kbps", bytes * 8 / window_ms)); + } + if let (Some(lost), Some(packets)) = ( + delta(self.last.packets_lost, self.first.packets_lost), + delta(self.last.packets, self.first.packets), + ) { + if lost + packets > 0 { + parts.push(format!("loss {:.1}%", lost as f64 * 100.0 / (lost + packets) as f64)); + } + } + if self.rtt.n > 0 { + parts.push(format!("rtt {:.0} ms", self.rtt.sum / self.rtt.n as f64)); + } + if self.fps.n > 0 { + parts.push(format!("{:.0} fps", self.fps.sum / self.fps.n as f64)); + } + if let Some(freezes) = + delta(self.last.freeze_count, self.first.freeze_count).filter(|f| *f > 0) + { + parts.push(format!("{freezes} freezes")); + } + format!( + "{} {}: {}", + self.last.kind.as_str(), + self.last.direction.as_str(), + if parts.is_empty() { "no data".to_owned() } else { parts.join(", ") } + ) + } + + /// The closed window as a queued record, with the owner it captured when it opened. + fn into_queued(self, end_ns: u64) -> Queued { + let (session, route, custom) = + (self.session.clone(), self.route.clone(), self.custom.clone()); + let mut event = self.into_event(end_ns); + for attribute in custom { + if !event.attributes.iter().any(|a| a.key == attribute.key) { + event.attributes.push(attribute); + } + } + Queued { event, session, route } + } + + fn add(&mut self, sample: RtcStatsSample) { + self.samples += 1; + self.jitter.add(sample.jitter_ms); + self.rtt.add(sample.rtt_ms); + self.fps.add(sample.frames_per_second); + self.audio_level.add(sample.audio_level); + self.last = sample; + } + + /// The `lk.rtc.stats.sample` event for this window, stamped at `end_ns`. Its body is the + /// one-line human summary a log view shows (OTel: an event's body is its display message); + /// the attributes carry the numbers. + fn into_event(self, end_ns: u64) -> TelemetryEvent { + let window_ms = end_ns.saturating_sub(self.start_ns) / 1_000_000; + let body = self.summary(window_ms); + let last = self.last; + let mut event = TelemetryEvent::new("lk.rtc.stats.sample") + .with_body(body) + .with_attribute("lk.track.sid", last.track_sid) + .with_attribute("lk.track.kind", last.kind.as_str()) + .with_attribute("lk.track.direction", last.direction.as_str()) + .with_attribute("lk.rtc.window_ms", window_ms as i64) + .with_attribute("lk.rtc.samples", self.samples as i64); + if let Some(codec) = last.codec { + event = event.with_attribute("lk.rtc.codec", codec); + } + let counters = [ + ("lk.rtc.bytes", last.bytes), + ("lk.rtc.packets", last.packets), + ("lk.rtc.packets_lost", last.packets_lost), + ("lk.rtc.freeze_count", last.freeze_count), + ("lk.rtc.freezes_duration_ms", last.freezes_duration_ms), + ("lk.rtc.concealed_samples", last.concealed_samples), + ("lk.rtc.concealment_events", last.concealment_events), + ("lk.rtc.jitter_buffer_delay_ms", last.jitter_buffer_delay_ms), + ("lk.rtc.jitter_buffer_emitted_count", last.jitter_buffer_emitted_count), + ("lk.rtc.quality_limitation.bandwidth_ms", last.quality_limitation_bandwidth_ms), + ("lk.rtc.quality_limitation.cpu_ms", last.quality_limitation_cpu_ms), + ("lk.rtc.quality_limitation.other_ms", last.quality_limitation_other_ms), + ("lk.rtc.pause_count", last.pause_count), + ("lk.rtc.pauses_duration_ms", last.pauses_duration_ms), + ("lk.rtc.silent_concealed_samples", last.silent_concealed_samples), + ("lk.rtc.interruption_count", last.interruption_count), + ("lk.rtc.interruptions_duration_ms", last.interruptions_duration_ms), + ]; + for (key, value) in counters { + if let Some(value) = value { + event = event.with_attribute(key, value as i64); + } + } + event = self.jitter.attach(event, "lk.rtc.jitter_ms"); + event = self.rtt.attach(event, "lk.rtc.rtt_ms"); + event = self.fps.attach(event, "lk.rtc.fps"); + event = self.audio_level.attach(event, "lk.rtc.audio_level"); + event.timestamp_ns = Some(end_ns); + event + } +} + +/// Open RTC stats windows, one per track and direction. +/// +/// The platform pushes raw `getStats()` readings when the core asks; the core ships one +/// `lk.rtc.stats.sample` per window (60 s by default, stretched under device pressure) — on +/// device: 1 Hz raw sampling across ~100k concurrent participants would be the same ~100k +/// records/s fleet-wide. +#[derive(Default)] +pub(crate) struct StatsWindows { + windows: HashMap, + /// Last cumulative `qualityLimitationDurations.cpu` per outbound track. + limitation: HashMap, + cpu_limited_until: Option, + /// Last cumulative counters per simulcast layer, per outbound track: the track's counters + /// are their sum, so a suspended top layer cannot freeze them. + layers: HashMap>, + /// The opt-out, checked under the lock that guards the windows (see `Store`). + revoked: Arc, +} + +/// One track in one direction *in one session*: two rooms in a process may subscribe to the same +/// published track (same sid, same direction), and their readings must never merge. +type TrackKey = ([u8; 16], String, StreamDirection); + +fn track_key(session: &ScopeState, sample: &RtcStatsSample) -> TrackKey { + (session.trace_id, sample.track_sid.clone(), sample.direction) +} + +#[derive(Debug, Clone, Copy, Default)] +struct LayerCounters { + bytes: Option, + packets: Option, + fps: Option, + limitation_bandwidth_ms: Option, + limitation_cpu_ms: Option, + limitation_other_ms: Option, +} + +fn sum(values: impl Iterator>) -> Option { + values.flatten().reduce(|a, b| a + b) +} + +fn max_u64(values: impl Iterator>) -> Option { + values.flatten().max() +} + +/// CPU starvation is sticky: stretch the cadence for a while after the last sign of it. +/// `qualityLimitationDurations.bandwidth` is deliberately not a signal: WebRTC reports it for +/// minutes during a normal ramp-up and for as long as an encoder stalls (an iPhone camera at +/// 0 kbps held uploads for 8 minutes); the congestion controller does not need our help. +const CPU_LIMITED_HOLD: Duration = Duration::from_secs(60); + +impl StatsWindows { + /// Tie the windows to a pipeline's opt-out. + pub fn with_consent(revoked: Arc) -> Self { + Self { revoked, ..Self::default() } + } + + pub fn record_in(&mut self, mut sample: RtcStatsSample, session: &Arc) { + if self.revoked.load(std::sync::atomic::Ordering::SeqCst) { + return; + } + let key = track_key(session, &sample); + self.fold_layers(&key, &mut sample); + self.track_limitation(&key, &sample); + let timestamp = *sample.timestamp_ns.get_or_insert_with(now_unix_nanos); + match self.windows.get_mut(&key) { + Some(window) => window.add(sample), + None => { + self.windows.insert(key, Window::open(timestamp, sample, session.clone())); + } + } + } + + /// Simulcast publishes one RTP stream per layer under one track sid. Remember each layer's + /// latest cumulative counters and rewrite the sample as the track's total, so the window + /// sees one monotonic series per track. + fn fold_layers(&mut self, key: &TrackKey, sample: &mut RtcStatsSample) { + let Some(layer) = sample.layer.clone() else { return }; + let layers = self.layers.entry(key.clone()).or_default(); + layers.insert( + layer, + LayerCounters { + bytes: sample.bytes, + packets: sample.packets, + fps: sample.frames_per_second, + limitation_bandwidth_ms: sample.quality_limitation_bandwidth_ms, + limitation_cpu_ms: sample.quality_limitation_cpu_ms, + limitation_other_ms: sample.quality_limitation_other_ms, + }, + ); + sample.bytes = sum(layers.values().map(|l| l.bytes)); + sample.packets = sum(layers.values().map(|l| l.packets)); + sample.frames_per_second = layers + .values() + .filter_map(|l| l.fps) + .fold(None, |m, v| Some(m.map_or(v, |m: f64| m.max(v)))); + sample.quality_limitation_bandwidth_ms = + max_u64(layers.values().map(|l| l.limitation_bandwidth_ms)); + sample.quality_limitation_cpu_ms = max_u64(layers.values().map(|l| l.limitation_cpu_ms)); + sample.quality_limitation_other_ms = + max_u64(layers.values().map(|l| l.limitation_other_ms)); + } + + /// Close every open window into its event, filed under the window's session, and start fresh. + /// A track that produced no reading for a whole window has left (unpublished, unsubscribed, + /// its room gone): its per-layer and CPU counters are retired with it, so memory follows the + /// live tracks rather than every track the process ever saw. + pub fn close(&mut self) -> Vec { + let windows = &self.windows; + self.layers.retain(|key, _| windows.contains_key(key)); + self.limitation.retain(|key, _| windows.contains_key(key)); + let end = now_unix_nanos(); + self.windows.drain().map(|(_, window)| window.into_queued(end)).collect() + } + + /// A track left `session` (`track_sid`), or the session ended (`None`): close its windows now + /// — the last partial window still ships — and forget everything kept for it. + pub fn retire(&mut self, session: &ScopeState, track_sid: Option<&str>) -> Vec { + let gone = |(trace_id, sid, _): &TrackKey| { + *trace_id == session.trace_id && track_sid.is_none_or(|t| t == sid) + }; + self.layers.retain(|key, _| !gone(key)); + self.limitation.retain(|key, _| !gone(key)); + let keys: Vec = self.windows.keys().filter(|k| gone(k)).cloned().collect(); + let end = now_unix_nanos(); + keys.into_iter() + .filter_map(|key| self.windows.remove(&key)) + .map(|window| window.into_queued(end)) + .collect() + } + + /// Close `session`'s open windows now — their records then carry the attributes they were + /// captured under — keeping each track's cumulative state for the windows that follow. + pub fn split(&mut self, session: &ScopeState) -> Vec { + let keys: Vec = self + .windows + .keys() + .filter(|(trace_id, ..)| *trace_id == session.trace_id) + .cloned() + .collect(); + let end = now_unix_nanos(); + keys.into_iter() + .filter_map(|key| self.windows.remove(&key)) + .map(|window| window.into_queued(end)) + .collect() + } + + /// How many tracks the accumulator holds state for (windows, layers, CPU counters). + #[cfg(test)] + pub fn tracked(&self) -> (usize, usize, usize) { + (self.windows.len(), self.layers.len(), self.limitation.len()) + } + + #[cfg(test)] + pub fn record(&mut self, sample: RtcStatsSample) { + self.record_in(sample, &test_session()); + } + + #[cfg(test)] + pub fn close_events(&mut self) -> Vec { + self.close().into_iter().map(|q| q.event).collect() + } + + /// The encoder was CPU-starved within the last minute: stretch the cadence, like thermal + /// pressure. + pub fn cpu_limited(&self) -> bool { + self.cpu_limited_until.is_some_and(|t| Instant::now() < t) + } + + /// A cpu limitation counter that grew since the previous reading means the encoder was + /// starved in between. + fn track_limitation(&mut self, key: &TrackKey, sample: &RtcStatsSample) { + if sample.direction != StreamDirection::Outbound { + return; + } + let current = sample.quality_limitation_cpu_ms.unwrap_or(0); + if let Some(previous) = self.limitation.insert(key.clone(), current) { + if current > previous { + self.cpu_limited_until = Some(Instant::now() + CPU_LIMITED_HOLD); + } + } + } + + /// Drop everything (opt-out); returns how many open windows went. + pub fn clear(&mut self) -> u64 { + let open = self.windows.len() as u64; + *self = Self::with_consent(self.revoked.clone()); + open + } +} + +/// The one session every `StatsWindows::record` in a unit test files under. +#[cfg(test)] +fn test_session() -> Arc { + ScopeState::with_trace_id([7; 16]) +} + +impl TrackKind { + fn as_str(self) -> &'static str { + match self { + TrackKind::Audio => "audio", + TrackKind::Video => "video", + } + } +} + +impl StreamDirection { + fn as_str(self) -> &'static str { + match self { + StreamDirection::Inbound => "inbound", + StreamDirection::Outbound => "outbound", + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn simulcast_layers_fold_into_one_track_series() { + let mut windows = StatsWindows::default(); + let layer = |id: &str, bytes: u64, fps: f64| RtcStatsSample { + bytes: Some(bytes), + frames_per_second: Some(fps), + layer: Some(id.into()), + ..RtcStatsSample::new("TR_1", TrackKind::Video, StreamDirection::Outbound) + }; + windows.record(layer("h", 1_000, 30.0)); + windows.record(layer("f", 4_000, 30.0)); + // The top layer stalls; the half layer keeps sending. + windows.record(layer("h", 3_000, 30.0)); + windows.record(layer("f", 4_000, 0.0)); + let events = windows.close_events(); + assert_eq!(events.len(), 1, "one window per track, not per layer"); + let bytes = + events[0].attributes.iter().find(|a| a.key == "lk.rtc.bytes").map(|a| a.value.clone()); + assert_eq!(bytes, Some(AttributeValue::Int(7_000)), "last folded total"); + let body = events[0].body.clone().unwrap_or_default(); + assert!(!body.contains(" 0 kbps"), "a stalled layer is not a stalled track: {body}"); + } + + #[tokio::test(start_paused = true)] + async fn cpu_limitation_counter_drives_cadence_pressure() { + let mut windows = StatsWindows::default(); + let reading = |bandwidth_ms, cpu_ms| RtcStatsSample { + quality_limitation_bandwidth_ms: Some(bandwidth_ms), + quality_limitation_cpu_ms: Some(cpu_ms), + ..RtcStatsSample::new("TR_1", TrackKind::Video, StreamDirection::Outbound) + }; + windows.record(reading(0, 0)); + assert!(!windows.cpu_limited(), "first reading: no delta"); + windows.record(reading(5_000, 0)); + assert!(!windows.cpu_limited(), "bandwidth limitation is not pressure"); + windows.record(reading(5_000, 250)); + assert!(windows.cpu_limited()); + tokio::time::advance(Duration::from_secs(59)).await; + assert!(windows.cpu_limited(), "cpu hold is sticky"); + tokio::time::advance(Duration::from_secs(2)).await; + assert!(!windows.cpu_limited(), "and expires"); + let mut inbound = RtcStatsSample::new("TR_2", TrackKind::Audio, StreamDirection::Inbound); + inbound.quality_limitation_cpu_ms = Some(9_999); + windows.record(inbound.clone()); + windows.record(inbound); + assert!(!windows.cpu_limited(), "inbound counters are ignored"); + } + use crate::AttributeValue; + + fn attr(event: &TelemetryEvent, key: &str) -> Option { + event.attributes.iter().find(|a| a.key == key).map(|a| a.value.clone()) + } + + #[test] + fn window_keeps_last_counter_and_summarises_gauges() { + let mut windows = StatsWindows::default(); + for (bytes, jitter) in [(100, 1.0), (200, 3.0), (300, 2.0)] { + let mut sample = + RtcStatsSample::new("TR_1", TrackKind::Audio, StreamDirection::Inbound); + sample.bytes = Some(bytes); + sample.jitter_ms = Some(jitter); + windows.record(sample); + } + let mut other = RtcStatsSample::new("TR_1", TrackKind::Audio, StreamDirection::Outbound); + other.packets = Some(7); + windows.record(other); + + let mut events = windows.close_events(); + assert!(windows.close_events().is_empty(), "closing again yields nothing"); + events.sort_by_key(|e| attr(e, "lk.track.direction").map(|v| format!("{v:?}"))); + assert_eq!(events.len(), 2, "one event per track and direction"); + let inbound = &events[0]; + assert_eq!(inbound.name, "lk.rtc.stats.sample"); + assert_eq!( + attr(inbound, "lk.track.direction"), + Some(AttributeValue::Str("inbound".into())) + ); + assert_eq!( + attr(inbound, "lk.rtc.bytes"), + Some(AttributeValue::Int(300)), + "cumulative: last value" + ); + assert_eq!(attr(inbound, "lk.rtc.samples"), Some(AttributeValue::Int(3))); + assert_eq!(attr(inbound, "lk.rtc.jitter_ms.min"), Some(AttributeValue::Double(1.0))); + assert_eq!(attr(inbound, "lk.rtc.jitter_ms.max"), Some(AttributeValue::Double(3.0))); + assert_eq!(attr(inbound, "lk.rtc.jitter_ms.avg"), Some(AttributeValue::Double(2.0))); + assert_eq!(attr(inbound, "lk.rtc.rtt_ms.avg"), None, "absent gauges are omitted"); + assert_eq!(attr(&events[1], "lk.rtc.packets"), Some(AttributeValue::Int(7))); + } + + #[test] + fn a_report_maps_to_samples_with_codec_rtt_layers_and_milliseconds() { + let stat = |kind: &str, id: &str, members: &[(&str, AttributeValue)]| RtcStat { + kind: kind.into(), + id: id.into(), + members: members.iter().map(|(k, v)| (k.to_string(), v.clone())).collect(), + }; + let s = |v: &str| AttributeValue::Str(v.into()); + let report = vec![ + stat("codec", "C1", &[("mimeType", s("video/VP8"))]), + stat( + "outbound-rtp", + "OUT_f", + &[ + ("rid", s("f")), + ("codecId", s("C1")), + ("bytesSent", AttributeValue::Int(7_000)), + ("packetsSent", s("70")), + ("framesPerSecond", AttributeValue::Double(29.5)), + ("qualityLimitationDurations.cpu", AttributeValue::Double(1.25)), + ], + ), + stat( + "outbound-rtp", + "OUT_h", + &[("rid", s("h")), ("bytesSent", AttributeValue::Int(500))], + ), + stat( + "remote-inbound-rtp", + "RI_f", + &[("localId", s("OUT_f")), ("roundTripTime", AttributeValue::Double(0.042))], + ), + ]; + let out = samples_from_report( + "TR_1", + TrackKind::Video, + StreamDirection::Outbound, + &report, + Some(5), + ); + assert_eq!(out.len(), 2); + let f = out.iter().find(|x| x.layer.as_deref() == Some("f")).expect("layer f"); + assert_eq!(f.codec.as_deref(), Some("video/VP8")); + assert_eq!((f.bytes, f.packets, f.frames_per_second), (Some(7_000), Some(70), Some(29.5))); + assert_eq!(f.quality_limitation_cpu_ms, Some(1_250)); + assert_eq!(f.rtt_ms, Some(42.0)); + assert_eq!(f.timestamp_ns, Some(5)); + assert_eq!( + out.iter().find(|x| x.layer.as_deref() == Some("h")).and_then(|x| x.rtt_ms), + None + ); + + let report = vec![ + stat( + "candidate-pair", + "CP", + &[ + ("nominated", AttributeValue::Bool(true)), + ("currentRoundTripTime", AttributeValue::Double(0.1)), + ], + ), + stat( + "inbound-rtp", + "IN", + &[ + ("bytesReceived", AttributeValue::Int(12)), + ("packetsLost", AttributeValue::Int(-3)), + ("jitter", AttributeValue::Double(0.02)), + ("totalFreezesDuration", AttributeValue::Double(2.5)), + ("jitterBufferDelay", AttributeValue::Double(3.0)), + ("audioLevel", AttributeValue::Double(0.5)), + ], + ), + ]; + let out = + samples_from_report("TR_2", TrackKind::Audio, StreamDirection::Inbound, &report, None); + assert_eq!(out.len(), 1); + let x = &out[0]; + assert_eq!((x.bytes, x.packets_lost, x.rtt_ms), (Some(12), Some(0), Some(100.0))); + assert_eq!( + (x.jitter_ms, x.freezes_duration_ms, x.jitter_buffer_delay_ms), + (Some(20.0), Some(2_500), Some(3_000)) + ); + assert_eq!(x.layer, None); + } + + /// Finding 19: two rooms in one process subscribed to the same published track keep two + /// windows, each filed under its own session. + #[test] + fn the_same_track_in_two_sessions_never_merges() { + let mut windows = StatsWindows::default(); + let (a, b) = (ScopeState::new(), ScopeState::new()); + let reading = |bytes| RtcStatsSample { + bytes: Some(bytes), + ..RtcStatsSample::new("TR_shared", TrackKind::Audio, StreamDirection::Inbound) + }; + windows.record_in(reading(100), &a); + windows.record_in(reading(900), &b); + windows.record_in(reading(200), &a); + let closed = windows.close(); + assert_eq!(closed.len(), 2, "one window per session"); + for (session, bytes) in [(&a, 200), (&b, 900)] { + let window = closed.iter().find(|q| q.session == *session).expect("own window"); + assert_eq!(attr(&window.event, "lk.rtc.bytes"), Some(AttributeValue::Int(bytes))); + } + } + + /// Finding 14: per-track state follows the live tracks — retired when a track leaves, when + /// its session ends, and when it goes a whole window without a reading. + #[test] + fn per_track_state_is_retired_with_the_track() { + let mut windows = StatsWindows::default(); + let (a, b) = (ScopeState::new(), ScopeState::new()); + let outbound = |sid: &str| RtcStatsSample { + bytes: Some(1), + quality_limitation_cpu_ms: Some(0), + layer: Some("f".into()), + ..RtcStatsSample::new(sid, TrackKind::Video, StreamDirection::Outbound) + }; + for sid in ["TR_1", "TR_2"] { + windows.record_in(outbound(sid), &a); + } + windows.record_in(outbound("TR_3"), &b); + assert_eq!(windows.tracked(), (3, 3, 3)); + + let last = windows.retire(&a, Some("TR_1")); + assert_eq!(last.len(), 1, "the unpublished track's partial window ships now"); + assert_eq!(windows.tracked(), (2, 2, 2)); + + let last = windows.retire(&a, None); + assert_eq!(last.len(), 1, "the session's last window ships at disconnect"); + assert_eq!(windows.tracked(), (1, 1, 1), "the other session keeps its track"); + + windows.close(); + assert_eq!(windows.tracked(), (0, 1, 1), "counters survive an ordinary window…"); + windows.close(); + assert_eq!(windows.tracked(), (0, 0, 0), "…not a whole window without a reading"); + } + + #[test] + fn a_peer_connection_report_maps_to_its_tracks() { + let stat = |kind: &str, id: &str, members: &[(&str, &str)]| RtcStat { + kind: kind.into(), + id: id.into(), + members: members + .iter() + .map(|(k, v)| (k.to_string(), AttributeValue::Str(v.to_string()))) + .collect(), + }; + let report = vec![ + stat("media-source", "MS1", &[("trackIdentifier", "local-cam")]), + stat( + "outbound-rtp", + "O1", + &[("kind", "video"), ("mediaSourceId", "MS1"), ("rid", "f"), ("bytesSent", "10")], + ), + stat( + "outbound-rtp", + "O2", + &[("kind", "video"), ("mediaSourceId", "MS1"), ("rid", "h"), ("bytesSent", "5")], + ), + stat( + "inbound-rtp", + "I1", + &[("kind", "audio"), ("trackIdentifier", "TR_remote"), ("bytesReceived", "7")], + ), + stat( + "inbound-rtp", + "I2", + &[("kind", "audio"), ("trackIdentifier", "unknown"), ("bytesReceived", "1")], + ), + ]; + let tracks = HashMap::from([ + ("local-cam".to_owned(), "TR_cam".to_owned()), + ("TR_remote".to_owned(), "TR_remote".to_owned()), + ]); + let out = samples_from_peer_report(&report, &tracks, None); + assert_eq!(out.len(), 3, "two camera layers and the known remote track"); + assert!(out + .iter() + .filter(|s| s.track_sid == "TR_cam") + .all(|s| { s.kind == TrackKind::Video && s.direction == StreamDirection::Outbound })); + let remote = out.iter().find(|s| s.track_sid == "TR_remote").expect("remote"); + assert_eq!((remote.kind, remote.bytes), (TrackKind::Audio, Some(7))); + } +} diff --git a/livekit-telemetry/src/scope.rs b/livekit-telemetry/src/scope.rs new file mode 100644 index 000000000..ab497c979 --- /dev/null +++ b/livekit-telemetry/src/scope.rs @@ -0,0 +1,566 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{ + collections::HashMap, + fmt, + sync::{Arc, Mutex}, + time::Duration, +}; + +use tokio::time::Instant; + +use crate::{ + rtc::RtcStat, Attribute, AttributeValue, LogRecord, RtcStatsSample, Span, SpanName, + SpanOutcome, SpanStep, SpanTrack, StreamDirection, Telemetry, TelemetryEvent, TrackKind, +}; + +/// One session's identity: the trace id every one of its records carries, and the attributes +/// attached to them at export time (`lk.room.sid`, `lk.participant.identity`, …). +pub(crate) struct ScopeState { + pub trace_id: [u8; 16], + /// SDK-owned attributes (`lk.room.*`, `lk.participant.*`), attached at export: late-known + /// identity (the room sid arrives after join) still reaches the records captured before. + attributes: Mutex>, + /// The app's correlation attributes, copied into each record when it is captured, so a + /// later change never rewrites what is already queued. + custom: Mutex>, + /// Open `lk.subscribe` spans by track sid, from intent to first media. + subscribes: Mutex, Instant)>>, + /// Published tracks awaiting their first outbound reading, and since when. + publishing: Mutex>, + /// The project host this session's batches go to (`Scope::set_server`); `None` until then. + route: Mutex>, + /// The last `(url, token)` handed over, so handing the same pair again costs a comparison; + /// `None` again once disconnected (see [`ScopeState::in_call`]). + server: Mutex>, +} + +impl ScopeState { + /// A fresh session: random, non-zero trace id (OTLP treats all-zero as absent). + pub fn new() -> Arc { + Self::with_trace_id(rand::random::().max(1).to_be_bytes()) + } + + pub fn with_trace_id(trace_id: [u8; 16]) -> Arc { + Arc::new(Self { + trace_id, + attributes: Mutex::new(Vec::new()), + custom: Mutex::new(Vec::new()), + subscribes: Mutex::new(HashMap::new()), + publishing: Mutex::new(HashMap::new()), + route: Mutex::new(None), + server: Mutex::new(None), + }) + } + + /// In a call: it has a server and has not disconnected since. + pub fn in_call(&self) -> bool { + self.server.lock().unwrap_or_else(|e| e.into_inner()).is_some() + } + + /// Whether `(url, token)` is what this session already has; remembers it if not. + pub fn same_server(&self, url: &str, token: &str) -> bool { + let mut server = self.server.lock().unwrap_or_else(|e| e.into_inner()); + if server.as_ref().is_some_and(|(u, t)| u == url && t == token) { + return true; + } + *server = Some((url.to_owned(), token.to_owned())); + false + } + + pub fn route(&self) -> Option { + self.route.lock().unwrap_or_else(|e| e.into_inner()).clone() + } + + pub fn set_route(&self, host: String) { + *self.route.lock().unwrap_or_else(|e| e.into_inner()) = Some(host); + } + + /// A track was published: poll fast until its first outbound reading (at most + /// [`Scope::SUBSCRIBE_TIMEOUT`]). + pub fn await_first_outbound(&self, sid: &str) { + self.publishing + .lock() + .unwrap_or_else(|e| e.into_inner()) + .insert(sid.to_owned(), Instant::now()); + } + + /// Whether a published track still waits for its first outbound reading (entries older than + /// the subscribe timeout are forgotten, so a track that never sends cannot keep polling fast). + fn awaiting_outbound(&self) -> bool { + let mut publishing = self.publishing.lock().unwrap_or_else(|e| e.into_inner()); + publishing.retain(|_, since| since.elapsed() < Scope::SUBSCRIBE_TIMEOUT); + !publishing.is_empty() + } + + /// When the oldest open subscribe runs out of time. + pub fn subscribe_deadline(&self) -> Option { + let open = self.subscribes.lock().unwrap_or_else(|e| e.into_inner()); + open.values().map(|(_, since)| *since + Scope::SUBSCRIBE_TIMEOUT).min() + } + + /// The pipeline stops: end every pending subscribe (`timed_out` past its deadline, + /// `cancelled` before) so it ships with the last batch, or, on opt-out, just let go of them. + pub fn end_subscribes(&self, export: bool) { + let open: Vec<_> = + self.subscribes.lock().unwrap_or_else(|e| e.into_inner()).drain().collect(); + if export { + for (_, (span, since)) in open { + end_unfinished(&span, since); + } + } + } + + /// End every subscribe past its deadline as `timed_out` — driven by the exporter's clock, so + /// a track that never produces a single RTP reading still times out. + pub fn expire_subscribes(&self) { + let expired: Vec> = { + let mut open = self.subscribes.lock().unwrap_or_else(|e| e.into_inner()); + let sids: Vec = open + .iter() + .filter(|(_, (_, since))| since.elapsed() >= Scope::SUBSCRIBE_TIMEOUT) + .map(|(sid, _)| sid.clone()) + .collect(); + sids.iter().filter_map(|sid| open.remove(sid)).map(|(span, _)| span).collect() + }; + for span in expired { + span.fail("timed_out".to_owned()); + } + } + + /// The trace id as 32 hex characters. + pub fn hex(&self) -> String { + format!("{:032x}", u128::from_be_bytes(self.trace_id)) + } + + pub fn set_attribute(&self, key: &str, value: Option) { + let mut attributes = self.attributes.lock().unwrap_or_else(|e| e.into_inner()); + attributes.retain(|a| a.key != key); + if let Some(value) = value { + attributes.push(Attribute::new(key, value)); + } + } + + /// Set or remove an app correlation attribute. `false` when rejected: over the limits, in + /// the SDK's namespace, or one attribute too many. + /// Whether `set_custom(key, value)` would be accepted: within the limits, outside the + /// SDK's namespace, not one attribute too many. + pub fn accepts_custom(&self, key: &str, value: Option<&AttributeValue>) -> bool { + if !crate::event::valid_custom(key, value) { + return false; + } + let custom = self.custom.lock().unwrap_or_else(|e| e.into_inner()); + value.is_none() + || custom.iter().any(|a| a.key == key) + || custom.len() < crate::event::MAX_CUSTOM_ATTRIBUTES + } + + /// Set or remove an app correlation attribute; `false` when rejected (see + /// [`accepts_custom`](Self::accepts_custom)). + pub fn set_custom(&self, key: &str, value: Option) -> bool { + if !self.accepts_custom(key, value.as_ref()) { + return false; + } + let mut custom = self.custom.lock().unwrap_or_else(|e| e.into_inner()); + custom.retain(|a| a.key != key); + if let Some(value) = value { + custom.push(Attribute::new(key, value)); + } + true + } + + /// The app's correlation attributes right now. + pub fn custom_snapshot(&self) -> Vec { + self.custom.lock().unwrap_or_else(|e| e.into_inner()).clone() + } + + /// Copy the app's correlation attributes into a record being captured; the record's own + /// attributes win. + pub fn snapshot_custom(&self, own: &mut Vec) { + let custom = self.custom.lock().unwrap_or_else(|e| e.into_inner()); + for attribute in custom.iter() { + if !own.iter().any(|a| a.key == attribute.key) { + own.push(attribute.clone()); + } + } + } + + /// At export: the session's SDK-owned attributes win over anything the record carries + /// (an app cannot spoof them), then the pipeline-wide ones (`global`) fill in, and + /// `session.id` (OTel semconv) — the trace id, so a record can be joined to its session even + /// where a backend drops trace ids from logs. + pub fn decorate(&self, own: &mut Vec, global: &[Attribute]) { + let session = self.attributes.lock().unwrap_or_else(|e| e.into_inner()); + for attribute in session.iter() { + own.retain(|a| a.key != attribute.key); + own.push(attribute.clone()); + } + for attribute in global { + if !own.iter().any(|a| a.key == attribute.key) { + own.push(attribute.clone()); + } + } + own.retain(|a| a.key != "session.id"); + own.push(Attribute::new("session.id", self.hex())); + } +} + +impl PartialEq for ScopeState { + fn eq(&self, other: &Self) -> bool { + self.trace_id == other.trace_id + } +} + +impl fmt::Debug for ScopeState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Scope({})", self.hex()) + } +} + +/// Who this session is: attached to every record once the room is joined. `None` clears. +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct RoomIdentity { + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub sid: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub name: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub participant_sid: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub participant_identity: Option, +} + +/// Why a session ended: the protocol's `DisconnectReason`, plus the client giving up. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DisconnectReason { + Unknown, + ClientInitiated, + DuplicateIdentity, + ServerShutdown, + ParticipantRemoved, + RoomDeleted, + StateMismatch, + JoinFailure, + Migration, + SignalClose, + RoomClosed, + UserUnavailable, + UserRejected, + SipTrunkFailure, + ConnectionTimeout, + MediaFailure, + AgentError, + /// The reconnect policy ran out of attempts. + ReconnectFailed, +} + +impl DisconnectReason { + /// The protocol's `DisconnectReason` number, so no SDK keeps its own switch. + pub fn from_proto(value: i32) -> Self { + match value { + 1 => Self::ClientInitiated, + 2 => Self::DuplicateIdentity, + 3 => Self::ServerShutdown, + 4 => Self::ParticipantRemoved, + 5 => Self::RoomDeleted, + 6 => Self::StateMismatch, + 7 => Self::JoinFailure, + 8 => Self::Migration, + 9 => Self::SignalClose, + 10 => Self::RoomClosed, + 11 => Self::UserUnavailable, + 12 => Self::UserRejected, + 13 => Self::SipTrunkFailure, + 14 => Self::ConnectionTimeout, + 15 => Self::MediaFailure, + 16 => Self::AgentError, + _ => Self::Unknown, + } + } +} + +/// A session on the shared pipeline: its own trace id and attributes, the same queue, cache, +/// cadence and exporter as every other session in the process. +/// +/// The pipeline starts once, at SDK init, so nothing that happens before the first room is +/// lost; a `Scope` is what a room — one call — gets from it, and what its spans, RTC windows +/// and events are filed under. Everything emitted outside a session (device state, pre-room +/// errors, self-telemetry) belongs to the pipeline's own process session. Cheap to clone. +#[derive(Clone)] +pub struct Scope { + pub(crate) telemetry: Telemetry, + pub(crate) state: Arc, +} + +impl Scope { + /// The session's trace id as 32 hex characters — print it (`lkt_…`) so support can find + /// the call. + pub fn trace_id(&self) -> String { + self.state.hex() + } + + /// The LiveKit server this room talks to and the participant token it holds. Call it at + /// connect and again with every refreshed token: the core derives the ingest URL (LiveKit + /// Cloud only), reads the token's grant and expiry, and routes this session's records to its + /// own project with its own token. Cheap and idempotent — the same pair again is a no-op. + pub fn set_server(&self, url: &str, token: &str) { + self.telemetry.set_server(&self.state, url, token); + } + + /// Queue an event or log record under this session. + pub fn emit(&self, event: TelemetryEvent) { + self.telemetry.emit_in(event, &self.state); + } + + /// An app-defined event, exported as `custom.` under this session with the session's + /// correlation attributes (its own attributes win over them). Rejected and counted as + /// `invalid` — never truncated — when the name is empty or over 128 bytes, an attribute is + /// over the limits or in the SDK's namespace (`lk.*`, `session.id`), or there are more than + /// 64 attributes. + pub fn emit_custom(&self, name: &str, attributes: Vec) { + let valid = !name.is_empty() + && name.len() <= crate::event::MAX_NAME_BYTES + && attributes.len() <= crate::event::MAX_CUSTOM_ATTRIBUTES + && attributes.iter().all(|a| crate::event::valid_custom(&a.key, Some(&a.value))); + if !valid { + self.telemetry.count_invalid(); + return; + } + self.emit(TelemetryEvent::custom(name, attributes)); + } + + /// A log record filed under this session even without an ambient span — for platforms with + /// no task-local context (Dart outside a zone). Same floor and filters as `Telemetry::log`. + pub fn log(&self, record: LogRecord) { + if let Some(event) = self.telemetry.log_event(record) { + self.emit(event); + } + } + + /// The session ended for good (not a reconnect): the `lk.room.disconnected` record. + pub fn disconnected(&self, reason: DisconnectReason) { + let open: Vec<_> = + self.state.subscribes.lock().unwrap_or_else(|e| e.into_inner()).drain().collect(); + for (_, (span, since)) in open { + end_unfinished(&span, since); + } + self.telemetry.retire_stats(&self.state, None); + // Out of the call: uploads no longer yield to it. + self.state.server.lock().unwrap_or_else(|e| e.into_inner()).take(); + let severity = if reason == DisconnectReason::ClientInitiated { + crate::Severity::Info + } else { + crate::Severity::Warn + }; + let reason = crate::device::snake(reason); + self.emit( + TelemetryEvent::new("lk.room.disconnected") + .with_severity(severity) + .with_body(format!("disconnected: {reason}")) + .with_attribute("lk.disconnect.reason", reason), + ); + } + + /// Set (or, with `None`, remove) an app correlation attribute — `app.call_id`, + /// `enduser.id` — on every record this session captures from now on: logs, spans, events and + /// RTC windows. Records already captured keep the value they had. Rejected and counted as + /// `invalid` when the key is empty or over 128 bytes, a string value is over 1024 bytes, the + /// key is the SDK's (`lk.*`, `session.id`), or the session already has 64. + pub fn set_attribute(&self, key: &str, value: Option) { + // Under the windows lock: the open windows close under the old value and the change + // lands before any reading can open a new one (readings record under the same lock). + let check = value.clone(); + let accepted = self.telemetry.with_windows_split( + &self.state, + || self.state.accepts_custom(key, check.as_ref()), + || { + self.state.set_custom(key, value); + }, + ); + if !accepted { + self.telemetry.count_invalid(); + } + } + + /// Push one `getStats()` reading; its window ships under this session. The first inbound + /// reading with bytes is a subscribed track's first media. + pub fn record_stats(&self, sample: RtcStatsSample) { + if sample.direction == StreamDirection::Inbound && sample.bytes.unwrap_or(0) > 0 { + self.first_media(&sample.track_sid); + } + if sample.direction == StreamDirection::Outbound { + self.state + .publishing + .lock() + .unwrap_or_else(|e| e.into_inner()) + .remove(&sample.track_sid); + } + self.telemetry.record_stats_in(sample, &self.state); + } + + /// A whole `getStats()` report for one track, as the platform got it. The core picks the RTP + /// streams, resolves codec and RTT, converts units and records one sample per stream (outbound + /// ones tagged with their layer), so no SDK maps stats fields itself. + pub fn record_stats_report( + &self, + track_sid: &str, + kind: TrackKind, + direction: StreamDirection, + report: Vec, + timestamp_ns: Option, + ) { + for sample in + crate::rtc::samples_from_report(track_sid, kind, direction, &report, timestamp_ns) + { + self.record_stats(sample); + } + } + + /// A whole peer connection's `getStats()` report, as the platform got it, and the tracks it + /// carries (MediaStreamTrack id → track sid, local and remote). The core finds each track's + /// RTP streams and records them as [`record_stats_report`](Self::record_stats_report) would: + /// one call per peer connection per poll, whatever the number of participants and tracks. + pub fn record_peer_stats( + &self, + report: Vec, + tracks: HashMap, + timestamp_ns: Option, + ) { + for sample in crate::rtc::samples_from_peer_report(&report, &tracks, timestamp_ns) { + self.record_stats(sample); + } + } + + /// How long the platform should wait before its next `getStats()` poll of this room: 1 s + /// while a subscribe waits for its first media (the core sees first media in the readings) or + /// a newly published track for its first outbound reading (known from its `lk.publish` + /// span's `set_track` with a sid; at most 30 s), else half the current stats window — two readings per window, at the pressure-stretched + /// cadence. Ask again after every poll. + pub fn stats_poll_interval_ms(&self) -> u64 { + let subscribing = + !self.state.subscribes.lock().unwrap_or_else(|e| e.into_inner()).is_empty(); + if subscribing || self.state.awaiting_outbound() { + return 1000; + } + self.telemetry.stats_window_ms() / 2 + } + + /// No media within this long ends `lk.subscribe` with `error.type = timed_out`. + pub const SUBSCRIBE_TIMEOUT: Duration = Duration::from_secs(30); + + /// The intent to subscribe exists (autoSubscribe: at the remote publish; manual: at the + /// subscribe call; a track already in the room at join: at connect): `lk.subscribe` opens, + /// once per track. Its end is an RTC fact the core + /// sees itself — the first inbound reading with bytes — so no SDK keeps this state. + pub fn subscribe_started(&self, track: SpanTrack) { + let Some(sid) = track.sid.clone() else { return }; + if self.telemetry.shared.revoked() { + return; + } + { + let mut open = self.state.subscribes.lock().unwrap_or_else(|e| e.into_inner()); + if open.contains_key(&sid) { + return; + } + let span = self.start(SpanName::Subscribe, None); + span.set_track(track); + open.insert(sid, (span, Instant::now())); + } + // The exporter owns the clock that enforces the deadline. + self.telemetry.wake(); + } + + /// The server confirmed the subscription: the `subscribed` step. Without an earlier intent + /// this is the intent (a fallback, measured from here): the span opens and + /// [`stats_poll_interval_ms`](Self::stats_poll_interval_ms) turns fast at once. + pub fn subscribed(&self, track: SpanTrack) { + self.subscribe_started(track.clone()); + let Some(sid) = &track.sid else { return }; + if let Some((span, _)) = + self.state.subscribes.lock().unwrap_or_else(|e| e.into_inner()).get(sid) + { + span.step(SpanStep::Subscribed); + } + } + + /// A track left this session — unpublished, unsubscribed, its publisher gone. A pending + /// `lk.subscribe` ends `cancelled` (or `timed_out` past its deadline), the track's last + /// partial RTC window ships now, and the core forgets everything it kept for the track. + pub fn track_ended(&self, sid: &str) { + self.state.publishing.lock().unwrap_or_else(|e| e.into_inner()).remove(sid); + if let Some((span, since)) = self.take_subscribe(sid) { + end_unfinished(&span, since); + } + self.telemetry.retire_stats(&self.state, Some(sid)); + } + + /// The subscription failed (`error.type` = the platform's error name). + pub fn subscribe_failed(&self, sid: &str, error_type: &str) { + if let Some((span, _)) = self.take_subscribe(sid) { + span.fail(error_type.to_owned()); + } + } + + /// First media ends the subscribe `ok` — unless its deadline already passed and the sweep + /// has not run yet: then it is `timed_out`, whoever looks first. + fn first_media(&self, sid: &str) { + let Some((span, since)) = self.take_subscribe(sid) else { return }; + if since.elapsed() >= Self::SUBSCRIBE_TIMEOUT { + span.fail("timed_out".to_owned()); + return; + } + span.step(SpanStep::FirstMedia); + span.end(SpanOutcome::Ok, None); + } + + fn take_subscribe(&self, sid: &str) -> Option<(Arc, Instant)> { + self.state.subscribes.lock().unwrap_or_else(|e| e.into_inner()).remove(sid) + } + + /// Start a typed span in this session's trace, stamped now. `parent` nests it. + pub fn start(&self, name: SpanName, parent: Option>) -> Arc { + let parent = parent.and_then(|p| p.context()).map(|c| c.span_id); + Span::bound(name, parent, self.telemetry.clone(), &self.state) + } + + /// The room and local participant, as `lk.room.*` / `lk.participant.*` on every record. + pub fn set_room(&self, room: RoomIdentity) { + for (key, value) in [ + ("lk.room.sid", room.sid), + ("lk.room.name", room.name), + ("lk.participant.sid", room.participant_sid), + ("lk.participant.identity", room.participant_identity), + ] { + // Identifiers, not free text: an over-long one is not attached (counted as invalid). + if value.as_ref().is_some_and(|v| v.len() > crate::event::MAX_VALUE_BYTES) { + self.telemetry.count_invalid(); + continue; + } + self.state.set_attribute(key, value.map(AttributeValue::Str)); + } + } +} + +/// A subscribe that ended without media: `timed_out` once past its deadline — even when the +/// cleanup comes later — `cancelled` before. +fn end_unfinished(span: &Span, since: Instant) { + if since.elapsed() >= Scope::SUBSCRIBE_TIMEOUT { + span.fail("timed_out".to_owned()); + } else { + span.cancel(); + } +} diff --git a/livekit-telemetry/src/span.rs b/livekit-telemetry/src/span.rs new file mode 100644 index 000000000..6a553f05f --- /dev/null +++ b/livekit-telemetry/src/span.rs @@ -0,0 +1,347 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::{HashMap, VecDeque}; +use std::sync::Arc; + +use crate::{event::now_unix_nanos, scope::ScopeState, Attribute, AttributeValue}; + +/// OTel span kind, restricted to what client operations need; implied by [`crate::SpanName`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SpanKind { + /// An operation inside the SDK (publish, subscribe). + Internal, + /// A call to the SFU that waits for its answer (connect, reconnect). + Client, +} + +/// How an attempt ended. OTel status knows only `Unset`/`Ok`/`Error`, so `Cancelled` travels as +/// `status = Unset` plus the `lk.outcome` attribute — every span carries `lk.outcome` so rollups +/// never have to infer it (a user hanging up mid-connect is not a failure). +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SpanOutcome { + Ok, + Error, + Cancelled, +} + +impl SpanOutcome { + pub(crate) fn as_str(self) -> &'static str { + match self { + SpanOutcome::Ok => "ok", + SpanOutcome::Error => "error", + SpanOutcome::Cancelled => "cancelled", + } + } +} + +/// A checkpoint inside a span (OTLP span event). Structural to one attempt — the connect +/// sequence's `ws_open → join_recv → pc_connected → …` — hence in the span's own envelope rather +/// than a standalone log record (OTEP 4430 keeps that legal). +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct SpanEvent { + pub name: String, + pub time_ns: u64, + pub attributes: Vec, +} + +/// One attempt at an operation, from `begin_span` to `end_span`. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct SpanRecord { + pub span_id: u64, + pub parent_span_id: Option, + pub name: String, + pub kind: SpanKind, + pub start_ns: u64, + pub end_ns: u64, + pub outcome: SpanOutcome, + pub error_type: Option, + pub attributes: Vec, + pub events: Vec, + /// The session (trace) the span belongs to. + pub session: Arc, + /// The session's project when the span ended (see `Queued::route`). + pub route: Option, +} + +/// OTel default span limits. +const MAX_EVENTS_PER_SPAN: usize = 128; +const MAX_ATTRIBUTES_PER_SPAN: usize = 128; +/// Spans a host can leave open before the oldest is abandoned (counted as dropped). +const MAX_OPEN_SPANS: usize = 256; + +/// Open spans by handle, plus the finished ones waiting for the exporter. +/// +/// Handles are opaque `u64`s minted here; the host keeps them (a Swift `Span` object, a Kotlin +/// value) and never sees ambient context — that is the platform's job (task-locals, coroutine +/// context, zones), not the FFI's. +pub(crate) struct Spans { + open: HashMap, + /// Insertion order of `open`, to abandon the oldest when the cap is hit. + open_order: VecDeque, + finished: Vec, + finished_capacity: usize, + /// Which session every recent span belongs to — open, finished or already exported — so a + /// log record that arrives after its span ended (a warning logged right before a failing + /// publish ends, delivered a hop later) is still filed under the right session. + sessions: HashMap>, + session_order: VecDeque, + next_id: u64, + pub dropped: u64, + /// The opt-out, checked under the lock that guards the registry (see `Store`). + revoked: Arc, +} + +/// Spans whose session stays resolvable after they ended. +// ponytail: a fixed ring; a time-based expiry if a long session ever opens more spans than this +// between a log and its export. +const REMEMBERED_SPANS: usize = 1024; + +impl Spans { + pub fn new(finished_capacity: usize) -> Self { + Self { + open: HashMap::new(), + open_order: VecDeque::new(), + finished: Vec::new(), + finished_capacity, + sessions: HashMap::new(), + session_order: VecDeque::new(), + // Span ids must be non-zero (OTLP treats all-zero as absent); start at 1 and mix in + // randomness so ids from two pipelines in one process never collide. 63 bits: a + // platform whose integers are signed (Dart) must be able to hand an id back. + next_id: (rand::random::() >> 1) | 1, + dropped: 0, + revoked: Arc::default(), + } + } + + /// Tie the registry to a pipeline's opt-out. + pub fn with_consent(mut self, revoked: Arc) -> Self { + self.revoked = revoked; + self + } + + fn revoked(&self) -> bool { + self.revoked.load(std::sync::atomic::Ordering::SeqCst) + } + + /// Open a span in `session`'s trace. + pub fn begin_in( + &mut self, + name: &str, + kind: SpanKind, + parent: Option, + session: Arc, + ) -> u64 { + if self.revoked() { + return 0; + } + let id = self.next_id; + self.next_id = ((self.next_id + 1) & (u64::MAX >> 1)).max(1); + if self.open.len() >= MAX_OPEN_SPANS { + if let Some(oldest) = self.open_order.pop_front() { + self.open.remove(&oldest); + self.dropped += 1; + } + } + self.sessions.insert(id, session.clone()); + self.session_order.push_back(id); + if self.session_order.len() > REMEMBERED_SPANS { + if let Some(old) = self.session_order.pop_front() { + self.sessions.remove(&old); + } + } + let record = SpanRecord { + span_id: id, + parent_span_id: parent.filter(|p| *p != 0), + name: name.to_owned(), + kind, + start_ns: now_unix_nanos(), + end_ns: 0, + outcome: SpanOutcome::Ok, + error_type: None, + attributes: Vec::new(), + events: Vec::new(), + session, + route: None, + }; + self.open.insert(id, record); + self.open_order.push_back(id); + id + } + + /// Whether a span with one of these names is still open (the exporter holds uploads while + /// `lk.connect` / `lk.reconnect` are). + /// The session a recent span belongs to — open, ended or exported (log records emitted inside + /// a span are filed there, and they may arrive after the span ended). + pub fn scope_of(&self, id: u64) -> Option> { + self.sessions.get(&id).cloned() + } + + #[cfg(test)] + pub fn begin(&mut self, name: &str, kind: SpanKind, parent: Option) -> u64 { + self.begin_in(name, kind, parent, ScopeState::new()) + } + + #[cfg(test)] + pub fn open_count(&self) -> usize { + self.open.len() + } + + pub fn any_open(&self, names: &[&str]) -> bool { + self.open.values().any(|span| names.contains(&span.name.as_str())) + } + + pub fn add_event(&mut self, id: u64, name: &str, attributes: Vec) { + if self.revoked() { + return; + } + let Some(span) = self.open.get_mut(&id) else { return }; + if span.events.len() >= MAX_EVENTS_PER_SPAN { + return; + } + span.events.push(SpanEvent { + name: name.to_owned(), + time_ns: now_unix_nanos(), + attributes, + }); + } + + /// Close a span; the finished record waits for the next export. Unknown ids are ignored + /// (double `end` is harmless, like OTel's). + pub fn end( + &mut self, + id: u64, + outcome: SpanOutcome, + error_type: Option, + mut attributes: Vec, + ) { + if self.revoked() { + return; + } + let Some(mut span) = self.open.remove(&id) else { return }; + self.open_order.retain(|open| *open != id); + span.end_ns = now_unix_nanos().max(span.start_ns); + span.outcome = outcome; + span.error_type = error_type; + span.session.snapshot_custom(&mut attributes); + span.route = span.session.route(); + attributes.truncate(MAX_ATTRIBUTES_PER_SPAN); + span.attributes = attributes; + if self.finished.len() >= self.finished_capacity { + self.finished.remove(0); + self.dropped += 1; + } + self.finished.push(span); + } + + /// Take the finished spans, oldest first. + /// At most `max` spans and about `max_bytes` (always at least one, so an oversized span + /// still ships). + pub fn drain(&mut self, max: usize, max_bytes: usize) -> Vec { + let (mut n, mut bytes) = (0, 0); + for span in self.finished.iter().take(max) { + let size = span.size_hint(); + if n > 0 && bytes + size > max_bytes { + break; + } + bytes += size; + n += 1; + } + self.finished.drain(..n).collect() + } + + /// Forget every open and finished span (opt-out); returns how many went. + pub fn clear(&mut self) -> u64 { + let n = self.open.len() + self.finished.len(); + self.open.clear(); + self.open_order.clear(); + self.finished.clear(); + self.sessions.clear(); + self.session_order.clear(); + n as u64 + } + + pub fn take_dropped(&mut self) -> u64 { + std::mem::take(&mut self.dropped) + } +} + +impl SpanRecord { + /// Rough encoded size, like `TelemetryEvent::size_hint`. + pub(crate) fn size_hint(&self) -> usize { + let attributes = |attributes: &[Attribute]| { + attributes.iter().map(|a| 4 + a.key.len() + a.value.size_hint()).sum::() + }; + 64 + self.name.len() + + attributes(&self.attributes) + + self + .events + .iter() + .map(|e| 16 + e.name.len() + attributes(&e.attributes)) + .sum::() + } + + /// `lk.outcome` and `error.type`, the attributes every span carries beyond the caller's. + pub(crate) fn outcome_attributes(&self) -> Vec { + let mut attributes = vec![Attribute::new("lk.outcome", self.outcome.as_str())]; + if let Some(error_type) = &self.error_type { + attributes.push(Attribute::new("error.type", AttributeValue::Str(error_type.clone()))); + } + attributes + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn spans_open_record_events_and_finish_in_order() { + let mut spans = Spans::new(8); + let parent = spans.begin("lk.connect", SpanKind::Client, None); + let child = spans.begin("lk.publish", SpanKind::Internal, Some(parent)); + spans.add_event(parent, "ws_open", vec![]); + spans.end(child, SpanOutcome::Cancelled, None, vec![]); + spans.end( + parent, + SpanOutcome::Error, + Some("timeout".into()), + vec![Attribute::new("lk.connect.attempt", 1i64)], + ); + spans.end(parent, SpanOutcome::Ok, None, vec![]); // double end: ignored + + let finished = spans.drain(10, usize::MAX); + assert_eq!(finished.len(), 2); + assert_eq!(finished[0].name, "lk.publish"); + assert_eq!(finished[0].parent_span_id, Some(parent)); + assert_eq!(finished[0].outcome, SpanOutcome::Cancelled); + assert_eq!(finished[1].events[0].name, "ws_open"); + assert_eq!(finished[1].error_type.as_deref(), Some("timeout")); + assert!(finished[1].end_ns >= finished[1].start_ns); + assert_eq!(spans.take_dropped(), 0); + } + + #[test] + fn finished_spans_are_bounded() { + let mut spans = Spans::new(1); + for _ in 0..2 { + let id = spans.begin("lk.publish", SpanKind::Internal, None); + spans.end(id, SpanOutcome::Ok, None, vec![]); + } + assert_eq!(spans.drain(10, usize::MAX).len(), 1); + assert_eq!(spans.take_dropped(), 1); + } +} diff --git a/livekit-telemetry/src/stats.rs b/livekit-telemetry/src/stats.rs new file mode 100644 index 000000000..75ab223ed --- /dev/null +++ b/livekit-telemetry/src/stats.rs @@ -0,0 +1,305 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::atomic::{AtomicU64, Ordering}; + +use crate::TelemetryEvent; + +/// Declares the pipeline's health counters once: the shared atomics ([`Counters`]), their +/// point-in-time copy ([`Snapshot`]) and the delta between two copies. +macro_rules! counters { + ($($(#[$doc:meta])* $name:ident,)*) => { + /// Pipeline health counters, shared by the store, the exporter and + /// [`Telemetry::stats`](crate::Telemetry::stats). Loss reasons follow the OpenTelemetry + /// SDK self-metrics conventions (`queue_full`, `rejected`, `timeout`) where one exists. + #[derive(Default)] + pub(crate) struct Counters { + $($(#[$doc])* pub $name: AtomicU64,)* + } + + /// A point-in-time copy of [`Counters`]. + #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] + pub(crate) struct Snapshot { + $(pub $name: u64,)* + } + + impl Counters { + pub fn snapshot(&self) -> Snapshot { + Snapshot { $($name: self.$name.load(Ordering::Relaxed),)* } + } + } + + impl Snapshot { + /// Counts accumulated since `earlier`. + pub fn since(&self, earlier: &Snapshot) -> Snapshot { + Snapshot { $($name: self.$name.saturating_sub(earlier.$name),)* } + } + } + }; +} + +counters! { + /// Records evicted from the in-memory queue (`max_queue_size`). + queue_full, + /// Records lost because no cache, not even memory, could take their batch. + cache_error, + /// Records evicted from the cache by its size or file-count bound. + cache_full, + /// Records in cached batches past the 24 h age limit. + expired, + /// Records in cached batches that failed their integrity check (truncated, corrupt). + corrupt, + /// Records the core refused at the door: a custom event or attribute over the limits. + invalid, + /// Records the collector rejected: a final 4xx/5xx, or refused in a partial success. + rejected, + /// Single records larger than the collector accepts (413 down to one record). + oversized, + /// Records evicted from the cache while the collector held uploads off (`Retry-After`). + throttled, + /// Records for a project that receives nothing (self-hosted, no grant, disabled, 404). + disabled, + /// Discrete events dropped by the flood guard (`max_events_per_10min`). + rate_limited, + /// Records deleted by the opt-out. + purged, + /// Batches the collector accepted. + uploads_sent, + /// Compressed bytes the collector accepted — what telemetry actually cost the uplink. + upload_bytes, + /// Upload attempts that failed transiently (no answer, 429, 5xx). + upload_failures, + /// Upload attempts that hit `export_timeout_ms` (a slow network, or a stalled collector). + upload_timeouts, + /// 401/403 answers: a token the collector refused (the batches wait for the next one). + auth_denied, + /// Soft holds that reached the cap and let one batch through: the policy was starving + /// telemetry, and data arrived late. + hold_cap_hits, + /// Batches the disk cache could not store (disk full, directory gone), kept in memory + /// instead: they survive a failed upload, not the process. + cache_write_errors, +} + +impl Counters { + pub fn add(counter: &AtomicU64, n: u64) { + counter.fetch_add(n, Ordering::Relaxed); + } +} + +impl Snapshot { + /// Everything that was lost, by any reason. + pub fn dropped(&self) -> u64 { + self.queue_full + + self.cache_error + + self.cache_full + + self.expired + + self.corrupt + + self.invalid + + self.rejected + + self.oversized + + self.throttled + + self.disabled + + self.rate_limited + + self.purged + } + + /// Anything worth telling the backend about: data lost (other than by policy: a project + /// that receives nothing, an opt-out), uploads failing or refused, holds starving uploads, + /// the disk refusing writes. + pub fn has_problems(&self) -> bool { + self.dropped() - self.disabled - self.purged + + self.upload_failures + + self.upload_timeouts + + self.auth_denied + + self.hold_cap_hits + + self.cache_write_errors + > 0 + } + + /// The `lk.telemetry.report` event: what this pipeline sent, dropped or failed to upload + /// since the previous report — deltas by reason, riding along with the next batch, never + /// persisted on its own, never an extra request — plus one at shutdown, so every session + /// leaves a summary the fleet's success rates can be computed from. + pub fn report(&self, cached_batches: u64) -> TelemetryEvent { + let mut event = TelemetryEvent::new("lk.telemetry.report") + .with_body(format!( + "telemetry: {} batches sent ({} B), {} failed, {} dropped, {} cached", + self.uploads_sent, + self.upload_bytes, + self.upload_failures + self.upload_timeouts, + self.dropped() - self.disabled - self.purged, + cached_batches + )) + .with_attribute("lk.telemetry.uploads.sent", self.uploads_sent as i64) + .with_attribute("lk.telemetry.uploads.bytes", self.upload_bytes as i64) + .with_attribute("lk.telemetry.uploads.failed", self.upload_failures as i64) + .with_attribute("lk.telemetry.cache.batches", cached_batches as i64); + for (key, value) in [ + ("lk.telemetry.uploads.timeouts", self.upload_timeouts), + ("lk.telemetry.uploads.unauthorized", self.auth_denied), + ("lk.telemetry.holds.capped", self.hold_cap_hits), + ("lk.telemetry.cache.write_errors", self.cache_write_errors), + ("lk.telemetry.dropped.queue_full", self.queue_full), + ("lk.telemetry.dropped.cache_error", self.cache_error), + ("lk.telemetry.dropped.cache_full", self.cache_full), + ("lk.telemetry.dropped.expired", self.expired), + ("lk.telemetry.dropped.corrupt", self.corrupt), + ("lk.telemetry.dropped.invalid", self.invalid), + ("lk.telemetry.dropped.rejected", self.rejected), + ("lk.telemetry.dropped.oversized", self.oversized), + ("lk.telemetry.dropped.throttled", self.throttled), + ("lk.telemetry.dropped.rate_limited", self.rate_limited), + ] { + if value > 0 { + event = event.with_attribute(key, value as i64); + } + } + event + } +} + +/// What the upload policy is doing right now. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TelemetryStatus { + /// Uploading as data arrives. + Ok, + /// Uploads wait for a connect/reconnect to finish or for the device (Low Data Mode, low + /// battery); capped at 60 s. + Held, + /// The last upload failed; retrying after a jittered exponential backoff. + Paused, + /// The collector asked for a pause (`Retry-After`). + Throttled, + /// No destination or no usable token yet (no room connected, the token expired, lacks the + /// grant or was refused); everything waits in the cache for the next token. + Waiting, + /// No project receives telemetry: every server this process talked to is self-hosted, never + /// granted observability, has no ingest, or disabled data recording. + Off, +} + +impl std::fmt::Display for TelemetryStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Ok => "ok", + Self::Held => "held", + Self::Paused => "paused", + Self::Throttled => "throttled", + Self::Waiting => "waiting", + Self::Off => "off", + }) + } +} + +/// Pipeline health as seen by the SDK: [`Telemetry::stats`](crate::Telemetry::stats). +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TelemetryStats { + /// What the upload policy is doing right now. + pub status: TelemetryStatus, + /// Records lost for any reason (sum of the `dropped_*` fields). + pub dropped: u64, + pub dropped_queue_full: u64, + pub dropped_cache_error: u64, + pub dropped_cache_full: u64, + pub dropped_expired: u64, + pub dropped_corrupt: u64, + pub dropped_invalid: u64, + pub dropped_rejected: u64, + pub dropped_oversized: u64, + pub dropped_throttled: u64, + pub dropped_disabled: u64, + pub dropped_rate_limited: u64, + pub dropped_purged: u64, + /// Batches the collector accepted. + pub uploads_sent: u64, + /// Compressed bytes the collector accepted. + pub upload_bytes: u64, + /// Upload attempts that failed transiently (no answer, 429, 5xx). + pub upload_failures: u64, + /// Upload attempts that timed out. + pub upload_timeouts: u64, + /// Tokens the collector refused (401/403). + pub uploads_unauthorized: u64, + /// Soft holds that reached the cap. + pub holds_capped: u64, + /// Batches the disk cache could not store, kept in memory instead. + pub cache_write_errors: u64, + /// Batches currently waiting in the cache. + pub cached_batches: u64, +} + +/// One line for a debug console: status, throughput, one backlog number, one loss number, then +/// the loss breakdown. +impl std::fmt::Display for TelemetryStats { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}, sent {} ({} B), backlog {}, failed {}, lost {} (queue {}, cache {}, expired {}, \ + corrupt {}, invalid {}, rejected {}, oversized {}, throttled {}, rate-limited {}, \ + disabled {}, purged {}; unauthorized {}, holds capped {}, write errors {})", + self.status, + self.uploads_sent, + self.upload_bytes, + self.cached_batches, + self.upload_failures + self.upload_timeouts, + self.dropped, + self.dropped_queue_full, + self.dropped_cache_full + self.dropped_cache_error, + self.dropped_expired, + self.dropped_corrupt, + self.dropped_invalid, + self.dropped_rejected, + self.dropped_oversized, + self.dropped_throttled, + self.dropped_rate_limited, + self.dropped_disabled, + self.dropped_purged, + self.uploads_unauthorized, + self.holds_capped, + self.cache_write_errors, + ) + } +} + +impl TelemetryStats { + pub(crate) fn new(snapshot: Snapshot, cached_batches: u64, status: TelemetryStatus) -> Self { + Self { + status, + dropped: snapshot.dropped(), + dropped_queue_full: snapshot.queue_full, + dropped_cache_error: snapshot.cache_error, + dropped_cache_full: snapshot.cache_full, + dropped_expired: snapshot.expired, + dropped_corrupt: snapshot.corrupt, + dropped_invalid: snapshot.invalid, + dropped_rejected: snapshot.rejected, + dropped_oversized: snapshot.oversized, + dropped_throttled: snapshot.throttled, + dropped_disabled: snapshot.disabled, + dropped_rate_limited: snapshot.rate_limited, + dropped_purged: snapshot.purged, + uploads_sent: snapshot.uploads_sent, + upload_bytes: snapshot.upload_bytes, + upload_failures: snapshot.upload_failures, + upload_timeouts: snapshot.upload_timeouts, + uploads_unauthorized: snapshot.auth_denied, + holds_capped: snapshot.hold_cap_hits, + cache_write_errors: snapshot.cache_write_errors, + cached_batches, + } + } +} diff --git a/livekit-telemetry/src/store.rs b/livekit-telemetry/src/store.rs new file mode 100644 index 000000000..88977b227 --- /dev/null +++ b/livekit-telemetry/src/store.rs @@ -0,0 +1,190 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{ + collections::VecDeque, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, + }, +}; + +use crate::{scope::ScopeState, stats::Counters, TelemetryEvent}; + +/// An event waiting for export, filed under the session whose trace id and attributes it +/// will carry. +pub(crate) struct Queued { + pub event: TelemetryEvent, + pub session: Arc, + /// The project the session was routed to when the record was captured: immutable, so a Room + /// that later reconnects to another project never takes queued records along. `None`: the + /// session had no server yet. + pub route: Option, +} + +impl Queued { + /// Capture a record for `session`, with its owner as of now: the session's project and its + /// correlation attributes (the record's own win). + pub fn new(mut event: TelemetryEvent, session: Arc) -> Self { + let route = session.route(); + session.snapshot_custom(&mut event.attributes); + Self { event, session, route } + } +} + +/// Bounded FIFO of events waiting for export. +/// +/// When full, the *oldest* event is dropped so the freshest context survives a burst +/// (the queue role of OTel's `BatchLogRecordProcessor`, with drop-oldest instead of +/// drop-newest); every eviction is counted as `queue_full`. Tracks its approximate size in +/// bytes so the exporter can flush early (design doc: every tick *or* at 256 KB) and bound a +/// request's size. +// ponytail: one mutex around a VecDeque; a lock-free ring only if `emit` shows up in a profile. +pub(crate) struct Store { + queue: Mutex, + /// The opt-out, checked under the queue lock `clear` also takes: nothing lands after a purge. + revoked: Arc, + /// Test-only: runs at the start of `push`, before the lock (to hold a producer there). + #[cfg(test)] + pub(crate) pause: Mutex>>, + capacity: usize, + flush_threshold: usize, + counters: Arc, +} + +#[derive(Default)] +struct Queue { + events: VecDeque, + bytes: usize, + /// The first eviction of an episode logs; the rest are counted. + full_warned: bool, +} + +impl Store { + pub fn new(capacity: usize, flush_threshold: usize, counters: Arc) -> Self { + Self { + queue: Mutex::new(Queue::default()), + revoked: Arc::default(), + #[cfg(test)] + pause: Mutex::new(None), + capacity, + flush_threshold, + counters, + } + } + + /// Tie the queue to a pipeline's opt-out. + pub fn with_consent(mut self, revoked: Arc) -> Self { + self.revoked = revoked; + self + } + + /// Queue an event. Returns `true` when this push carried the queue across + /// `flush_threshold` bytes — the caller should wake the exporter. + pub fn push(&self, queued: Queued) -> bool { + #[cfg(test)] + { + let pause = self.pause.lock().unwrap_or_else(|e| e.into_inner()).clone(); + if let Some(pause) = pause { + pause(); + } + } + let mut queue = self.queue.lock().unwrap_or_else(|e| e.into_inner()); + if self.revoked.load(Ordering::SeqCst) { + return false; + } + if queue.events.len() >= self.capacity { + if let Some(oldest) = queue.events.pop_front() { + queue.bytes = queue.bytes.saturating_sub(oldest.event.size_hint()); + } + Counters::add(&self.counters.queue_full, 1); + if !queue.full_warned { + queue.full_warned = true; + log::warn!( + "queue full ({} records): dropping oldest until the exporter drains", + self.capacity + ); + } + } + let before = queue.bytes; + queue.bytes += queued.event.size_hint(); + queue.events.push_back(queued); + before < self.flush_threshold && queue.bytes >= self.flush_threshold + } + + /// Remove and return the oldest events: at most `max` of them and about `max_bytes` in total + /// (always at least one, so an oversized event still ships). + pub fn drain(&self, max: usize, max_bytes: usize) -> Vec { + let mut queue = self.queue.lock().unwrap_or_else(|e| e.into_inner()); + queue.full_warned = false; + let mut out = Vec::new(); + let mut bytes = 0; + while out.len() < max { + let Some(next) = queue.events.front() else { break }; + let size = next.event.size_hint(); + if !out.is_empty() && bytes + size > max_bytes { + break; + } + bytes += size; + queue.bytes = queue.bytes.saturating_sub(size); + out.extend(queue.events.pop_front()); + } + out + } + + pub fn is_empty(&self) -> bool { + self.queue.lock().unwrap_or_else(|e| e.into_inner()).events.is_empty() + } + + /// Drop everything queued (opt-out); returns how many records went. + pub fn clear(&self) -> u64 { + let mut queue = self.queue.lock().unwrap_or_else(|e| e.into_inner()); + queue.bytes = 0; + queue.events.drain(..).count() as u64 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn queued(event: TelemetryEvent) -> Queued { + Queued::new(event, ScopeState::new()) + } + + #[test] + fn drops_oldest_when_full() { + let counters = Arc::new(Counters::default()); + let store = Store::new(2, usize::MAX, counters.clone()); + for name in ["a", "b", "c"] { + store.push(queued(TelemetryEvent::new(name))); + } + let names: Vec<_> = store.drain(10, usize::MAX).into_iter().map(|q| q.event.name).collect(); + assert_eq!(names, ["b", "c"]); + assert_eq!(counters.snapshot().queue_full, 1); + assert!(store.drain(10, usize::MAX).is_empty()); + } + + #[test] + fn reports_the_threshold_crossing_once_and_drains_by_bytes() { + let store = Store::new(100, 100, Arc::default()); + let event = || queued(TelemetryEvent::new("e").with_body("x".repeat(30))); // 63 bytes + assert!(!store.push(event()), "63 < 100"); + assert!(store.push(event()), "126 crosses 100"); + assert!(!store.push(event()), "already above: no second wake-up"); + assert_eq!(store.drain(10, 130).len(), 2, "two fit in 130 bytes"); + assert_eq!(store.drain(10, 1).len(), 1, "an oversized event still ships alone"); + assert!(store.drain(10, usize::MAX).is_empty()); + } +} diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs new file mode 100644 index 000000000..a72359419 --- /dev/null +++ b/livekit-telemetry/src/telemetry.rs @@ -0,0 +1,892 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{ + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, Weak, + }, + time::Duration, +}; + +use tokio::sync::{mpsc, oneshot}; +use tokio::time::{timeout, Instant}; + +use crate::span::SpanKind; +use crate::{ + cache::SpillCache, + destination::{Destinations, ENDPOINT_OVERRIDE_ENV}, + event::now_unix_nanos, + exporter::Command, + rtc::StatsWindows, + scope::{Scope, ScopeState}, + span::Spans, + stats::{Counters, TelemetryStatus}, + store::{Queued, Store}, + Attribute, AttributeValue, BatchCache, DeviceState, Exporter, FileCache, LogRecord, LogSource, + MemoryCache, RtcStatsSample, Severity, SpanOutcome, TelemetryEvent, TelemetryStats, + TelemetryTransport, +}; +use crate::{DeviceEvent, Span, SpanName}; + +/// Pipeline configuration: storage and tuning. Where batches go is not configurable — the core +/// derives it from the server URL and token each room hands over ([`Scope::set_server`]). +/// +/// Defaults are conservative, sized so a fleet cannot overload the collector: one export and +/// one RTC stats window per minute, OTel's `BatchLogRecordProcessor` queue (2048) and batch (512). +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone)] +pub struct TelemetryConfig { + /// Resource attributes describing the emitter (`service.name`, `os.name`, + /// `device.model.identifier`, `session.id`, …). `telemetry.sdk.*` are filled in by the core. + #[cfg_attr(feature = "uniffi", uniffi(default = []))] + pub resource: Vec, + /// Who is reporting, typed; the core owns the semconv keys. Extra attributes go in `resource`. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub sdk: Option, + /// Directory for the on-disk batch cache (created if missing; its parent must exist). + /// `None` keeps batches in memory only: they survive failed uploads, not the process. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub storage_dir: Option, + /// Cap on cached batches, in memory or on disk; the oldest are evicted first. + #[cfg_attr(feature = "uniffi", uniffi(default = 4194304))] + pub max_cache_bytes: u64, + /// Base export cadence; stretched up to 4× by [`DeviceState::cadence_factor`]. + #[cfg_attr(feature = "uniffi", uniffi(default = 60000))] + pub flush_interval_ms: u64, + /// Events buffered before the oldest are dropped. + #[cfg_attr(feature = "uniffi", uniffi(default = 2048))] + pub max_queue_size: u32, + /// Events per export request. + #[cfg_attr(feature = "uniffi", uniffi(default = 512))] + pub max_batch_size: u32, + /// Bound on a single transport attempt, and on `shutdown`. + #[cfg_attr(feature = "uniffi", uniffi(default = 10000))] + pub export_timeout_ms: u64, + /// RTC stats window: readings pushed with [`Scope::record_stats`] are summarised into one + /// `lk.rtc.stats.sample` per track and direction every window (stretched like the cadence). + #[cfg_attr(feature = "uniffi", uniffi(default = 60000))] + pub stats_window_ms: u64, + /// Flood guard for discrete events: beyond this many `emit`s per 10 minutes the rest are + /// dropped and counted as `rate_limited`. RTC windows and self-telemetry are exempt; 0 = off. + #[cfg_attr(feature = "uniffi", uniffi(default = 300))] + pub max_events_per_10min: u32, + /// Cached batches uploaded per tick while a session may be live — bounds how fast a backlog + /// (offline period, previous launch) replays next to a call: 4 × ~20 KB gzipped per minute + /// is ~10 kbps. `shutdown` drains without the budget. + #[cfg_attr(feature = "uniffi", uniffi(default = 4))] + pub max_batches_per_upload: u32, + /// Export as soon as the queue holds about this many bytes, without waiting for the tick + /// (design doc: "flush on the tick or at 256 KB"). + #[cfg_attr(feature = "uniffi", uniffi(default = 262144))] + pub flush_threshold_bytes: u64, + /// Cap on one request's payload before compression (design doc: "single POST ≤ 1 MB"). + #[cfg_attr(feature = "uniffi", uniffi(default = 1048576))] + pub max_batch_bytes: u64, + /// Lowest severity a plain log record (an event with no name) needs to leave the device. + /// Events are not subject to it. Design doc: warn. + /// `None` is `Warn`. (Optional because UniFFI 0.31 cannot default an enum literal.) + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub log_severity: Option, + /// Platform instruments not to run; all run by default. + #[cfg_attr(feature = "uniffi", uniffi(default = []))] + pub disabled_instruments: Vec, +} + +/// The platform instruments a config can switch off. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Instrument { + /// Thermal, power, memory, network, battery, audio session. + Device, + /// Warn/error lines from the SDK, the core and WebRTC. + Logs, + /// `getStats` windows per track and subscribe spans. + Rtc, + /// Connect, reconnect and publish spans. + Room, +} + +impl Default for TelemetryConfig { + fn default() -> Self { + Self { + resource: Vec::new(), + sdk: None, + storage_dir: None, + max_cache_bytes: 4 * 1024 * 1024, + flush_interval_ms: 60_000, + max_queue_size: 2048, + max_batch_size: 512, + export_timeout_ms: 10_000, + stats_window_ms: 60_000, + max_events_per_10min: 300, + max_batches_per_upload: 4, + flush_threshold_bytes: 256 * 1024, + max_batch_bytes: 1024 * 1024, + log_severity: None, + disabled_instruments: Vec::new(), + } + } +} + +/// Entry point: the synchronous, never-blocking side SDKs push into. +/// +/// Fail-open by design: [`emit`](Self::emit) cannot fail or block — when the queue is full the +/// oldest event is dropped and counted in [`stats`](Self::stats). Cheap to clone; every clone +/// feeds the same pipeline. +/// +/// ``` +/// # use std::sync::Arc; +/// # use livekit_telemetry::*; +/// # struct Discard; +/// # #[async_trait::async_trait] +/// # impl TelemetryTransport for Discard { +/// # async fn send(&self, _: ExportRequest) -> Result { +/// # Ok(ExportResponse::accepted()) +/// # } +/// # } +/// # #[tokio::main(flavor = "current_thread")] async fn main() { +/// let (telemetry, exporter) = Telemetry::new(TelemetryConfig::default(), Arc::new(Discard)); +/// tokio::spawn(exporter.run()); +/// +/// // One scope per room: the server URL and token route its records to the room's project. +/// let room = telemetry.begin_scope(); +/// room.set_server("wss://my-project.livekit.cloud", ""); +/// room.emit_custom("checkout", vec![Attribute::new("plan", "pro")]); +/// telemetry.emit(TelemetryEvent::new("lk.ping")); +/// telemetry.shutdown().await; +/// # } +/// ``` +#[derive(Clone)] +pub struct Telemetry { + pub(crate) shared: Arc, + guard: Arc>, + commands: mpsc::UnboundedSender, +} + +/// How much longer than `export_timeout_ms` [`Telemetry::shutdown`] waits for the exporter to +/// confirm it stopped. +const SHUTDOWN_GRACE: Duration = Duration::from_secs(1); + +/// Everything the synchronous side ([`Telemetry`]) and the exporter actor share. +pub(crate) struct Shared { + pub store: Store, + pub config: TelemetryConfig, + pub cache: Arc, + pub counters: Arc, + /// Read synchronously by the exporter, so a state pushed right before `emit` already governs + /// the flush that follows. + pub device: Mutex>, + pub windows: Mutex, + pub spans: Mutex, + /// The pipeline's own session: whatever is emitted outside a room session. + pub process: Arc, + /// Attributes attached to every record of every session. + pub global: Mutex>, + /// Where each session's batches go, and with which token. + pub destinations: Mutex, + pub status: Mutex, + /// Every room session, for the subscribe deadlines the exporter enforces. + pub scopes: Mutex>>, + /// The queue crossed `flush_threshold_bytes` since the exporter last looked: export now. + pub overflowed: AtomicBool, + /// A destination, a token or a hold changed: re-evaluate what waits (nothing is encoded). + pub released: AtomicBool, + /// Test-only: runs after a capture's early opt-out check, before it takes the lock it commits + /// under (to hold a producer exactly where a racing purge could slip in). + #[cfg(test)] + pub pause: Mutex>>, + /// Upload passes started, for tests that must see the exporter stay idle. + #[cfg(test)] + pub passes: std::sync::atomic::AtomicU64, + /// Coalesced wake-ups for the exporter (see [`Exporter`]): notifications never queue up. + pub wake: tokio::sync::Notify, + /// The app went to the background since the exporter last looked. + pub backgrounded: AtomicBool, + /// The app opted out ([`Telemetry::purge`]). One-way: checked before anything is captured, + /// written or sent, so a request still in flight cannot bring anything back. + pub revoked: Arc, +} + +impl Shared { + /// The test hook between a capture's early check and its commit; nothing in production. + fn before_commit(&self) { + #[cfg(test)] + { + let pause = self.pause.lock().unwrap_or_else(|e| e.into_inner()).clone(); + if let Some(pause) = pause { + pause(); + } + } + } + + /// The queue crossed its threshold: wake the exporter to export now. + pub fn overflow(&self) { + self.overflowed.store(true, Ordering::SeqCst); + self.wake.notify_one(); + } + + /// Something that held batches back changed: wake the exporter to re-evaluate. + pub fn release(&self) { + self.released.store(true, Ordering::SeqCst); + self.wake.notify_one(); + } + + /// Record a checkpoint inside an open span (`ws_open`, `join_recv`, …), stamped now. + pub fn add_span_event(&self, span: u64, name: &str, attributes: Vec) { + if self.revoked() { + return; + } + self.spans.lock().unwrap_or_else(|e| e.into_inner()).add_event(span, name, attributes); + } + + /// End a span with its outcome; `error_type` becomes `error.type` and the status message. + /// The span is exported with the next batch. Ending twice, or an unknown handle, is a no-op. + pub fn end_span( + &self, + span: u64, + outcome: SpanOutcome, + error_type: Option, + attributes: Vec, + ) { + if self.revoked() { + return; + } + self.spans + .lock() + .unwrap_or_else(|e| e.into_inner()) + .end(span, outcome, error_type, attributes); + } + + /// End every Room's pending subscribes (shutdown) or just forget them (opt-out): nothing + /// is left referencing a span once the pipeline stops. + pub fn end_pending_subscribes(&self, export: bool) { + let scopes: Vec> = { + let scopes = self.scopes.lock().unwrap_or_else(|e| e.into_inner()); + scopes.iter().filter_map(|s| s.upgrade()).collect() + }; + for scope in scopes { + scope.end_subscribes(export); + } + } + + /// Device pressure, doubled again while WebRTC reports the encoder CPU-limited; capped at 4×. + pub fn cadence_factor(&self) -> u32 { + let device = self.device.lock().unwrap_or_else(|e| e.into_inner()).unwrap_or_default(); + let cpu = self.windows.lock().unwrap_or_else(|e| e.into_inner()).cpu_limited(); + (device.cadence_factor() * if cpu { 2 } else { 1 }).min(4) + } + + pub fn revoked(&self) -> bool { + self.revoked.load(Ordering::SeqCst) + } + + /// Drop every record and batch, in memory and on disk, counted as `purged`. `false` when + /// the storage refused to delete something: that data is still on disk. + pub fn clear(&self) -> bool { + self.end_pending_subscribes(false); + let queued = self.store.clear(); + let spans = self.spans.lock().unwrap_or_else(|e| e.into_inner()).clear(); + let windows = self.windows.lock().unwrap_or_else(|e| e.into_inner()).clear(); + // Count what really went, once: what was there before and is gone after (an + // unreadable directory lists nothing, so nothing is counted for it). + let before = self.cache.pending(); + let deleted = self.cache.clear(); + let after: std::collections::HashSet = self.cache.pending().into_iter().collect(); + let cached: u64 = before + .iter() + .filter(|id| !after.contains(*id)) + .map(|id| crate::exporter::events_in(id)) + .sum(); + if let Err(err) = &deleted { + log::warn!("opt-out: cached telemetry could not all be deleted ({err})"); + } + Counters::add(&self.counters.purged, queued + spans + windows + cached); + deleted.is_ok() + } +} + +/// Fixed-window cap on discrete events (the design doc's ~300 per 10 min). +struct FloodGuard { + max: u32, + window_start: Instant, + count: u32, + /// The first drop of a window logs; the rest are counted. + warned: bool, +} + +impl FloodGuard { + const WINDOW: Duration = Duration::from_secs(10 * 60); + + fn new(max: u32) -> Self { + Self { max, window_start: Instant::now(), count: 0, warned: false } + } + + fn admit(&mut self) -> bool { + if self.max == 0 { + return true; + } + let now = Instant::now(); + if now.duration_since(self.window_start) >= Self::WINDOW { + self.window_start = now; + self.count = 0; + self.warned = false; + } + if self.count >= self.max { + if !self.warned { + self.warned = true; + log::warn!( + "flood: {} records in 10 min; dropping until the window moves", + self.max + ); + } + return false; + } + self.count += 1; + true + } +} + +impl Telemetry { + /// Build the pipeline with the cache the config asks for: a [`FileCache`] in `storage_dir`, + /// or a [`MemoryCache`] when unset — or when the directory is unusable (logged, never an + /// error). Spawn the returned [`Exporter`] with `exporter.run()` on your runtime. + pub fn new( + config: TelemetryConfig, + transport: Arc, + ) -> (Self, Exporter) { + let mut fell_back = false; + let cache: Arc = match config.storage_dir.as_deref() { + Some(dir) => match FileCache::open(dir, config.max_cache_bytes) { + Ok(cache) => Arc::new(cache), + Err(err) => { + log::warn!("cannot use storage dir {dir}: {err}; caching in memory"); + fell_back = true; + Arc::new(MemoryCache::new(config.max_cache_bytes)) + } + }, + None => Arc::new(MemoryCache::new(config.max_cache_bytes)), + }; + let (telemetry, exporter) = Self::with_cache(config, transport, cache); + if fell_back { + // Reported like any other refused write: batches survive failures, not the process. + Counters::add(&telemetry.shared.counters.cache_write_errors, 1); + } + (telemetry, exporter) + } + + /// Build the pipeline around a caller-provided [`BatchCache`] (`storage_dir` is ignored). + pub fn with_cache( + mut config: TelemetryConfig, + transport: Arc, + cache: Arc, + ) -> (Self, Exporter) { + add_sdk_resource(&mut config.resource, config.sdk.as_ref()); + let counters = Arc::new(Counters::default()); + let revoked = Arc::new(AtomicBool::new(false)); + let cache: Arc = Arc::new(SpillCache::new( + cache, + config.max_cache_bytes, + counters.clone(), + revoked.clone(), + )); + // Batches the cache evicted while opening (past the age or size limit) are lost too. + let evicted: u64 = + cache.take_evicted().iter().map(|id| crate::exporter::events_in(id)).sum(); + Counters::add(&counters.cache_full, evicted); + let endpoint_override = std::env::var(ENDPOINT_OVERRIDE_ENV).ok(); + if let Some(endpoint) = endpoint_override.as_deref().filter(|e| !e.is_empty()) { + log::info!("{ENDPOINT_OVERRIDE_ENV} set: every upload goes to {endpoint}"); + } + // Unbounded, but only request/response commands travel here (flush, shutdown, purge), + // each with a caller awaiting its answer: the queue is bounded by the callers waiting. + // Notifications go through the coalescing `Shared::wake` instead. + let (commands, receiver) = mpsc::unbounded_channel(); + let shared = Arc::new(Shared { + store: Store::new( + config.max_queue_size.max(1) as usize, + usize::try_from(config.flush_threshold_bytes.max(1)).unwrap_or(usize::MAX), + counters.clone(), + ) + .with_consent(revoked.clone()), + cache, + counters, + device: Mutex::new(None), + windows: Mutex::new(StatsWindows::with_consent(revoked.clone())), + spans: Mutex::new( + Spans::new(config.max_queue_size.max(1) as usize).with_consent(revoked.clone()), + ), + // One pipeline per process; sessions (rooms) carry their own trace ids. This is the + // pipeline's own session, for everything emitted outside a room. + process: ScopeState::new(), + global: Mutex::new(Vec::new()), + destinations: Mutex::new(Destinations::new(endpoint_override.as_deref())), + status: Mutex::new(TelemetryStatus::Ok), + scopes: Mutex::new(Vec::new()), + revoked, + #[cfg(test)] + passes: Default::default(), + #[cfg(test)] + pause: Mutex::new(None), + wake: tokio::sync::Notify::new(), + overflowed: AtomicBool::new(false), + released: AtomicBool::new(false), + backgrounded: AtomicBool::new(false), + config, + }); + let guard = Arc::new(Mutex::new(FloodGuard::new(shared.config.max_events_per_10min))); + let exporter = Exporter::new(shared.clone(), transport, receiver); + (Self { shared, guard, commands }, exporter) + } + + /// Route a room's records: its server URL names the project (LiveKit Cloud only), its token + /// authorizes the upload. Called at connect and again with every refreshed token; cheap and + /// idempotent. Everything cached for the project starts uploading once the token allows. + pub(crate) fn set_server(&self, session: &ScopeState, url: &str, token: &str) { + if session.same_server(url, token) { + // A reconnect with the same pair still retries an ingest that answered 404 — one + // map lookup, no parsing. + let route = session.route(); + let revived = route.is_some_and(|host| { + self.shared.destinations.lock().unwrap_or_else(|e| e.into_inner()).revive(&host) + }); + if revived { + self.shared.release(); + } + return; + } + let host = self.shared.destinations.lock().unwrap_or_else(|e| e.into_inner()).set( + url, + token, + &session.hex(), + ); + let Some(host) = host else { + log::warn!("server url has no host; this session's telemetry stays cached"); + return; + }; + // A project change closes the session's RTC windows first: a window belongs to one project. + if session.route().as_deref() != Some(host.as_str()) { + self.with_windows_split(session, || true, || session.set_route(host.clone())); + } + // A new route or a new token may release what waits in the cache. + self.shared.release(); + } + + /// The RTC stats window right now: `stats_window_ms` stretched by device pressure and a + /// CPU-limited encoder (at most 4×). + pub(crate) fn stats_window_ms(&self) -> u64 { + self.shared.config.stats_window_ms.max(1) * u64::from(self.shared.cadence_factor()) + } + + /// An app handed over something over the limits (a custom event or attribute). + pub(crate) fn count_invalid(&self) { + Counters::add(&self.shared.counters.invalid, 1); + } + + /// Nudge the exporter to re-read its deadlines (a subscribe started). + pub(crate) fn wake(&self) { + self.shared.wake.notify_one(); + } + + /// Point every upload at `endpoint`, as [`ENDPOINT_OVERRIDE_ENV`] does. + #[cfg(test)] + pub(crate) fn override_endpoint(&self, endpoint: &str) { + *self.shared.destinations.lock().unwrap_or_else(|e| e.into_inner()) = + Destinations::new(Some(endpoint)); + } + + /// Start a session — one room, one call — with its own trace id and attributes on this + /// pipeline. Sessions do not need ending: a room's last record is simply its last. + pub fn begin_scope(&self) -> Scope { + let state = ScopeState::new(); + let mut scopes = self.shared.scopes.lock().unwrap_or_else(|e| e.into_inner()); + scopes.retain(|scope| scope.strong_count() > 0); + scopes.push(Arc::downgrade(&state)); + Scope { telemetry: self.clone(), state } + } + + /// A span in the pipeline's own trace: app-defined work outside any room, or the SDK before a + /// room exists. Stamped now; `parent` nests it. + pub fn start(&self, name: SpanName, parent: Option>) -> Arc { + let parent = parent.and_then(|p| p.context()).map(|c| c.span_id); + Span::bound(name, parent, self.clone(), &self.shared.process) + } + + /// Queue an event or log record for export. Stamps it with the current time unless it + /// carries one. + /// + /// A record with an empty `name` is a plain log line: only `Warn` and `Error` ones leave the + /// device (design doc: debug/info logs never do). Discrete events are subject to the flood + /// guard (`max_events_per_10min`); what it drops is counted as `rate_limited`. + /// Something happened to the device mid-call (audio route, interruption, a denied permission): + /// a process-level record with a display body, built here so every platform files it alike. + pub fn device_event(&self, event: DeviceEvent) { + self.emit_in(event.into_event(), &self.shared.process); + } + + /// A captured log line. WebRTC only counts at error; the SDK and the core at the configured + /// floor; the core's own telemetry module never (a rejected batch that produced a record that + /// produced a batch would never end). + pub fn log(&self, record: LogRecord) { + if let Some(event) = self.log_event(record) { + self.emit(event); + } + } + + /// The record as an event, or nothing when it is below the floor (WebRTC: error only) or is + /// telemetry's own. + pub(crate) fn log_event(&self, record: LogRecord) -> Option { + let floor = match record.source { + LogSource::WebRtc => self.log_severity().max(Severity::Error), + _ => self.log_severity(), + }; + if record.severity < floor { + return None; + } + if record.source == LogSource::Ffi + && record.logger.as_deref().is_some_and(|l| l.starts_with("livekit_telemetry")) + { + return None; + } + Some(record.into()) + } + + /// Queue an event or log record, filed under the session of the span it names (`span_id`), + /// else under the pipeline's own process session. + pub fn emit(&self, event: TelemetryEvent) { + // A record emitted inside a room's span belongs to that room's session; anything else + // is the process's own. + let session = event + .span_id + .and_then(|id| self.shared.spans.lock().unwrap_or_else(|e| e.into_inner()).scope_of(id)) + .unwrap_or_else(|| self.shared.process.clone()); + self.emit_in(event, &session); + } + + fn log_severity(&self) -> Severity { + self.shared.config.log_severity.unwrap_or(Severity::Warn) + } + + pub(crate) fn emit_in(&self, mut event: TelemetryEvent, session: &Arc) { + if event.name.is_empty() && event.severity < self.log_severity() { + return; + } + if self.shared.revoked() { + return; + } + if !self.collects(session) { + Counters::add(&self.shared.counters.disabled, 1); + return; + } + if !self.guard.lock().unwrap_or_else(|e| e.into_inner()).admit() { + Counters::add(&self.shared.counters.rate_limited, 1); + return; + } + if event.timestamp_ns.is_none() { + event.timestamp_ns = Some(now_unix_nanos()); + } + if self.shared.store.push(Queued::new(event, session.clone())) { + self.shared.overflow(); + } + } + + /// Queue a consumer-defined event, exported as `custom.` (see + /// [`TelemetryEvent::custom`]): the stringly-typed escape hatch next to the `lk.*` + /// catalogue. Same flood guard, same pipeline. + pub fn emit_custom(&self, name: &str, attributes: Vec) { + Scope { telemetry: self.clone(), state: self.shared.process.clone() } + .emit_custom(name, attributes); + } + + /// Set a pipeline-wide attribute (a consumer's `enduser.id`, an `acme.tenant`), attached to + /// every record of every session from now on unless the record — or its session — already + /// carries the key. `None` removes it. Scope-level identity goes through + /// [`Scope::set_attribute`]. + pub fn set_attribute(&self, key: &str, value: Option) { + let mut global = self.shared.global.lock().unwrap_or_else(|e| e.into_inner()); + global.retain(|a| a.key != key); + if let Some(value) = value { + global.push(Attribute::new(key, value)); + } + } + + /// The session's trace id as 32 hex characters — what every span and log record of this + /// pipeline carries. Print it (`lkt_…`) so support can find the session. + pub fn trace_id(&self) -> String { + self.shared.process.hex() + } + + /// Open a span: one attempt at an operation (`lk.connect`, `lk.publish`, …). Returns the + /// handle to record checkpoints and to end it with; `parent` nests it under another open span. + #[cfg(test)] + pub(crate) fn begin_span(&self, name: &str, kind: SpanKind, parent: Option) -> u64 { + self.begin_span_in(name, kind, parent, &self.shared.process) + } + + pub(crate) fn begin_span_in( + &self, + name: &str, + kind: SpanKind, + parent: Option, + session: &Arc, + ) -> u64 { + // After the opt-out nothing opens: 0 is never a span id, so the span is detached. + if self.shared.revoked() { + return 0; + } + self.shared.before_commit(); + self.shared.spans.lock().unwrap_or_else(|e| e.into_inner()).begin_in( + name, + kind, + parent, + session.clone(), + ) + } + + #[cfg(test)] + pub(crate) fn add_span_event(&self, span: u64, name: &str, attributes: Vec) { + self.shared.add_span_event(span, name, attributes); + } + + #[cfg(test)] + pub(crate) fn end_span( + &self, + span: u64, + outcome: SpanOutcome, + error_type: Option, + attributes: Vec, + ) { + self.shared.end_span(span, outcome, error_type, attributes); + } + + /// Push one `getStats()` reading. Readings are windowed on device into `lk.rtc.stats.sample` + /// events (see `stats_window_ms`); they never count against the flood guard. + pub fn record_stats(&self, sample: RtcStatsSample) { + self.record_stats_in(sample, &self.shared.process); + } + + pub(crate) fn record_stats_in(&self, sample: RtcStatsSample, session: &Arc) { + if self.shared.revoked() || !self.collects(session) { + return; + } + self.shared.before_commit(); + self.shared.windows.lock().unwrap_or_else(|e| e.into_inner()).record_in(sample, session); + } + + /// Whether `session` still collects: not once its project is known to receive nothing + /// (self-hosted server, no observability grant, disabled by the collector) — its records are + /// dropped at the door, never written to the cache. + fn collects(&self, session: &ScopeState) -> bool { + let route = session.route(); + self.shared.destinations.lock().unwrap_or_else(|e| e.into_inner()).alive(route.as_deref()) + } + + /// Change something a session's RTC windows captured when they opened (its correlation + /// attributes, its project), atomically with respect to readings, which record under the same + /// lock: under the windows lock, check the change is `accept`ed, close the session's open + /// windows, then `apply` it. Returns whether it was accepted. + pub(crate) fn with_windows_split( + &self, + session: &ScopeState, + accept: impl FnOnce() -> bool, + apply: impl FnOnce(), + ) -> bool { + let closed = { + let mut windows = self.shared.windows.lock().unwrap_or_else(|e| e.into_inner()); + if !accept() { + return false; + } + let closed = windows.split(session); + apply(); + closed + }; + for window in closed { + if self.shared.store.push(window) { + self.shared.overflow(); + } + } + true + } + + /// Close and forget the RTC windows of one track (or, with `None`, of the whole session): + /// the last partial window is queued now instead of at the next tick. + pub(crate) fn retire_stats(&self, session: &Arc, track_sid: Option<&str>) { + let closed = self + .shared + .windows + .lock() + .unwrap_or_else(|e| e.into_inner()) + .retire(session, track_sid); + for window in closed { + // Windows bypass the flood guard: they are the pipeline's own, bounded output. + if self.shared.store.push(window) { + self.shared.overflow(); + } + } + } + + /// Tell the pipeline what the device looks like. Emits the `lk.device.*.changed` events for + /// whatever differs from the last state (everything, the first time) and re-tunes the + /// pipeline: pressure stretches the cadence up to 4× ([`DeviceState::cadence_factor`]), a + /// constrained network or a nearly empty battery holds uploads + /// ([`DeviceState::holds_uploads`]), and entering the background flushes once right away. + pub fn set_device_state(&self, state: DeviceState) { + let mut previous = self.shared.device.lock().unwrap_or_else(|e| e.into_inner()); + for event in state.change_events(previous.as_ref()) { + self.emit(event); + } + let before = previous.replace(state).unwrap_or_default(); + drop(previous); + // Only three things are worth a wake-up: entering the background (export now, the app + // may be suspended), a hold changing (re-evaluate; nothing new is encoded for it) and the + // cadence changing (the pending tick is re-derived: relief brings it forward). + let offline = |s: &DeviceState| s.network == crate::NetworkType::Unavailable; + if state.app_state == crate::AppState::Background && before.app_state != state.app_state { + self.shared.backgrounded.store(true, Ordering::SeqCst); + self.shared.wake.notify_one(); + } else if before.holds_uploads() != state.holds_uploads() + || offline(&before) != offline(&state) + { + self.shared.release(); + } else if before.cadence_factor() != state.cadence_factor() { + self.shared.wake.notify_one(); + } + } + + /// Cache everything queued and upload the whole cache — no per-pass budget — as far as the + /// holds and pauses in force allow. Returns when that pass is over. + pub async fn flush(&self) { + self.command(Command::Flush).await; + } + + /// Flush, then stop the exporter, and return once it has stopped. The exporter gives the + /// network `export_timeout_ms`, then cancels the request in flight and exits; what did not + /// go out stays cached (with a [`FileCache`], for the next launch). Events emitted afterwards + /// are never exported. If the exporter is not running at all, this gives up a second after + /// that bound. + pub async fn shutdown(&self) { + let bound = Duration::from_millis(self.shared.config.export_timeout_ms.max(1)); + let _ = timeout(bound + SHUTDOWN_GRACE, self.command(Command::Shutdown)).await; + } + + /// Pipeline health: drops by reason, uploads, cached batches. The same numbers ride to the + /// backend as `lk.telemetry.report` events whenever something went wrong. + pub fn stats(&self) -> TelemetryStats { + TelemetryStats::new( + self.shared.counters.snapshot(), + self.shared.cache.pending().len() as u64, + *self.shared.status.lock().unwrap_or_else(|e| e.into_inner()), + ) + } + + /// Opt-out: stop the exporter without another upload and delete everything it holds — the + /// queue, open and finished spans, RTC windows and every cached batch, on disk included. + /// Events emitted afterwards go nowhere. Returns `false` when the storage refused to delete + /// something (it is logged; the files stay until a later purge or configure succeeds). + pub async fn purge(&self) -> bool { + self.shared.revoked.store(true, Ordering::SeqCst); + self.shared.clear(); + let bound = Duration::from_millis(self.shared.config.export_timeout_ms.max(1)); + let _ = timeout(bound, self.command(Command::Purge)).await; + // Whatever an upload in flight put back while the exporter wound down. + self.shared.clear() + } + + /// A handle that can reach the exporter without keeping it alive (the opt-out uses it to + /// cancel and await every generation). + pub(crate) fn weak_commands(&self) -> mpsc::WeakUnboundedSender { + self.commands.downgrade() + } + + async fn command(&self, make: impl FnOnce(oneshot::Sender<()>) -> Command) { + let (done, wait) = oneshot::channel(); + if self.commands.send(make(done)).is_ok() { + let _ = wait.await; + } + } +} + +/// Which LiveKit client SDK is reporting: `service.name` becomes `livekit-client-`. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Sdk { + Swift, + Android, + Flutter, + ReactNative, + Unity, + Rust, +} + +impl Sdk { + fn as_str(self) -> &'static str { + match self { + Self::Swift => "swift", + Self::Android => "android", + Self::Flutter => "flutter", + Self::ReactNative => "react-native", + Self::Unity => "unity", + Self::Rust => "rust", + } + } +} + +/// The reporting SDK and the device it runs on. Lowered to semconv: `service.name`, +/// `service.version`, `os.name`, `os.version`, `device.model.identifier`. +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TelemetryResource { + pub sdk: Sdk, + pub sdk_version: String, + pub os_name: String, + pub os_version: String, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub device_model: Option, +} + +impl TelemetryResource { + pub(crate) fn attributes(&self) -> Vec { + let mut out = vec![ + Attribute::new("service.name", format!("livekit-client-{}", self.sdk.as_str())), + Attribute::new("service.version", self.sdk_version.as_str()), + Attribute::new("os.name", self.os_name.as_str()), + Attribute::new("os.version", self.os_version.as_str()), + ]; + if let Some(model) = &self.device_model { + out.push(Attribute::new("device.model.identifier", model.as_str())); + } + out + } +} + +/// Lower the typed resource, then fill in the `telemetry.sdk.*` attributes and a fallback +/// `service.name`. Attributes already present (the open bag) win. +fn add_sdk_resource(resource: &mut Vec, sdk: Option<&TelemetryResource>) { + for attribute in sdk.map(TelemetryResource::attributes).unwrap_or_default() { + if !resource.iter().any(|a| a.key == attribute.key) { + resource.push(attribute); + } + } + let defaults = [ + ("service.name", "livekit-client"), + ("telemetry.sdk.name", env!("CARGO_PKG_NAME")), + ("telemetry.sdk.language", "rust"), + ("telemetry.sdk.version", env!("CARGO_PKG_VERSION")), + ]; + for (key, value) in defaults { + if !resource.iter().any(|a| a.key == key) { + resource.push(Attribute::new(key, value)); + } + } +} diff --git a/livekit-telemetry/src/trace.rs b/livekit-telemetry/src/trace.rs new file mode 100644 index 000000000..72aaf97a5 --- /dev/null +++ b/livekit-telemetry/src/trace.rs @@ -0,0 +1,525 @@ +//! The SDK's span vocabulary and the span itself, owned by the core so every platform names, +//! times and describes an operation the same way. Every call is synchronous and stamps the clock +//! inside, so the only skew is the FFI call itself; context propagation (the "current" span) +//! stays with the platform runtime, which is the one thing a core cannot do. + +use crate::device::snake; +use std::{ + sync::{Arc, Mutex, Weak}, + time::Duration, +}; + +use tokio::time::Instant; + +use crate::span::SpanKind; +use crate::{ + scope::ScopeState, telemetry::Shared, Attribute, AttributeValue, SpanOutcome, Telemetry, + TrackKind, +}; + +/// What triggered a reconnect cycle: the protocol's `ReconnectReason` values, plus the ones only +/// a client knows. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReconnectReason { + /// The signaling socket closed (`RR_SIGNAL_DISCONNECTED`). + SignalDisconnected, + PublisherFailed, + SubscriberFailed, + /// A peer connection failed and the platform did not say which. + TransportFailed, + SwitchCandidate, + /// The device moved to another network (Wi-Fi ↔ cellular): the client noticed first. + NetworkChanged, + /// A test or a debug menu asked for it. + Debug, + Unknown, +} + +impl ReconnectReason { + /// The protocol's `ReconnectReason` number (`RR_*`). + pub fn from_proto(value: i32) -> Self { + match value { + 1 => Self::SignalDisconnected, + 2 => Self::PublisherFailed, + 3 => Self::SubscriberFailed, + 4 => Self::SwitchCandidate, + _ => Self::Unknown, + } + } +} + +/// What an SDK operation is. The kind follows from the name: connects talk to the server +/// (`client`), the rest is internal work. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SpanName { + Connect, + Reconnect { + reason: ReconnectReason, + }, + Publish, + Subscribe, + /// An app-defined span; its name is the consumer's. + Custom { + name: String, + }, +} + +impl SpanName { + /// The span name on the wire (`lk.connect`, …). + pub fn label(&self) -> &str { + match self { + Self::Connect => "lk.connect", + Self::Reconnect { .. } => "lk.reconnect", + Self::Publish => "lk.publish", + Self::Subscribe => "lk.subscribe", + Self::Custom { name } => name, + } + } + + fn kind(&self) -> SpanKind { + match self { + Self::Connect | Self::Reconnect { .. } => SpanKind::Client, + _ => SpanKind::Internal, + } + } + + fn attributes(&self) -> Vec { + match self { + Self::Reconnect { reason } => { + vec![Attribute::new("lk.reconnect.reason", snake(reason))] + } + // The user-initiated connect is attempt 1; a platform that retries sets it again. + Self::Connect => vec![Attribute::new("lk.connect.attempt", 1i64)], + _ => Vec::new(), + } + } +} + +/// A checkpoint inside a span. One vocabulary for all spans; the core does not police which step +/// belongs to which span, the dashboard does. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SpanStep { + WsOpen, + Signal, + JoinRecv, + PcCreated, + OfferSent, + AnswerSent, + Engine, + PcConnected, + RoomConnected, + Subscribed, + FirstMedia, + /// One reconnect attempt; also sets `lk.reconnect.attempts` and `lk.reconnect.mode`. + Attempt { + number: u32, + full: bool, + }, + Custom { + name: String, + }, +} + +impl SpanStep { + fn label(&self) -> String { + match self { + Self::WsOpen => "ws_open".into(), + Self::Signal => "signal".into(), + Self::JoinRecv => "join_recv".into(), + Self::PcCreated => "pc_created".into(), + Self::OfferSent => "offer_sent".into(), + Self::AnswerSent => "answer_sent".into(), + Self::Engine => "engine".into(), + Self::PcConnected => "pc_connected".into(), + Self::RoomConnected => "room_connected".into(), + Self::Subscribed => "subscribed".into(), + Self::FirstMedia => "first_media".into(), + Self::Attempt { number, full } => { + format!("attempt {number} {}", if *full { "full" } else { "quick" }) + } + Self::Custom { name } => name.clone(), + } + } + + fn attributes(&self) -> Vec { + match self { + Self::Attempt { number, full } => vec![ + Attribute::new("lk.reconnect.attempts", *number as i64), + Attribute::new("lk.reconnect.mode", if *full { "full" } else { "quick" }), + ], + _ => Vec::new(), + } + } +} + +/// The protocol's `TrackSource`. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TrackSource { + Camera, + Microphone, + ScreenShare, + ScreenShareAudio, + Unknown, +} + +/// The track a publish or subscribe span is about. +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, PartialEq)] +pub struct SpanTrack { + /// Unknown until the server assigns it (publish): set the track again once it is. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub sid: Option, + pub kind: TrackKind, + pub source: TrackSource, + /// The publisher, for subscribe spans. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub remote_identity: Option, +} + +impl SpanTrack { + fn attributes(&self) -> Vec { + let mut out = vec![ + Attribute::new("lk.track.kind", format!("{:?}", self.kind).to_lowercase()), + Attribute::new("lk.track.source", snake(self.source)), + ]; + if let Some(sid) = &self.sid { + out.push(Attribute::new("lk.track.sid", sid.as_str())); + } + if let Some(identity) = &self.remote_identity { + out.push(Attribute::new("lk.participant.remote_identity", identity.as_str())); + } + out + } +} + +/// A span's identity in its session's trace, for log correlation. +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TraceContext { + pub trace_id: String, + pub span_id: u64, +} + +/// A span's tie to its pipeline. Weak: a span a Room still holds (a pending subscribe) never +/// keeps the pipeline — and its exporter — alive. +struct Bound { + shared: Weak, + /// The session the span belongs to (weak, like `shared`). + session: Weak, + trace_id: String, + id: u64, +} + +#[derive(Default)] +struct State { + /// (label, offset from start) + steps: Vec<(String, Duration)>, + attributes: Vec, + ended: Option<(SpanOutcome, Duration)>, +} + +/// One attempt at an SDK operation: a timed interval with checkpoints, attributes and an outcome. +/// Bound to a session it is exported as an OTLP span when it ends; detached it still times and +/// describes itself, so the console line looks the same with telemetry off. +pub struct Span { + name: SpanName, + started: Instant, + bound: Option, + state: Mutex, +} + +/// What a span keeps before it is exported: the OTel default limits (128 events, 128 +/// attributes), and no label or value longer than an app may hand over elsewhere. +const MAX_STEPS: usize = 128; +const MAX_ATTRIBUTES: usize = 128; + +/// Set an attribute; a new key beyond [`MAX_ATTRIBUTES`] is ignored (the OTel limit). Returns +/// `false` when the key or a string value is over its length limit: not kept, the caller counts it. +fn upsert(attributes: &mut Vec, attribute: Attribute) -> bool { + let too_long = attribute.key.len() > crate::event::MAX_KEY_BYTES + || matches!(&attribute.value, AttributeValue::Str(v) if v.len() > crate::event::MAX_VALUE_BYTES); + if too_long { + return false; + } + let known = attributes.iter().any(|a| a.key == attribute.key); + if !known && attributes.len() >= MAX_ATTRIBUTES { + return true; + } + attributes.retain(|a| a.key != attribute.key); + attributes.push(attribute); + true +} + +impl Span { + /// Timings and a description only; nothing is exported. + pub fn detached(name: SpanName) -> Arc { + Arc::new(Self::new(name, None)) + } + + pub(crate) fn bound( + name: SpanName, + parent: Option, + telemetry: Telemetry, + session: &Arc, + ) -> Arc { + // A caller-named span over the name limit is not recorded (counted as invalid). + if name.label().len() > crate::event::MAX_NAME_BYTES { + telemetry.count_invalid(); + return Arc::new(Self::new(name, None)); + } + let id = telemetry.begin_span_in(name.label(), name.kind(), parent, session); + if id == 0 { + return Arc::new(Self::new(name, None)); // opted out: detached + } + let shared = Arc::downgrade(&telemetry.shared); + let bound = Bound { shared, session: Arc::downgrade(session), trace_id: session.hex(), id }; + Arc::new(Self::new(name, Some(bound))) + } + + fn new(name: SpanName, bound: Option) -> Self { + // A name over the limit is not kept, detached spans included: only a fixed placeholder. + let name = match name { + SpanName::Custom { name } if name.len() > crate::event::MAX_NAME_BYTES => { + SpanName::Custom { name: "invalid".to_owned() } + } + name => name, + }; + let state = State { attributes: name.attributes(), ..State::default() }; + Self { name, started: Instant::now(), bound, state: Mutex::new(state) } + } + + fn lock(&self) -> std::sync::MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(|e| e.into_inner()) + } + + /// What this span is. + pub fn name(&self) -> SpanName { + self.name.clone() + } + + /// The span name on the wire. + pub fn label(&self) -> String { + self.name.label().to_owned() + } + + /// A checkpoint, stamped now. Ignored once the span has ended. + pub fn step(&self, step: SpanStep) { + let at = self.started.elapsed(); + let label = step.label(); + if label.len() > crate::event::MAX_NAME_BYTES { + return self.count_invalid(1); // a caller-named step over the name limit: not kept + } + { + let mut state = self.lock(); + if state.ended.is_some() || state.steps.len() >= MAX_STEPS { + return; + } + state.steps.push((label.clone(), at)); + for attribute in step.attributes() { + upsert(&mut state.attributes, attribute); // the core's own: always within limits + } + } + if let Some(bound) = &self.bound { + if let Some(shared) = bound.shared.upgrade() { + shared.add_span_event(bound.id, &label, Vec::new()); + } + } + } + + /// The open bag, for app-defined spans and one-off details. Replaces an existing key. + pub fn set_attribute(&self, key: String, value: AttributeValue) { + if !upsert(&mut self.lock().attributes, Attribute::new(key, value)) { + self.count_invalid(1); + } + } + + /// Count caller strings this span did not keep (over their length limits) as `invalid`. + fn count_invalid(&self, n: u64) { + if n == 0 { + return; + } + if let Some(shared) = self.bound.as_ref().and_then(|b| b.shared.upgrade()) { + crate::stats::Counters::add(&shared.counters.invalid, n); + } + } + + /// The track the operation is about (its sid may arrive later: set it again). + pub fn set_track(&self, track: SpanTrack) { + let invalid = { + let mut state = self.lock(); + track + .attributes() + .into_iter() + .filter(|a| !upsert(&mut state.attributes, a.clone())) + .count() + }; + self.count_invalid(invalid as u64); + // A published track's sid is known: its session polls fast until the first outbound + // reading arrives (see `Scope::stats_poll_interval_ms`). + if let (SpanName::Publish, Some(sid)) = (&self.name, &track.sid) { + if let Some(session) = self.bound.as_ref().and_then(|b| b.session.upgrade()) { + session.await_first_outbound(sid); + } + } + } + + /// End once; later calls are no-ops. `error` becomes `error.type` and the status message. + pub fn end(&self, outcome: SpanOutcome, error: Option) { + let at = self.started.elapsed(); + let attributes = { + let mut state = self.lock(); + if state.ended.is_some() { + return; + } + state.ended = Some((outcome, at)); + state.attributes.clone() + }; + if let Some(bound) = &self.bound { + if let Some(shared) = bound.shared.upgrade() { + // `error.type` is a type name, not a message: an over-long one is replaced. + let error = error.map(|e| { + if e.len() > crate::event::MAX_NAME_BYTES { + crate::stats::Counters::add(&shared.counters.invalid, 1); + "invalid".to_owned() + } else { + e + } + }); + shared.end_span(bound.id, outcome, error, attributes); + } + // The platform's console line, the same on every SDK (debug: FFI log path). + log::debug!("{}", self.describe()); + } + } + + /// End with an error; `error` becomes `error.type`. + pub fn fail(&self, error: String) { + self.end(SpanOutcome::Error, Some(error)); + } + + /// End as cancelled (status unset, `lk.outcome = cancelled`). + pub fn cancel(&self) { + self.end(SpanOutcome::Cancelled, None); + } + + /// Whether the span has ended. + pub fn is_ended(&self) -> bool { + self.lock().ended.is_some() + } + + /// How it ended, once it has. + pub fn outcome(&self) -> Option { + self.lock().ended.map(|(outcome, _)| outcome) + } + + /// `None` for a detached span. + pub fn context(&self) -> Option { + self.bound.as_ref().map(|b| TraceContext { trace_id: b.trace_id.clone(), span_id: b.id }) + } + + /// Seconds from start to the end, or to the last step while still running. + pub fn total_secs(&self) -> f64 { + let state = self.lock(); + state + .ended + .map(|(_, at)| at) + .or_else(|| state.steps.last().map(|(_, at)| *at)) + .unwrap_or_default() + .as_secs_f64() + } + + /// `lk.connect: ws_open +1.49s, signal +0.03s, total 1.83s, ok` — the same line on every + /// platform, for the console when a span ends. + pub fn describe(&self) -> String { + let state = self.lock(); + let mut parts = Vec::with_capacity(state.steps.len() + 2); + let mut previous = Duration::ZERO; + for (label, at) in &state.steps { + parts.push(format!("{label} +{:.2}s", at.saturating_sub(previous).as_secs_f64())); + previous = *at; + } + let total = state.ended.map(|(_, at)| at).unwrap_or(previous); + parts.push(format!("total {:.2}s", total.as_secs_f64())); + if let Some((outcome, _)) = state.ended { + parts.push(outcome.as_str().to_owned()); + } + format!("{}: {}", self.name.label(), parts.join(", ")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test(start_paused = true)] + async fn a_detached_span_times_and_describes_itself() { + let span = + Span::detached(SpanName::Reconnect { reason: ReconnectReason::SignalDisconnected }); + tokio::time::advance(Duration::from_millis(1490)).await; + span.step(SpanStep::Attempt { number: 1, full: false }); + tokio::time::advance(Duration::from_millis(30)).await; + span.step(SpanStep::WsOpen); + assert_eq!( + span.describe(), + "lk.reconnect: attempt 1 quick +1.49s, ws_open +0.03s, total 1.52s" + ); + tokio::time::advance(Duration::from_millis(10)).await; + span.end(SpanOutcome::Ok, None); + span.fail("late".into()); + assert_eq!(span.outcome(), Some(SpanOutcome::Ok), "ends once"); + assert_eq!( + span.describe(), + "lk.reconnect: attempt 1 quick +1.49s, ws_open +0.03s, total 1.53s, ok" + ); + assert!(span.context().is_none()); + let attributes = span.lock().attributes.clone(); + let get = |key: &str| attributes.iter().find(|a| a.key == key).map(|a| a.value.clone()); + assert_eq!( + get("lk.reconnect.reason"), + Some(AttributeValue::Str("signal_disconnected".into())) + ); + assert_eq!(get("lk.reconnect.attempts"), Some(AttributeValue::Int(1))); + assert_eq!(get("lk.reconnect.mode"), Some(AttributeValue::Str("quick".into()))); + } + + #[test] + fn a_track_sets_its_attributes_and_can_learn_its_sid_later() { + let span = Span::detached(SpanName::Publish); + span.set_track(SpanTrack { + sid: None, + kind: TrackKind::Video, + source: TrackSource::Camera, + remote_identity: None, + }); + span.set_track(SpanTrack { + sid: Some("TR_1".into()), + kind: TrackKind::Video, + source: TrackSource::Camera, + remote_identity: None, + }); + let attributes = span.lock().attributes.clone(); + let get = |key: &str| attributes.iter().find(|a| a.key == key).map(|a| a.value.clone()); + assert_eq!(get("lk.track.kind"), Some(AttributeValue::Str("video".into()))); + assert_eq!(get("lk.track.sid"), Some(AttributeValue::Str("TR_1".into()))); + assert_eq!(attributes.iter().filter(|a| a.key == "lk.track.source").count(), 1); + } + + /// Finding r1-12: what a span retains before export is bounded, however long it lives. + #[test] + fn retained_span_state_is_bounded() { + let span = Span::detached(SpanName::Custom { name: "work".into() }); + for n in 0..1000 { + span.step(SpanStep::Custom { name: format!("step {n}") }); + span.set_attribute(format!("k{n}"), AttributeValue::Str("v".into())); + } + span.step(SpanStep::Custom { name: "x".repeat(1000) }); + span.set_attribute("k0".into(), AttributeValue::Str("x".repeat(2000))); + let state = span.lock(); + assert_eq!((state.steps.len(), state.attributes.len()), (128, 128)); + assert!(state.attributes.iter().all(|a| a.value.size_hint() <= 1024)); + } +} diff --git a/livekit-telemetry/src/transport.rs b/livekit-telemetry/src/transport.rs new file mode 100644 index 000000000..060d5eb36 --- /dev/null +++ b/livekit-telemetry/src/transport.rs @@ -0,0 +1,568 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; + +use prost::Message; + +/// One fully composed OTLP/HTTP export request. +/// +/// The core fills in the URL, the headers (content type, auth, …) and the protobuf body; +/// a transport only moves the bytes. Non-HTTP transports (e.g. a data channel) may ignore +/// `url`/`headers` and forward `body`, which is a standard `ExportLogsServiceRequest`. +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, PartialEq)] +pub struct ExportRequest { + pub url: String, + pub headers: HashMap, + pub body: Vec, +} + +/// What the collector answered. A transport moves bytes both ways and never interprets them: +/// status classification, `Retry-After`, the `google.rpc.Status` body and the Cloud "disabled" +/// contract are read once, here, for every platform ([`ExportError::from_response`]). +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, PartialEq, Default)] +pub struct ExportResponse { + pub status: u16, + pub headers: HashMap, + pub body: Vec, +} + +impl ExportResponse { + /// `2xx`, nothing else: what an accepting collector answers. + pub fn accepted() -> Self { + Self { status: 200, ..Self::default() } + } +} + +/// LiveKit Cloud's answer when the project owner switched data recording off (401 or 403, plain +/// text, no machine-readable code yet). Matched as a phrase, not a word. +const DISABLED_ANSWER: &str = "data recording is disabled by owner"; + +/// How long a 429 that names no delay holds uploads: LiveKit Cloud's quota answer names none. +const THROTTLE_DEFAULT_MS: u64 = 60_000; + +/// Why a transport could not get an answer. A 4xx/5xx is an answer, not an error: return it as an +/// [`ExportResponse`] and the core classifies it. +#[cfg_attr(feature = "uniffi", derive(uniffi::Error))] +#[derive(Debug, Clone, PartialEq, thiserror::Error)] +pub enum ExportError { + /// No response: connection refused or reset, DNS or TLS failure, offline, timed out. The + /// batch stays cached and is retried with backoff. `retry_after_ms` is honored when set. + #[error("retryable export error: {reason}")] + Retryable { reason: String, retry_after_ms: Option }, + /// The request can never succeed (an invalid URL): the batch is dropped. + #[error("export rejected: {reason}")] + Rejected { reason: String }, + /// Telemetry is off for this project: the project goes silent. + #[error("telemetry disabled by the collector")] + Disabled, +} + +/// A foreign transport threw something its binding did not declare (a Swift `Error`, a Kotlin +/// `RuntimeException`): no answer, retried with backoff — never a panic in the exporter. +#[cfg(feature = "uniffi")] +impl From for ExportError { + fn from(error: uniffi::UnexpectedUniFFICallbackError) -> Self { + Self::Retryable { + reason: format!("transport threw: {}", error.reason), + retry_after_ms: None, + } + } +} + +/// What a collector's answer means for the batch: OTLP/HTTP failure semantics, plus the answers +/// LiveKit Cloud gives on top of them. +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum Verdict { + /// 2xx. `rejected` counts the records an OTLP partial success refused (never retried). + Accepted { rejected: u64, reason: String }, + /// 400, 3xx (a credential-bearing request is never redirected), and every other 4xx/5xx the + /// OTLP spec does not call retryable: drop the batch. + Rejected(String), + /// 413: split the batch and retry the halves, down to single records. + TooLarge, + /// 401/403 that is not the "disabled" answer: the credential is the problem, not the data — + /// hold the project's batches until the platform hands over a new token. + Unauthorized(String), + /// 404: the host has no client ingest; the project goes silent. + NotFound, + /// 401/403 "data recording is disabled by owner": the project goes silent and its cache is + /// purged. + Disabled, + /// 429, or 503 naming a delay: pause every upload for `delay_ms`, keep collecting. + Throttled { delay_ms: u64, reason: String }, + /// 502/503/504, or any error carrying `RetryInfo`: the server failed transiently. Retry the + /// batch after `delay_ms` or a backoff; a batch that keeps failing is dropped eventually. + Retry { delay_ms: Option, reason: String }, +} + +impl Verdict { + /// Classify a collector response. + pub(crate) fn of(response: &ExportResponse) -> Self { + let status = response.status; + if (200..300).contains(&status) { + return match PartialSuccessResponse::decode(response.body.as_slice()) { + Ok(PartialSuccessResponse { partial_success: Some(p) }) if p.rejected > 0 => { + Self::Accepted { rejected: p.rejected as u64, reason: p.error_message } + } + _ => Self::Accepted { rejected: 0, reason: String::new() }, + }; + } + // OTLP/HTTP errors are a protobuf `google.rpc.Status`; Cloud's auth answers are text. + let rpc = RpcStatus::decode(response.body.as_slice()).ok(); + let text = match rpc.as_ref().map(|s| s.message.trim()).filter(|m| !m.is_empty()) { + Some(message) => message.to_owned(), + None => String::from_utf8_lossy(&response.body).trim().chars().take(200).collect(), + }; + // `reason`, not `message`: a UniFFI field named `message` collides with + // `Throwable.message` in Kotlin. + let reason = if text.is_empty() { + format!("HTTP {status}") + } else { + format!("HTTP {status}: {text}") + }; + if matches!(status, 401 | 403) { + return if text.to_ascii_lowercase().contains(DISABLED_ANSWER) { + Self::Disabled + } else { + Self::Unauthorized(reason) + }; + } + let header = response + .headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("retry-after")) + .and_then(|(_, value)| retry_after_ms(value)); + let retry_info = rpc.as_ref().and_then(retry_info_ms); + let delay = header.or(retry_info); + // The status decides whether a batch is retried; a delay hint only says when. A hint on a + // final status (400, 422, 501, a redirect) changes nothing. + match status { + 404 => Self::NotFound, + 413 => Self::TooLarge, + // A rate limit is an instruction to stop, so 429 always yields a wait: LiveKit Cloud + // answers a quota check with a bare `ResourceExhausted` — no `Retry-After`, no + // `RetryInfo` — and without one the exporter would spend its retries hammering the + // endpoint that just asked for quiet. + 429 => Self::Throttled { delay_ms: delay.unwrap_or(THROTTLE_DEFAULT_MS), reason }, + 503 if delay.is_some() => { + Self::Throttled { delay_ms: delay.unwrap_or_default(), reason } + } + 502..=504 => Self::Retry { delay_ms: delay, reason }, + // LiveKit Cloud's one retryable 500 carries `RetryInfo` (custom: OTLP retries only + // 429/502/503/504). + 500 if retry_info.is_some() => Self::Retry { delay_ms: retry_info, reason }, + _ => Self::Rejected(reason), + } + } +} + +/// `ExportLogsServiceResponse` and `ExportTraceServiceResponse` share one shape: field 1 is the +/// partial success, whose field 1 counts the rejected records and field 2 says why. +#[derive(Clone, PartialEq, prost::Message)] +struct PartialSuccessResponse { + #[prost(message, optional, tag = "1")] + partial_success: Option, +} + +#[derive(Clone, PartialEq, prost::Message)] +struct PartialSuccess { + #[prost(int64, tag = "1")] + rejected: i64, + #[prost(string, tag = "2")] + error_message: String, +} + +/// `google.rpc.Status`, the OTLP/HTTP error body. Two messages, three fields: not worth a +/// generated crate. +#[derive(Clone, PartialEq, prost::Message)] +struct RpcStatus { + #[prost(int32, tag = "1")] + code: i32, + #[prost(string, tag = "2")] + message: String, + #[prost(message, repeated, tag = "3")] + details: Vec, +} + +/// `google.rpc.RetryInfo`. +#[derive(Clone, PartialEq, prost::Message)] +struct RetryInfo { + #[prost(message, optional, tag = "1")] + retry_delay: Option, +} + +/// RFC 9110 §10.2.3 `Retry-After`: delay-seconds or an IMF-fixdate. Untrusted input: garbage is +/// ignored, a date in the past means now; the caller clamps the far end. +fn retry_after_ms(value: &str) -> Option { + let value = value.trim(); + // delay-seconds: digits only; more than a u64 holds saturates (the exporter clamps it). + if !value.is_empty() && value.bytes().all(|b| b.is_ascii_digit()) { + return Some(value.parse::().unwrap_or(u64::MAX).saturating_mul(1000)); + } + let at = imf_fixdate_secs(value)?; + let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).ok()?.as_secs(); + Some(at.saturating_sub(now).saturating_mul(1000)) +} + +/// `Sun, 06 Nov 1994 08:49:37 GMT` → unix seconds. The obsolete RFC 850 / asctime forms are not +/// accepted (RFC 9110 lets a recipient ignore them). +fn imf_fixdate_secs(value: &str) -> Option { + let mut parts = value.split_ascii_whitespace(); + let (_weekday, day, month, year, time, zone) = + (parts.next()?, parts.next()?, parts.next()?, parts.next()?, parts.next()?, parts.next()?); + if zone != "GMT" || parts.next().is_some() { + return None; + } + const MONTHS: [&str; 12] = + ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; + let month = MONTHS.iter().position(|m| *m == month)? as i64 + 1; + let (day, year): (i64, i64) = (day.parse().ok()?, year.parse().ok()?); + let mut hms = time.split(':').map(|n| n.parse::().ok()); + let (h, m, s) = (hms.next()??, hms.next()??, hms.next()??); + // IMF-fixdate has a four-digit year; bounding it keeps the arithmetic below in range. + if !(1970..=9999).contains(&year) + || hms.next().is_some() + || !(1..=31).contains(&day) + || !(0..24).contains(&h) + || !(0..60).contains(&m) + || !(0..61).contains(&s) + { + return None; + } + // Days from the civil date (proleptic Gregorian), after Howard Hinnant's algorithm. + let y = if month <= 2 { year - 1 } else { year }; + let era = y.div_euclid(400); + let yoe = y - era * 400; + let doy = (153 * (month + if month > 2 { -3 } else { 9 }) + 2) / 5 + day - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + let days = era * 146_097 + doe - 719_468; + u64::try_from(days * 86_400 + h * 3600 + m * 60 + s).ok() +} + +fn retry_info_ms(status: &RpcStatus) -> Option { + status + .details + .iter() + .filter(|detail| detail.type_url.ends_with("google.rpc.RetryInfo")) + .find_map(|detail| RetryInfo::decode(detail.value.as_slice()).ok()?.retry_delay) + .map(|delay| { + (delay.seconds.max(0) as u64) + .saturating_mul(1000) + .saturating_add(delay.nanos.clamp(0, 999_999_999) as u64 / 1_000_000) + }) +} + +/// Moves an encoded batch off the device and hands back whatever came back. +/// +/// Implemented in Rust ([`NetTransport`], feature `net`) or by the host platform through UniFFI. +/// The contract: +/// +/// - POST `url` with `headers` and `body` as given, and return the response whatever its status; +/// fail only when there was no response (connection, DNS, TLS, timeout) — +/// [`ExportError::Retryable`] — or the request cannot be made at all +/// ([`ExportError::Rejected`], e.g. an invalid URL). +/// - Never retry: the core owns the retry policy. +/// - Never forward the `Authorization` header across origins: do not follow a redirect to +/// another scheme, host or port with it (returning the 3xx is fine — the core drops the batch). +/// `NetTransport` over `livekit-net`'s native client strips it on cross-origin redirects. +/// - Keep the request off the call's critical path: lowest priority where the platform allows it. +#[cfg_attr(feature = "uniffi", uniffi::export(with_foreign))] +#[async_trait::async_trait] +pub trait TelemetryTransport: Send + Sync { + async fn send(&self, request: ExportRequest) -> Result; +} + +#[cfg(feature = "net")] +mod net { + use std::sync::Arc; + + use livekit_net::{Header, HttpClient, HttpClientExt, TransportError}; + + use super::{ExportError, ExportRequest, ExportResponse, TelemetryTransport}; + + /// Default transport: HTTP POST through a `livekit-net` client — the native backend, or + /// whatever the host registered with `livekit_net::set_http_client`. + pub struct NetTransport(Arc); + + impl NetTransport { + /// Post through `client`. + pub fn new(client: Arc) -> Self { + Self(client) + } + + /// Resolve the process-wide `livekit-net` client; `None` when none is available. + pub fn from_registry() -> Option { + livekit_net::http_client().map(Self) + } + } + + #[async_trait::async_trait] + impl TelemetryTransport for NetTransport { + async fn send(&self, request: ExportRequest) -> Result { + let headers = + request.headers.into_iter().map(|(name, value)| Header { name, value }).collect(); + match self.0.post(request.url, headers, request.body).await { + Ok(response) => Ok(ExportResponse { + status: response.status, + headers: response.headers.into_iter().map(|h| (h.name, h.value)).collect(), + body: response.body, + }), + // The client swallowed the body with the status; the core still classifies it. + Err(TransportError::Http { status }) => { + Ok(ExportResponse { status, ..Default::default() }) + } + Err(other) => { + Err(ExportError::Retryable { reason: other.to_string(), retry_after_ms: None }) + } + } + } + } +} + +#[cfg(feature = "net")] +pub use net::NetTransport; + +#[cfg(test)] +mod tests { + use prost::Message; + + use super::*; + + fn status_body(message: &str, retry: Option<(i64, i32)>) -> Vec { + let details = retry + .map(|(seconds, nanos)| prost_types::Any { + type_url: "type.googleapis.com/google.rpc.RetryInfo".into(), + value: RetryInfo { retry_delay: Some(prost_types::Duration { seconds, nanos }) } + .encode_to_vec(), + }) + .into_iter() + .collect(); + RpcStatus { code: 8, message: message.into(), details }.encode_to_vec() + } + + fn response(status: u16, headers: &[(&str, &str)], body: Vec) -> ExportResponse { + ExportResponse { + status, + headers: headers.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect(), + body, + } + } + + fn verdict(status: u16, headers: &[(&str, &str)], body: Vec) -> Verdict { + Verdict::of(&response(status, headers, body)) + } + + #[test] + fn success_and_partial_success() { + assert_eq!( + Verdict::of(&ExportResponse::accepted()), + Verdict::Accepted { rejected: 0, reason: String::new() } + ); + assert!(matches!(verdict(204, &[], vec![]), Verdict::Accepted { rejected: 0, .. })); + let partial = PartialSuccessResponse { + partial_success: Some(PartialSuccess { rejected: 3, error_message: "too old".into() }), + } + .encode_to_vec(); + assert_eq!( + verdict(200, &[], partial), + Verdict::Accepted { rejected: 3, reason: "too old".into() } + ); + assert!(matches!(verdict(200, &[], b"not protobuf".to_vec()), Verdict::Accepted { .. })); + } + + #[test] + fn client_errors() { + assert_eq!( + verdict(400, &[], status_body("bad batch", None)), + Verdict::Rejected("HTTP 400: bad batch".into()) + ); + assert_eq!(verdict(413, &[], vec![]), Verdict::TooLarge); + assert!(matches!(verdict(307, &[], vec![]), Verdict::Rejected(_)), "never follow"); + assert!(matches!(verdict(422, &[], vec![]), Verdict::Rejected(_))); + assert_eq!(verdict(404, &[], vec![]), Verdict::NotFound); + for body in ["invalid token", "operation requires observability write grant"] { + assert_eq!( + verdict(401, &[], body.as_bytes().to_vec()), + Verdict::Unauthorized(format!("HTTP 401: {body}")) + ); + } + assert!(matches!(verdict(403, &[], vec![]), Verdict::Unauthorized(_))); + } + + #[test] + fn disabled_by_owner() { + let text = b"project data recording is disabled by owner".to_vec(); + assert_eq!(verdict(401, &[], text), Verdict::Disabled); + let rpc = status_body("Project data recording is disabled by owner", None); + assert_eq!(verdict(403, &[], rpc), Verdict::Disabled); + // Only the documented phrase: another answer that merely mentions "disabled" is a + // credential problem, not a project switch. + for body in ["account disabled", "token disabled for this room"] { + assert!(matches!( + verdict(401, &[], body.as_bytes().to_vec()), + Verdict::Unauthorized(_) + )); + } + } + + /// Finding r1-9: the status decides; a delay hint on a final status is ignored. Only 429, + /// 502–504 and Cloud's 500-with-RetryInfo are retried. + #[test] + fn delay_hints_never_make_a_final_status_retryable() { + type Hint<'a> = (&'a [(&'a str, &'a str)], Vec); + let hints: [Hint; 3] = [ + (&[("Retry-After", "5")], vec![]), + (&[], status_body("later", Some((5, 0)))), + (&[("Retry-After", "5")], status_body("later", Some((5, 0)))), + ]; + for (headers, body) in &hints { + for status in [300, 302, 307, 400, 405, 409, 410, 422, 501, 505, 599] { + assert!( + matches!(verdict(status, headers, body.clone()), Verdict::Rejected(_)), + "{status} with {headers:?}" + ); + } + assert_eq!(verdict(413, headers, body.clone()), Verdict::TooLarge); + assert_eq!(verdict(404, headers, body.clone()), Verdict::NotFound); + assert!(matches!(verdict(429, headers, body.clone()), Verdict::Throttled { .. })); + assert!(matches!(verdict(503, headers, body.clone()), Verdict::Throttled { .. })); + for status in [502, 504] { + assert!(matches!( + verdict(status, headers, body.clone()), + Verdict::Retry { delay_ms: Some(5_000), .. } + )); + } + } + // 500: only with RetryInfo in the body, never on a Retry-After header alone. + assert!(matches!(verdict(500, &[("Retry-After", "5")], vec![]), Verdict::Rejected(_))); + assert!(matches!( + verdict(500, &[], status_body("later", Some((5, 0)))), + Verdict::Retry { delay_ms: Some(5_000), .. } + )); + } + + /// Finding r1-11: untrusted time values never panic or wrap; oversized ones saturate. + #[test] + fn untrusted_time_values_are_bounded() { + assert_eq!(retry_after_ms("99999999999999999999999"), Some(u64::MAX)); + assert_eq!(retry_after_ms("18446744073709551615"), Some(u64::MAX)); + assert_eq!(retry_after_ms("-5"), None); + assert_eq!(retry_after_ms(""), None); + for date in [ + "Sun, 06 Nov 99999999999999 08:49:37 GMT", + "Sun, 06 Nov -9223372036854775808 08:49:37 GMT", + "Sun, 06 Nov 1969 08:49:37 GMT", + "Sun, 06 Nov 10000 08:49:37 GMT", + "Sun, 32 Nov 2026 08:49:37 GMT", + "Sun, 06 Nov 2026 08:49:37:99 GMT", + "Sun, 06 Nov 2026 99:49:37 GMT", + ] { + assert_eq!(imf_fixdate_secs(date), None, "{date}"); + } + assert_eq!(imf_fixdate_secs("Fri, 31 Dec 9999 23:59:59 GMT"), Some(253_402_300_799)); + let huge = verdict(429, &[("Retry-After", "99999999999999999999")], vec![]); + assert!(matches!(huge, Verdict::Throttled { delay_ms: u64::MAX, .. }), "clamped later"); + let far = verdict(429, &[], status_body("q", Some((i64::MAX, 999_999_999)))); + assert!(matches!(far, Verdict::Throttled { delay_ms: u64::MAX, .. })); + } + + #[test] + fn throttling_takes_the_header_then_the_body_then_the_default() { + let both = verdict(429, &[("Retry-After", "7")], status_body("quota", Some((30, 0)))); + assert!(matches!(both, Verdict::Throttled { delay_ms: 7_000, .. })); + let body = verdict(429, &[], status_body("quota", Some((30, 500_000_000)))); + assert!(matches!(body, Verdict::Throttled { delay_ms: 30_500, .. })); + // What LiveKit Cloud answers over quota: ResourceExhausted, no header, no RetryInfo. + assert_eq!( + verdict(429, &[], status_body("QuotaStatusExceeded", None)), + Verdict::Throttled { delay_ms: 60_000, reason: "HTTP 429: QuotaStatusExceeded".into() } + ); + let unavailable = verdict(503, &[("retry-after", " 12 ")], vec![]); + assert!(matches!(unavailable, Verdict::Throttled { delay_ms: 12_000, .. })); + let past = verdict(429, &[("Retry-After", "Wed, 21 Oct 2015 07:28:00 GMT")], vec![]); + assert!(matches!(past, Verdict::Throttled { delay_ms: 0, .. }), "a past date: now"); + let garbage = verdict(429, &[("Retry-After", "soon-ish")], vec![]); + assert!(matches!(garbage, Verdict::Throttled { delay_ms: 60_000, .. }), "ignored"); + let negative = verdict(429, &[], status_body("q", Some((-5, -1)))); + assert!(matches!(negative, Verdict::Throttled { delay_ms: 0, .. }), "clamped"); + } + + #[test] + fn retry_after_http_dates() { + assert_eq!(imf_fixdate_secs("Sun, 06 Nov 1994 08:49:37 GMT"), Some(784_111_777)); + assert_eq!(imf_fixdate_secs("Thu, 01 Jan 1970 00:00:00 GMT"), Some(0)); + assert_eq!(imf_fixdate_secs("Tue, 29 Feb 2028 12:00:00 GMT"), Some(1_835_438_400)); + assert_eq!(imf_fixdate_secs("Sunday, 06-Nov-94 08:49:37 GMT"), None, "obsolete form"); + assert_eq!(imf_fixdate_secs("Sun, 06 Nov 1994 08:49:37 PST"), None); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_secs(); + let date = |secs: u64| { + let days = secs / 86_400; + let (h, m, s) = ((secs % 86_400) / 3600, (secs % 3600) / 60, secs % 60); + // Civil from days (inverse of the parser's algorithm) for the fixture only. + let z = days as i64 + 719_468; + let era = z.div_euclid(146_097); + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let mo = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = yoe + era * 400 + if mo <= 2 { 1 } else { 0 }; + const M: [&str; 12] = [ + "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", + ]; + format!("Mon, {d:02} {} {y} {h:02}:{m:02}:{s:02} GMT", M[mo as usize - 1]) + }; + let ms = retry_after_ms(&date(now + 120)).expect("date"); + assert!((118_000..=120_000).contains(&ms), "{ms}"); + } + + #[test] + fn server_errors() { + for status in [502, 503, 504] { + assert!(matches!(verdict(status, &[], vec![]), Verdict::Retry { delay_ms: None, .. })); + } + assert!(matches!( + verdict(502, &[("Retry-After", "3")], vec![]), + Verdict::Retry { delay_ms: Some(3_000), .. } + )); + // Cloud's retryable 500 carries RetryInfo; a bare 500 is final, per OTLP/HTTP. + let internal = verdict(500, &[], status_body("try later", Some((5, 0)))); + assert!(matches!(internal, Verdict::Retry { delay_ms: Some(5_000), .. })); + assert!(matches!(verdict(500, &[], vec![]), Verdict::Rejected(_))); + assert!(matches!(verdict(501, &[], vec![]), Verdict::Rejected(_))); + } + + /// Finding r1-10: an exception the foreign transport did not declare becomes a retryable + /// transport error. Without the conversion UniFFI panics on the exporter's thread. + #[cfg(feature = "uniffi")] + #[test] + fn an_undeclared_foreign_exception_is_a_retryable_failure() { + let lifted = as uniffi::LiftReturn< + crate::UniFfiTag, + >>::handle_callback_unexpected_error( + uniffi::UnexpectedUniFFICallbackError::new("java.lang.IllegalStateException: boom"), + ); + assert!(matches!( + lifted, + Err(ExportError::Retryable { retry_after_ms: None, ref reason }) if reason.contains("boom") + )); + } +} diff --git a/livekit-telemetry/uniffi.toml b/livekit-telemetry/uniffi.toml new file mode 100644 index 000000000..bb41bd4cc --- /dev/null +++ b/livekit-telemetry/uniffi.toml @@ -0,0 +1,7 @@ +[bindings.swift] +ffi_module_name = "RustLiveKitTelemetry" + +[bindings.kotlin] +# The Kotlin checksum test is broken on ARM in every UniFFI release this workspace can use; see +# livekit-uniffi/uniffi.toml for the full explanation. +omit_checksums = true