From 7e947bbd738ccabff7ddae18cee326a4570f7a5c Mon Sep 17 00:00:00 2001 From: Jason Lernerman Date: Mon, 14 Sep 2026 15:55:07 -0400 Subject: [PATCH 01/13] simulate: move job ordering out of the TUI sortedJobs is the one place a run's jobs get a stable order, so the report can share it. --- cmd/lk/simulate.go | 10 ++++++++++ cmd/lk/simulate_tui.go | 7 +------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/cmd/lk/simulate.go b/cmd/lk/simulate.go index 0ea1c178..2c81bcfb 100644 --- a/cmd/lk/simulate.go +++ b/cmd/lk/simulate.go @@ -23,6 +23,7 @@ import ( "math/rand" "os" "path/filepath" + "sort" "strings" "time" @@ -933,6 +934,15 @@ func cancelSimulationRun(client *lksdk.AgentSimulationClient, runID string) { } } +// sortedJobs orders a run's jobs by job ID: the backend's ordering shuffles +// rows as statuses change. +func sortedJobs(run *livekit.SimulationRun) []*livekit.SimulationRun_Job { + jobs := make([]*livekit.SimulationRun_Job, len(run.Jobs)) + copy(jobs, run.Jobs) + sort.Slice(jobs, func(i, j int) bool { return jobs[i].GetId() < jobs[j].GetId() }) + return jobs +} + func simulationJobCounts(run *livekit.SimulationRun) (total, done, passed, failed int) { if run == nil { return diff --git a/cmd/lk/simulate_tui.go b/cmd/lk/simulate_tui.go index c7360db0..2ac5c337 100644 --- a/cmd/lk/simulate_tui.go +++ b/cmd/lk/simulate_tui.go @@ -20,7 +20,6 @@ import ( "os" "os/signal" "path/filepath" - "sort" "strings" "time" @@ -1026,11 +1025,7 @@ func (m *simulateModel) filteredJobs() []indexedJob { if m.run == nil { return nil } - // sort by job ID: the backend's ordering shuffles rows as statuses change - jobs := make([]*livekit.SimulationRun_Job, len(m.run.Jobs)) - copy(jobs, m.run.Jobs) - sort.Slice(jobs, func(i, j int) bool { return jobs[i].GetId() < jobs[j].GetId() }) - + jobs := sortedJobs(m.run) result := make([]indexedJob, 0, len(jobs)) for i, j := range jobs { result = append(result, indexedJob{origIdx: i + 1, job: j}) From 34077c2790f2f3e92796a9bdd71dcd2a2cb2b35e Mon Sep 17 00:00:00 2001 From: Jason Lernerman Date: Mon, 14 Sep 2026 15:56:56 -0400 Subject: [PATCH 02/13] deps: bump protocol for simulation repeats --- go.mod | 6 +++++- go.sum | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2ee88fd6..11dc2bd3 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/google/go-querystring v1.2.0 github.com/joho/godotenv v1.5.1 github.com/klauspost/compress v1.20.0 - github.com/livekit/protocol v1.51.1-0.20260915182407-3c0fcdbd25df + github.com/livekit/protocol v1.51.1-0.20260914194419-c7d18f5376cf github.com/livekit/server-sdk-go/v2 v2.18.2-0.20260904062056-1da58cd7b795 github.com/mattn/go-isatty v0.0.22 github.com/moby/moby/client v0.4.1 @@ -264,4 +264,8 @@ require ( sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect ) +// TEMP (local dev): use local protocol with the WorkerInfo dev message. +// Drop once github.com/livekit/protocol publishes it. +// replace github.com/livekit/protocol => ../protocol + tool github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen diff --git a/go.sum b/go.sum index 35e1ab3c..d63be428 100644 --- a/go.sum +++ b/go.sum @@ -370,8 +370,8 @@ github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5AT github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095 h1:BcliKAXoMhl/nWmzQweQ5kmh4Qqagxl4s3Z5pvM/7AY= github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095/go.mod h1:o8CFmAdrVwzJNOCsQCLUzXRjokkufNshnQHOe4fRaqU= -github.com/livekit/protocol v1.51.1-0.20260915182407-3c0fcdbd25df h1:GpBiP7S7Vloz8hqHTpeWdMpIfoY+vu6Abm118bzaDY8= -github.com/livekit/protocol v1.51.1-0.20260915182407-3c0fcdbd25df/go.mod h1:vlreOPeiOr2pSCVCHDr2xIbSGf+2++y4QFaTgZ6rjp4= +github.com/livekit/protocol v1.51.1-0.20260914194419-c7d18f5376cf h1:B7RXQRyxB9nnWkUI/C/WHE7qfjTrKj0scuOM37w98rs= +github.com/livekit/protocol v1.51.1-0.20260914194419-c7d18f5376cf/go.mod h1:vlreOPeiOr2pSCVCHDr2xIbSGf+2++y4QFaTgZ6rjp4= github.com/livekit/psrpc v0.7.6 h1:YG07lUMTtf+eaYI2goT9zcVZ0kGJNWN1K6ETNFtv1HQ= github.com/livekit/psrpc v0.7.6/go.mod h1:DMw15RO7x5XmcgfwzWJYk2In605kx+wu1QRVbPfzf8M= github.com/livekit/server-sdk-go/v2 v2.18.2-0.20260904062056-1da58cd7b795 h1:0gljvZ5rt8vSLgoyaQl3ocD9D6RQt4Iyn4kXjVr4un0= From f6add3ff4346ecd094e7179f389edab271b8b80f Mon Sep 17 00:00:00 2001 From: Jason Lernerman Date: Mon, 14 Sep 2026 15:56:57 -0400 Subject: [PATCH 03/13] simulate: --repeats runs every scenario k times for pass@k / pass^k One attempt per scenario cannot tell a flaky scenario from a broken one. --repeats K asks the server for K attempts of every scenario in one run; the CLI groups attempts under their scenario in the job list and report, labels each attempt, and adds a pass@k (passed at least once) / pass^k (passed every time) line to the counts header and summaries, computed over the scenarios whose attempts have all finished. The exit code is unchanged: any failed attempt fails the run. --- cmd/lk/simulate.go | 95 +++++++++++++++++++++++++++++++-- cmd/lk/simulate_ci.go | 3 ++ cmd/lk/simulate_repeats_test.go | 68 +++++++++++++++++++++++ cmd/lk/simulate_report.go | 6 ++- cmd/lk/simulate_tui.go | 9 ++-- 5 files changed, 174 insertions(+), 7 deletions(-) create mode 100644 cmd/lk/simulate_repeats_test.go diff --git a/cmd/lk/simulate.go b/cmd/lk/simulate.go index 2c81bcfb..9be7cb82 100644 --- a/cmd/lk/simulate.go +++ b/cmd/lk/simulate.go @@ -82,6 +82,11 @@ var simulateCommand = &cli.Command{ Name: "concurrency", Usage: "Max simulations running in parallel (default: server-side limit)", }, + &cli.IntFlag{ + Name: "repeats", + Aliases: []string{"k"}, + Usage: "Run every scenario `K` times and report pass@k (passed at least once) and pass^k (passed every time). Requires --scenarios", + }, &cli.StringFlag{ Name: "scenarios", Usage: "Path to a scenarios `FILE` (yaml). If omitted, scenarios are generated from the agent's source", @@ -382,6 +387,7 @@ type simulateConfig struct { pc *config.ProjectConfig numSimulations int32 concurrency int32 + repeats int32 mode simulateMode simulationMode livekit.SimulationMode agentName string @@ -488,11 +494,15 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S numSimulations := int32(cmd.Int("num-simulations")) concurrency := int32(cmd.Int("concurrency")) + repeats := int32(cmd.Int("repeats")) liveAgentName := cmd.String("agent-name") // never auto-discovered: an explicit --scenarios file is the source of // truth, otherwise scenarios are generated from the agent's source scenariosPath := cmd.String("scenarios") + if repeats > 1 && scenariosPath == "" { + return fmt.Errorf("--repeats requires --scenarios (generated scenarios are not repeated)") + } var ( agentName string @@ -565,6 +575,7 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S pc: pc, numSimulations: numSimulations, concurrency: concurrency, + repeats: repeats, mode: mode, simulationMode: simulationMode, agentName: agentName, @@ -761,6 +772,7 @@ func createSimulationRun(ctx context.Context, c *simulateConfig) (string, *livek } if c.mode == modeScenarios { req.ScenarioGroup = c.scenarioGroup + req.Repeats = c.repeats } resp, err := c.client.CreateSimulationRun(ctx, req) @@ -934,15 +946,92 @@ func cancelSimulationRun(client *lksdk.AgentSimulationClient, runID string) { } } -// sortedJobs orders a run's jobs by job ID: the backend's ordering shuffles -// rows as statuses change. +// sortedJobs orders a run's jobs so every attempt of a scenario sits together, +// scenarios in scenarios.yaml order, then by job ID: the backend's ordering +// shuffles rows as statuses change. func sortedJobs(run *livekit.SimulationRun) []*livekit.SimulationRun_Job { + order := make(map[string]int, len(run.GetScenarioGroup().GetScenarios())) + for i, sc := range run.GetScenarioGroup().GetScenarios() { + order[sc.GetId()] = i + } jobs := make([]*livekit.SimulationRun_Job, len(run.Jobs)) copy(jobs, run.Jobs) - sort.Slice(jobs, func(i, j int) bool { return jobs[i].GetId() < jobs[j].GetId() }) + sort.Slice(jobs, func(i, j int) bool { + a, b := jobs[i], jobs[j] + if a.GetScenarioId() != b.GetScenarioId() { + return order[a.GetScenarioId()] < order[b.GetScenarioId()] + } + if a.GetAttempt() != b.GetAttempt() { + return a.GetAttempt() < b.GetAttempt() + } + return a.GetId() < b.GetId() + }) return jobs } +// scenarioPassCounts folds a repeated run's jobs into scenarios: how many +// have every attempt finished, and of those how many passed at least once +// (pass@k) and every time (pass^k). Zero scenarios when the run did not +// repeat, so callers can skip the line. +func scenarioPassCounts(run *livekit.SimulationRun) (scenarios, passAny, passAll int) { + k := int(run.GetRepeats()) + if k < 2 { + return + } + type tally struct{ terminal, passed int } + tallies := map[string]*tally{} + for _, j := range run.Jobs { + if j.GetScenarioId() == "" { + continue + } + t := tallies[j.GetScenarioId()] + if t == nil { + t = &tally{} + tallies[j.GetScenarioId()] = t + } + if isTerminalJobStatus(j.Status) { + t.terminal++ + } + if j.Status == livekit.SimulationRun_Job_STATUS_COMPLETED { + t.passed++ + } + } + for _, t := range tallies { + if t.terminal < k { + continue + } + scenarios++ + if t.passed > 0 { + passAny++ + } + if t.passed == k { + passAll++ + } + } + return +} + +// passRateLine is the pass@k / pass^k summary, or "" when the run did not +// repeat or no scenario has finished every attempt. +func passRateLine(run *livekit.SimulationRun) string { + scenarios, passAny, passAll := scenarioPassCounts(run) + if scenarios == 0 { + return "" + } + k := run.GetRepeats() + return fmt.Sprintf("pass@%d %d/%d (%.2f), pass^%d %d/%d (%.2f)", + k, passAny, scenarios, float64(passAny)/float64(scenarios), + k, passAll, scenarios, float64(passAll)/float64(scenarios)) +} + +// attemptSuffix marks a job's attempt when the run repeated scenarios. +func attemptSuffix(run *livekit.SimulationRun, job *livekit.SimulationRun_Job) string { + if run.GetRepeats() < 2 || job.GetAttempt() == 0 { + return "" + } + return fmt.Sprintf(" (attempt %d/%d)", job.GetAttempt(), run.GetRepeats()) +} + func simulationJobCounts(run *livekit.SimulationRun) (total, done, passed, failed int) { if run == nil { return diff --git a/cmd/lk/simulate_ci.go b/cmd/lk/simulate_ci.go index 1514bde8..f88a320d 100644 --- a/cmd/lk/simulate_ci.go +++ b/cmd/lk/simulate_ci.go @@ -207,6 +207,9 @@ func runSimulateCI(ctx context.Context, config *simulateConfig) error { } total, _, passed, failedN := simulationJobCounts(run) fmt.Fprintf(out.ResultWriter(), "%d total, %d passed, %d failed\n", total, passed, failedN) + if line := passRateLine(run); line != "" { + fmt.Fprintln(out.ResultWriter(), line) + } } if brokenAgent && agent != nil { diff --git a/cmd/lk/simulate_repeats_test.go b/cmd/lk/simulate_repeats_test.go new file mode 100644 index 00000000..78003021 --- /dev/null +++ b/cmd/lk/simulate_repeats_test.go @@ -0,0 +1,68 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "github.com/livekit/protocol/livekit" + "github.com/stretchr/testify/require" +) + +func attemptJob(id, scenario string, attempt int32, status livekit.SimulationRun_Job_Status) *livekit.SimulationRun_Job { + return &livekit.SimulationRun_Job{Id: id, ScenarioId: scenario, Attempt: attempt, Status: status} +} + +func TestScenarioPassCounts(t *testing.T) { + const ( + done = livekit.SimulationRun_Job_STATUS_COMPLETED + failed = livekit.SimulationRun_Job_STATUS_FAILED + running = livekit.SimulationRun_Job_STATUS_RUNNING + ) + run := &livekit.SimulationRun{Repeats: 2, Jobs: []*livekit.SimulationRun_Job{ + attemptJob("SRJ_1", "SCN_a", 1, done), attemptJob("SRJ_2", "SCN_a", 2, done), // pass^k + attemptJob("SRJ_3", "SCN_b", 1, done), attemptJob("SRJ_4", "SCN_b", 2, failed), // pass@k only + attemptJob("SRJ_5", "SCN_c", 1, failed), attemptJob("SRJ_6", "SCN_c", 2, failed), // neither + attemptJob("SRJ_7", "SCN_d", 1, done), attemptJob("SRJ_8", "SCN_d", 2, running), // not finished + }} + + scenarios, passAny, passAll := scenarioPassCounts(run) + require.Equal(t, 3, scenarios) + require.Equal(t, 2, passAny) + require.Equal(t, 1, passAll) + require.Equal(t, "pass@2 2/3 (0.67), pass^2 1/3 (0.33)", passRateLine(run)) + + run.Repeats = 1 + require.Equal(t, "", passRateLine(run)) +} + +func TestSortedJobs_GroupsAttemptsInScenarioOrder(t *testing.T) { + run := &livekit.SimulationRun{ + Repeats: 2, + ScenarioGroup: &livekit.ScenarioGroup{Scenarios: []*livekit.Scenario{{Id: "SCN_b"}, {Id: "SCN_a"}}}, + Jobs: []*livekit.SimulationRun_Job{ + attemptJob("SRJ_1", "SCN_a", 2, 0), + attemptJob("SRJ_2", "SCN_b", 2, 0), + attemptJob("SRJ_3", "SCN_a", 1, 0), + attemptJob("SRJ_4", "SCN_b", 1, 0), + }, + } + var ids []string + for _, j := range sortedJobs(run) { + ids = append(ids, j.Id) + } + require.Equal(t, []string{"SRJ_4", "SRJ_2", "SRJ_3", "SRJ_1"}, ids) + require.Equal(t, " (attempt 1/2)", attemptSuffix(run, run.Jobs[3])) +} diff --git a/cmd/lk/simulate_report.go b/cmd/lk/simulate_report.go index 5344ab58..c1e7230a 100644 --- a/cmd/lk/simulate_report.go +++ b/cmd/lk/simulate_report.go @@ -173,7 +173,7 @@ func writeRunResults(w io.Writer, run *livekit.SimulationRun, ap *AgentProcess) return } - for i, job := range run.Jobs { + for i, job := range sortedJobs(run) { icon := "⏺" switch job.Status { case livekit.SimulationRun_Job_STATUS_COMPLETED: @@ -186,6 +186,7 @@ func writeRunResults(w io.Writer, run *livekit.SimulationRun, ap *AgentProcess) if label == "" { label = fmt.Sprintf("Job %d", i+1) } + label += attemptSuffix(run, job) fmt.Fprintf(w, "::group::%s %s (%s)\n", icon, label, job.Id) @@ -251,6 +252,9 @@ func writeRunSummary(w io.Writer, run *livekit.SimulationRun, summary *livekit.S fmt.Fprintln(w) fmt.Fprintln(w, "::group::Summary") fmt.Fprintf(w, "%d total, %d passed, %d failed\n", total, passed, failed) + if line := passRateLine(run); line != "" { + fmt.Fprintln(w, line) + } if summary.GoingWell != "" { fmt.Fprintln(w) diff --git a/cmd/lk/simulate_tui.go b/cmd/lk/simulate_tui.go index 2ac5c337..87148640 100644 --- a/cmd/lk/simulate_tui.go +++ b/cmd/lk/simulate_tui.go @@ -1409,6 +1409,9 @@ func (m *simulateModel) renderCounts() string { if running > 0 { parts = append(parts, yellowStyle().Render(fmt.Sprintf("%d running", running))) } + if line := passRateLine(m.run); line != "" { + parts = append(parts, boldStyle.Render(line)) + } elapsed := "" if !m.startTime.IsZero() { @@ -1474,7 +1477,7 @@ func (m *simulateModel) renderJobList() string { maxWidth := 0 for i := winStart; i < winEnd; i++ { ij := jobs[i] - row := rowData{ij: ij, label: jobLabel(ij.job)} + row := rowData{ij: ij, label: jobLabel(ij.job) + attemptSuffix(m.run, ij.job)} rows = append(rows, row) w := lipgloss.Width(fmt.Sprintf(" ⏺ %3d. %s %s", ij.origIdx, ij.job.Id, row.label)) if w > maxWidth { @@ -1556,7 +1559,7 @@ func (m *simulateModel) buildMatrixRows() []matrixRow { } for i := winStart; i < winEnd; i++ { ij := jobs[i] - label := jobLabel(ij.job) + label := jobLabel(ij.job) + attemptSuffix(m.run, ij.job) iconCh, iconStyle := jobStatusIcon(ij.job) line := fmt.Sprintf(" %c %3d. %s %s", iconCh, ij.origIdx, ij.job.Id, label) rows = append(rows, matrixRow{ @@ -1598,7 +1601,7 @@ func (m *simulateModel) renderDetail() string { b.WriteString("\n") fmt.Fprintf(&b, " %s %s %s\n", jobIcon(job), - boldStyle.Render(fmt.Sprintf("Job %d", origIdx)), + boldStyle.Render(fmt.Sprintf("Job %d", origIdx)+attemptSuffix(m.run, job)), dimStyle.Render(job.Id), ) if url := simulationJobDashboardURL(m.projectID(), m.runID, job.Id); url != "" { From 7cdc882d2b98fdc4d88186a17bbe96a46a350ef0 Mon Sep 17 00:00:00 2001 From: Jason Lernerman Date: Mon, 14 Sep 2026 16:25:41 -0400 Subject: [PATCH 04/13] simulate tui: rename detailJobID to detailID The open detail is about to be a scenario as well as a job. --- cmd/lk/simulate_tui.go | 50 ++++++++++++++++---------------- cmd/lk/simulate_tui_keys_test.go | 10 +++---- cmd/lk/simulate_tui_vrt_test.go | 2 +- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/cmd/lk/simulate_tui.go b/cmd/lk/simulate_tui.go index 87148640..9a9b2551 100644 --- a/cmd/lk/simulate_tui.go +++ b/cmd/lk/simulate_tui.go @@ -216,7 +216,7 @@ type simulateModel struct { spinnerIdx int cursor int - detailJobID string + detailID string // The summary's citations, in the order their numbers were rendered, so a // digit key resolves to the turn its label points at. refItemID is the chat // item a jump cited, marked when the job view prints because printed @@ -285,7 +285,7 @@ func (m *simulateModel) hasDescription() bool { } func (m *simulateModel) descriptionExpanded() bool { - return m.detailJobID == "" && m.showDescription && m.hasDescription() + return m.detailID == "" && m.showDescription && m.hasDescription() } func (m *simulateModel) quotaModalActive() bool { @@ -873,7 +873,7 @@ func (m *simulateModel) handleKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { } return m, tea.Quit case "m": - if m.detailJobID != "" || m.run == nil || !m.setupDone || len(m.run.Jobs) == 0 || m.width < 10 { + if m.detailID != "" || m.run == nil || !m.setupDone || len(m.run.Jobs) == 0 || m.width < 10 { return m, nil } rows := m.buildMatrixRows() @@ -916,16 +916,16 @@ func (m *simulateModel) handleKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { m.logScrollOff = 0 m.logPinned = false case "t": - if m.detailJobID != "" { + if m.detailID != "" { m.showToolDetail = !m.showToolDetail } case "d": - if m.detailJobID == "" && m.hasDescription() { + if m.detailID == "" && m.hasDescription() { m.showDescription = !m.showDescription m.descScrollOff = 0 } case "s": - if m.canExportScenarios() && m.detailJobID == "" { + if m.canExportScenarios() && m.detailID == "" { m.saving = true m.saveErr = "" m.toast = "" @@ -934,14 +934,14 @@ func (m *simulateModel) handleKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { return m, m.saveInput.Focus() } case "c": - if m.detailJobID != "" { - text, ok := m.copyScenario(m.detailJobID) + if m.detailID != "" { + text, ok := m.copyScenario(m.detailID) return m, m.showToast(text, ok) } case "up", "down", "pgup", "pgdown": // The open job's view is scrolled by the terminal, so these keys must // not disturb the list underneath it. - if m.detailJobID != "" { + if m.detailID != "" { return m, nil } switch key { @@ -972,9 +972,9 @@ func (m *simulateModel) handleKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { // A citation's number opens the turn it cites. Only live on the list view, // which is where the numbered summary is on screen to read them off. case "1", "2", "3", "4", "5", "6", "7", "8", "9": - if m.detailJobID == "" { + if m.detailID == "" { if ref, ok := m.summaryRef(key); ok { - m.detailJobID = ref.job + m.detailID = ref.job m.refItemID = ref.item return m, m.openDetailCmd() } @@ -982,15 +982,15 @@ func (m *simulateModel) handleKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { // j and l sit either side of k on the home row, so they double for the // left/right arrows without reaching for them. case "enter", "right", "l": - if m.detailJobID == "" { + if m.detailID == "" { jobs := m.filteredJobs() if m.cursor >= 0 && m.cursor < len(jobs) { - m.detailJobID = jobs[m.cursor].job.Id + m.detailID = jobs[m.cursor].job.Id return m, m.openDetailCmd() } } case "esc", "left", "backspace", "j": - if m.detailJobID != "" { + if m.detailID != "" { return m, m.closeDetailCmd() } else if m.showDescription { m.showDescription = false @@ -998,7 +998,7 @@ func (m *simulateModel) handleKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { } case "q": switch { - case m.detailJobID != "": + case m.detailID != "": return m, m.closeDetailCmd() case m.showDescription: m.showDescription = false @@ -1055,7 +1055,7 @@ func (m *simulateModel) render() string { if !m.setupDone || m.run == nil || m.run.Status == livekit.SimulationRun_STATUS_GENERATING { return m.viewSetup() } - if m.detailJobID != "" { + if m.detailID != "" { return m.viewDetailLive() } switch m.run.Status { @@ -1586,14 +1586,14 @@ func (m *simulateModel) renderDetail() string { var job *livekit.SimulationRun_Job origIdx := 0 for i, j := range m.run.Jobs { - if j.Id == m.detailJobID { + if j.Id == m.detailID { job = j origIdx = i + 1 break } } if job == nil { - m.detailJobID = "" + m.detailID = "" return dimStyle.Render(" (job not found)\n") } @@ -1709,7 +1709,7 @@ func (m *simulateModel) openDetailCmd() tea.Cmd { // closeDetailCmd returns to the list view. The printed job stays in the // scrollback until the next one replaces it. func (m *simulateModel) closeDetailCmd() tea.Cmd { - m.detailJobID = "" + m.detailID = "" m.detailPrinted = "" m.refItemID = "" m.altScreen = true @@ -1738,12 +1738,12 @@ const clearScrollback = ansi.CursorHomePosition + ansi.EraseEntireScreen + ansi. // gained since the last call, or nil when it has gained nothing. Callers do not // need to know whether anything changed. func (m *simulateModel) flushDetail() tea.Cmd { - if m.detailJobID == "" { + if m.detailID == "" { return nil } rendered := strings.TrimRight(m.renderDetail(), "\n") - // renderDetail clears detailJobID when the job is gone from the run - if m.detailJobID == "" || rendered == "" { + // renderDetail clears detailID when the job is gone from the run + if m.detailID == "" || rendered == "" { return nil } if m.width != m.detailWidth { @@ -1786,7 +1786,7 @@ func detailTail(printed, rendered string) (string, bool) { // the job, which is the only part of it that is still moving. func (m *simulateModel) viewDetailLive() string { var b strings.Builder - if job := m.findJob(m.detailJobID); job != nil && !isTerminalJobStatus(job.Status) { + if job := m.findJob(m.detailID); job != nil && !isTerminalJobStatus(job.Status) { fmt.Fprintf(&b, "\n %s %s %s\n", jobIcon(job), dimStyle.Render(jobLabel(job)), m.spinner()) } else { b.WriteString("\n") @@ -2181,10 +2181,10 @@ func (m *simulateModel) renderHint() string { } var parts []string switch { - case m.detailJobID != "": + case m.detailID != "": // the job view is in the terminal's scrollback, which scrolls itself parts = append(parts, "c copy scenario · ←/ESC back to list") - if m.hasToolDetail(m.detailJobID) { + if m.hasToolDetail(m.detailID) { if m.showToolDetail { parts = append(parts, "t hide tool detail") } else { diff --git a/cmd/lk/simulate_tui_keys_test.go b/cmd/lk/simulate_tui_keys_test.go index b83c10b4..1a29cb98 100644 --- a/cmd/lk/simulate_tui_keys_test.go +++ b/cmd/lk/simulate_tui_keys_test.go @@ -111,7 +111,7 @@ func TestSimulateKeyDispatch(t *testing.T) { key: "t", setup: func() *simulateModel { m := runningFixture() - m.detailJobID = "SRJ_aaaaaaaa" + m.detailID = "SRJ_aaaaaaaa" return m }, check: func(t *testing.T, m *simulateModel) { require.True(t, m.showToolDetail) }, @@ -132,17 +132,17 @@ func TestSimulateKeyDispatch(t *testing.T) { name: "enter opens the job detail", key: "enter", setup: runningFixture, - check: func(t *testing.T, m *simulateModel) { require.Equal(t, "SRJ_bbbbbbbb", m.detailJobID) }, + check: func(t *testing.T, m *simulateModel) { require.Equal(t, "SRJ_bbbbbbbb", m.detailID) }, }, { name: "esc closes the job detail", key: "esc", setup: func() *simulateModel { m := runningFixture() - m.detailJobID = "SRJ_aaaaaaaa" + m.detailID = "SRJ_aaaaaaaa" return m }, - check: func(t *testing.T, m *simulateModel) { require.Empty(t, m.detailJobID) }, + check: func(t *testing.T, m *simulateModel) { require.Empty(t, m.detailID) }, }, { name: "q asks before quitting a live run", @@ -168,7 +168,7 @@ func TestSimulateDetailLeavesAltScreen(t *testing.T) { m := runningFixture() require.True(t, m.View().AltScreen, "the list view runs on the alt screen") - m.detailJobID = "SRJ_aaaaaaaa" + m.detailID = "SRJ_aaaaaaaa" m.openDetailCmd() require.False(t, m.View().AltScreen, "opening a job must leave the alt screen before it prints") diff --git a/cmd/lk/simulate_tui_vrt_test.go b/cmd/lk/simulate_tui_vrt_test.go index ad15aa71..730d9f13 100644 --- a/cmd/lk/simulate_tui_vrt_test.go +++ b/cmd/lk/simulate_tui_vrt_test.go @@ -194,7 +194,7 @@ func TestVRTSimulateFrames(t *testing.T) { }}, {"simulate_detail_live", func() *simulateModel { m := runningFixture() - m.detailJobID = "SRJ_aaaaaaaa" + m.detailID = "SRJ_aaaaaaaa" m.detailWidth = m.width return m }}, From b956e47ba0a5cd45a0142ff5826e3a28d08563c3 Mon Sep 17 00:00:00 2001 From: Jason Lernerman Date: Mon, 14 Sep 2026 16:27:56 -0400 Subject: [PATCH 05/13] simulate tui: nest a repeated scenario's attempts under one row A repeated run listed every attempt as its own numbered job, so a 4-scenario run read as 12 rows. Each scenario is now one row with its pass count and its attempts branch under it. Opening the scenario row shows the brief once and every attempt's verdict and transcript; opening an attempt shows that job as before. --- cmd/lk/simulate_repeats_test.go | 50 +++++++ cmd/lk/simulate_tui.go | 238 +++++++++++++++++++++++++++----- 2 files changed, 257 insertions(+), 31 deletions(-) diff --git a/cmd/lk/simulate_repeats_test.go b/cmd/lk/simulate_repeats_test.go index 78003021..cefed94a 100644 --- a/cmd/lk/simulate_repeats_test.go +++ b/cmd/lk/simulate_repeats_test.go @@ -66,3 +66,53 @@ func TestSortedJobs_GroupsAttemptsInScenarioOrder(t *testing.T) { require.Equal(t, []string{"SRJ_4", "SRJ_2", "SRJ_3", "SRJ_1"}, ids) require.Equal(t, " (attempt 1/2)", attemptSuffix(run, run.Jobs[3])) } + +func repeatedFixture() *simulateModel { + m := runningFixture() + m.run = &livekit.SimulationRun{ + Id: "SR_fixture0001", Status: livekit.SimulationRun_STATUS_RUNNING, Repeats: 2, + ScenarioGroup: &livekit.ScenarioGroup{Scenarios: []*livekit.Scenario{{Id: "SCN_a"}, {Id: "SCN_b"}}}, + Jobs: []*livekit.SimulationRun_Job{ + attemptJob("SRJ_a1", "SCN_a", 1, livekit.SimulationRun_Job_STATUS_COMPLETED), + attemptJob("SRJ_a2", "SCN_a", 2, livekit.SimulationRun_Job_STATUS_FAILED), + attemptJob("SRJ_b1", "SCN_b", 1, livekit.SimulationRun_Job_STATUS_COMPLETED), + attemptJob("SRJ_b2", "SCN_b", 2, livekit.SimulationRun_Job_STATUS_RUNNING), + }, + } + for _, j := range m.run.Jobs { + j.Label = "scenario " + j.ScenarioId + } + return m +} + +func TestFilteredJobs_RepeatedRunNestsAttempts(t *testing.T) { + rows := repeatedFixture().filteredJobs() + var ids []string + for _, r := range rows { + ids = append(ids, r.id()) + } + require.Equal(t, []string{"SCN_a", "SRJ_a1", "SRJ_a2", "SCN_b", "SRJ_b1", "SRJ_b2"}, ids) + require.Equal(t, 1, rows[0].origIdx) + require.Equal(t, 0, rows[1].origIdx, "attempt rows carry no number") + require.True(t, rows[2].last) + require.Equal(t, 2, rows[3].origIdx) + + icon, _ := scenarioStatusIcon(rows[0].scenario.attempts) + require.Equal(t, '✗', icon, "a scenario with a failed attempt is not passed") + icon, _ = scenarioStatusIcon(rows[3].scenario.attempts) + require.Equal(t, '⏺', icon, "a scenario still runs while any attempt does") +} + +func TestEnterOnScenarioRowOpensTheScenario(t *testing.T) { + m := repeatedFixture() + m.cursor = 0 + m.Update(keyPress("enter")) + require.Equal(t, "SCN_a", m.detailID) + require.NotNil(t, m.findScenario(m.detailID)) + require.Contains(t, m.renderDetail(), "1/2 attempts passed") + + m.closeDetailCmd() + m.cursor = 2 + m.Update(keyPress("enter")) + require.Equal(t, "SRJ_a2", m.detailID) +} diff --git a/cmd/lk/simulate_tui.go b/cmd/lk/simulate_tui.go index 9a9b2551..e4eb04b6 100644 --- a/cmd/lk/simulate_tui.go +++ b/cmd/lk/simulate_tui.go @@ -215,7 +215,7 @@ type simulateModel struct { spinnerIdx int - cursor int + cursor int detailID string // The summary's citations, in the order their numbers were rendered, so a // digit key resolves to the turn its label points at. refItemID is the chat @@ -356,6 +356,11 @@ func (m *simulateModel) saveScenarios(name string) (string, bool) { // copyScenario puts the job's scenario on the clipboard as a one-entry scenarios.yaml. func (m *simulateModel) copyScenario(jobID string) (string, bool) { job := m.findJob(jobID) + if job == nil { + if sc := m.findScenario(jobID); sc != nil { + job = sc.attempts[0] + } + } if job == nil { return "Copy failed: scenario not found", false } @@ -985,7 +990,7 @@ func (m *simulateModel) handleKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { if m.detailID == "" { jobs := m.filteredJobs() if m.cursor >= 0 && m.cursor < len(jobs) { - m.detailID = jobs[m.cursor].job.Id + m.detailID = jobs[m.cursor].id() return m, m.openDetailCmd() } } @@ -1016,9 +1021,31 @@ func (m *simulateModel) handleKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { return m, nil } +// indexedJob is one row of the job list. An unrepeated run has a row per job. +// A repeated run has a row per scenario with its attempts nested under it: +// +// ✓ 1. Book a table for two 3/3 +// ├─ ✓ SRJ_01 attempt 1 +// └─ ✓ SRJ_09 attempt 3 type indexedJob struct { - origIdx int - job *livekit.SimulationRun_Job + origIdx int // 1-based number of the scenario or, unrepeated, the job; 0 on an attempt row + job *livekit.SimulationRun_Job // nil on a scenario row + scenario *scenarioRow // set on a scenario row only + last bool // attempt row: the scenario's final attempt (└ rather than ├) +} + +type scenarioRow struct { + id string + label string + attempts []*livekit.SimulationRun_Job +} + +// id is what opening the row shows: a job, or a whole scenario. +func (ij indexedJob) id() string { + if ij.scenario != nil { + return ij.scenario.id + } + return ij.job.GetId() } func (m *simulateModel) filteredJobs() []indexedJob { @@ -1027,12 +1054,108 @@ func (m *simulateModel) filteredJobs() []indexedJob { } jobs := sortedJobs(m.run) result := make([]indexedJob, 0, len(jobs)) - for i, j := range jobs { - result = append(result, indexedJob{origIdx: i + 1, job: j}) + if m.run.GetRepeats() < 2 { + for i, j := range jobs { + result = append(result, indexedJob{origIdx: i + 1, job: j}) + } + return result + } + n := 0 + for i := 0; i < len(jobs); { + j := jobs[i] + n++ + if j.GetScenarioId() == "" { + result = append(result, indexedJob{origIdx: n, job: j}) + i++ + continue + } + sc := &scenarioRow{id: j.GetScenarioId(), label: jobLabel(j)} + for i < len(jobs) && jobs[i].GetScenarioId() == sc.id { + sc.attempts = append(sc.attempts, jobs[i]) + i++ + } + result = append(result, indexedJob{origIdx: n, scenario: sc}) + for k, a := range sc.attempts { + result = append(result, indexedJob{job: a, last: k == len(sc.attempts)-1}) + } } return result } +func (m *simulateModel) findScenario(id string) *scenarioRow { + for _, ij := range m.filteredJobs() { + if ij.scenario != nil && ij.scenario.id == id { + return ij.scenario + } + } + return nil +} + +// scenarioStatusIcon folds a scenario's attempts: running while any attempt +// runs, pending while any waits, then passed only if every attempt did. +func scenarioStatusIcon(attempts []*livekit.SimulationRun_Job) (rune, *lipgloss.Style) { + passed := 0 + pending := false + for _, a := range attempts { + switch a.Status { + case livekit.SimulationRun_Job_STATUS_RUNNING: + s := yellowStyle() + return '⏺', &s + case livekit.SimulationRun_Job_STATUS_COMPLETED: + passed++ + case livekit.SimulationRun_Job_STATUS_FAILED, livekit.SimulationRun_Job_STATUS_CANCELLED: + default: + pending = true + } + } + if pending { + return '⏺', &dimStyle + } + if passed == len(attempts) { + s := greenStyle() + return '✓', &s + } + s := redStyle() + return '✗', &s +} + +func scenarioPassed(attempts []*livekit.SimulationRun_Job) int { + n := 0 + for _, a := range attempts { + if a.Status == livekit.SimulationRun_Job_STATUS_COMPLETED { + n++ + } + } + return n +} + +// listRowParts splits a row into the indent before its icon, the icon, and the +// text after it, in plain and styled forms, so the cursor highlight and the +// matrix rain can lay out the same line. +func listRowParts(ij indexedJob) (indent string, iconCh rune, iconStyle *lipgloss.Style, text, styled string) { + switch { + case ij.scenario != nil: + iconCh, iconStyle = scenarioStatusIcon(ij.scenario.attempts) + text = fmt.Sprintf(" %3d. %s %d/%d", ij.origIdx, ij.scenario.label, scenarioPassed(ij.scenario.attempts), len(ij.scenario.attempts)) + return " ", iconCh, iconStyle, text, text + case ij.origIdx == 0: + connector := "├─ " + if ij.last { + connector = "└─ " + } + iconCh, iconStyle = jobStatusIcon(ij.job) + text = fmt.Sprintf(" %s attempt %d", ij.job.Id, ij.job.GetAttempt()) + styled = fmt.Sprintf(" %s attempt %d", dimStyle.Render(ij.job.Id), ij.job.GetAttempt()) + return " " + dimStyle.Render(connector), iconCh, iconStyle, text, styled + default: + iconCh, iconStyle = jobStatusIcon(ij.job) + label := jobLabel(ij.job) + text = fmt.Sprintf(" %3d. %s %s", ij.origIdx, ij.job.Id, label) + styled = fmt.Sprintf(" %3d. %s %s", ij.origIdx, dimStyle.Render(ij.job.Id), label) + return " ", iconCh, iconStyle, text, styled + } +} + func (m *simulateModel) findJob(id string) *livekit.SimulationRun_Job { if m.run == nil { return nil @@ -1468,37 +1591,26 @@ func (m *simulateModel) renderJobList() string { b.WriteString("\n") } - // Compute labels and max width for consistent hover highlight - type rowData struct { - ij indexedJob - label string - } - rows := make([]rowData, 0, winEnd-winStart) + // The cursor highlight pads to the widest row so the reverse bar is one width. maxWidth := 0 for i := winStart; i < winEnd; i++ { - ij := jobs[i] - row := rowData{ij: ij, label: jobLabel(ij.job) + attemptSuffix(m.run, ij.job)} - rows = append(rows, row) - w := lipgloss.Width(fmt.Sprintf(" ⏺ %3d. %s %s", ij.origIdx, ij.job.Id, row.label)) - if w > maxWidth { + indent, iconCh, _, text, _ := listRowParts(jobs[i]) + if w := lipgloss.Width(indent + string(iconCh) + text); w > maxWidth { maxWidth = w } } - for i, row := range rows { - idx := winStart + i - pr, _ := jobStatusIcon(row.ij.job) - plainIcon := string(pr) + for i := winStart; i < winEnd; i++ { + indent, iconCh, iconStyle, text, styled := listRowParts(jobs[i]) var line string - if idx == m.cursor { - line = fmt.Sprintf(" %s %3d. %s %s", plainIcon, row.ij.origIdx, row.ij.job.Id, row.label) + if i == m.cursor { + line = ansi.Strip(indent) + string(iconCh) + text if pad := maxWidth - lipgloss.Width(line); pad > 0 { line += strings.Repeat(" ", pad) } line = reverseStyle.Render(line) } else { - icon := jobIcon(row.ij.job) - line = fmt.Sprintf(" %s %3d. %s %s", icon, row.ij.origIdx, dimStyle.Render(row.ij.job.Id), row.label) + line = indent + iconStyle.Render(string(iconCh)) + styled } b.WriteString(line) b.WriteString("\n") @@ -1558,13 +1670,12 @@ func (m *simulateModel) buildMatrixRows() []matrixRow { }) } for i := winStart; i < winEnd; i++ { - ij := jobs[i] - label := jobLabel(ij.job) + attemptSuffix(m.run, ij.job) - iconCh, iconStyle := jobStatusIcon(ij.job) - line := fmt.Sprintf(" %c %3d. %s %s", iconCh, ij.origIdx, ij.job.Id, label) + indent, iconCh, iconStyle, text, _ := listRowParts(jobs[i]) + indent = ansi.Strip(indent) + line := indent + string(iconCh) + text rows = append(rows, matrixRow{ text: []rune(line), - iconCol: 2, + iconCol: len([]rune(indent)), iconCh: iconCh, iconStyle: iconStyle, cursorMarker: i == m.cursor, @@ -1593,15 +1704,22 @@ func (m *simulateModel) renderDetail() string { } } if job == nil { + if sc := m.findScenario(m.detailID); sc != nil { + return m.renderScenarioDetail(sc) + } m.detailID = "" return dimStyle.Render(" (job not found)\n") } + title := fmt.Sprintf("Job %d", origIdx) + if m.run.GetRepeats() >= 2 { + title = jobLabel(job) + attemptSuffix(m.run, job) + } var b strings.Builder b.WriteString("\n") fmt.Fprintf(&b, " %s %s %s\n", jobIcon(job), - boldStyle.Render(fmt.Sprintf("Job %d", origIdx)+attemptSuffix(m.run, job)), + boldStyle.Render(title), dimStyle.Render(job.Id), ) if url := simulationJobDashboardURL(m.projectID(), m.runID, job.Id); url != "" { @@ -1682,6 +1800,64 @@ func (m *simulateModel) renderDetail() string { return b.String() } +// renderScenarioDetail is a repeated scenario as a whole: its brief once, then +// every attempt's verdict and transcript in attempt order. +func (m *simulateModel) renderScenarioDetail(sc *scenarioRow) string { + var b strings.Builder + iconCh, iconStyle := scenarioStatusIcon(sc.attempts) + b.WriteString("\n") + fmt.Fprintf(&b, " %s %s %s\n", + iconStyle.Render(string(iconCh)), + boldStyle.Render(sc.label), + dimStyle.Render(fmt.Sprintf("%d/%d attempts passed", scenarioPassed(sc.attempts), len(sc.attempts))), + ) + b.WriteString("\n") + + wrapWidth := proseWidth(m.width, 6) + first := sc.attempts[0] + + b.WriteString(boldStyle.Render(" Instructions:")) + b.WriteString("\n") + instr := first.Instructions + if instr == "" { + instr = "—" + } + for _, line := range wrapLines(instr, wrapWidth) { + b.WriteString(" " + line + "\n") + } + b.WriteString("\n") + + b.WriteString(dimStyle.Bold(true).Render(" Expected:")) + b.WriteString("\n") + expect := first.AgentExpectations + if expect == "" { + expect = "—" + } + for _, line := range wrapLines(expect, wrapWidth) { + b.WriteString(dimStyle.Render(" "+line) + "\n") + } + + for _, job := range sc.attempts { + b.WriteString("\n") + fmt.Fprintf(&b, " %s %s %s\n", + jobIcon(job), + boldStyle.Render(fmt.Sprintf("Attempt %d", job.GetAttempt())), + dimStyle.Render(job.Id), + ) + if job.Error != "" { + style := redStyle() + if job.Status == livekit.SimulationRun_Job_STATUS_COMPLETED { + style = greenStyle() + } + for _, line := range wrapLines(job.Error, wrapWidth) { + b.WriteString(style.Render(" "+line) + "\n") + } + } + b.WriteString(m.renderChatTranscript(job.Id)) + } + return b.String() +} + // --- Job detail (native scrollback) --- // // The detail view is printed into the terminal below the alt screen rather than From 66963f17ffb56cf3df570ac4c23f11f814abc96f Mon Sep 17 00:00:00 2001 From: Jason Lernerman Date: Tue, 15 Sep 2026 14:26:30 -0400 Subject: [PATCH 06/13] simulate: rename --repeats to --samples Follows livekit/protocol#1799 renaming the proto field. --- cmd/lk/simulate.go | 22 +++++++++---------- ...peats_test.go => simulate_samples_test.go} | 8 +++---- cmd/lk/simulate_tui.go | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 5 files changed, 20 insertions(+), 20 deletions(-) rename cmd/lk/{simulate_repeats_test.go => simulate_samples_test.go} (97%) diff --git a/cmd/lk/simulate.go b/cmd/lk/simulate.go index 9be7cb82..e9fd7d61 100644 --- a/cmd/lk/simulate.go +++ b/cmd/lk/simulate.go @@ -83,7 +83,7 @@ var simulateCommand = &cli.Command{ Usage: "Max simulations running in parallel (default: server-side limit)", }, &cli.IntFlag{ - Name: "repeats", + Name: "samples", Aliases: []string{"k"}, Usage: "Run every scenario `K` times and report pass@k (passed at least once) and pass^k (passed every time). Requires --scenarios", }, @@ -387,7 +387,7 @@ type simulateConfig struct { pc *config.ProjectConfig numSimulations int32 concurrency int32 - repeats int32 + samples int32 mode simulateMode simulationMode livekit.SimulationMode agentName string @@ -494,14 +494,14 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S numSimulations := int32(cmd.Int("num-simulations")) concurrency := int32(cmd.Int("concurrency")) - repeats := int32(cmd.Int("repeats")) + samples := int32(cmd.Int("samples")) liveAgentName := cmd.String("agent-name") // never auto-discovered: an explicit --scenarios file is the source of // truth, otherwise scenarios are generated from the agent's source scenariosPath := cmd.String("scenarios") - if repeats > 1 && scenariosPath == "" { - return fmt.Errorf("--repeats requires --scenarios (generated scenarios are not repeated)") + if samples > 1 && scenariosPath == "" { + return fmt.Errorf("--samples requires --scenarios (generated scenarios are not sampled more than once)") } var ( @@ -575,7 +575,7 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S pc: pc, numSimulations: numSimulations, concurrency: concurrency, - repeats: repeats, + samples: samples, mode: mode, simulationMode: simulationMode, agentName: agentName, @@ -772,7 +772,7 @@ func createSimulationRun(ctx context.Context, c *simulateConfig) (string, *livek } if c.mode == modeScenarios { req.ScenarioGroup = c.scenarioGroup - req.Repeats = c.repeats + req.Samples = c.samples } resp, err := c.client.CreateSimulationRun(ctx, req) @@ -974,7 +974,7 @@ func sortedJobs(run *livekit.SimulationRun) []*livekit.SimulationRun_Job { // (pass@k) and every time (pass^k). Zero scenarios when the run did not // repeat, so callers can skip the line. func scenarioPassCounts(run *livekit.SimulationRun) (scenarios, passAny, passAll int) { - k := int(run.GetRepeats()) + k := int(run.GetSamples()) if k < 2 { return } @@ -1018,7 +1018,7 @@ func passRateLine(run *livekit.SimulationRun) string { if scenarios == 0 { return "" } - k := run.GetRepeats() + k := run.GetSamples() return fmt.Sprintf("pass@%d %d/%d (%.2f), pass^%d %d/%d (%.2f)", k, passAny, scenarios, float64(passAny)/float64(scenarios), k, passAll, scenarios, float64(passAll)/float64(scenarios)) @@ -1026,10 +1026,10 @@ func passRateLine(run *livekit.SimulationRun) string { // attemptSuffix marks a job's attempt when the run repeated scenarios. func attemptSuffix(run *livekit.SimulationRun, job *livekit.SimulationRun_Job) string { - if run.GetRepeats() < 2 || job.GetAttempt() == 0 { + if run.GetSamples() < 2 || job.GetAttempt() == 0 { return "" } - return fmt.Sprintf(" (attempt %d/%d)", job.GetAttempt(), run.GetRepeats()) + return fmt.Sprintf(" (attempt %d/%d)", job.GetAttempt(), run.GetSamples()) } func simulationJobCounts(run *livekit.SimulationRun) (total, done, passed, failed int) { diff --git a/cmd/lk/simulate_repeats_test.go b/cmd/lk/simulate_samples_test.go similarity index 97% rename from cmd/lk/simulate_repeats_test.go rename to cmd/lk/simulate_samples_test.go index cefed94a..0b25ce0b 100644 --- a/cmd/lk/simulate_repeats_test.go +++ b/cmd/lk/simulate_samples_test.go @@ -31,7 +31,7 @@ func TestScenarioPassCounts(t *testing.T) { failed = livekit.SimulationRun_Job_STATUS_FAILED running = livekit.SimulationRun_Job_STATUS_RUNNING ) - run := &livekit.SimulationRun{Repeats: 2, Jobs: []*livekit.SimulationRun_Job{ + run := &livekit.SimulationRun{Samples: 2, Jobs: []*livekit.SimulationRun_Job{ attemptJob("SRJ_1", "SCN_a", 1, done), attemptJob("SRJ_2", "SCN_a", 2, done), // pass^k attemptJob("SRJ_3", "SCN_b", 1, done), attemptJob("SRJ_4", "SCN_b", 2, failed), // pass@k only attemptJob("SRJ_5", "SCN_c", 1, failed), attemptJob("SRJ_6", "SCN_c", 2, failed), // neither @@ -44,13 +44,13 @@ func TestScenarioPassCounts(t *testing.T) { require.Equal(t, 1, passAll) require.Equal(t, "pass@2 2/3 (0.67), pass^2 1/3 (0.33)", passRateLine(run)) - run.Repeats = 1 + run.Samples = 1 require.Equal(t, "", passRateLine(run)) } func TestSortedJobs_GroupsAttemptsInScenarioOrder(t *testing.T) { run := &livekit.SimulationRun{ - Repeats: 2, + Samples: 2, ScenarioGroup: &livekit.ScenarioGroup{Scenarios: []*livekit.Scenario{{Id: "SCN_b"}, {Id: "SCN_a"}}}, Jobs: []*livekit.SimulationRun_Job{ attemptJob("SRJ_1", "SCN_a", 2, 0), @@ -70,7 +70,7 @@ func TestSortedJobs_GroupsAttemptsInScenarioOrder(t *testing.T) { func repeatedFixture() *simulateModel { m := runningFixture() m.run = &livekit.SimulationRun{ - Id: "SR_fixture0001", Status: livekit.SimulationRun_STATUS_RUNNING, Repeats: 2, + Id: "SR_fixture0001", Status: livekit.SimulationRun_STATUS_RUNNING, Samples: 2, ScenarioGroup: &livekit.ScenarioGroup{Scenarios: []*livekit.Scenario{{Id: "SCN_a"}, {Id: "SCN_b"}}}, Jobs: []*livekit.SimulationRun_Job{ attemptJob("SRJ_a1", "SCN_a", 1, livekit.SimulationRun_Job_STATUS_COMPLETED), diff --git a/cmd/lk/simulate_tui.go b/cmd/lk/simulate_tui.go index e4eb04b6..3cd8675f 100644 --- a/cmd/lk/simulate_tui.go +++ b/cmd/lk/simulate_tui.go @@ -1054,7 +1054,7 @@ func (m *simulateModel) filteredJobs() []indexedJob { } jobs := sortedJobs(m.run) result := make([]indexedJob, 0, len(jobs)) - if m.run.GetRepeats() < 2 { + if m.run.GetSamples() < 2 { for i, j := range jobs { result = append(result, indexedJob{origIdx: i + 1, job: j}) } @@ -1712,7 +1712,7 @@ func (m *simulateModel) renderDetail() string { } title := fmt.Sprintf("Job %d", origIdx) - if m.run.GetRepeats() >= 2 { + if m.run.GetSamples() >= 2 { title = jobLabel(job) + attemptSuffix(m.run, job) } var b strings.Builder diff --git a/go.mod b/go.mod index 11dc2bd3..1094c8a7 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/google/go-querystring v1.2.0 github.com/joho/godotenv v1.5.1 github.com/klauspost/compress v1.20.0 - github.com/livekit/protocol v1.51.1-0.20260914194419-c7d18f5376cf + github.com/livekit/protocol v1.51.1-0.20260915182316-f9e48c3cf763 github.com/livekit/server-sdk-go/v2 v2.18.2-0.20260904062056-1da58cd7b795 github.com/mattn/go-isatty v0.0.22 github.com/moby/moby/client v0.4.1 diff --git a/go.sum b/go.sum index d63be428..fbcf321e 100644 --- a/go.sum +++ b/go.sum @@ -370,8 +370,8 @@ github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5AT github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095 h1:BcliKAXoMhl/nWmzQweQ5kmh4Qqagxl4s3Z5pvM/7AY= github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095/go.mod h1:o8CFmAdrVwzJNOCsQCLUzXRjokkufNshnQHOe4fRaqU= -github.com/livekit/protocol v1.51.1-0.20260914194419-c7d18f5376cf h1:B7RXQRyxB9nnWkUI/C/WHE7qfjTrKj0scuOM37w98rs= -github.com/livekit/protocol v1.51.1-0.20260914194419-c7d18f5376cf/go.mod h1:vlreOPeiOr2pSCVCHDr2xIbSGf+2++y4QFaTgZ6rjp4= +github.com/livekit/protocol v1.51.1-0.20260915182316-f9e48c3cf763 h1:VfQ2psqfg2GOrAkSv5OrnEu890rqJNEaI3oKcJfPlPQ= +github.com/livekit/protocol v1.51.1-0.20260915182316-f9e48c3cf763/go.mod h1:vlreOPeiOr2pSCVCHDr2xIbSGf+2++y4QFaTgZ6rjp4= github.com/livekit/psrpc v0.7.6 h1:YG07lUMTtf+eaYI2goT9zcVZ0kGJNWN1K6ETNFtv1HQ= github.com/livekit/psrpc v0.7.6/go.mod h1:DMw15RO7x5XmcgfwzWJYk2In605kx+wu1QRVbPfzf8M= github.com/livekit/server-sdk-go/v2 v2.18.2-0.20260904062056-1da58cd7b795 h1:0gljvZ5rt8vSLgoyaQl3ocD9D6RQt4Iyn4kXjVr4un0= From 58c226814cdb5962f7ac77ce67ea6e535612a6ac Mon Sep 17 00:00:00 2001 From: Jason Lernerman Date: Tue, 15 Sep 2026 14:32:49 -0400 Subject: [PATCH 07/13] simulate: rename attempt to sample Follows livekit/protocol#1799 renaming Job.attempt. --- cmd/lk/simulate.go | 18 +++++----- cmd/lk/simulate_report.go | 2 +- cmd/lk/simulate_samples_test.go | 46 ++++++++++++------------- cmd/lk/simulate_tui.go | 60 ++++++++++++++++----------------- go.mod | 2 +- go.sum | 4 +-- 6 files changed, 66 insertions(+), 66 deletions(-) diff --git a/cmd/lk/simulate.go b/cmd/lk/simulate.go index e9fd7d61..f1a16575 100644 --- a/cmd/lk/simulate.go +++ b/cmd/lk/simulate.go @@ -946,7 +946,7 @@ func cancelSimulationRun(client *lksdk.AgentSimulationClient, runID string) { } } -// sortedJobs orders a run's jobs so every attempt of a scenario sits together, +// sortedJobs orders a run's jobs so every sample of a scenario sits together, // scenarios in scenarios.yaml order, then by job ID: the backend's ordering // shuffles rows as statuses change. func sortedJobs(run *livekit.SimulationRun) []*livekit.SimulationRun_Job { @@ -961,8 +961,8 @@ func sortedJobs(run *livekit.SimulationRun) []*livekit.SimulationRun_Job { if a.GetScenarioId() != b.GetScenarioId() { return order[a.GetScenarioId()] < order[b.GetScenarioId()] } - if a.GetAttempt() != b.GetAttempt() { - return a.GetAttempt() < b.GetAttempt() + if a.GetSample() != b.GetSample() { + return a.GetSample() < b.GetSample() } return a.GetId() < b.GetId() }) @@ -970,7 +970,7 @@ func sortedJobs(run *livekit.SimulationRun) []*livekit.SimulationRun_Job { } // scenarioPassCounts folds a repeated run's jobs into scenarios: how many -// have every attempt finished, and of those how many passed at least once +// have every sample finished, and of those how many passed at least once // (pass@k) and every time (pass^k). Zero scenarios when the run did not // repeat, so callers can skip the line. func scenarioPassCounts(run *livekit.SimulationRun) (scenarios, passAny, passAll int) { @@ -1012,7 +1012,7 @@ func scenarioPassCounts(run *livekit.SimulationRun) (scenarios, passAny, passAll } // passRateLine is the pass@k / pass^k summary, or "" when the run did not -// repeat or no scenario has finished every attempt. +// repeat or no scenario has finished every sample. func passRateLine(run *livekit.SimulationRun) string { scenarios, passAny, passAll := scenarioPassCounts(run) if scenarios == 0 { @@ -1024,12 +1024,12 @@ func passRateLine(run *livekit.SimulationRun) string { k, passAll, scenarios, float64(passAll)/float64(scenarios)) } -// attemptSuffix marks a job's attempt when the run repeated scenarios. -func attemptSuffix(run *livekit.SimulationRun, job *livekit.SimulationRun_Job) string { - if run.GetSamples() < 2 || job.GetAttempt() == 0 { +// sampleSuffix marks a job's sample when the run repeated scenarios. +func sampleSuffix(run *livekit.SimulationRun, job *livekit.SimulationRun_Job) string { + if run.GetSamples() < 2 || job.GetSample() == 0 { return "" } - return fmt.Sprintf(" (attempt %d/%d)", job.GetAttempt(), run.GetSamples()) + return fmt.Sprintf(" (sample %d/%d)", job.GetSample(), run.GetSamples()) } func simulationJobCounts(run *livekit.SimulationRun) (total, done, passed, failed int) { diff --git a/cmd/lk/simulate_report.go b/cmd/lk/simulate_report.go index c1e7230a..b4557b75 100644 --- a/cmd/lk/simulate_report.go +++ b/cmd/lk/simulate_report.go @@ -186,7 +186,7 @@ func writeRunResults(w io.Writer, run *livekit.SimulationRun, ap *AgentProcess) if label == "" { label = fmt.Sprintf("Job %d", i+1) } - label += attemptSuffix(run, job) + label += sampleSuffix(run, job) fmt.Fprintf(w, "::group::%s %s (%s)\n", icon, label, job.Id) diff --git a/cmd/lk/simulate_samples_test.go b/cmd/lk/simulate_samples_test.go index 0b25ce0b..ca7822f5 100644 --- a/cmd/lk/simulate_samples_test.go +++ b/cmd/lk/simulate_samples_test.go @@ -21,8 +21,8 @@ import ( "github.com/stretchr/testify/require" ) -func attemptJob(id, scenario string, attempt int32, status livekit.SimulationRun_Job_Status) *livekit.SimulationRun_Job { - return &livekit.SimulationRun_Job{Id: id, ScenarioId: scenario, Attempt: attempt, Status: status} +func sampleJob(id, scenario string, sample int32, status livekit.SimulationRun_Job_Status) *livekit.SimulationRun_Job { + return &livekit.SimulationRun_Job{Id: id, ScenarioId: scenario, Sample: sample, Status: status} } func TestScenarioPassCounts(t *testing.T) { @@ -32,10 +32,10 @@ func TestScenarioPassCounts(t *testing.T) { running = livekit.SimulationRun_Job_STATUS_RUNNING ) run := &livekit.SimulationRun{Samples: 2, Jobs: []*livekit.SimulationRun_Job{ - attemptJob("SRJ_1", "SCN_a", 1, done), attemptJob("SRJ_2", "SCN_a", 2, done), // pass^k - attemptJob("SRJ_3", "SCN_b", 1, done), attemptJob("SRJ_4", "SCN_b", 2, failed), // pass@k only - attemptJob("SRJ_5", "SCN_c", 1, failed), attemptJob("SRJ_6", "SCN_c", 2, failed), // neither - attemptJob("SRJ_7", "SCN_d", 1, done), attemptJob("SRJ_8", "SCN_d", 2, running), // not finished + sampleJob("SRJ_1", "SCN_a", 1, done), sampleJob("SRJ_2", "SCN_a", 2, done), // pass^k + sampleJob("SRJ_3", "SCN_b", 1, done), sampleJob("SRJ_4", "SCN_b", 2, failed), // pass@k only + sampleJob("SRJ_5", "SCN_c", 1, failed), sampleJob("SRJ_6", "SCN_c", 2, failed), // neither + sampleJob("SRJ_7", "SCN_d", 1, done), sampleJob("SRJ_8", "SCN_d", 2, running), // not finished }} scenarios, passAny, passAll := scenarioPassCounts(run) @@ -48,15 +48,15 @@ func TestScenarioPassCounts(t *testing.T) { require.Equal(t, "", passRateLine(run)) } -func TestSortedJobs_GroupsAttemptsInScenarioOrder(t *testing.T) { +func TestSortedJobs_GroupsSamplesInScenarioOrder(t *testing.T) { run := &livekit.SimulationRun{ Samples: 2, ScenarioGroup: &livekit.ScenarioGroup{Scenarios: []*livekit.Scenario{{Id: "SCN_b"}, {Id: "SCN_a"}}}, Jobs: []*livekit.SimulationRun_Job{ - attemptJob("SRJ_1", "SCN_a", 2, 0), - attemptJob("SRJ_2", "SCN_b", 2, 0), - attemptJob("SRJ_3", "SCN_a", 1, 0), - attemptJob("SRJ_4", "SCN_b", 1, 0), + sampleJob("SRJ_1", "SCN_a", 2, 0), + sampleJob("SRJ_2", "SCN_b", 2, 0), + sampleJob("SRJ_3", "SCN_a", 1, 0), + sampleJob("SRJ_4", "SCN_b", 1, 0), }, } var ids []string @@ -64,7 +64,7 @@ func TestSortedJobs_GroupsAttemptsInScenarioOrder(t *testing.T) { ids = append(ids, j.Id) } require.Equal(t, []string{"SRJ_4", "SRJ_2", "SRJ_3", "SRJ_1"}, ids) - require.Equal(t, " (attempt 1/2)", attemptSuffix(run, run.Jobs[3])) + require.Equal(t, " (sample 1/2)", sampleSuffix(run, run.Jobs[3])) } func repeatedFixture() *simulateModel { @@ -73,10 +73,10 @@ func repeatedFixture() *simulateModel { Id: "SR_fixture0001", Status: livekit.SimulationRun_STATUS_RUNNING, Samples: 2, ScenarioGroup: &livekit.ScenarioGroup{Scenarios: []*livekit.Scenario{{Id: "SCN_a"}, {Id: "SCN_b"}}}, Jobs: []*livekit.SimulationRun_Job{ - attemptJob("SRJ_a1", "SCN_a", 1, livekit.SimulationRun_Job_STATUS_COMPLETED), - attemptJob("SRJ_a2", "SCN_a", 2, livekit.SimulationRun_Job_STATUS_FAILED), - attemptJob("SRJ_b1", "SCN_b", 1, livekit.SimulationRun_Job_STATUS_COMPLETED), - attemptJob("SRJ_b2", "SCN_b", 2, livekit.SimulationRun_Job_STATUS_RUNNING), + sampleJob("SRJ_a1", "SCN_a", 1, livekit.SimulationRun_Job_STATUS_COMPLETED), + sampleJob("SRJ_a2", "SCN_a", 2, livekit.SimulationRun_Job_STATUS_FAILED), + sampleJob("SRJ_b1", "SCN_b", 1, livekit.SimulationRun_Job_STATUS_COMPLETED), + sampleJob("SRJ_b2", "SCN_b", 2, livekit.SimulationRun_Job_STATUS_RUNNING), }, } for _, j := range m.run.Jobs { @@ -85,7 +85,7 @@ func repeatedFixture() *simulateModel { return m } -func TestFilteredJobs_RepeatedRunNestsAttempts(t *testing.T) { +func TestFilteredJobs_RepeatedRunNestsSamples(t *testing.T) { rows := repeatedFixture().filteredJobs() var ids []string for _, r := range rows { @@ -93,14 +93,14 @@ func TestFilteredJobs_RepeatedRunNestsAttempts(t *testing.T) { } require.Equal(t, []string{"SCN_a", "SRJ_a1", "SRJ_a2", "SCN_b", "SRJ_b1", "SRJ_b2"}, ids) require.Equal(t, 1, rows[0].origIdx) - require.Equal(t, 0, rows[1].origIdx, "attempt rows carry no number") + require.Equal(t, 0, rows[1].origIdx, "sample rows carry no number") require.True(t, rows[2].last) require.Equal(t, 2, rows[3].origIdx) - icon, _ := scenarioStatusIcon(rows[0].scenario.attempts) - require.Equal(t, '✗', icon, "a scenario with a failed attempt is not passed") - icon, _ = scenarioStatusIcon(rows[3].scenario.attempts) - require.Equal(t, '⏺', icon, "a scenario still runs while any attempt does") + icon, _ := scenarioStatusIcon(rows[0].scenario.samples) + require.Equal(t, '✗', icon, "a scenario with a failed sample is not passed") + icon, _ = scenarioStatusIcon(rows[3].scenario.samples) + require.Equal(t, '⏺', icon, "a scenario still runs while any sample does") } func TestEnterOnScenarioRowOpensTheScenario(t *testing.T) { @@ -109,7 +109,7 @@ func TestEnterOnScenarioRowOpensTheScenario(t *testing.T) { m.Update(keyPress("enter")) require.Equal(t, "SCN_a", m.detailID) require.NotNil(t, m.findScenario(m.detailID)) - require.Contains(t, m.renderDetail(), "1/2 attempts passed") + require.Contains(t, m.renderDetail(), "1/2 samples passed") m.closeDetailCmd() m.cursor = 2 diff --git a/cmd/lk/simulate_tui.go b/cmd/lk/simulate_tui.go index 3cd8675f..a1deb2c2 100644 --- a/cmd/lk/simulate_tui.go +++ b/cmd/lk/simulate_tui.go @@ -358,7 +358,7 @@ func (m *simulateModel) copyScenario(jobID string) (string, bool) { job := m.findJob(jobID) if job == nil { if sc := m.findScenario(jobID); sc != nil { - job = sc.attempts[0] + job = sc.samples[0] } } if job == nil { @@ -1022,22 +1022,22 @@ func (m *simulateModel) handleKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { } // indexedJob is one row of the job list. An unrepeated run has a row per job. -// A repeated run has a row per scenario with its attempts nested under it: +// A repeated run has a row per scenario with its samples nested under it: // // ✓ 1. Book a table for two 3/3 -// ├─ ✓ SRJ_01 attempt 1 -// └─ ✓ SRJ_09 attempt 3 +// ├─ ✓ SRJ_01 sample 1 +// └─ ✓ SRJ_09 sample 3 type indexedJob struct { - origIdx int // 1-based number of the scenario or, unrepeated, the job; 0 on an attempt row + origIdx int // 1-based number of the scenario or, unrepeated, the job; 0 on a sample row job *livekit.SimulationRun_Job // nil on a scenario row scenario *scenarioRow // set on a scenario row only - last bool // attempt row: the scenario's final attempt (└ rather than ├) + last bool // sample row: the scenario's final sample (└ rather than ├) } type scenarioRow struct { - id string - label string - attempts []*livekit.SimulationRun_Job + id string + label string + samples []*livekit.SimulationRun_Job } // id is what opening the row shows: a job, or a whole scenario. @@ -1071,12 +1071,12 @@ func (m *simulateModel) filteredJobs() []indexedJob { } sc := &scenarioRow{id: j.GetScenarioId(), label: jobLabel(j)} for i < len(jobs) && jobs[i].GetScenarioId() == sc.id { - sc.attempts = append(sc.attempts, jobs[i]) + sc.samples = append(sc.samples, jobs[i]) i++ } result = append(result, indexedJob{origIdx: n, scenario: sc}) - for k, a := range sc.attempts { - result = append(result, indexedJob{job: a, last: k == len(sc.attempts)-1}) + for k, a := range sc.samples { + result = append(result, indexedJob{job: a, last: k == len(sc.samples)-1}) } } return result @@ -1091,12 +1091,12 @@ func (m *simulateModel) findScenario(id string) *scenarioRow { return nil } -// scenarioStatusIcon folds a scenario's attempts: running while any attempt -// runs, pending while any waits, then passed only if every attempt did. -func scenarioStatusIcon(attempts []*livekit.SimulationRun_Job) (rune, *lipgloss.Style) { +// scenarioStatusIcon folds a scenario's samples: running while any sample +// runs, pending while any waits, then passed only if every sample did. +func scenarioStatusIcon(samples []*livekit.SimulationRun_Job) (rune, *lipgloss.Style) { passed := 0 pending := false - for _, a := range attempts { + for _, a := range samples { switch a.Status { case livekit.SimulationRun_Job_STATUS_RUNNING: s := yellowStyle() @@ -1111,7 +1111,7 @@ func scenarioStatusIcon(attempts []*livekit.SimulationRun_Job) (rune, *lipgloss. if pending { return '⏺', &dimStyle } - if passed == len(attempts) { + if passed == len(samples) { s := greenStyle() return '✓', &s } @@ -1119,9 +1119,9 @@ func scenarioStatusIcon(attempts []*livekit.SimulationRun_Job) (rune, *lipgloss. return '✗', &s } -func scenarioPassed(attempts []*livekit.SimulationRun_Job) int { +func scenarioPassed(samples []*livekit.SimulationRun_Job) int { n := 0 - for _, a := range attempts { + for _, a := range samples { if a.Status == livekit.SimulationRun_Job_STATUS_COMPLETED { n++ } @@ -1135,8 +1135,8 @@ func scenarioPassed(attempts []*livekit.SimulationRun_Job) int { func listRowParts(ij indexedJob) (indent string, iconCh rune, iconStyle *lipgloss.Style, text, styled string) { switch { case ij.scenario != nil: - iconCh, iconStyle = scenarioStatusIcon(ij.scenario.attempts) - text = fmt.Sprintf(" %3d. %s %d/%d", ij.origIdx, ij.scenario.label, scenarioPassed(ij.scenario.attempts), len(ij.scenario.attempts)) + iconCh, iconStyle = scenarioStatusIcon(ij.scenario.samples) + text = fmt.Sprintf(" %3d. %s %d/%d", ij.origIdx, ij.scenario.label, scenarioPassed(ij.scenario.samples), len(ij.scenario.samples)) return " ", iconCh, iconStyle, text, text case ij.origIdx == 0: connector := "├─ " @@ -1144,8 +1144,8 @@ func listRowParts(ij indexedJob) (indent string, iconCh rune, iconStyle *lipglos connector = "└─ " } iconCh, iconStyle = jobStatusIcon(ij.job) - text = fmt.Sprintf(" %s attempt %d", ij.job.Id, ij.job.GetAttempt()) - styled = fmt.Sprintf(" %s attempt %d", dimStyle.Render(ij.job.Id), ij.job.GetAttempt()) + text = fmt.Sprintf(" %s sample %d", ij.job.Id, ij.job.GetSample()) + styled = fmt.Sprintf(" %s sample %d", dimStyle.Render(ij.job.Id), ij.job.GetSample()) return " " + dimStyle.Render(connector), iconCh, iconStyle, text, styled default: iconCh, iconStyle = jobStatusIcon(ij.job) @@ -1713,7 +1713,7 @@ func (m *simulateModel) renderDetail() string { title := fmt.Sprintf("Job %d", origIdx) if m.run.GetSamples() >= 2 { - title = jobLabel(job) + attemptSuffix(m.run, job) + title = jobLabel(job) + sampleSuffix(m.run, job) } var b strings.Builder b.WriteString("\n") @@ -1801,20 +1801,20 @@ func (m *simulateModel) renderDetail() string { } // renderScenarioDetail is a repeated scenario as a whole: its brief once, then -// every attempt's verdict and transcript in attempt order. +// every sample's verdict and transcript in sample order. func (m *simulateModel) renderScenarioDetail(sc *scenarioRow) string { var b strings.Builder - iconCh, iconStyle := scenarioStatusIcon(sc.attempts) + iconCh, iconStyle := scenarioStatusIcon(sc.samples) b.WriteString("\n") fmt.Fprintf(&b, " %s %s %s\n", iconStyle.Render(string(iconCh)), boldStyle.Render(sc.label), - dimStyle.Render(fmt.Sprintf("%d/%d attempts passed", scenarioPassed(sc.attempts), len(sc.attempts))), + dimStyle.Render(fmt.Sprintf("%d/%d samples passed", scenarioPassed(sc.samples), len(sc.samples))), ) b.WriteString("\n") wrapWidth := proseWidth(m.width, 6) - first := sc.attempts[0] + first := sc.samples[0] b.WriteString(boldStyle.Render(" Instructions:")) b.WriteString("\n") @@ -1837,11 +1837,11 @@ func (m *simulateModel) renderScenarioDetail(sc *scenarioRow) string { b.WriteString(dimStyle.Render(" "+line) + "\n") } - for _, job := range sc.attempts { + for _, job := range sc.samples { b.WriteString("\n") fmt.Fprintf(&b, " %s %s %s\n", jobIcon(job), - boldStyle.Render(fmt.Sprintf("Attempt %d", job.GetAttempt())), + boldStyle.Render(fmt.Sprintf("Sample %d", job.GetSample())), dimStyle.Render(job.Id), ) if job.Error != "" { diff --git a/go.mod b/go.mod index 1094c8a7..66a2c98a 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/google/go-querystring v1.2.0 github.com/joho/godotenv v1.5.1 github.com/klauspost/compress v1.20.0 - github.com/livekit/protocol v1.51.1-0.20260915182316-f9e48c3cf763 + github.com/livekit/protocol v1.51.1-0.20260915183127-a58538af6196 github.com/livekit/server-sdk-go/v2 v2.18.2-0.20260904062056-1da58cd7b795 github.com/mattn/go-isatty v0.0.22 github.com/moby/moby/client v0.4.1 diff --git a/go.sum b/go.sum index fbcf321e..86774159 100644 --- a/go.sum +++ b/go.sum @@ -370,8 +370,8 @@ github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5AT github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095 h1:BcliKAXoMhl/nWmzQweQ5kmh4Qqagxl4s3Z5pvM/7AY= github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095/go.mod h1:o8CFmAdrVwzJNOCsQCLUzXRjokkufNshnQHOe4fRaqU= -github.com/livekit/protocol v1.51.1-0.20260915182316-f9e48c3cf763 h1:VfQ2psqfg2GOrAkSv5OrnEu890rqJNEaI3oKcJfPlPQ= -github.com/livekit/protocol v1.51.1-0.20260915182316-f9e48c3cf763/go.mod h1:vlreOPeiOr2pSCVCHDr2xIbSGf+2++y4QFaTgZ6rjp4= +github.com/livekit/protocol v1.51.1-0.20260915183127-a58538af6196 h1:xWDQTyZ4pb2ZMckIK44kp164DEHH9iWhcjBAVV/BsmA= +github.com/livekit/protocol v1.51.1-0.20260915183127-a58538af6196/go.mod h1:vlreOPeiOr2pSCVCHDr2xIbSGf+2++y4QFaTgZ6rjp4= github.com/livekit/psrpc v0.7.6 h1:YG07lUMTtf+eaYI2goT9zcVZ0kGJNWN1K6ETNFtv1HQ= github.com/livekit/psrpc v0.7.6/go.mod h1:DMw15RO7x5XmcgfwzWJYk2In605kx+wu1QRVbPfzf8M= github.com/livekit/server-sdk-go/v2 v2.18.2-0.20260904062056-1da58cd7b795 h1:0gljvZ5rt8vSLgoyaQl3ocD9D6RQt4Iyn4kXjVr4un0= From d053337f8a0683926833ff2eee5b9e1a541d4422 Mon Sep 17 00:00:00 2001 From: Jason Lernerman Date: Tue, 15 Sep 2026 15:15:15 -0400 Subject: [PATCH 08/13] deps: bump protocol to the rebased simulation-repeats branch --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 66a2c98a..7f68866c 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/google/go-querystring v1.2.0 github.com/joho/godotenv v1.5.1 github.com/klauspost/compress v1.20.0 - github.com/livekit/protocol v1.51.1-0.20260915183127-a58538af6196 + github.com/livekit/protocol v1.51.1-0.20260915193415-be5b074d9407 github.com/livekit/server-sdk-go/v2 v2.18.2-0.20260904062056-1da58cd7b795 github.com/mattn/go-isatty v0.0.22 github.com/moby/moby/client v0.4.1 diff --git a/go.sum b/go.sum index 86774159..a56c8c62 100644 --- a/go.sum +++ b/go.sum @@ -370,8 +370,8 @@ github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5AT github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095 h1:BcliKAXoMhl/nWmzQweQ5kmh4Qqagxl4s3Z5pvM/7AY= github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095/go.mod h1:o8CFmAdrVwzJNOCsQCLUzXRjokkufNshnQHOe4fRaqU= -github.com/livekit/protocol v1.51.1-0.20260915183127-a58538af6196 h1:xWDQTyZ4pb2ZMckIK44kp164DEHH9iWhcjBAVV/BsmA= -github.com/livekit/protocol v1.51.1-0.20260915183127-a58538af6196/go.mod h1:vlreOPeiOr2pSCVCHDr2xIbSGf+2++y4QFaTgZ6rjp4= +github.com/livekit/protocol v1.51.1-0.20260915193415-be5b074d9407 h1:Cg9gZfRL761Gu5IOvVXE8zaibG39bPO4NBMnDTHhlUI= +github.com/livekit/protocol v1.51.1-0.20260915193415-be5b074d9407/go.mod h1:vlreOPeiOr2pSCVCHDr2xIbSGf+2++y4QFaTgZ6rjp4= github.com/livekit/psrpc v0.7.6 h1:YG07lUMTtf+eaYI2goT9zcVZ0kGJNWN1K6ETNFtv1HQ= github.com/livekit/psrpc v0.7.6/go.mod h1:DMw15RO7x5XmcgfwzWJYk2In605kx+wu1QRVbPfzf8M= github.com/livekit/server-sdk-go/v2 v2.18.2-0.20260904062056-1da58cd7b795 h1:0gljvZ5rt8vSLgoyaQl3ocD9D6RQt4Iyn4kXjVr4un0= From e867b8ec32cda6bebe601cb2c1efcc056ea1001d Mon Sep 17 00:00:00 2001 From: Jason Lernerman Date: Fri, 18 Sep 2026 11:25:13 -0400 Subject: [PATCH 09/13] deps: follow samples into the protocol's Sampling message No behaviour change: the field moved, the run still requires every sample. --- cmd/lk/simulate.go | 20 +++++++++++++++----- cmd/lk/simulate_samples_test.go | 12 ++++++++---- cmd/lk/simulate_tui.go | 4 ++-- go.mod | 2 +- go.sum | 2 ++ 5 files changed, 28 insertions(+), 12 deletions(-) diff --git a/cmd/lk/simulate.go b/cmd/lk/simulate.go index f1a16575..b4bbcd89 100644 --- a/cmd/lk/simulate.go +++ b/cmd/lk/simulate.go @@ -772,7 +772,9 @@ func createSimulationRun(ctx context.Context, c *simulateConfig) (string, *livek } if c.mode == modeScenarios { req.ScenarioGroup = c.scenarioGroup - req.Samples = c.samples + if c.samples > 1 { + req.Sampling = &livekit.SimulationRun_Sampling{Samples: c.samples} + } } resp, err := c.client.CreateSimulationRun(ctx, req) @@ -973,8 +975,16 @@ func sortedJobs(run *livekit.SimulationRun) []*livekit.SimulationRun_Job { // have every sample finished, and of those how many passed at least once // (pass@k) and every time (pass^k). Zero scenarios when the run did not // repeat, so callers can skip the line. +// runSamples is the run's samples per scenario, 1 when it did not repeat. +func runSamples(run *livekit.SimulationRun) int32 { + if k := run.GetSampling().GetSamples(); k > 1 { + return k + } + return 1 +} + func scenarioPassCounts(run *livekit.SimulationRun) (scenarios, passAny, passAll int) { - k := int(run.GetSamples()) + k := int(runSamples(run)) if k < 2 { return } @@ -1018,7 +1028,7 @@ func passRateLine(run *livekit.SimulationRun) string { if scenarios == 0 { return "" } - k := run.GetSamples() + k := runSamples(run) return fmt.Sprintf("pass@%d %d/%d (%.2f), pass^%d %d/%d (%.2f)", k, passAny, scenarios, float64(passAny)/float64(scenarios), k, passAll, scenarios, float64(passAll)/float64(scenarios)) @@ -1026,10 +1036,10 @@ func passRateLine(run *livekit.SimulationRun) string { // sampleSuffix marks a job's sample when the run repeated scenarios. func sampleSuffix(run *livekit.SimulationRun, job *livekit.SimulationRun_Job) string { - if run.GetSamples() < 2 || job.GetSample() == 0 { + if runSamples(run) < 2 || job.GetSample() == 0 { return "" } - return fmt.Sprintf(" (sample %d/%d)", job.GetSample(), run.GetSamples()) + return fmt.Sprintf(" (sample %d/%d)", job.GetSample(), runSamples(run)) } func simulationJobCounts(run *livekit.SimulationRun) (total, done, passed, failed int) { diff --git a/cmd/lk/simulate_samples_test.go b/cmd/lk/simulate_samples_test.go index ca7822f5..a9ace58a 100644 --- a/cmd/lk/simulate_samples_test.go +++ b/cmd/lk/simulate_samples_test.go @@ -25,13 +25,17 @@ func sampleJob(id, scenario string, sample int32, status livekit.SimulationRun_J return &livekit.SimulationRun_Job{Id: id, ScenarioId: scenario, Sample: sample, Status: status} } +func sampling(k int32, rate float64) *livekit.SimulationRun_Sampling { + return &livekit.SimulationRun_Sampling{Samples: k, PassRate: rate} +} + func TestScenarioPassCounts(t *testing.T) { const ( done = livekit.SimulationRun_Job_STATUS_COMPLETED failed = livekit.SimulationRun_Job_STATUS_FAILED running = livekit.SimulationRun_Job_STATUS_RUNNING ) - run := &livekit.SimulationRun{Samples: 2, Jobs: []*livekit.SimulationRun_Job{ + run := &livekit.SimulationRun{Sampling: sampling(2, 1), Jobs: []*livekit.SimulationRun_Job{ sampleJob("SRJ_1", "SCN_a", 1, done), sampleJob("SRJ_2", "SCN_a", 2, done), // pass^k sampleJob("SRJ_3", "SCN_b", 1, done), sampleJob("SRJ_4", "SCN_b", 2, failed), // pass@k only sampleJob("SRJ_5", "SCN_c", 1, failed), sampleJob("SRJ_6", "SCN_c", 2, failed), // neither @@ -44,13 +48,13 @@ func TestScenarioPassCounts(t *testing.T) { require.Equal(t, 1, passAll) require.Equal(t, "pass@2 2/3 (0.67), pass^2 1/3 (0.33)", passRateLine(run)) - run.Samples = 1 + run.Sampling = nil require.Equal(t, "", passRateLine(run)) } func TestSortedJobs_GroupsSamplesInScenarioOrder(t *testing.T) { run := &livekit.SimulationRun{ - Samples: 2, + Sampling: sampling(2, 1), ScenarioGroup: &livekit.ScenarioGroup{Scenarios: []*livekit.Scenario{{Id: "SCN_b"}, {Id: "SCN_a"}}}, Jobs: []*livekit.SimulationRun_Job{ sampleJob("SRJ_1", "SCN_a", 2, 0), @@ -70,7 +74,7 @@ func TestSortedJobs_GroupsSamplesInScenarioOrder(t *testing.T) { func repeatedFixture() *simulateModel { m := runningFixture() m.run = &livekit.SimulationRun{ - Id: "SR_fixture0001", Status: livekit.SimulationRun_STATUS_RUNNING, Samples: 2, + Id: "SR_fixture0001", Status: livekit.SimulationRun_STATUS_RUNNING, Sampling: sampling(2, 1), ScenarioGroup: &livekit.ScenarioGroup{Scenarios: []*livekit.Scenario{{Id: "SCN_a"}, {Id: "SCN_b"}}}, Jobs: []*livekit.SimulationRun_Job{ sampleJob("SRJ_a1", "SCN_a", 1, livekit.SimulationRun_Job_STATUS_COMPLETED), diff --git a/cmd/lk/simulate_tui.go b/cmd/lk/simulate_tui.go index a1deb2c2..b39b8841 100644 --- a/cmd/lk/simulate_tui.go +++ b/cmd/lk/simulate_tui.go @@ -1054,7 +1054,7 @@ func (m *simulateModel) filteredJobs() []indexedJob { } jobs := sortedJobs(m.run) result := make([]indexedJob, 0, len(jobs)) - if m.run.GetSamples() < 2 { + if runSamples(m.run) < 2 { for i, j := range jobs { result = append(result, indexedJob{origIdx: i + 1, job: j}) } @@ -1712,7 +1712,7 @@ func (m *simulateModel) renderDetail() string { } title := fmt.Sprintf("Job %d", origIdx) - if m.run.GetSamples() >= 2 { + if runSamples(m.run) >= 2 { title = jobLabel(job) + sampleSuffix(m.run, job) } var b strings.Builder diff --git a/go.mod b/go.mod index 7f68866c..cf95d1ba 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/google/go-querystring v1.2.0 github.com/joho/godotenv v1.5.1 github.com/klauspost/compress v1.20.0 - github.com/livekit/protocol v1.51.1-0.20260915193415-be5b074d9407 + github.com/livekit/protocol v1.51.1-0.20260918145636-43aaf61f0f22 github.com/livekit/server-sdk-go/v2 v2.18.2-0.20260904062056-1da58cd7b795 github.com/mattn/go-isatty v0.0.22 github.com/moby/moby/client v0.4.1 diff --git a/go.sum b/go.sum index a56c8c62..77aa10f8 100644 --- a/go.sum +++ b/go.sum @@ -372,6 +372,8 @@ github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095 h1:Bcli github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095/go.mod h1:o8CFmAdrVwzJNOCsQCLUzXRjokkufNshnQHOe4fRaqU= github.com/livekit/protocol v1.51.1-0.20260915193415-be5b074d9407 h1:Cg9gZfRL761Gu5IOvVXE8zaibG39bPO4NBMnDTHhlUI= github.com/livekit/protocol v1.51.1-0.20260915193415-be5b074d9407/go.mod h1:vlreOPeiOr2pSCVCHDr2xIbSGf+2++y4QFaTgZ6rjp4= +github.com/livekit/protocol v1.51.1-0.20260918145636-43aaf61f0f22 h1:4ijb2Q6hWZHG2lMjiUWMkNALXPx+jsL78VPalyM1Eys= +github.com/livekit/protocol v1.51.1-0.20260918145636-43aaf61f0f22/go.mod h1:vlreOPeiOr2pSCVCHDr2xIbSGf+2++y4QFaTgZ6rjp4= github.com/livekit/psrpc v0.7.6 h1:YG07lUMTtf+eaYI2goT9zcVZ0kGJNWN1K6ETNFtv1HQ= github.com/livekit/psrpc v0.7.6/go.mod h1:DMw15RO7x5XmcgfwzWJYk2In605kx+wu1QRVbPfzf8M= github.com/livekit/server-sdk-go/v2 v2.18.2-0.20260904062056-1da58cd7b795 h1:0gljvZ5rt8vSLgoyaQl3ocD9D6RQt4Iyn4kXjVr4un0= From 4dc8b12d29b332386a3c64af576bdb2725cd71bc Mon Sep 17 00:00:00 2001 From: Jason Lernerman Date: Fri, 18 Sep 2026 11:25:37 -0400 Subject: [PATCH 10/13] simulate: --pass-rate decides a scenario's verdict A scenario was all-or-nothing, so sampling it could only ever make it harder to pass and a single flaky sample failed the run. --pass-rate is the share of a scenario's samples that must pass, rounded to whole samples and floored at one, defaulting to 0.75 so a scenario may flake once in three. The run fails on scenarios below the bar rather than on failed samples; pass@k and pass^k still only report. The TUI settles a scenario's icon as soon as the remaining samples cannot change the verdict, so 2 of 3 goes green on the second pass. --- cmd/lk/simulate.go | 71 ++++++++++++++++++++++++++++++--- cmd/lk/simulate_ci.go | 16 +++++--- cmd/lk/simulate_samples_test.go | 55 +++++++++++++++++++++++-- cmd/lk/simulate_tui.go | 36 ++++++++++------- 4 files changed, 149 insertions(+), 29 deletions(-) diff --git a/cmd/lk/simulate.go b/cmd/lk/simulate.go index b4bbcd89..0a6f09d0 100644 --- a/cmd/lk/simulate.go +++ b/cmd/lk/simulate.go @@ -20,6 +20,7 @@ import ( "encoding/json" "fmt" "io" + "math" "math/rand" "os" "path/filepath" @@ -87,6 +88,11 @@ var simulateCommand = &cli.Command{ Aliases: []string{"k"}, Usage: "Run every scenario `K` times and report pass@k (passed at least once) and pass^k (passed every time). Requires --scenarios", }, + &cli.FloatFlag{ + Name: "pass-rate", + Value: defaultPassRate, + Usage: "Share of a scenario's samples that must pass for it to pass, as a `RATE` from 0 to 1. Rounds to whole samples, so 0.75 of 3 needs 2. Requires --samples", + }, &cli.StringFlag{ Name: "scenarios", Usage: "Path to a scenarios `FILE` (yaml). If omitted, scenarios are generated from the agent's source", @@ -388,6 +394,7 @@ type simulateConfig struct { numSimulations int32 concurrency int32 samples int32 + passRate float64 mode simulateMode simulationMode livekit.SimulationMode agentName string @@ -495,6 +502,7 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S numSimulations := int32(cmd.Int("num-simulations")) concurrency := int32(cmd.Int("concurrency")) samples := int32(cmd.Int("samples")) + passRate := cmd.Float("pass-rate") liveAgentName := cmd.String("agent-name") // never auto-discovered: an explicit --scenarios file is the source of @@ -503,6 +511,12 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S if samples > 1 && scenariosPath == "" { return fmt.Errorf("--samples requires --scenarios (generated scenarios are not sampled more than once)") } + if cmd.IsSet("pass-rate") && samples < 2 { + return fmt.Errorf("--pass-rate requires --samples (a single sample must always pass)") + } + if passRate <= 0 || passRate > 1 { + return fmt.Errorf("--pass-rate must be greater than 0 and at most 1 (got %g)", passRate) + } var ( agentName string @@ -576,6 +590,7 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S numSimulations: numSimulations, concurrency: concurrency, samples: samples, + passRate: passRate, mode: mode, simulationMode: simulationMode, agentName: agentName, @@ -773,7 +788,7 @@ func createSimulationRun(ctx context.Context, c *simulateConfig) (string, *livek if c.mode == modeScenarios { req.ScenarioGroup = c.scenarioGroup if c.samples > 1 { - req.Sampling = &livekit.SimulationRun_Sampling{Samples: c.samples} + req.Sampling = &livekit.SimulationRun_Sampling{Samples: c.samples, PassRate: c.passRate} } } @@ -971,10 +986,10 @@ func sortedJobs(run *livekit.SimulationRun) []*livekit.SimulationRun_Job { return jobs } -// scenarioPassCounts folds a repeated run's jobs into scenarios: how many -// have every sample finished, and of those how many passed at least once -// (pass@k) and every time (pass^k). Zero scenarios when the run did not -// repeat, so callers can skip the line. +// defaultPassRate lets a scenario flake once in three before it fails. At +// three samples every rate in [0.5, 0.833] rounds to the same two samples. +const defaultPassRate = 0.75 + // runSamples is the run's samples per scenario, 1 when it did not repeat. func runSamples(run *livekit.SimulationRun) int32 { if k := run.GetSampling().GetSamples(); k > 1 { @@ -983,6 +998,52 @@ func runSamples(run *livekit.SimulationRun) int32 { return 1 } +// requiredSamples is how many of a scenario's samples must pass for the +// scenario to pass. Always at least one: a rate low enough to round to zero +// would pass a scenario that never succeeded. +func requiredSamples(run *livekit.SimulationRun) int { + k := runSamples(run) + if k < 2 { + return 1 + } + return max(1, int(math.Round(run.GetSampling().GetPassRate()*float64(k)))) +} + +// scenarioFailureCounts is the run's verdict: how many scenarios it holds and +// how many of them finished below the pass rate. An unsampled run has one job +// per scenario, so this counts failed jobs exactly as it always did. +func scenarioFailureCounts(run *livekit.SimulationRun) (scenarios, failed int) { + required := requiredSamples(run) + byScenario := map[string][]*livekit.SimulationRun_Job{} + for _, j := range run.GetJobs() { + key := j.GetScenarioId() + if key == "" { + key = j.GetId() + } + byScenario[key] = append(byScenario[key], j) + } + for _, samples := range byScenario { + scenarios++ + passed, unfinished := 0, 0 + for _, j := range samples { + switch { + case j.Status == livekit.SimulationRun_Job_STATUS_COMPLETED: + passed++ + case !isTerminalJobStatus(j.Status): + unfinished++ + } + } + if passed < required && passed+unfinished < required { + failed++ + } + } + return +} + +// scenarioPassCounts folds a repeated run's jobs into scenarios: how many +// have every sample finished, and of those how many passed at least once +// (pass@k) and every time (pass^k). Zero scenarios when the run did not +// repeat, so callers can skip the line. func scenarioPassCounts(run *livekit.SimulationRun) (scenarios, passAny, passAll int) { k := int(runSamples(run)) if k < 2 { diff --git a/cmd/lk/simulate_ci.go b/cmd/lk/simulate_ci.go index f88a320d..4bf9c2ac 100644 --- a/cmd/lk/simulate_ci.go +++ b/cmd/lk/simulate_ci.go @@ -228,12 +228,16 @@ func runSimulateCI(ctx context.Context, config *simulateConfig) error { // full dump above already carries the detail. Returns nil when everything // passed. func runFailureError(run *livekit.SimulationRun) error { - _, _, _, failed := simulationJobCounts(run) - if failed > 0 || run.Status == livekit.SimulationRun_STATUS_FAILED { - if run.Status == livekit.SimulationRun_STATUS_FAILED && len(run.Jobs) == 0 { - return fmt.Errorf("simulation failed: %s", run.Error) - } - return fmt.Errorf("%d of %d simulations failed", failed, len(run.Jobs)) + if run.Status == livekit.SimulationRun_STATUS_FAILED && len(run.Jobs) == 0 { + return fmt.Errorf("simulation failed: %s", run.Error) + } + // A sampled scenario fails only below its pass rate, so a failed sample no + // longer fails the run on its own. + if scenarios, failed := scenarioFailureCounts(run); failed > 0 { + return fmt.Errorf("%d of %d scenarios failed", failed, scenarios) + } + if run.Status == livekit.SimulationRun_STATUS_FAILED { + return fmt.Errorf("simulation failed: %s", run.Error) } return nil diff --git a/cmd/lk/simulate_samples_test.go b/cmd/lk/simulate_samples_test.go index a9ace58a..6161bbe5 100644 --- a/cmd/lk/simulate_samples_test.go +++ b/cmd/lk/simulate_samples_test.go @@ -54,7 +54,7 @@ func TestScenarioPassCounts(t *testing.T) { func TestSortedJobs_GroupsSamplesInScenarioOrder(t *testing.T) { run := &livekit.SimulationRun{ - Sampling: sampling(2, 1), + Sampling: sampling(2, defaultPassRate), ScenarioGroup: &livekit.ScenarioGroup{Scenarios: []*livekit.Scenario{{Id: "SCN_b"}, {Id: "SCN_a"}}}, Jobs: []*livekit.SimulationRun_Job{ sampleJob("SRJ_1", "SCN_a", 2, 0), @@ -74,7 +74,7 @@ func TestSortedJobs_GroupsSamplesInScenarioOrder(t *testing.T) { func repeatedFixture() *simulateModel { m := runningFixture() m.run = &livekit.SimulationRun{ - Id: "SR_fixture0001", Status: livekit.SimulationRun_STATUS_RUNNING, Sampling: sampling(2, 1), + Id: "SR_fixture0001", Status: livekit.SimulationRun_STATUS_RUNNING, Sampling: sampling(2, defaultPassRate), ScenarioGroup: &livekit.ScenarioGroup{Scenarios: []*livekit.Scenario{{Id: "SCN_a"}, {Id: "SCN_b"}}}, Jobs: []*livekit.SimulationRun_Job{ sampleJob("SRJ_a1", "SCN_a", 1, livekit.SimulationRun_Job_STATUS_COMPLETED), @@ -101,9 +101,10 @@ func TestFilteredJobs_RepeatedRunNestsSamples(t *testing.T) { require.True(t, rows[2].last) require.Equal(t, 2, rows[3].origIdx) - icon, _ := scenarioStatusIcon(rows[0].scenario.samples) + required := requiredSamples(repeatedFixture().run) + icon, _ := scenarioStatusIcon(rows[0].scenario.samples, required) require.Equal(t, '✗', icon, "a scenario with a failed sample is not passed") - icon, _ = scenarioStatusIcon(rows[3].scenario.samples) + icon, _ = scenarioStatusIcon(rows[3].scenario.samples, required) require.Equal(t, '⏺', icon, "a scenario still runs while any sample does") } @@ -120,3 +121,49 @@ func TestEnterOnScenarioRowOpensTheScenario(t *testing.T) { m.Update(keyPress("enter")) require.Equal(t, "SRJ_a2", m.detailID) } + +func TestRequiredSamples_RoundsToWholeSamples(t *testing.T) { + require.Equal(t, 1, requiredSamples(nil), "an unsampled run needs its one sample") + require.Equal(t, 1, requiredSamples(&livekit.SimulationRun{Sampling: sampling(1, 0.5)})) + + // Every rate in [0.5, 0.833] is the same gate at three samples. + for _, rate := range []float64{0.5, 0.67, 0.75, 0.8} { + require.Equal(t, 2, requiredSamples(&livekit.SimulationRun{Sampling: sampling(3, rate)}), + "rate %v of 3", rate) + } + require.Equal(t, 1, requiredSamples(&livekit.SimulationRun{Sampling: sampling(3, 0.34)})) + require.Equal(t, 3, requiredSamples(&livekit.SimulationRun{Sampling: sampling(3, 1)})) + + // A rate that rounds to zero would pass a scenario that never succeeded. + require.Equal(t, 1, requiredSamples(&livekit.SimulationRun{Sampling: sampling(3, 0.01)})) +} + +func TestScenarioFailureCounts_GatesOnThePassRate(t *testing.T) { + const ( + done = livekit.SimulationRun_Job_STATUS_COMPLETED + failed = livekit.SimulationRun_Job_STATUS_FAILED + running = livekit.SimulationRun_Job_STATUS_RUNNING + ) + // 0.75 of 3 needs 2: SCN_a flakes once and passes, SCN_b does not. + run := &livekit.SimulationRun{Sampling: sampling(3, defaultPassRate), Jobs: []*livekit.SimulationRun_Job{ + sampleJob("SRJ_1", "SCN_a", 1, done), sampleJob("SRJ_2", "SCN_a", 2, failed), sampleJob("SRJ_3", "SCN_a", 3, done), + sampleJob("SRJ_4", "SCN_b", 1, done), sampleJob("SRJ_5", "SCN_b", 2, failed), sampleJob("SRJ_6", "SCN_b", 3, failed), + }} + scenarios, failedScenarios := scenarioFailureCounts(run) + require.Equal(t, 2, scenarios) + require.Equal(t, 1, failedScenarios) + require.EqualError(t, runFailureError(run), "1 of 2 scenarios failed") + + // A scenario that can still reach the bar is not yet a failure. + run.Jobs[5].Status = running + _, failedScenarios = scenarioFailureCounts(run) + require.Equal(t, 0, failedScenarios) + + // Without sampling every job is its own scenario and one failure fails it. + plain := &livekit.SimulationRun{Jobs: []*livekit.SimulationRun_Job{ + {Id: "SRJ_1", Status: done}, {Id: "SRJ_2", Status: failed}, + }} + scenarios, failedScenarios = scenarioFailureCounts(plain) + require.Equal(t, 2, scenarios) + require.Equal(t, 1, failedScenarios) +} diff --git a/cmd/lk/simulate_tui.go b/cmd/lk/simulate_tui.go index b39b8841..25a7e2ba 100644 --- a/cmd/lk/simulate_tui.go +++ b/cmd/lk/simulate_tui.go @@ -1093,9 +1093,8 @@ func (m *simulateModel) findScenario(id string) *scenarioRow { // scenarioStatusIcon folds a scenario's samples: running while any sample // runs, pending while any waits, then passed only if every sample did. -func scenarioStatusIcon(samples []*livekit.SimulationRun_Job) (rune, *lipgloss.Style) { - passed := 0 - pending := false +func scenarioStatusIcon(samples []*livekit.SimulationRun_Job, required int) (rune, *lipgloss.Style) { + passed, unfinished := 0, 0 for _, a := range samples { switch a.Status { case livekit.SimulationRun_Job_STATUS_RUNNING: @@ -1105,16 +1104,18 @@ func scenarioStatusIcon(samples []*livekit.SimulationRun_Job) (rune, *lipgloss.S passed++ case livekit.SimulationRun_Job_STATUS_FAILED, livekit.SimulationRun_Job_STATUS_CANCELLED: default: - pending = true + unfinished++ } } - if pending { - return '⏺', &dimStyle - } - if passed == len(samples) { + // The verdict is settled as soon as the remaining samples cannot change it, + // so a scenario needing 2 of 3 goes green on its second pass. + if passed >= required { s := greenStyle() return '✓', &s } + if passed+unfinished >= required { + return '⏺', &dimStyle + } s := redStyle() return '✗', &s } @@ -1132,10 +1133,10 @@ func scenarioPassed(samples []*livekit.SimulationRun_Job) int { // listRowParts splits a row into the indent before its icon, the icon, and the // text after it, in plain and styled forms, so the cursor highlight and the // matrix rain can lay out the same line. -func listRowParts(ij indexedJob) (indent string, iconCh rune, iconStyle *lipgloss.Style, text, styled string) { +func listRowParts(ij indexedJob, required int) (indent string, iconCh rune, iconStyle *lipgloss.Style, text, styled string) { switch { case ij.scenario != nil: - iconCh, iconStyle = scenarioStatusIcon(ij.scenario.samples) + iconCh, iconStyle = scenarioStatusIcon(ij.scenario.samples, required) text = fmt.Sprintf(" %3d. %s %d/%d", ij.origIdx, ij.scenario.label, scenarioPassed(ij.scenario.samples), len(ij.scenario.samples)) return " ", iconCh, iconStyle, text, text case ij.origIdx == 0: @@ -1532,6 +1533,13 @@ func (m *simulateModel) renderCounts() string { if running > 0 { parts = append(parts, yellowStyle().Render(fmt.Sprintf("%d running", running))) } + // The counts above are samples; with a pass rate the run's verdict is the + // scenario one, and the two disagree whenever a scenario flakes within it. + if runSamples(m.run) > 1 { + if scenarios, failedScenarios := scenarioFailureCounts(m.run); failedScenarios > 0 { + parts = append(parts, redStyle().Render(fmt.Sprintf("%d/%d scenarios failed", failedScenarios, scenarios))) + } + } if line := passRateLine(m.run); line != "" { parts = append(parts, boldStyle.Render(line)) } @@ -1594,14 +1602,14 @@ func (m *simulateModel) renderJobList() string { // The cursor highlight pads to the widest row so the reverse bar is one width. maxWidth := 0 for i := winStart; i < winEnd; i++ { - indent, iconCh, _, text, _ := listRowParts(jobs[i]) + indent, iconCh, _, text, _ := listRowParts(jobs[i], requiredSamples(m.run)) if w := lipgloss.Width(indent + string(iconCh) + text); w > maxWidth { maxWidth = w } } for i := winStart; i < winEnd; i++ { - indent, iconCh, iconStyle, text, styled := listRowParts(jobs[i]) + indent, iconCh, iconStyle, text, styled := listRowParts(jobs[i], requiredSamples(m.run)) var line string if i == m.cursor { line = ansi.Strip(indent) + string(iconCh) + text @@ -1670,7 +1678,7 @@ func (m *simulateModel) buildMatrixRows() []matrixRow { }) } for i := winStart; i < winEnd; i++ { - indent, iconCh, iconStyle, text, _ := listRowParts(jobs[i]) + indent, iconCh, iconStyle, text, _ := listRowParts(jobs[i], requiredSamples(m.run)) indent = ansi.Strip(indent) line := indent + string(iconCh) + text rows = append(rows, matrixRow{ @@ -1804,7 +1812,7 @@ func (m *simulateModel) renderDetail() string { // every sample's verdict and transcript in sample order. func (m *simulateModel) renderScenarioDetail(sc *scenarioRow) string { var b strings.Builder - iconCh, iconStyle := scenarioStatusIcon(sc.samples) + iconCh, iconStyle := scenarioStatusIcon(sc.samples, requiredSamples(m.run)) b.WriteString("\n") fmt.Fprintf(&b, " %s %s %s\n", iconStyle.Render(string(iconCh)), From 631bd7f21847a14352c3aa7b7ad193d08370f1f9 Mon Sep 17 00:00:00 2001 From: Jason Lernerman Date: Mon, 21 Sep 2026 14:08:51 -0400 Subject: [PATCH 11/13] deps: bump protocol to the rebased simulation-repeats branch --- go.mod | 8 ++------ go.sum | 14 ++++---------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index cf95d1ba..30fe9b91 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/google/go-querystring v1.2.0 github.com/joho/godotenv v1.5.1 github.com/klauspost/compress v1.20.0 - github.com/livekit/protocol v1.51.1-0.20260918145636-43aaf61f0f22 + github.com/livekit/protocol v1.52.1-0.20260921180805-34e27aec11ac github.com/livekit/server-sdk-go/v2 v2.18.2-0.20260904062056-1da58cd7b795 github.com/mattn/go-isatty v0.0.22 github.com/moby/moby/client v0.4.1 @@ -156,7 +156,7 @@ require ( github.com/lithammer/shortuuid/v4 v4.3.0 // indirect github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 // indirect github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095 // indirect - github.com/livekit/psrpc v0.7.6 // indirect + github.com/livekit/psrpc v0.8.0 // indirect github.com/lucasb-eyer/go-colorful v1.4.1 // indirect github.com/magefile/mage v1.17.2 // indirect github.com/mattn/go-colorable v0.1.14 // indirect @@ -171,9 +171,6 @@ require ( github.com/morikuni/aec v1.1.0 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/nats-io/nats.go v1.53.1 // indirect - github.com/nats-io/nkeys v0.4.16 // indirect - github.com/nats-io/nuid v1.0.1 // indirect github.com/oapi-codegen/oapi-codegen/v2 v2.8.0 // indirect github.com/oasdiff/yaml v0.1.1 // indirect github.com/oasdiff/yaml3 v0.0.14 // indirect @@ -201,7 +198,6 @@ require ( github.com/prometheus/common v0.71.0 // indirect github.com/prometheus/procfs v0.22.0 // indirect github.com/puzpuzpuz/xsync/v4 v4.5.0 // indirect - github.com/redis/go-redis/v9 v9.22.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sajari/fuzzy v1.0.0 // indirect github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect diff --git a/go.sum b/go.sum index 77aa10f8..6af6a03f 100644 --- a/go.sum +++ b/go.sum @@ -124,10 +124,6 @@ github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4= github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs= -github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= -github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= -github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= -github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/catppuccin/go v0.3.0 h1:d+0/YicIq+hSTo5oPuRi5kOpqkVA5tAsU6dNhvRu+aY= github.com/catppuccin/go v0.3.0/go.mod h1:8IHJuMGaUUjQM82qBrGNBv7LFq6JI3NnQCF6MOlZjpc= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= @@ -370,12 +366,10 @@ github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5AT github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095 h1:BcliKAXoMhl/nWmzQweQ5kmh4Qqagxl4s3Z5pvM/7AY= github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095/go.mod h1:o8CFmAdrVwzJNOCsQCLUzXRjokkufNshnQHOe4fRaqU= -github.com/livekit/protocol v1.51.1-0.20260915193415-be5b074d9407 h1:Cg9gZfRL761Gu5IOvVXE8zaibG39bPO4NBMnDTHhlUI= -github.com/livekit/protocol v1.51.1-0.20260915193415-be5b074d9407/go.mod h1:vlreOPeiOr2pSCVCHDr2xIbSGf+2++y4QFaTgZ6rjp4= -github.com/livekit/protocol v1.51.1-0.20260918145636-43aaf61f0f22 h1:4ijb2Q6hWZHG2lMjiUWMkNALXPx+jsL78VPalyM1Eys= -github.com/livekit/protocol v1.51.1-0.20260918145636-43aaf61f0f22/go.mod h1:vlreOPeiOr2pSCVCHDr2xIbSGf+2++y4QFaTgZ6rjp4= -github.com/livekit/psrpc v0.7.6 h1:YG07lUMTtf+eaYI2goT9zcVZ0kGJNWN1K6ETNFtv1HQ= -github.com/livekit/psrpc v0.7.6/go.mod h1:DMw15RO7x5XmcgfwzWJYk2In605kx+wu1QRVbPfzf8M= +github.com/livekit/protocol v1.52.1-0.20260921180805-34e27aec11ac h1:TVGfP2c7rcdGkGwRKwNfuvCmfDniECf3927VTBJgxXg= +github.com/livekit/protocol v1.52.1-0.20260921180805-34e27aec11ac/go.mod h1:t5CV/LCsWbI9zsSTHTJKDyLtByqvvVkioSP7jgqpIGA= +github.com/livekit/psrpc v0.8.0 h1:uo3x+DL8pK2OIaCxg/s9sLUO2cIocS5HwdPhhI+m+fQ= +github.com/livekit/psrpc v0.8.0/go.mod h1:6KnU8vULoizX0Er9NNfeKJDV8ug87j3fuo19M0bhj6Q= github.com/livekit/server-sdk-go/v2 v2.18.2-0.20260904062056-1da58cd7b795 h1:0gljvZ5rt8vSLgoyaQl3ocD9D6RQt4Iyn4kXjVr4un0= github.com/livekit/server-sdk-go/v2 v2.18.2-0.20260904062056-1da58cd7b795/go.mod h1:XjoifafCULiHBrmrefkzHX+YEhgraLykU4UAuT8Cf3U= github.com/lucasb-eyer/go-colorful v1.4.1 h1:1EO+WB73+EH8EVbzlrG3KLAfEypQWVHIBqlTf+2hNss= From e3b09b10f2771f70d0699b403af17a5708f12249 Mon Sep 17 00:00:00 2001 From: Jason Lernerman Date: Wed, 23 Sep 2026 11:23:45 -0400 Subject: [PATCH 12/13] deps: pin protocol to main now that livekit/protocol#1799 merged --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 30fe9b91..868cca91 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/google/go-querystring v1.2.0 github.com/joho/godotenv v1.5.1 github.com/klauspost/compress v1.20.0 - github.com/livekit/protocol v1.52.1-0.20260921180805-34e27aec11ac + github.com/livekit/protocol v1.52.1-0.20260923151923-935e69b482ab github.com/livekit/server-sdk-go/v2 v2.18.2-0.20260904062056-1da58cd7b795 github.com/mattn/go-isatty v0.0.22 github.com/moby/moby/client v0.4.1 diff --git a/go.sum b/go.sum index 6af6a03f..6154e437 100644 --- a/go.sum +++ b/go.sum @@ -366,8 +366,8 @@ github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5AT github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095 h1:BcliKAXoMhl/nWmzQweQ5kmh4Qqagxl4s3Z5pvM/7AY= github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095/go.mod h1:o8CFmAdrVwzJNOCsQCLUzXRjokkufNshnQHOe4fRaqU= -github.com/livekit/protocol v1.52.1-0.20260921180805-34e27aec11ac h1:TVGfP2c7rcdGkGwRKwNfuvCmfDniECf3927VTBJgxXg= -github.com/livekit/protocol v1.52.1-0.20260921180805-34e27aec11ac/go.mod h1:t5CV/LCsWbI9zsSTHTJKDyLtByqvvVkioSP7jgqpIGA= +github.com/livekit/protocol v1.52.1-0.20260923151923-935e69b482ab h1:NZqJwz3y//GExVL+njYVN4Z2SpG7uXxKpdMSZJdoM2A= +github.com/livekit/protocol v1.52.1-0.20260923151923-935e69b482ab/go.mod h1:t5CV/LCsWbI9zsSTHTJKDyLtByqvvVkioSP7jgqpIGA= github.com/livekit/psrpc v0.8.0 h1:uo3x+DL8pK2OIaCxg/s9sLUO2cIocS5HwdPhhI+m+fQ= github.com/livekit/psrpc v0.8.0/go.mod h1:6KnU8vULoizX0Er9NNfeKJDV8ug87j3fuo19M0bhj6Q= github.com/livekit/server-sdk-go/v2 v2.18.2-0.20260904062056-1da58cd7b795 h1:0gljvZ5rt8vSLgoyaQl3ocD9D6RQt4Iyn4kXjVr4un0= From 5197d242e95234e09ea6d65338c05aab30e7c1a9 Mon Sep 17 00:00:00 2001 From: Jason Lernerman Date: Mon, 5 Oct 2026 18:03:02 -0400 Subject: [PATCH 13/13] simulate: follow Sampling's rename to SamplingOptions Pins protocol to main now that livekit/protocol#1832 merged. --- cmd/lk/simulate.go | 2 +- cmd/lk/simulate_samples_test.go | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/lk/simulate.go b/cmd/lk/simulate.go index 0a6f09d0..10aa1f38 100644 --- a/cmd/lk/simulate.go +++ b/cmd/lk/simulate.go @@ -788,7 +788,7 @@ func createSimulationRun(ctx context.Context, c *simulateConfig) (string, *livek if c.mode == modeScenarios { req.ScenarioGroup = c.scenarioGroup if c.samples > 1 { - req.Sampling = &livekit.SimulationRun_Sampling{Samples: c.samples, PassRate: c.passRate} + req.Sampling = &livekit.SimulationRun_SamplingOptions{Samples: c.samples, PassRate: c.passRate} } } diff --git a/cmd/lk/simulate_samples_test.go b/cmd/lk/simulate_samples_test.go index 6161bbe5..7abf6946 100644 --- a/cmd/lk/simulate_samples_test.go +++ b/cmd/lk/simulate_samples_test.go @@ -25,8 +25,8 @@ func sampleJob(id, scenario string, sample int32, status livekit.SimulationRun_J return &livekit.SimulationRun_Job{Id: id, ScenarioId: scenario, Sample: sample, Status: status} } -func sampling(k int32, rate float64) *livekit.SimulationRun_Sampling { - return &livekit.SimulationRun_Sampling{Samples: k, PassRate: rate} +func sampling(k int32, rate float64) *livekit.SimulationRun_SamplingOptions { + return &livekit.SimulationRun_SamplingOptions{Samples: k, PassRate: rate} } func TestScenarioPassCounts(t *testing.T) { diff --git a/go.mod b/go.mod index 868cca91..c4d538eb 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/google/go-querystring v1.2.0 github.com/joho/godotenv v1.5.1 github.com/klauspost/compress v1.20.0 - github.com/livekit/protocol v1.52.1-0.20260923151923-935e69b482ab + github.com/livekit/protocol v1.52.2-0.20261005220542-1efe2b4d9ed5 github.com/livekit/server-sdk-go/v2 v2.18.2-0.20260904062056-1da58cd7b795 github.com/mattn/go-isatty v0.0.22 github.com/moby/moby/client v0.4.1 diff --git a/go.sum b/go.sum index 6154e437..b790c7a0 100644 --- a/go.sum +++ b/go.sum @@ -366,8 +366,8 @@ github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5AT github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095 h1:BcliKAXoMhl/nWmzQweQ5kmh4Qqagxl4s3Z5pvM/7AY= github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095/go.mod h1:o8CFmAdrVwzJNOCsQCLUzXRjokkufNshnQHOe4fRaqU= -github.com/livekit/protocol v1.52.1-0.20260923151923-935e69b482ab h1:NZqJwz3y//GExVL+njYVN4Z2SpG7uXxKpdMSZJdoM2A= -github.com/livekit/protocol v1.52.1-0.20260923151923-935e69b482ab/go.mod h1:t5CV/LCsWbI9zsSTHTJKDyLtByqvvVkioSP7jgqpIGA= +github.com/livekit/protocol v1.52.2-0.20261005220542-1efe2b4d9ed5 h1:8HJsjdFcYqB+rEF2kHZq5Ur3qz5GH+8Ul+FkhZXxsqY= +github.com/livekit/protocol v1.52.2-0.20261005220542-1efe2b4d9ed5/go.mod h1:t5CV/LCsWbI9zsSTHTJKDyLtByqvvVkioSP7jgqpIGA= github.com/livekit/psrpc v0.8.0 h1:uo3x+DL8pK2OIaCxg/s9sLUO2cIocS5HwdPhhI+m+fQ= github.com/livekit/psrpc v0.8.0/go.mod h1:6KnU8vULoizX0Er9NNfeKJDV8ug87j3fuo19M0bhj6Q= github.com/livekit/server-sdk-go/v2 v2.18.2-0.20260904062056-1da58cd7b795 h1:0gljvZ5rt8vSLgoyaQl3ocD9D6RQt4Iyn4kXjVr4un0=