From 7d68002718d3dafd153d6c2b6e6593e744596cc7 Mon Sep 17 00:00:00 2001 From: yyjeqhc <1772413353@qq.com> Date: Tue, 27 Oct 2026 18:32:26 +0800 Subject: [PATCH 1/3] feat(runner): hot apply Job capacity through fenced config reload --- .../src-tauri/src/state/workspace_settings.rs | 6 +- .../src/state/workspace_settings/tests.rs | 59 ++++++++ .../src-tauri/src/webcodex/settings.rs | 14 +- .../src-tauri/src/webcodex/settings/tests.rs | 10 +- .../settings/RunnerCapacityPanel.test.tsx | 40 ++++-- .../features/settings/RunnerCapacityPanel.tsx | 11 +- apps/desktop/src/i18n/runtime-shell.ts | 10 +- crates/webcodex-core/src/runner_protocol.rs | 69 +++++++++- .../src/runner_protocol/transport.rs | 2 + .../webcodex-runner-registry/src/runners.rs | 52 +++++++ crates/webcodex-runner-registry/src/state.rs | 3 +- .../src/tests/registration_projection.rs | 70 ++++++++++ .../src/main_tests/config_reload.rs | 109 ++++++++++++++- .../src/webcodex_runner/config.rs | 15 +- .../src/webcodex_runner/job_manager.rs | 17 ++- .../webcodex_runner/job_manager/admission.rs | 12 +- .../webcodex_runner/job_manager/shutdown.rs | 17 ++- .../src/webcodex_runner/job_manager_tests.rs | 2 + .../job_manager_tests/capacity.rs | 129 ++++++++++++++++++ .../src/webcodex_runner/transport.rs | 4 +- .../webcodex_runner/transport/bootstrap.rs | 4 + .../transport/poll_dispatch.rs | 4 + .../webcodex_runner/transport/registration.rs | 5 +- .../src/webcodex_runner/transport/runtime.rs | 6 +- .../transport_tests/stream_telemetry.rs | 25 ++++ docs/DEPLOYMENT.md | 2 +- docs/DEPLOYMENT.zh-CN.md | 2 +- docs/RUNNER.md | 12 +- docs/RUNNER.zh-CN.md | 9 +- docs/agent/claude-code-mcp-provider.md | 2 +- docs/agent/job-reliability-and-concurrency.md | 7 + docs/agent/runtime-tuning-inventory.md | 4 + docs/desktop-guide.md | 11 +- docs/desktop-guide.zh-CN.md | 4 +- scripts/test-runner-config-reload-e2e.sh | 78 +++++++++-- src/runner_http/handlers.rs | 8 ++ src/runner_quic.rs | 96 +++++++++++++ src/runner_session.rs | 9 ++ src/runner_ws.rs | 5 + 39 files changed, 851 insertions(+), 93 deletions(-) create mode 100644 crates/webcodex-runner/src/webcodex_runner/job_manager_tests/capacity.rs diff --git a/apps/desktop/src-tauri/src/state/workspace_settings.rs b/apps/desktop/src-tauri/src/state/workspace_settings.rs index ccb894cec..542669f77 100644 --- a/apps/desktop/src-tauri/src/state/workspace_settings.rs +++ b/apps/desktop/src-tauri/src/state/workspace_settings.rs @@ -112,11 +112,13 @@ impl AppState { .runtime .clone() .ok_or_else(|| desktop_state_unavailable("Configure a Runner first"))?; - tokio::task::spawn_blocking(move || { - crate::webcodex::settings::save_job_concurrency(&runtime, request) + let edit = tokio::task::spawn_blocking({ + let runtime = runtime.clone(); + move || crate::webcodex::settings::stage_job_concurrency(&runtime, request) }) .await .map_err(|_| desktop_state_unavailable("Settings worker stopped"))??; + apply_staged_settings(&runtime, &edit).await?; core.get_state().await } .await; diff --git a/apps/desktop/src-tauri/src/state/workspace_settings/tests.rs b/apps/desktop/src-tauri/src/state/workspace_settings/tests.rs index 3c97cd6cd..040d17c28 100644 --- a/apps/desktop/src-tauri/src/state/workspace_settings/tests.rs +++ b/apps/desktop/src-tauri/src/state/workspace_settings/tests.rs @@ -187,3 +187,62 @@ async fn job_concurrency_save_rejects_unowned_runner_without_changing_the_file() before ); } + +#[tokio::test] +async fn job_capacity_uses_the_shared_staged_transaction_for_all_outcomes() { + for (responses, restored, applied) in [ + (vec![Ok(check(4)), Ok(reload(5))], false, true), + (vec![Err(desktop_state_unavailable("offline"))], true, false), + ( + vec![ + Ok(check(4)), + Ok( + serde_json::json!({"success":false,"output":{"execution_state":"not_started","error_code":"config_generation_conflict"}}), + ), + Ok(check(5)), + ], + false, + false, + ), + ( + vec![ + Ok(check(4)), + Err(desktop_state_unavailable("response lost")), + Ok(check(5)), + ], + false, + false, + ), + ] { + let (_root, runtime, previous_edit, _original) = fixture(); + let baseline = std::fs::read_to_string(runtime.runner_config.as_ref().unwrap()).unwrap(); + assert!(previous_edit.candidate_unchanged().unwrap()); + let edit = settings::stage_job_concurrency( + &runtime, + settings::JobConcurrencyUpdate { + target: settings::target(&runtime).unwrap(), + expected: None, + limit: 6, + }, + ) + .unwrap(); + let mut responses = VecDeque::from(responses); + let mut reloads = 0; + let result = apply_with_control("local", &edit, |tool, params| { + if tool == "reload_runner_config" { + reloads += 1; + assert_eq!(params["expected_generation"], 4); + } + ready(responses.pop_front().unwrap()) + }) + .await; + assert_eq!(result.is_ok(), applied); + assert!(reloads <= 1); + assert!(responses.is_empty()); + let current = std::fs::read_to_string(runtime.runner_config.as_ref().unwrap()).unwrap(); + assert_eq!(current == baseline, restored); + if !restored { + assert!(edit.candidate_unchanged().unwrap()); + } + } +} diff --git a/apps/desktop/src-tauri/src/webcodex/settings.rs b/apps/desktop/src-tauri/src/webcodex/settings.rs index 732c87cda..f4b22e817 100644 --- a/apps/desktop/src-tauri/src/webcodex/settings.rs +++ b/apps/desktop/src-tauri/src/webcodex/settings.rs @@ -345,12 +345,12 @@ fn configured_job_concurrency(doc: &DocumentMut) -> DesktopResult> .transpose() } -/// Startup-only setting: saving does not reload or restart the active Runner. +/// Stage capacity through the same checked reload transaction as other hot settings. /// Preserve unrelated TOML and fence both the displayed value and full-file write. -pub fn save_job_concurrency( +pub fn stage_job_concurrency( runtime: &StoredRuntime, request: JobConcurrencyUpdate, -) -> DesktopResult<()> { +) -> DesktopResult { verify_target(runtime, &request.target)?; if !valid_job_concurrency(request.limit) { return Err(job_concurrency_error()); @@ -362,7 +362,13 @@ pub fn save_job_concurrency( return Err(job_concurrency_error()); } doc["max_concurrent_jobs"] = toml_edit::value(request.limit as i64); - persist(path, &original, &doc) + let candidate = doc.to_string(); + persist_text(path, &original, &candidate)?; + Ok(PendingSettingsEdit { + path: path.clone(), + original, + candidate, + }) } pub fn stage_allowed_roots_update( diff --git a/apps/desktop/src-tauri/src/webcodex/settings/tests.rs b/apps/desktop/src-tauri/src/webcodex/settings/tests.rs index e308ba6ea..c32eda10d 100644 --- a/apps/desktop/src-tauri/src/webcodex/settings/tests.rs +++ b/apps/desktop/src-tauri/src/webcodex/settings/tests.rs @@ -358,7 +358,7 @@ fn job_concurrency_save_preserves_config_and_requires_exact_saved_value() { expected, limit, }; - save_job_concurrency(&f.runtime, request(None, 12)).unwrap(); + stage_job_concurrency(&f.runtime, request(None, 12)).unwrap(); assert_eq!( inspect(&f.runtime, true).unwrap().max_concurrent_jobs, Some(12) @@ -372,20 +372,20 @@ fn job_concurrency_save_preserves_config_and_requires_exact_saved_value() { ] { assert!(text.contains(preserved)); } - assert!(save_job_concurrency(&f.runtime, request(None, 8)).is_err()); + assert!(stage_job_concurrency(&f.runtime, request(None, 8)).is_err()); for limit in [0, 65] { - assert!(save_job_concurrency(&f.runtime, request(Some(12), limit)).is_err()); + assert!(stage_job_concurrency(&f.runtime, request(Some(12), limit)).is_err()); } let mut wrong = request(Some(12), 8); wrong.target.client_id = "other".into(); - assert!(save_job_concurrency(&f.runtime, wrong).is_err()); + assert!(stage_job_concurrency(&f.runtime, wrong).is_err()); assert_eq!( read(f.runtime.runner_config.as_ref().unwrap()).unwrap(), text ); for limit in [1, 64] { let expected = inspect(&f.runtime, true).unwrap().max_concurrent_jobs; - save_job_concurrency(&f.runtime, request(expected, limit)).unwrap(); + stage_job_concurrency(&f.runtime, request(expected, limit)).unwrap(); } } diff --git a/apps/desktop/src/features/settings/RunnerCapacityPanel.test.tsx b/apps/desktop/src/features/settings/RunnerCapacityPanel.test.tsx index 62f01e8ec..90e57ce8f 100644 --- a/apps/desktop/src/features/settings/RunnerCapacityPanel.test.tsx +++ b/apps/desktop/src/features/settings/RunnerCapacityPanel.test.tsx @@ -30,12 +30,34 @@ describe("local Runner concurrency settings", () => { expect(input).toHaveValue(32); expect(screen.getByText("4 running · 2 queued · limit 4")).toBeInTheDocument(); expect(screen.getByText("Saved limit: — · Default: 32")).toBeInTheDocument(); expect(screen.queryByText("Saved limit is in effect.")).not.toBeInTheDocument(); - fireEvent.change(input, { target: { value: "12" } }); fireEvent.click(screen.getByRole("button", { name: "Save for next restart" })); + fireEvent.change(input, { target: { value: "12" } }); fireEvent.click(screen.getByRole("button", { name: "Save and apply" })); await waitFor(() => expect(api.saveRunnerJobConcurrency).toHaveBeenCalledExactlyOnceWith(target, null, 12)); expect(api.restartOwnedRunner).not.toHaveBeenCalled(); - expect(await screen.findByText("Restart required to apply the saved limit.")).toBeInTheDocument(); + expect(await screen.findByText("Saved limit is not yet reported as active.")).toBeInTheDocument(); expect(screen.getByText("4 running · 2 queued · limit 4")).toBeInTheDocument(); }); + it("shows applied capacity from the server even while running exceeds a shrunken limit", async () => { + settings.max_concurrent_jobs = 6; + runner = { ...runner, jobs_running: 6, jobs_queued: 2, job_concurrency_limit: 6 }; + api.saveRunnerJobConcurrency.mockImplementationOnce(async () => { + settings.max_concurrent_jobs = 2; + runner = { ...runner, job_concurrency_limit: 2 }; + return state; + }); + render(view()); + fireEvent.change(await screen.findByRole("spinbutton"), { target: { value: "2" } }); + fireEvent.click(screen.getByRole("button", { name: "Save and apply" })); + expect(await screen.findByText("6 running · 2 queued · limit 2")).toBeInTheDocument(); + expect(screen.getByText("Saved limit is in effect.")).toBeInTheDocument(); + expect(api.restartOwnedRunner).not.toHaveBeenCalled(); + }); + it("allows an explicit apply of a saved limit awaiting activation", async () => { + settings.max_concurrent_jobs = 6; + render(view()); + fireEvent.click(await screen.findByRole("button", { name: "Save and apply" })); + await waitFor(() => expect(api.saveRunnerJobConcurrency).toHaveBeenCalledExactlyOnceWith(target, 6, 6)); + expect(api.restartOwnedRunner).not.toHaveBeenCalled(); + }); it("requires an interruption warning and supports Cancel", async () => { settings.max_concurrent_jobs = 12; render(view()); fireEvent.click(await screen.findByRole("button", { name: "Restart Runner…" })); @@ -44,13 +66,13 @@ describe("local Runner concurrency settings", () => { fireEvent.click(screen.getByRole("button", { name: "Restart Runner…" })); dialog = await screen.findByRole("dialog"); fireEvent.click(within(dialog).getByRole("button", { name: "Restart Runner now" })); await waitFor(() => expect(api.restartOwnedRunner).toHaveBeenCalledExactlyOnceWith(target)); - expect(await screen.findByText("Restart required to apply the saved limit.")).toBeInTheDocument(); + expect(await screen.findByText("Saved limit is not yet reported as active.")).toBeInTheDocument(); }); it("rejects invalid input and does not replay a failed save", async () => { render(view()); const input = await screen.findByRole("spinbutton"); - for (const value of ["", "0", "65", "1.5"]) { fireEvent.change(input, { target: { value } }); expect(screen.getByRole("button", { name: "Save for next restart" })).toBeDisabled(); } + for (const value of ["", "0", "65", "1.5"]) { fireEvent.change(input, { target: { value } }); expect(screen.getByRole("button", { name: "Save and apply" })).toBeDisabled(); } api.saveRunnerJobConcurrency.mockRejectedValue({ code: "runner_job_concurrency_invalid", message: "Saved value changed", next_action: "Reload settings" }); - fireEvent.change(input, { target: { value: "12" } }); fireEvent.click(screen.getByRole("button", { name: "Save for next restart" })); + fireEvent.change(input, { target: { value: "12" } }); fireEvent.click(screen.getByRole("button", { name: "Save and apply" })); expect(await screen.findByRole("alert")).toHaveTextContent("Saved value changed"); expect(api.saveRunnerJobConcurrency).toHaveBeenCalledTimes(1); expect(api.restartOwnedRunner).not.toHaveBeenCalled(); }); @@ -93,7 +115,7 @@ describe("local Runner concurrency settings", () => { api.saveRunnerJobConcurrency.mockReturnValue(new Promise(done => { resolve = done; })); render(view()); const input = await screen.findByRole("spinbutton"); fireEvent.change(input, { target: { value: "12" } }); - const save = screen.getByRole("button", { name: "Save for next restart" }); + const save = screen.getByRole("button", { name: "Save and apply" }); fireEvent.click(save); fireEvent.click(save); expect(api.saveRunnerJobConcurrency).toHaveBeenCalledTimes(1); await act(async () => resolve(state)); @@ -105,7 +127,7 @@ describe("local Runner concurrency settings", () => { settings.max_concurrent_jobs = 8; fireEvent(document, new Event("visibilitychange")); await screen.findByText("Saved limit: 8 · Default: 32"); - fireEvent.click(screen.getByRole("button", { name: "Save for next restart" })); + fireEvent.click(screen.getByRole("button", { name: "Save and apply" })); expect(await screen.findByRole("alert")).toHaveTextContent("Saved value changed"); expect(api.saveRunnerJobConcurrency).toHaveBeenCalledExactlyOnceWith(target, null, 12); }); @@ -113,14 +135,14 @@ describe("local Runner concurrency settings", () => { render(view()); const input = await screen.findByRole("spinbutton"); fireEvent.change(input, { target: { value: "12" } }); settings.max_concurrent_jobs = 8; - fireEvent.click(screen.getByRole("button", { name: "Save for next restart" })); + fireEvent.click(screen.getByRole("button", { name: "Save and apply" })); expect(await screen.findByRole("alert")).toHaveTextContent("Saved value changed"); await waitFor(() => expect(screen.getByRole("button", { name: "Refresh" })).toBeEnabled()); fireEvent.click(screen.getByRole("button", { name: "Refresh" })); await waitFor(() => expect(input).toHaveValue(8)); expect(screen.queryByRole("alert")).not.toBeInTheDocument(); fireEvent.change(input, { target: { value: "12" } }); - fireEvent.click(screen.getByRole("button", { name: "Save for next restart" })); + fireEvent.click(screen.getByRole("button", { name: "Save and apply" })); await waitFor(() => expect(api.saveRunnerJobConcurrency).toHaveBeenLastCalledWith(target, 8, 12)); expect(api.saveRunnerJobConcurrency).toHaveBeenCalledTimes(2); }); diff --git a/apps/desktop/src/features/settings/RunnerCapacityPanel.tsx b/apps/desktop/src/features/settings/RunnerCapacityPanel.tsx index 67f54a629..819ad4821 100644 --- a/apps/desktop/src/features/settings/RunnerCapacityPanel.tsx +++ b/apps/desktop/src/features/settings/RunnerCapacityPanel.tsx @@ -86,6 +86,7 @@ function RunnerCapacitySettings({ state, onState, active }: Props) { const canEdit = state.topology?.runner?.kind === "local" && Boolean(settings?.can_restart) && saved !== undefined; const disabled = busy || operationBusy || !canEdit; const changed = limit !== null && limit !== savedLimit; + const canApply = changed || (saved != null && limit === saved && capacity.state === "available" && capacity.limit !== saved); const run = async (work: () => Promise) => { if (action.current || disabled) return; action.current = true; setBusy(true); setError(null); @@ -99,7 +100,7 @@ function RunnerCapacitySettings({ state, onState, active }: Props) { } }; const save = () => run(async () => { - if (!settings || saved === undefined || limit === null || !changed) return; + if (!settings || saved === undefined || limit === null || !canApply) return; const next = await desktopApi.saveRunnerJobConcurrency(settings.target, editBase.current === undefined ? saved : editBase.current, limit); editBase.current = undefined; if (alive.current) onState(next); @@ -138,17 +139,17 @@ function RunnerCapacitySettings({ state, onState, active }: Props) { } {loading ?

{s("Loading…")}

: <>

{s("Saved limit: {saved} · Default: {default}").replace("{saved}", String(savedDisplay)).replace("{default}", "32")}

- {saved != null && savedLimit !== null &&

{s(capacity.state !== "available" ? "Saved for the next Runner start; the effective limit is unknown." : capacity.limit !== savedLimit ? "Restart required to apply the saved limit." : "Saved limit is in effect.")}

} + {saved != null && savedLimit !== null &&

{s(capacity.state !== "available" ? "The effective limit is unknown. Reconnect and refresh before applying." : capacity.limit !== savedLimit ? "Saved limit is not yet reported as active." : "Saved limit is in effect.")}

} {saved !== undefined &&
{ editBase.current = event.target.value === String(savedLimit) ? undefined : editBase.current === undefined ? saved : editBase.current; setDraft(event.target.value); }} disabled={disabled} aria-describedby="runner-job-concurrency-help" /> -

{s("Choose a whole number from 1 to 64. Saving does not restart the Runner.")}

+

{s("Choose a whole number from 1 to 64. Apply without restarting; running jobs continue.")}

} - {!canEdit && !settingsUnavailable &&

{s("Read-only. Change and restart this Runner through its actual process owner.")}

} + {!canEdit && !settingsUnavailable &&

{s("Read-only. Change this Runner through its actual process owner.")}

}
- {saved !== undefined && } + {saved !== undefined && } {canEdit && }
diff --git a/apps/desktop/src/i18n/runtime-shell.ts b/apps/desktop/src/i18n/runtime-shell.ts index 8636a2105..ffe47a8e8 100644 --- a/apps/desktop/src/i18n/runtime-shell.ts +++ b/apps/desktop/src/i18n/runtime-shell.ts @@ -20,13 +20,13 @@ const zh: Record = { "Saved limit: {saved} · Default: {default}": "已保存的上限:{saved} · 默认值:{default}", - "Saved for the next Runner start; the effective limit is unknown.": "已保存,将在 Runner 下次启动时使用;当前生效上限未知。", - "Restart required to apply the saved limit.": "需要重启 Runner,已保存的上限才会生效。", + "The effective limit is unknown. Reconnect and refresh before applying.": "当前生效上限未知。请重新连接并刷新后再应用。", + "Saved limit is not yet reported as active.": "服务端尚未报告已保存的上限生效。", "Saved limit is in effect.": "已保存的上限正在生效。", "Maximum concurrent jobs": "最大并发任务数", - "Choose a whole number from 1 to 64. Saving does not restart the Runner.": "请输入 1–64 的整数。保存不会重启 Runner。", - "Read-only. Change and restart this Runner through its actual process owner.": "此处只读。请通过实际管理此 Runner 的程序修改配置并重启。", - "Save for next restart": "保存,下次重启时生效", + "Choose a whole number from 1 to 64. Apply without restarting; running jobs continue.": "请输入 1–64 的整数。热应用无需重启,正在运行的任务会继续执行。", + "Read-only. Change this Runner through its actual process owner.": "此处只读。请通过实际管理此 Runner 的程序修改配置。", + "Save and apply": "保存并热应用", "Restart Runner…": "重启 Runner…", "Restart Runner?": "确定重启 Runner?", "Restart Runner now": "立即重启 Runner", diff --git a/crates/webcodex-core/src/runner_protocol.rs b/crates/webcodex-core/src/runner_protocol.rs index 99b339609..be8d35026 100644 --- a/crates/webcodex-core/src/runner_protocol.rs +++ b/crates/webcodex-core/src/runner_protocol.rs @@ -1709,7 +1709,7 @@ pub struct RunnerRegisterRequest { /// for mixed-version diagnostics; never carries paths or environment. #[serde(default, skip_serializing_if = "Option::is_none")] pub build: Option, - /// Effective static Job execution concurrency configured for this Runner + /// Effective Job execution concurrency configured for this Runner /// process. Older Runners omit it, so the Server must preserve `None` /// rather than infer a value from capabilities or observed Jobs. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -1917,7 +1917,7 @@ pub struct RunnerView { /// Runner-reported build identity, when available. #[serde(default, skip_serializing_if = "Option::is_none")] pub build: Option, - /// Runner-reported effective static Job execution concurrency. `None` for + /// Runner-reported effective Job execution concurrency. `None` for /// older Runners; the Server never infers this value. #[serde(default, skip_serializing_if = "Option::is_none")] pub job_concurrency_limit: Option, @@ -2267,8 +2267,17 @@ pub struct RunnerPollRequest { /// Polling transport wrapper. Current Runners may attach changed-only sanitized /// runtime metadata and one bounded project-inventory page. +/// Applied scheduler capacity, fenced by the process-local config generation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct RunnerJobCapacity { + pub generation: u64, + pub limit: usize, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RunnerPollPayload { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub job_capacity: Option, #[serde(flatten)] pub request: RunnerPollRequest, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -4194,6 +4203,7 @@ mod envelope_tests { #[test] fn runtime_metadata_and_legacy_poll_payloads_round_trip() { let env = RunnerEnvelope::RuntimeMetadata { + job_capacity: None, tool_providers: sample_tool_providers(), computer_session_availability: None, mcp_gateway_providers: Some(vec![crate::mcp_gateway::McpGatewayProvider { @@ -4213,10 +4223,65 @@ mod envelope_tests { let legacy = r#"{"client_id":"oe","agent_instance_id":"inst","projects":null}"#; let payload: RunnerPollPayload = serde_json::from_str(legacy).unwrap(); assert_eq!(payload.request.client_id, "oe"); + assert!(payload.job_capacity.is_none()); assert!(payload.tool_providers.is_none()); assert!(payload.mcp_gateway_providers.is_none()); } + #[test] + fn capacity_metadata_is_optional_and_legacy_decoders_ignore_it() { + #[derive(Deserialize)] + struct LegacyMetadata { + tool_providers: ToolProvidersStatus, + } + let envelope = RunnerEnvelope::RuntimeMetadata { + job_capacity: Some(RunnerJobCapacity { + generation: 7, + limit: 32, + }), + tool_providers: sample_tool_providers(), + mcp_gateway_providers: None, + computer_session_availability: None, + }; + let mut json = serde_json::to_value(&envelope).unwrap(); + let old: LegacyMetadata = serde_json::from_value(json.clone()).unwrap(); + assert_eq!( + old.tool_providers.config_reload.generation, + sample_tool_providers().config_reload.generation + ); + json.as_object_mut().unwrap().remove("job_capacity"); + assert!(matches!( + serde_json::from_value::(json).unwrap(), + RunnerEnvelope::RuntimeMetadata { + job_capacity: None, + .. + } + )); + let poll = RunnerPollPayload { + job_capacity: Some(RunnerJobCapacity { + generation: 7, + limit: 32, + }), + request: RunnerPollRequest { + client_id: "runner".into(), + runner_instance_id: "instance".into(), + }, + tool_providers: None, + mcp_gateway_providers: None, + computer_session_availability: None, + project_inventory_page: None, + }; + let json = serde_json::to_value(&poll).unwrap(); + let old: RunnerPollRequest = serde_json::from_value(json.clone()).unwrap(); + assert_eq!(old.client_id, "runner"); + assert_eq!( + serde_json::from_value::(json) + .unwrap() + .job_capacity, + poll.job_capacity + ); + } + #[test] fn goodbye_envelope_round_trips_and_reason_is_optional() { let env = RunnerEnvelope::Goodbye { diff --git a/crates/webcodex-core/src/runner_protocol/transport.rs b/crates/webcodex-core/src/runner_protocol/transport.rs index 448adef7e..238e5c31d 100644 --- a/crates/webcodex-core/src/runner_protocol/transport.rs +++ b/crates/webcodex-core/src/runner_protocol/transport.rs @@ -79,6 +79,8 @@ pub enum RunnerEnvelope { /// Runner -> server changed-only sanitized runtime metadata. It reuses the /// active transport and never requires an acknowledgement round trip. RuntimeMetadata { + #[serde(default, skip_serializing_if = "Option::is_none")] + job_capacity: Option, tool_providers: ToolProvidersStatus, #[serde(default, skip_serializing_if = "Option::is_none")] mcp_gateway_providers: Option>, diff --git a/crates/webcodex-runner-registry/src/runners.rs b/crates/webcodex-runner-registry/src/runners.rs index 9fd06bd1c..5ce2c45a0 100644 --- a/crates/webcodex-runner-registry/src/runners.rs +++ b/crates/webcodex-runner-registry/src/runners.rs @@ -298,6 +298,11 @@ impl RunnerRegistry { // exclusively through the bounded paged inventory protocol. let projects = Vec::new(); let project_inventory = pending_inventory_state(0); + let job_capacity_generation = policy + .as_ref() + .and_then(|p| p.tool_providers.as_ref()) + .map(|p| p.config_reload.generation) + .unwrap_or(0); let record = RunnerRecord { client_id: client_id.clone(), runner_instance_id: runner_instance_id.clone(), @@ -327,6 +332,7 @@ impl RunnerRegistry { process_started_at: body.process_started_at, build: body.build, job_concurrency_limit: body.job_concurrency_limit, + job_capacity_generation, coding_agent_providers: coding_agent_providers.clone(), coding_agent_inventory, projected_structured_terminal_suppressions: VecDeque::new(), @@ -408,6 +414,15 @@ impl RunnerRegistry { client_id )); } + if let Some(existing) = inner.runners.get(&client_id) { + if existing.runner_instance_id == runner_instance_id + && (record.job_capacity_generation < existing.job_capacity_generation + || (record.job_capacity_generation == existing.job_capacity_generation + && record.job_concurrency_limit != existing.job_concurrency_limit)) + { + return Err("stale or conflicting Runner capacity generation".into()); + } + } reject_same_instance_feature_downgrade( inner.runners.get(&client_id), &runner_instance_id, @@ -798,6 +813,43 @@ impl RunnerRegistry { Ok(()) } + /// Capacity is observational metadata, accepted only for the active instance + /// and stream lease. A delayed same-instance poll cannot regress generation. + pub async fn update_job_capacity( + &self, + client_id: &str, + runner_instance_id: &str, + connection_id: Option<&str>, + capacity: Option, + ) -> Result<(), String> { + let Some(capacity) = capacity else { + return Ok(()); + }; + if capacity.generation == 0 + || !(RUNNER_JOB_CONCURRENCY_MIN..=RUNNER_JOB_CONCURRENCY_MAX).contains(&capacity.limit) + { + return Err("invalid Runner capacity".into()); + } + let mut inner = self.inner.lock().await; + let runner = inner.runners.get_mut(client_id).ok_or("unknown Runner")?; + if runner.runner_instance_id != runner_instance_id + || connection_id.is_some_and(|id| runner.connection_id.as_deref() != Some(id)) + { + return Err("stale Runner capacity lease".into()); + } + if capacity.generation < runner.job_capacity_generation + || (capacity.generation == runner.job_capacity_generation + && runner + .job_concurrency_limit + .is_some_and(|limit| limit != capacity.limit)) + { + return Err("stale or conflicting Runner capacity generation".into()); + } + runner.job_concurrency_limit = Some(capacity.limit); + runner.job_capacity_generation = capacity.generation; + Ok(()) + } + /// Update only brokered Computer availability for the exact active lease. /// A missing field leaves legacy/transient behavior unchanged. pub async fn update_computer_session_availability( diff --git a/crates/webcodex-runner-registry/src/state.rs b/crates/webcodex-runner-registry/src/state.rs index 60731fb2d..8939acdb4 100644 --- a/crates/webcodex-runner-registry/src/state.rs +++ b/crates/webcodex-runner-registry/src/state.rs @@ -105,9 +105,10 @@ pub(super) struct RunnerRecord { pub(super) process_started_at: Option, /// Runner-reported build identity (register payload). pub(super) build: Option, - /// Runner-reported effective static Job execution concurrency. This is + /// Runner-reported effective Job execution concurrency. This is /// safe operational metadata and remains unknown for older Runners. pub(super) job_concurrency_limit: Option, + pub(super) job_capacity_generation: u64, /// Sanitized startup-owned ACP providers for this exact Runner process. pub(super) coding_agent_providers: Vec, /// Authoritative active/recent-terminal CodingAgentRun inventory from this diff --git a/crates/webcodex-runner-registry/src/tests/registration_projection.rs b/crates/webcodex-runner-registry/src/tests/registration_projection.rs index a3ab2286a..4c6d49415 100644 --- a/crates/webcodex-runner-registry/src/tests/registration_projection.rs +++ b/crates/webcodex-runner-registry/src/tests/registration_projection.rs @@ -72,3 +72,73 @@ async fn registry_registers_and_lists_client() { assert!(runners[0].connected); assert_eq!(runners[0].pending_requests, 0); } + +#[tokio::test] +async fn hot_job_capacity_is_bounded_and_fenced_by_instance_connection_and_generation() { + use webcodex_core::runner_protocol::RunnerJobCapacity; + for transport in [ + RunnerTransport::WebSocket, + RunnerTransport::Quic, + RunnerTransport::Polling, + ] { + let registry = RunnerRegistry::default(); + let mut request = runner_registration("capacity", "instance", Vec::new()); + request.job_concurrency_limit = Some(2); + let connection = if transport == RunnerTransport::Polling { + registry.register(request.clone()).await.unwrap(); + None + } else { + registry + .register_streaming_session( + request.clone(), + None, + "current", + transport, + Arc::new(Notify::new()), + ) + .await + .unwrap(); + Some("current") + }; + let capacity = |generation, limit| Some(RunnerJobCapacity { generation, limit }); + registry + .update_job_capacity("capacity", "instance", connection, capacity(2, 6)) + .await + .unwrap(); + for (instance, conn, generation, limit) in [ + ("old", connection, 3, 32), + ("instance", Some("old"), 3, 32), + ("instance", connection, 1, 32), + ("instance", connection, 2, 32), + ("instance", connection, 3, 0), + ("instance", connection, 3, 65), + ] { + assert!(registry + .update_job_capacity("capacity", instance, conn, capacity(generation, limit)) + .await + .is_err()); + assert_eq!( + registry + .get_runner_view("capacity") + .await + .unwrap() + .job_concurrency_limit, + Some(6) + ); + } + // An obsolete registration from this same process must not regress capacity. + assert!(registry.register(request).await.is_err()); + registry + .update_job_capacity("capacity", "instance", connection, None) + .await + .unwrap(); + registry + .update_job_capacity("capacity", "instance", connection, capacity(3, 2)) + .await + .unwrap(); + assert_eq!( + registry.list_runners().await[0].job_concurrency_limit, + Some(2) + ); + } +} diff --git a/crates/webcodex-runner/src/main_tests/config_reload.rs b/crates/webcodex-runner/src/main_tests/config_reload.rs index 496c51081..49aa0afce 100644 --- a/crates/webcodex-runner/src/main_tests/config_reload.rs +++ b/crates/webcodex-runner/src/main_tests/config_reload.rs @@ -70,6 +70,7 @@ fn reload_fixture() -> (tempfile::TempDir, PathBuf, ReloadableRunnerConfig) { fn reload_field_classification_is_exhaustive_and_allowlisted() { let startup = test_config(PathBuf::from("projects-a")); let mut hot_only = startup.clone(); + hot_only.max_concurrent_jobs = Some(6); hot_only.policy.max_timeout_secs += 1; hot_only.shell.program = "bash".to_string(); hot_only.skills.roots.push(PathBuf::from("live-skill-root")); @@ -104,7 +105,7 @@ fn reload_field_classification_is_exhaustive_and_allowlisted() { changed.quic = Some(quic_client_config()); assert_eq!( webcodex_runner::config::restart_required_fields(&startup, &changed).join(" "), - "capabilities client_id display_name hostname host_context max_concurrent_jobs owner poll_interval_ms project_registry_dir quic server_url token transport websocket_connect_timeout_secs" + "capabilities client_id display_name hostname host_context owner poll_interval_ms project_registry_dir quic server_url token transport websocket_connect_timeout_secs" ); } @@ -608,10 +609,7 @@ fn mixed_reload_applies_hot_fields_and_reports_static_restart_fields() { let active = runtime.snapshot(); assert_eq!(status.last_reload_result, "partial"); assert!(status.restart_required); - assert_eq!( - status.restart_required_fields, - ["client_id", "max_concurrent_jobs"] - ); + assert_eq!(status.restart_required_fields, ["client_id"]); assert_eq!( ( active.policy.max_timeout_secs, @@ -653,7 +651,7 @@ fn first_class_check_validates_candidate_without_mutating_active_config() { assert!(checked.restart_required); assert_eq!( checked.restart_required_fields, - vec!["client_id".to_string(), "max_concurrent_jobs".to_string()] + vec!["client_id".to_string()] ); let after = runtime.snapshot(); @@ -830,7 +828,7 @@ fn first_class_partial_reload_reports_exact_restart_only_fields() { assert!(result.restart_required); assert_eq!( result.restart_required_fields, - vec!["client_id".to_string(), "max_concurrent_jobs".to_string()] + vec!["client_id".to_string()] ); let active = runtime.snapshot(); assert_eq!(active.policy.max_timeout_secs, 180); @@ -913,3 +911,100 @@ fn unix_sighup_listener_still_triggers_authoritative_reload() { runtime.begin_shutdown(); listener.join().unwrap(); } + +#[test] +fn capacity_reload_applies_to_shared_scheduler_and_keeps_failed_generations() { + let (_tmp, path, runtime) = reload_fixture(); + let worker = runtime.jobs.clone(); + assert_eq!(worker.concurrency_limit(), 32); + for (generation, limit) in [(1, 2), (2, 6), (3, 2)] { + std::fs::write( + &path, + reload_toml( + "oe", + Some(limit), + 60, + 1024, + "sh", + "native", + false, + "claude", + "search", + ), + ) + .unwrap(); + let checked = runtime.check_config(); + assert_eq!(checked.valid, Some(true)); + assert!(!checked.restart_required); + let applied = runtime.reload_config(generation); + assert_eq!(applied.valid, Some(true)); + assert_eq!(worker.concurrency_limit(), limit); + assert_eq!(runtime.snapshot().max_concurrent_jobs, limit); + assert_eq!( + runtime.reload_config(generation).error_code, + Some(runner_protocol::RunnerConfigErrorCode::ConfigGenerationConflict) + ); + assert_eq!(worker.concurrency_limit(), limit); + } + std::fs::write(&path, "invalid toml").unwrap(); + assert_eq!(runtime.reload_config(4).valid, Some(false)); + assert_eq!(worker.concurrency_limit(), 2); + runtime.begin_shutdown(); + assert_eq!( + runtime.reload_config(4).error_code, + Some(runner_protocol::RunnerConfigErrorCode::RunnerUnavailable) + ); + assert_eq!(worker.concurrency_limit(), 2); +} + +#[test] +fn concurrent_capacity_reloads_commit_one_generation_and_shutdown_fences_activation() { + let (_tmp, path, runtime) = reload_fixture(); + std::fs::write( + &path, + reload_toml( + "oe", + Some(6), + 60, + 1024, + "sh", + "native", + false, + "claude", + "search", + ), + ) + .unwrap(); + let barrier = std::sync::Barrier::new(3); + let responses = std::thread::scope(|scope| { + let one = scope.spawn(|| { + barrier.wait(); + runtime.reload_config(1) + }); + let two = scope.spawn(|| { + barrier.wait(); + runtime.reload_config(1) + }); + barrier.wait(); + [one.join().unwrap(), two.join().unwrap()] + }); + assert_eq!( + responses.iter().filter(|r| r.valid == Some(true)).count(), + 1 + ); + assert_eq!( + responses + .iter() + .filter(|r| r.error_code + == Some(runner_protocol::RunnerConfigErrorCode::ConfigGenerationConflict)) + .count(), + 1 + ); + assert_eq!(runtime.jobs.concurrency_limit(), 6); + runtime.begin_shutdown(); + assert_eq!( + runtime.reload_config(2).error_code, + Some(runner_protocol::RunnerConfigErrorCode::RunnerUnavailable) + ); + assert_eq!(runtime.snapshot().generation, 2); +} diff --git a/crates/webcodex-runner/src/webcodex_runner/config.rs b/crates/webcodex-runner/src/webcodex_runner/config.rs index 4478028a0..a04a3dca6 100644 --- a/crates/webcodex-runner/src/webcodex_runner/config.rs +++ b/crates/webcodex-runner/src/webcodex_runner/config.rs @@ -534,6 +534,7 @@ pub(crate) struct ShellProfileConfig { pub(crate) struct HotRunnerConfig { pub(crate) generation: u64, + pub(crate) max_concurrent_jobs: usize, pub(crate) policy: RunnerPolicy, pub(crate) shell: ShellConfig, pub(crate) skills: SkillsConfig, @@ -557,6 +558,7 @@ impl HotRunnerConfig { let ssh = ManagedSshResourceStore::merge_active(&cfg.ssh, startup_managed_ssh)?; Ok(Self { generation, + max_concurrent_jobs: max_concurrent_jobs(cfg), policy: cfg.policy.clone(), shell: cfg.shell.clone(), skills: cfg.skills.clone(), @@ -574,6 +576,7 @@ impl HotRunnerConfig { } pub(crate) struct ReloadableRunnerConfig { + pub(crate) jobs: super::job_manager::JobManager, startup: RunnerConfig, startup_managed_ssh: SshConfig, managed_ssh: Arc, @@ -617,6 +620,8 @@ impl ReloadableRunnerConfig { } }; Self { + jobs: super::job_manager::JobManager::new(max_concurrent_jobs(&startup)) + .with_detached_profile_identity(&startup.server_url), mcp_gateway: Arc::new(McpGatewayManager::new(&startup.mcp_gateway)), plugins: Arc::new(PluginManager::new(&startup, path.clone())), coding_agents, @@ -645,6 +650,7 @@ impl ReloadableRunnerConfig { // after their managers have entered stopping state. let _reload_guard = lock_unpoison(&self.reload_lock); self.stopping.store(true, Ordering::SeqCst); + self.jobs.stop_accepting_work(); self.mcp_gateway.shutdown(); self.plugins.shutdown(); if let Some(manager) = &self.coding_agents { @@ -888,6 +894,8 @@ impl ReloadableRunnerConfig { // through this Hot config swap, so a specialized Plugin reload // cannot interleave and create contradictory active truths. let mut current = self.current.write().unwrap(); + self.jobs + .commit_concurrency_limit(next_for_commit.max_concurrent_jobs); *current = next_for_commit; }) { Ok(()) => {} @@ -952,6 +960,10 @@ impl ReloadableRunnerConfig { ); } } + // Activation is committed. Spawn/I/O must run outside configuration and + // provider gates. Concurrent reload/shutdown is rechecked by admission. + drop(_reload_guard); + self.jobs.start_available_queued(); eprintln!( "webcodex-runner config reload {}", status.last_reload_result @@ -1112,7 +1124,7 @@ pub(crate) fn restart_required_fields( macro_rules! classify { ($($field:ident),+ $(,)?) => {{ let RunnerConfig { - server_url_override: _, policy: _, shell: _, skills: _, instructions: _, ssh: _, plugins: _, tool_providers: _, mcp_gateway: _, + server_url_override: _, max_concurrent_jobs: _, policy: _, shell: _, skills: _, instructions: _, ssh: _, plugins: _, tool_providers: _, mcp_gateway: _, $($field: _),+ } = candidate; [$((stringify!($field), startup.$field != candidate.$field)),+] @@ -1127,7 +1139,6 @@ pub(crate) fn restart_required_fields( display_name, hostname, host_context, - max_concurrent_jobs, acp, owner, poll_interval_ms, diff --git a/crates/webcodex-runner/src/webcodex_runner/job_manager.rs b/crates/webcodex-runner/src/webcodex_runner/job_manager.rs index 486bdb558..216e64624 100644 --- a/crates/webcodex-runner/src/webcodex_runner/job_manager.rs +++ b/crates/webcodex-runner/src/webcodex_runner/job_manager.rs @@ -33,7 +33,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::io::Write; use std::path::{Path, PathBuf}; use std::process::Stdio; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{mpsc, Arc, Condvar, Mutex, Weak}; use std::time::{Duration, Instant}; use webcodex_core::runner_job_lifecycle::RunnerJobLifecycle; @@ -149,7 +149,7 @@ impl PendingJobStart { #[derive(Debug, Clone)] pub(crate) struct JobManager { - max_concurrent: usize, + max_concurrent: Arc, jobs: Arc>>, detached_jobs: Arc>>, queued: Arc>>, @@ -202,7 +202,7 @@ impl JobManager { delivery_signal: Arc::clone(&delivery_signal), }); Self { - max_concurrent: max_concurrent.max(1), + max_concurrent: Arc::new(AtomicUsize::new(max_concurrent.max(1))), jobs, detached_jobs, queued: Arc::new(Mutex::new(VecDeque::new())), @@ -229,6 +229,17 @@ impl JobManager { self } + // All reservations and capacity changes serialize on lifecycle. The atomic + // only permits cheap observations; it does not authorize admission alone. + pub(crate) fn concurrency_limit(&self) -> usize { + self.max_concurrent.load(Ordering::Relaxed) + } + + pub(crate) fn commit_concurrency_limit(&self, limit: usize) { + let _lifecycle = lock_unpoison(&self.lifecycle); + self.max_concurrent.store(limit, Ordering::Relaxed); + } + pub(crate) fn prepared_profiles(&self) -> &PreparedShellProfileCache { &self.prepared_profiles } diff --git a/crates/webcodex-runner/src/webcodex_runner/job_manager/admission.rs b/crates/webcodex-runner/src/webcodex_runner/job_manager/admission.rs index 19d50049d..fc96e07a5 100644 --- a/crates/webcodex-runner/src/webcodex_runner/job_manager/admission.rs +++ b/crates/webcodex-runner/src/webcodex_runner/job_manager/admission.rs @@ -371,7 +371,9 @@ impl JobManager { } else { None }; - let queue_locally = immediate_failure.is_none() && reserved >= self.max_concurrent; + let queue_locally = immediate_failure.is_none() + && (reserved >= self.concurrency_limit() + || !lock_unpoison(&self.queued).is_empty()); let slot_reserved = immediate_failure.is_none() && !queue_locally; let now = chrono::Utc::now() .timestamp() @@ -435,6 +437,7 @@ impl JobManager { }, ); if queue_locally { + self.start_available_queued(); return; } self.start_now(start); @@ -462,16 +465,15 @@ impl JobManager { } } - pub(super) fn start_available_queued(&self) { + pub(crate) fn start_available_queued(&self) { loop { if self.shutting_down.load(Ordering::SeqCst) { - lock_unpoison(&self.queued).clear(); + // Shutdown owns cancellation and terminalization of these entries. return; } let next = { let _lifecycle = lock_unpoison(&self.lifecycle); if self.shutting_down.load(Ordering::SeqCst) { - lock_unpoison(&self.queued).clear(); return; } let mut jobs = lock_unpoison(&self.jobs); @@ -486,7 +488,7 @@ impl JobManager { && runner_job_is_active(&job.snapshot.status) }) .count(); - if reserved < self.max_concurrent { + if reserved < self.concurrency_limit() { selected = Some(idx); break; } diff --git a/crates/webcodex-runner/src/webcodex_runner/job_manager/shutdown.rs b/crates/webcodex-runner/src/webcodex_runner/job_manager/shutdown.rs index c3a48ec07..c3942945a 100644 --- a/crates/webcodex-runner/src/webcodex_runner/job_manager/shutdown.rs +++ b/crates/webcodex-runner/src/webcodex_runner/job_manager/shutdown.rs @@ -120,12 +120,17 @@ impl JobManager { } pub(in crate::webcodex_runner) fn cancel_queued_for_shutdown(&self) -> usize { - let _lifecycle = lock_unpoison(&self.lifecycle); - self.shutting_down.store(true, Ordering::SeqCst); - let mut queued = lock_unpoison(&self.queued); - let cancelled = queued.len(); - queued.clear(); - cancelled + let pending = { + let _lifecycle = lock_unpoison(&self.lifecycle); + self.shutting_down.store(true, Ordering::SeqCst); + lock_unpoison(&self.queued).drain(..).collect::>() + }; + // Preserve terminal evidence for every accepted queued identity. Delivery + // and cleanup happen outside admission locks, just as for an explicit stop. + for start in &pending { + self.shutdown_rejection(&start.operation); + } + pending.len() } pub(in crate::webcodex_runner) fn signal_all_for_shutdown(&self) -> JobShutdownBatch { diff --git a/crates/webcodex-runner/src/webcodex_runner/job_manager_tests.rs b/crates/webcodex-runner/src/webcodex_runner/job_manager_tests.rs index 5245e60c9..ae24d9741 100644 --- a/crates/webcodex-runner/src/webcodex_runner/job_manager_tests.rs +++ b/crates/webcodex-runner/src/webcodex_runner/job_manager_tests.rs @@ -19,6 +19,8 @@ use std::sync::OnceLock; use tempfile::TempDir; use webcodex_core::runner_protocol::{RunnerEnvelope, VALIDATION_STEP_WAIT_FAILED_CODE}; +#[path = "job_manager_tests/capacity.rs"] +mod capacity; #[path = "job_manager_tests/heartbeat.rs"] mod heartbeat; #[path = "job_manager_tests/node_json.rs"] diff --git a/crates/webcodex-runner/src/webcodex_runner/job_manager_tests/capacity.rs b/crates/webcodex-runner/src/webcodex_runner/job_manager_tests/capacity.rs new file mode 100644 index 000000000..014ce38a7 --- /dev/null +++ b/crates/webcodex-runner/src/webcodex_runner/job_manager_tests/capacity.rs @@ -0,0 +1,129 @@ +use super::*; + +#[test] +fn hot_capacity_growth_wakes_fifo_and_shrink_drains_without_interrupting_jobs() { + let temp = tempfile::tempdir().unwrap(); + let helper = structured_process_helper(); + let (sink, _rx) = structured_test_sink("structured-agent", "structured-instance"); + let manager = JobManager::new(2); + let clone = manager.clone(); + let jobs = (0..8) + .map(|i| GatedStructuredJob::new(temp.path(), &format!("resize-{i}"))) + .collect::>(); + for job in &jobs { + enqueue_gated_structured_job(&manager, &sink, temp.path(), &helper, job); + } + wait_for_all_started(&jobs[..2]); + assert!(jobs[2..].iter().all(|job| !job.started.exists())); + let before = manager.inventory(); + clone.commit_concurrency_limit(6); + clone.start_available_queued(); + assert_eq!(manager.concurrency_limit(), 6); + // No original child has been released: growth itself must start the queue. + wait_for_all_started(&jobs[..6]); + assert!(!jobs[6].started.exists()); + clone.commit_concurrency_limit(2); + clone.start_available_queued(); + assert_eq!(manager.concurrency_limit(), 2); + assert_eq!( + lock_unpoison(&manager.jobs) + .values() + .filter(|job| job.slot_reserved) + .count(), + 6 + ); + for job in &jobs[..4] { + job.release(); + } + assert!(wait_until(Duration::from_secs(10), || manager + .inventory() + .jobs + .iter() + .filter(|j| j.status == "completed") + .count() + == 4)); + assert!(!jobs[6].started.exists()); + jobs[4].release(); + wait_for_all_started(&jobs[6..7]); + assert!(!jobs[7].started.exists()); + let after = manager.inventory(); + for original in before.jobs { + let current = after + .jobs + .iter() + .find(|j| j.job_id == original.job_id) + .unwrap(); + assert_eq!(current.request_id, original.request_id); + assert_eq!(current.context, original.context); + } + for job in &jobs[5..] { + job.release(); + } + wait_for_job_workers(&manager); + for job in &jobs { + assert_gated_job_started_once(job); + } +} + +#[test] +fn capacity_changes_race_enqueue_completion_stop_and_shutdown_without_replay() { + let temp = tempfile::tempdir().unwrap(); + let helper = structured_process_helper(); + let (sink, _rx) = structured_test_sink("structured-agent", "structured-instance"); + let manager = JobManager::new(2); + let jobs = (0..12) + .map(|i| GatedStructuredJob::new(temp.path(), &format!("race-{i}"))) + .collect::>(); + for job in &jobs[..6] { + enqueue_gated_structured_job(&manager, &sink, temp.path(), &helper, job); + } + wait_for_all_started(&jobs[..2]); + let barrier = std::sync::Barrier::new(4); + std::thread::scope(|scope| { + scope.spawn(|| { + barrier.wait(); + for limit in [6, 2, 6, 2] { + manager.commit_concurrency_limit(limit); + manager.start_available_queued(); + } + }); + scope.spawn(|| { + barrier.wait(); + for job in &jobs[6..] { + enqueue_gated_structured_job(&manager, &sink, temp.path(), &helper, job); + } + }); + scope.spawn(|| { + barrier.wait(); + let _ = manager.stop(&jobs[5].job_id); + jobs[0].release(); + jobs[1].release(); + }); + barrier.wait(); + }); + std::thread::scope(|scope| { + scope.spawn(|| { + manager.commit_concurrency_limit(6); + manager.start_available_queued(); + }); + scope.spawn(|| { + manager.stop_accepting_work(); + manager.cancel_queued_for_shutdown(); + }); + }); + for job in &jobs { + job.release(); + } + wait_for_job_workers(&manager); + assert!(lock_unpoison(&manager.queued).is_empty()); + assert!(manager + .inventory() + .jobs + .iter() + .all(|j| !runner_job_is_active(&j.status))); + for job in &jobs { + if job.started.exists() { + assert_gated_job_started_once(job); + } + } +} diff --git a/crates/webcodex-runner/src/webcodex_runner/transport.rs b/crates/webcodex-runner/src/webcodex_runner/transport.rs index 090a7b492..61a89d855 100644 --- a/crates/webcodex-runner/src/webcodex_runner/transport.rs +++ b/crates/webcodex-runner/src/webcodex_runner/transport.rs @@ -1,8 +1,8 @@ //! Runner transport bootstrap, supervision, and shared resource ownership. use super::config::{ - max_concurrent_jobs, project_registry_dir, validate_quic_config, QuicClientConfig, - ReloadableRunnerConfig, RunnerConfig, + project_registry_dir, validate_quic_config, QuicClientConfig, ReloadableRunnerConfig, + RunnerConfig, }; use super::contains_any; #[cfg(any(target_os = "linux", target_os = "macos", windows))] diff --git a/crates/webcodex-runner/src/webcodex_runner/transport/bootstrap.rs b/crates/webcodex-runner/src/webcodex_runner/transport/bootstrap.rs index 6a392dd19..4302c74fc 100644 --- a/crates/webcodex-runner/src/webcodex_runner/transport/bootstrap.rs +++ b/crates/webcodex-runner/src/webcodex_runner/transport/bootstrap.rs @@ -33,6 +33,10 @@ pub(super) fn send_provider_metadata( transport, tx, RunnerEnvelope::RuntimeMetadata { + job_capacity: Some(webcodex_core::runner_protocol::RunnerJobCapacity { + generation: config.generation, + limit: config.max_concurrent_jobs, + }), tool_providers: status, mcp_gateway_providers: Some(runtime.mcp_gateway().provider_inventory()), computer_session_availability: availability_update, diff --git a/crates/webcodex-runner/src/webcodex_runner/transport/poll_dispatch.rs b/crates/webcodex-runner/src/webcodex_runner/transport/poll_dispatch.rs index ee04b5e5f..e459985ce 100644 --- a/crates/webcodex-runner/src/webcodex_runner/transport/poll_dispatch.rs +++ b/crates/webcodex-runner/src/webcodex_runner/transport/poll_dispatch.rs @@ -525,6 +525,10 @@ pub(crate) fn handle_one_poll( }); let computer_session_update = webcodex_runner::computer_session::changed_availability(); let poll = RunnerPollPayload { + job_capacity: Some(webcodex_core::runner_protocol::RunnerJobCapacity { + generation: metadata_config.generation, + limit: metadata_config.max_concurrent_jobs, + }), request: RunnerPollRequest { client_id: cfg.client_id.clone(), runner_instance_id: runner_instance_id.to_string(), diff --git a/crates/webcodex-runner/src/webcodex_runner/transport/registration.rs b/crates/webcodex-runner/src/webcodex_runner/transport/registration.rs index d83e1e475..9276c1643 100644 --- a/crates/webcodex-runner/src/webcodex_runner/transport/registration.rs +++ b/crates/webcodex-runner/src/webcodex_runner/transport/registration.rs @@ -5,8 +5,7 @@ use super::http_client::{ }; use crate::webcodex_runner; use crate::webcodex_runner::config::{ - hostname, max_concurrent_jobs, HotRunnerConfig, ReloadableRunnerConfig, RunnerConfig, - ShellConfig, + hostname, HotRunnerConfig, ReloadableRunnerConfig, RunnerConfig, ShellConfig, }; use crate::webcodex_runner::job_manager::JobManager; use crate::webcodex_runner::projects::RunnerProjectCache; @@ -547,7 +546,7 @@ pub(crate) fn build_register_request_with_provider_status( )), process_started_at: Some(process_started_at()), build: Some(runner_build_info()), - job_concurrency_limit: Some(max_concurrent_jobs(cfg)), + job_concurrency_limit: Some(hot.max_concurrent_jobs), // A Runner with reconciliation disabled for E2E must not send a job // inventory: the server rejects inventory without the capability and // vice-versa. diff --git a/crates/webcodex-runner/src/webcodex_runner/transport/runtime.rs b/crates/webcodex-runner/src/webcodex_runner/transport/runtime.rs index b8f7314c7..c9e83f2c2 100644 --- a/crates/webcodex-runner/src/webcodex_runner/transport/runtime.rs +++ b/crates/webcodex-runner/src/webcodex_runner/transport/runtime.rs @@ -12,8 +12,8 @@ impl RunnerRuntimeState { path: PathBuf, budget: Duration, ) -> Self { - let jobs = JobManager::new(max_concurrent_jobs(cfg)) - .with_detached_profile_identity(&cfg.server_url); + let config = Arc::new(ReloadableRunnerConfig::new(cfg.clone(), path)); + let jobs = config.jobs.clone(); // Persistent shells reuse the same authenticated OpenSSH multiplex pool // as async jobs: one transport per (session, resource, generation), // never a second SSH configuration or connection pool. @@ -30,7 +30,7 @@ impl RunnerRuntimeState { BrowserSupervisor::new_with_extension_bridge() } }, - config: Arc::new(ReloadableRunnerConfig::new(cfg.clone(), path)), + config, jobs, persistent_shells, coordinator: Arc::new(ShutdownCoordinator::new(budget)), diff --git a/crates/webcodex-runner/src/webcodex_runner/transport_tests/stream_telemetry.rs b/crates/webcodex-runner/src/webcodex_runner/transport_tests/stream_telemetry.rs index 4751093ab..06d761612 100644 --- a/crates/webcodex-runner/src/webcodex_runner/transport_tests/stream_telemetry.rs +++ b/crates/webcodex-runner/src/webcodex_runner/transport_tests/stream_telemetry.rs @@ -480,3 +480,28 @@ fn wait_for_path(path: &Path, deadline: Instant, context: &str) { thread::sleep(Duration::from_millis(5)); } } + +#[test] +fn hot_capacity_metadata_retries_backpressure_and_registration_uses_active_generation() { + for transport in [StreamTransport::WebSocket, StreamTransport::Quic] { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("runner.toml"); + let cfg = test_runner_config("http://127.0.0.1:1".into()); + let runtime = ReloadableRunnerConfig::new(cfg.clone(), path.clone()); + std::fs::write(&path, "server_url='http://127.0.0.1:1'\ntoken='test-token'\nclient_id='oe'\nmax_concurrent_jobs=6\n").unwrap(); + assert_eq!(runtime.reload_config(1).valid, Some(true)); + let (tx, mut rx) = mpsc::channel(1); + tx.try_send(RunnerEnvelope::Pong { ts: 1 }).unwrap(); + send_provider_metadata(transport, &tx, &runtime, Some(1)); + assert!(matches!(rx.try_recv(), Ok(RunnerEnvelope::Pong { .. }))); + send_provider_metadata(transport, &tx, &runtime, Some(1)); + let RunnerEnvelope::RuntimeMetadata { job_capacity: Some(capacity), .. } = rx.try_recv().unwrap() else { panic!("missing capacity"); }; + assert_eq!(capacity.generation, 2); + assert_eq!(capacity.limit, 6); + assert_eq!(runtime.jobs.concurrency_limit(), 6); + // Reconnect reconstructs registration from active state even though cfg + // remains the immutable startup/transport identity. + let (registered, _, _) = build_register_request_with_provider_status(&cfg, &runtime, "instance", 0, runtime.jobs.inventory()); + assert_eq!(registered.job_concurrency_limit, Some(6)); + } +} diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index e8188cf08..de2d32788 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -426,7 +426,7 @@ writes the file. Invalid candidates preserve the active snapshot/generation, and `restart_required_fields` names startup-only changes that are not claimed live until restart. Unix service reload/SIGHUP remains a compatibility trigger for the same reload primitive, but is not required for first-class config control. Identity, -server/auth, project source, concurrency, capabilities, and transport changes +server/auth, project source, capabilities, and transport changes remain restart-only where reported. `[instructions].files` is explicitly hot-reloadable: after check/reload, new diff --git a/docs/DEPLOYMENT.zh-CN.md b/docs/DEPLOYMENT.zh-CN.md index 01967e7b8..87e7fa92c 100644 --- a/docs/DEPLOYMENT.zh-CN.md +++ b/docs/DEPLOYMENT.zh-CN.md @@ -377,7 +377,7 @@ max_output_bytes = 262144 无效 candidate 保留旧 active snapshot/generation,`restart_required_fields` 明确列出仍需 重启、且不会假装已经在线生效的 startup-only 变更。Unix service reload/SIGHUP 仍保留为 调用同一 reload primitive 的兼容 trigger,但 first-class config control 不依赖它。身份、 -server/auth、项目来源、并发、能力与传输等字段在被报告为 restart-only 时仍需要重启。 +server/auth、项目来源、能力与传输等字段在被报告为 restart-only 时仍需要重启。 `[instructions].files` 明确属于 hot-reloadable 字段:完成 check/reload 后,新 Project bootstrap 会立即使用新的路径列表,不需要重启 Runner。已配置 instruction 文件的内容 diff --git a/docs/RUNNER.md b/docs/RUNNER.md index 203639180..82181b613 100644 --- a/docs/RUNNER.md +++ b/docs/RUNNER.md @@ -548,13 +548,19 @@ restarted. The Runner executes up to `max_concurrent_jobs` Jobs at once (default 32, valid range 1..64). Values outside that range are rejected as configuration errors. -This is an operational tuning control, not a security boundary, and requires a -Runner restart to change: +This operational tuning control supports `check_runner_config` followed by +`reload_runner_config` (or Unix SIGHUP), without restarting: ```toml max_concurrent_jobs = 32 ``` +Increasing the limit immediately schedules existing queued Jobs. Decreasing it +leaves running Jobs intact; new starts wait until occupied slots fall below the +new limit. Running may temporarily exceed the limit. Registration and runtime +metadata report the applied capacity with a config-generation fence, including +after reconnect. Older Servers retain startup-only capacity observations. + When all slots are occupied, an accepted Job remains the same queryable Job with the same `job_id` and reports `agent_queued`. @@ -862,7 +868,7 @@ of finding its PID or sending signals manually: `reload_runner_config` never writes `runner.toml`; it only activates the candidate already on disk. Hot-reloadable policy, shell, configured Skill roots, configured -instruction files, Native Plugin, and static SSH-resource changes can become active +instruction files, Job capacity, Native Plugin, and static SSH-resource changes can become active immediately, while fields reported in `restart_required_fields` remain startup-only until the Runner restarts. Invalid candidates leave the active snapshot and generation unchanged. Managed `manage_ssh_resource` mutations are different: diff --git a/docs/RUNNER.zh-CN.md b/docs/RUNNER.zh-CN.md index b30df905d..2ccbbae24 100644 --- a/docs/RUNNER.zh-CN.md +++ b/docs/RUNNER.zh-CN.md @@ -446,12 +446,17 @@ stdout/stderr 尾部,并且可以停止。结构化执行(`run_process`、`r 运行——绝不会被重启。 Runner 同时最多执行 `max_concurrent_jobs` 个 Job(默认 32,合法范围 1..64)。超出范围 -的值会作为配置错误被拒绝。这个参数用于运维调优,不是安全边界,修改需要重启 Runner: +的值会作为配置错误被拒绝。此运维参数支持先 `check_runner_config` 再 +`reload_runner_config`(或 Unix SIGHUP),无需重启: ```toml max_concurrent_jobs = 32 ``` +扩容会立即调度原队列;缩容不会中断正在运行的 Job,只在已占槽位降至新上限以下后 +放行新任务,期间运行数可以高于上限。注册和运行中 Metadata 使用配置 Generation +上报实际容量,重连后重新同步;旧 Server 仍只能观察注册时的容量。 + 当所有槽位都被占用时,已接受的 Job 仍是同一个可查询 Job(同一 `job_id`),并 报告 `agent_queued`。 @@ -653,7 +658,7 @@ User scope 使用 `systemctl --user`;system scope 使用 `/etc/systemd/system` 4. reload 后调用 `get_runtime_status(client_id=...)`(或 `list_runners`)检查当前运行状态。 `reload_runner_config` 不写 `runner.toml`,只激活磁盘上已经存在的 candidate。policy、 -shell、configured Skill roots、configured instruction files、Native Plugin 与静态 SSH resource +shell、configured Skill roots、configured instruction files、Job 并发、Native Plugin 与静态 SSH resource 中可热加载的字段可以立即生效;`restart_required_fields` 报告的字段仍保持 startup-only,重启前不会假装已在线生效。无效 candidate 保留旧 active snapshot 与 generation。`ssh_resource` managed mutation 不同:它使用 frozen startup diff --git a/docs/agent/claude-code-mcp-provider.md b/docs/agent/claude-code-mcp-provider.md index 98613bb7e..0d39383fb 100644 --- a/docs/agent/claude-code-mcp-provider.md +++ b/docs/agent/claude-code-mcp-provider.md @@ -130,7 +130,7 @@ Provider. Identity, server/auth, registration, project source, concurrency, and transport fields still require restart: `server_url`, `token`, `client_id`, `display_name`, `owner`, `hostname`, `project_registry_dir`, `poll_interval_ms`, -`capabilities`, `max_concurrent_jobs`, `transport`, `websocket_connect_timeout_secs`, +`capabilities`, `transport`, `websocket_connect_timeout_secs`, and `quic.*`. A mixed reload applies the hot sections and reports these field names as `restart_required_fields`; it never reports their values. Read, parse, validation, or Provider-config failure keeps diff --git a/docs/agent/job-reliability-and-concurrency.md b/docs/agent/job-reliability-and-concurrency.md index b7b52d387..233617090 100644 --- a/docs/agent/job-reliability-and-concurrency.md +++ b/docs/agent/job-reliability-and-concurrency.md @@ -396,6 +396,13 @@ final drain and durable commit of the final retained tails. range 1..64; out-of-range configuration is rejected). It is not allocated per ChatGPT window, Workflow Session, or Project. +Capacity reload commits under the existing config-generation transaction and +JobManager lifecycle lock. Clone handles share the limit; reservation and dequeue +use that same lock. Growth drains the original FIFO after releasing config locks. +Shrink leaves reservations and running processes intact, so running can exceed +limit until natural completion. Runtime metadata is fenced by instance, connection +and generation, and reconnect registration uses the active config snapshot. + Opening multiple model windows consumes no Job slot by itself. A slot is consumed while a Job-backed execution owns Runner execution capacity. Therefore several windows using different Projects on the same Runner can contend for the same diff --git a/docs/agent/runtime-tuning-inventory.md b/docs/agent/runtime-tuning-inventory.md index 29f31ca45..b940e2645 100644 --- a/docs/agent/runtime-tuning-inventory.md +++ b/docs/agent/runtime-tuning-inventory.md @@ -109,3 +109,7 @@ This supersedes historical Job-default facts without changing the audit baseline Polling ordinary-request dispatch remains independently bounded at 4; WebSocket and QUIC dispatch share the same JobManager execution pool for Job-backed work. ACP run admission and persistent-shell population retain their own bounds. + +Job capacity also supports the existing generation-fenced config reload transaction. +Growth schedules the original queue; shrink leaves active Jobs running. Server and +Desktop observations use applied metadata, never an unactivated on-disk candidate. diff --git a/docs/desktop-guide.md b/docs/desktop-guide.md index 2ae06e3ad..92e94d940 100644 --- a/docs/desktop-guide.md +++ b/docs/desktop-guide.md @@ -44,15 +44,16 @@ never mean zero usage; ordinary request-dispatch queues are separate. In **Settings → Runtime & services → Runner job capacity**, Desktop-managed local Runners can save **Maximum concurrent jobs** as a whole number from **1 to 64**. The default for an omitted setting is **32**; explicit saved values are preserved. The saved value and current effective limit are shown -separately. **Save for next restart** only writes the exact local Runner's -configuration, preserves unrelated settings, and rejects a conflicting edit. -It does not hot-reload or restart services. +separately. **Save and apply** stages the exact local Runner configuration, preserves +unrelated settings, then validates and reloads with a generation fence. A definite +rejection restores only an unchanged candidate; an uncertain result requires +reconciliation and is never blindly retried or rolled back. Running Jobs continue. -Choose **Restart Runner…** when ready and confirm the interruption warning. +**Restart Runner…** remains a separate operation with an interruption warning. Restarting applies the saved Runner configuration and may interrupt Jobs, browser sessions, and handoffs. Desktop reuses its existing owned-process or persistent-service restart path. Remote and independently managed Runners remain -read-only; their actual operator must edit the configuration and restart them. +read-only; their actual operator must edit and apply their configuration. A restart is not reported as applying the saved limit until fresh Runner status reports that limit. diff --git a/docs/desktop-guide.zh-CN.md b/docs/desktop-guide.zh-CN.md index 076fedd8a..625e219cc 100644 --- a/docs/desktop-guide.zh-CN.md +++ b/docs/desktop-guide.zh-CN.md @@ -24,9 +24,9 @@ Runner 文件系统 policy 仍然是权限边界。全新 Desktop 的本机 Runn **项目 → 设备** 显示每个 Runner 上报的持久任务运行数、排队数和当前生效的并发上限。计数来自 Server 的容量聚合;已请求停止的任务在真正终止前仍计入运行数,因为它们仍占用执行槽位。离线、过期、不可用或未上报的状态不会显示为零使用量;普通请求调度队列单独计算。 -打开 **设置 → Runtime 与服务 → Runner 任务容量**,可为 Desktop 管理的本地 Runner 保存 **1–64** 的整数并发上限。未配置时默认值为 **32**,已有显式配置值保持不变,已保存的配置与实际生效值分别显示。**保存,下次重启时生效** 只修改身份匹配的本地 Runner 配置,保留其他内容并拒绝冲突修改,不会热重载或自动重启。 +打开 **设置 → Runtime 与服务 → Runner 任务容量**,可为 Desktop 管理的本地 Runner 保存 **1–64** 的整数并发上限。未配置时默认值为 **32**,已有显式配置值保持不变,已保存的配置与实际生效值分别显示。**保存并热应用** 暂存身份匹配的本地 Runner 配置,保留其他内容,随后使用 Check → Reload 与 Generation Fence 验证生效。明确拒绝时只回滚未被外部修改的候选;结果不确定时要求核对,不盲目重试或回滚。运行中的 Job 继续执行。 -准备好后选择 **重启 Runner…**,确认中断警告。重启会应用已保存的 Runner 配置,可能中断任务、浏览器会话和用户接管。Desktop 复用现有进程所有权或持久服务重启机制。远程及独立管理的 Runner 保持只读,需由实际管理者修改配置并重启。只有刷新后的 Runner 状态上报新上限,界面才会确认它已生效。 +**重启 Runner…** 保留为独立操作,仍需确认中断警告。重启会应用已保存的 Runner 配置,可能中断任务、浏览器会话和用户接管。Desktop 复用现有进程所有权或持久服务重启机制。远程及独立管理的 Runner 保持只读,需由实际管理者修改并应用配置。只有刷新后的 Runner 状态上报新上限,界面才会确认它已生效。 ## Windows 启动与升级 diff --git a/scripts/test-runner-config-reload-e2e.sh b/scripts/test-runner-config-reload-e2e.sh index 1dd30e9d4..1a285f8db 100755 --- a/scripts/test-runner-config-reload-e2e.sh +++ b/scripts/test-runner-config-reload-e2e.sh @@ -122,7 +122,7 @@ display_name = "${display}" project_registry_dir = "${PROJECTS_DIR}" poll_interval_ms = 100 max_concurrent_jobs = ${max_jobs} -transport = "polling" +transport = "${WEBCODEX_E2E_TRANSPORT:-polling}" [policy] allow_raw_shell = true allow_cwd_anywhere = false @@ -162,6 +162,7 @@ r = p["config_reload"] expected_error = None if sys.argv[6] == "null" else sys.argv[6] expected_fields = [] if sys.argv[8] == "-" else sys.argv[8].split(",") assert r["generation"] == int(sys.argv[4]) and r["last_reload_result"] == sys.argv[5] +assert c["job_concurrency"]["limit"] == (2 if int(sys.argv[4]) == 3 else 1) assert r.get("last_reload_error_code") == expected_error assert r["restart_required"] is (sys.argv[7] == "true") assert r["restart_required_fields"] == expected_fields == sorted(set(expected_fields)) @@ -201,7 +202,7 @@ run_shell_request() { } assert_marker() { run_shell_request 'printf %s "$WEBCODEX_RELOAD_MARKER"' - python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); assert d["success"] and d["output"]["stdout_tail"]==sys.argv[2] and d["output"]["stderr_tail"]=="" and d["output"]["exit_code"]==0' \ + python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); assert d["success"] and d["output"]["stdout_tail"]==sys.argv[2] and d["output"].get("stderr_tail", "")=="" and d["output"].get("exit_code", 0)==0 and not d["output"].get("job_id")' \ "$RESPONSE_FILE" "$1" || fail "shell marker was not $1" } start_job() { @@ -212,14 +213,14 @@ job_status() { runtime_tool_call "observe_jobs" "{\"items\":[{\"job_id\":\"$1\"}],\"tail_lines\":1}" | python3 -c \ 'import json,sys; d=json.load(sys.stdin); assert d["success"]; item=d["output"]["items"][0]; assert item["job_id"]==sys.argv[1]; print(item["status"])' "$1" } -for command in awk curl git mv python3 setsid tail "$CARGO_BIN"; do require_command "$command"; done +for command in awk curl git mv python3 tail "$CARGO_BIN"; do require_command "$command"; done cd "$ROOT" REPO_STATUS_BEFORE="$(git status --short)" if [ "${WEBCODEX_E2E_SKIP_BUILD:-0}" != "1" ]; then - "$CARGO_BIN" build --quiet -p webcodex -p webcodex-runner --bins + "$CARGO_BIN" build --profile dogfood --quiet -p webcodex -p webcodex-runner --bins fi -[ -x target/debug/webcodex-server ] && [ -x target/debug/webcodex-runner ] \ - || fail "debug server/agent binaries are unavailable" +[ -x target/dogfood/webcodex-server ] && [ -x target/dogfood/webcodex-runner ] \ + || fail "dogfood server/agent binaries are unavailable" TMP_ROOT="$(mktemp -d -t webcodex-runner-reload-e2e-XXXXXX)" DATA_DIR="$TMP_ROOT/data" PROJECTS_DIR="$TMP_ROOT/project-registry" @@ -254,19 +255,19 @@ EOF PORT="$(find_port)" write_runner_config generation-1 5 65536 native false 30 "$STARTUP_DISPLAY" 1 \ project_search_generation_1 -setsid env -i PATH="$PATH" LANG=C HOME="$ISOLATED_HOME" \ +python3 -c 'import os,sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' env -i PATH="$PATH" LANG=C HOME="$ISOLATED_HOME" \ XDG_CONFIG_HOME="$ISOLATED_HOME/.config" XDG_DATA_HOME="$ISOLATED_HOME/.local/share" \ XDG_STATE_HOME="$ISOLATED_HOME/.local/state" XDG_CACHE_HOME="$ISOLATED_HOME/.cache" \ TMPDIR="$RUNTIME_TMP" WEBCODEX_ENV_FILE="$TMP_ROOT/empty.env" \ WEBCODEX_ADDR="127.0.0.1:${PORT}" WEBCODEX_DATA="$DATA_DIR" WEBCODEX_TOKEN="$TOKEN" \ - RUST_LOG=warn target/debug/webcodex-server >"$SERVER_LOG" 2>&1 & + RUST_LOG=warn target/dogfood/webcodex-server < /dev/null >"$SERVER_LOG" 2>&1 & SERVER_PID=$! wait_for_port open || fail "isolated server port did not open" -setsid env -i PATH="$PATH" LANG=C HOME="$ISOLATED_HOME" \ +python3 -c 'import os,sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' env -i PATH="$PATH" LANG=C HOME="$ISOLATED_HOME" \ XDG_CONFIG_HOME="$ISOLATED_HOME/.config" XDG_DATA_HOME="$ISOLATED_HOME/.local/share" \ XDG_STATE_HOME="$ISOLATED_HOME/.local/state" XDG_CACHE_HOME="$ISOLATED_HOME/.cache" \ TMPDIR="$RUNTIME_TMP" WEBCODEX_ENV_FILE="$TMP_ROOT/empty.env" RUST_LOG=warn \ - target/debug/webcodex-runner --config "$RUNNER_CONFIG" >"$RUNNER_LOG" 2>&1 & + target/dogfood/webcodex-runner --config "$RUNNER_CONFIG" < /dev/null >"$RUNNER_LOG" 2>&1 & RUNNER_PID=$! START_RUNNER_PID="$RUNNER_PID" STAGE="generation 1 baseline" @@ -305,7 +306,7 @@ STAGE="generation 3 mixed reload" write_runner_config generation-3 5 24576 native false 11 'Restart-Only Display' 2 \ project_search_generation_3 kill -HUP "$RUNNER_PID" -wait_for_status 3 partial null true display_name,max_concurrent_jobs native false \ +wait_for_status 3 partial null true display_name native false \ || fail "mixed reload status did not arrive" assert_runner_pid command='sleep 3; printf %s "$WEBCODEX_RELOAD_MARKER"' @@ -318,11 +319,11 @@ queued_seen=false for _ in $(seq 1 50); do one="$(job_status "$JOB_ONE" 2>/dev/null || true)" two="$(job_status "$JOB_TWO" 2>/dev/null || true)" - if [ "$one" = running ] && [ "$two" = agent_queued ]; then queued_seen=true; break; fi + if [ "$one" = running ] && [ "$two" = completed ]; then queued_seen=true; break; fi [ "$two" != completed ] || break sleep 0.1 done -[ "$queued_seen" = true ] || fail "startup max_concurrent_jobs=1 was not retained" +[ "$queued_seen" = true ] || fail "hot max_concurrent_jobs=2 did not execute the second Job while the first remained gated" touch "$GATE" for _ in $(seq 1 80); do [ "$(job_status "$JOB_ONE" 2>/dev/null || true)" = completed ] && \ @@ -331,7 +332,7 @@ for _ in $(seq 1 80); do done [ "$(job_status "$JOB_ONE")" = completed ] && [ "$(job_status "$JOB_TWO")" = completed ] \ || fail "mixed-reload concurrency probe jobs did not complete" -ok "generation 3 applied hot fields and retained startup identity/concurrency" +ok "generation 3 applied hot fields and retained startup identity and applied concurrency" STAGE="generation 4 recovery" write_runner_config generation-4 3 16384 claude_code_then_native true 13 "$STARTUP_DISPLAY" 1 \ @@ -347,6 +348,55 @@ STAGE="shutdown and cleanup" [ -z "$(git -C "$FIXTURE" status --porcelain)" ] || fail "fixture worktree is dirty" status_matches 4 success null false - claude_code_then_native true \ || fail "final status changed or Claude was started" +STAGE="live capacity growth and shrink" +python3 - "$PORT" "$TOKEN" "$CLIENT_ID" "$RUNTIME_PROJECT" "$RUNNER_CONFIG" "$TMP_ROOT" <<'CAPACITY_PY' +import json, pathlib, re, sys, time, urllib.request +port, token, client, project, config, root = sys.argv[1:] +def post(path, body): + req = urllib.request.Request(f"http://127.0.0.1:{port}{path}", json.dumps(body).encode(), {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=15) as response: return json.load(response) +def tool(name, **params): + value = post("/api/tools/call", {"tool": name, "params": params}) + assert value["success"], (name, value) + return value["output"] +def capacity(): + clients = post("/api/runtime/status", {})["output"]["runners"]["clients"] + return next(c for c in clients if c["client_id"] == client)["job_concurrency"] +def until(predicate): + deadline = time.monotonic() + 15 + while not predicate(): + assert time.monotonic() < deadline, capacity() + time.sleep(.05) +def resize(limit): + path = pathlib.Path(config) + text = re.sub(r"max_concurrent_jobs = \d+", f"max_concurrent_jobs = {limit}", path.read_text()) + text = re.sub(r"max_timeout_secs = \d+", "max_timeout_secs = 120", text) + path.write_text(text) + checked = tool("check_runner_config", client_id=client) + assert checked["valid"] and not checked["restart_required"] + applied = tool("reload_runner_config", client_id=client, expected_generation=checked["current_generation"]) + assert applied["current_generation"] == checked["current_generation"] + 1 + until(lambda: capacity()["limit"] == limit) +resize(2) +gates = [pathlib.Path(root) / f"capacity-gate-{i}" for i in range(8)] +ids = [tool("run_job", project=project, command=f"while [ ! -e '{gate}' ]; do sleep 0.05; done", timeout_secs=120)["job_id"] for gate in gates] +until(lambda: capacity()["running"] == 2 and capacity()["queued"] == 6) +resize(6) +until(lambda: capacity()["running"] == 6 and capacity()["queued"] == 2) +resize(2) +assert capacity()["running"] == 6 and capacity()["queued"] == 2 +for gate in gates[:4]: gate.touch() +until(lambda: capacity()["running"] == 2 and capacity()["queued"] == 2) +gates[4].touch() +until(lambda: capacity()["running"] == 2 and capacity()["queued"] == 1) +for gate in gates[5:]: gate.touch() +until(lambda: capacity()["running"] == 0 and capacity()["queued"] == 0) +observed = tool("observe_jobs", items=[{"job_id": jid} for jid in ids], tail_lines=1)["items"] +assert {j["job_id"] for j in observed} == set(ids) +assert all(j["status"] == "completed" for j in observed) +CAPACITY_PY +assert_runner_pid +ok "real Runner 2 -> 6 -> 2 preserves eight Job identities and drains without restart" stop_group "$RUNNER_PID" kill -0 -- "-$RUNNER_PID" 2>/dev/null && fail "agent process group remained" || true RUNNER_PID="" diff --git a/src/runner_http/handlers.rs b/src/runner_http/handlers.rs index 3da86133d..736165248 100644 --- a/src/runner_http/handlers.rs +++ b/src/runner_http/handlers.rs @@ -206,6 +206,14 @@ pub async fn runner_poll(req: &mut Request, depot: &mut Depot, res: &mut Respons })); return; } + let _ = registry + .update_job_capacity( + &body.request.client_id, + &body.request.runner_instance_id, + None, + body.job_capacity, + ) + .await; let _ = registry .update_runtime_metadata( &body.request.client_id, diff --git a/src/runner_quic.rs b/src/runner_quic.rs index 9085c3089..3d2f2e9c4 100644 --- a/src/runner_quic.rs +++ b/src/runner_quic.rs @@ -893,6 +893,102 @@ mod tests { conn.close(quinn::VarInt::from_u32(0), b"done"); } + #[tokio::test] + async fn quic_capacity_metadata_and_reconnect_preserve_effective_generation() { + use crate::runner_protocol::{RunnerJobCapacity, RunnerPolicySummary, ToolProvidersStatus}; + let (cert, key) = self_signed_cert(); + let registry = Arc::new(RunnerRegistry::default()); + let addr = start_quic_server(registry.clone(), test_config(None), cert.clone(), key).await; + let status = |generation| { + let mut status = ToolProvidersStatus { + strategy: "native".into(), + claude_code: crate::runner_protocol::ClaudeCodeProviderStatus { + enabled: false, + version: None, + available: false, + process_state: "not_started".into(), + discovered_tool_names: Vec::new(), + capabilities: Default::default(), + last_error_code: None, + last_call: None, + }, + config_reload: Default::default(), + }; + status.config_reload.generation = generation; + status + }; + for (generation, limit) in [(1, 2), (2, 6)] { + let (endpoint, conn, mut send, mut recv) = connect_quic_client(&cert, addr).await; + let mut register = register_envelope("quic-capacity", "same-instance"); + register.payload_mut().job_concurrency_limit = Some(limit); + register.payload_mut().policy = Some(RunnerPolicySummary { + tool_providers: Some(status(generation)), + ..Default::default() + }); + write_quic_register_frame(&mut send, ®ister) + .await + .unwrap(); + let ack = tokio::time::timeout(Duration::from_secs(5), read_quic_frame(&mut recv)) + .await + .unwrap() + .unwrap(); + assert!(matches!( + ack, + RunnerEnvelope::Registered { success: true, .. } + )); + assert_eq!( + registry + .get_runner_view("quic-capacity") + .await + .unwrap() + .job_concurrency_limit, + Some(limit) + ); + for (update_generation, update_limit, expected) in [(2, 6, 6), (1, 32, 6)] { + write_quic_frame( + &mut send, + &RunnerEnvelope::RuntimeMetadata { + job_capacity: Some(RunnerJobCapacity { + generation: update_generation, + limit: update_limit, + }), + tool_providers: status(update_generation), + mcp_gateway_providers: None, + computer_session_availability: None, + }, + ) + .await + .unwrap(); + // Ordered Ping/Pong is a processing barrier, not a timing guess. + write_quic_frame(&mut send, &RunnerEnvelope::Ping { ts: 99 }) + .await + .unwrap(); + loop { + let envelope = + tokio::time::timeout(Duration::from_secs(5), read_quic_frame(&mut recv)) + .await + .unwrap() + .unwrap(); + if matches!(envelope, RunnerEnvelope::Pong { ts: 99 }) { + break; + } + } + assert_eq!( + registry + .get_runner_view("quic-capacity") + .await + .unwrap() + .job_concurrency_limit, + Some(expected) + ); + } + send.finish().unwrap(); + wait_for_quic_client_connected(®istry, "quic-capacity", false).await; + conn.close(quinn::VarInt::from_u32(0), b"done"); + endpoint.close(quinn::VarInt::from_u32(0), b"done"); + } + } + #[tokio::test] async fn quic_request_result_roundtrip() { let (cert_der, key_der) = self_signed_cert(); diff --git a/src/runner_session.rs b/src/runner_session.rs index f5beb2293..e40567679 100644 --- a/src/runner_session.rs +++ b/src/runner_session.rs @@ -602,10 +602,19 @@ async fn dispatch_inbound( } } RunnerEnvelope::RuntimeMetadata { + job_capacity, tool_providers, mcp_gateway_providers, computer_session_availability, } => { + let _ = registry + .update_job_capacity( + client_id, + runner_instance_id, + Some(connection_id), + job_capacity, + ) + .await; let _ = registry .update_runtime_metadata_for_connection( client_id, diff --git a/src/runner_ws.rs b/src/runner_ws.rs index c0634755f..12f0e35ef 100644 --- a/src/runner_ws.rs +++ b/src/runner_ws.rs @@ -813,6 +813,10 @@ mod tests { ws.send(TungsteniteMessage::Text( RunnerEnvelope::RuntimeMetadata { + job_capacity: Some(crate::runner_protocol::RunnerJobCapacity { + generation: 2, + limit: 6, + }), computer_session_availability: None, tool_providers: provider_status(), mcp_gateway_providers: Some(vec![crate::mcp_gateway::McpGatewayProvider { @@ -846,6 +850,7 @@ mod tests { tokio::time::sleep(Duration::from_millis(25)).await; } let view = registry.get_runner_view("ws-roundtrip").await.unwrap(); + assert_eq!(view.job_concurrency_limit, Some(6)); let call = view .policy .unwrap() From 04045ee2c2040ec023161af2f73f0b15f4638dbe Mon Sep 17 00:00:00 2001 From: yyjeqhc <1772413353@qq.com> Date: Tue, 27 Oct 2026 19:01:00 +0800 Subject: [PATCH 2/3] fix(runner): reject capacity updates from retired transport leases --- .../webcodex-runner-registry/src/runners.rs | 6 +++- .../src/tests/registration_projection.rs | 29 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/crates/webcodex-runner-registry/src/runners.rs b/crates/webcodex-runner-registry/src/runners.rs index 5ce2c45a0..f9a8e7382 100644 --- a/crates/webcodex-runner-registry/src/runners.rs +++ b/crates/webcodex-runner-registry/src/runners.rs @@ -833,7 +833,11 @@ impl RunnerRegistry { let mut inner = self.inner.lock().await; let runner = inner.runners.get_mut(client_id).ok_or("unknown Runner")?; if runner.runner_instance_id != runner_instance_id - || connection_id.is_some_and(|id| runner.connection_id.as_deref() != Some(id)) + || runner.disconnected_at.is_some() + || match connection_id { + Some(id) => runner.connection_id.as_deref() != Some(id), + None => runner.transport != RunnerTransport::Polling, + } { return Err("stale Runner capacity lease".into()); } diff --git a/crates/webcodex-runner-registry/src/tests/registration_projection.rs b/crates/webcodex-runner-registry/src/tests/registration_projection.rs index 4c6d49415..5cea1d0af 100644 --- a/crates/webcodex-runner-registry/src/tests/registration_projection.rs +++ b/crates/webcodex-runner-registry/src/tests/registration_projection.rs @@ -126,6 +126,22 @@ async fn hot_job_capacity_is_bounded_and_fenced_by_instance_connection_and_gener Some(6) ); } + // A delayed poll cannot mutate a same-instance stream lease, even if + // it claims a higher generation than the current stream has reported. + if connection.is_some() { + assert!(registry + .update_job_capacity("capacity", "instance", None, capacity(3, 32)) + .await + .is_err()); + assert_eq!( + registry + .get_runner_view("capacity") + .await + .unwrap() + .job_concurrency_limit, + Some(6) + ); + } // An obsolete registration from this same process must not regress capacity. assert!(registry.register(request).await.is_err()); registry @@ -140,5 +156,18 @@ async fn hot_job_capacity_is_bounded_and_fenced_by_instance_connection_and_gener registry.list_runners().await[0].job_concurrency_limit, Some(2) ); + registry.reconcile_disconnect("capacity", "instance").await; + assert!(registry + .update_job_capacity("capacity", "instance", connection, capacity(4, 32)) + .await + .is_err()); + assert_eq!( + registry + .get_runner_view("capacity") + .await + .unwrap() + .job_concurrency_limit, + Some(2) + ); } } From 61e5b767d6ec032c95cd307fab1ccbabb6e79165 Mon Sep 17 00:00:00 2001 From: yyjeqhc <1772413353@qq.com> Date: Tue, 27 Oct 2026 19:05:52 +0800 Subject: [PATCH 3/3] test(runner): preserve mixed Job lifecycles across capacity reload --- .../job_manager_tests/capacity.rs | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/crates/webcodex-runner/src/webcodex_runner/job_manager_tests/capacity.rs b/crates/webcodex-runner/src/webcodex_runner/job_manager_tests/capacity.rs index 014ce38a7..b9370b45b 100644 --- a/crates/webcodex-runner/src/webcodex_runner/job_manager_tests/capacity.rs +++ b/crates/webcodex-runner/src/webcodex_runner/job_manager_tests/capacity.rs @@ -127,3 +127,96 @@ fn capacity_changes_race_enqueue_completion_stop_and_shutdown_without_replay() { } } } + +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn hot_capacity_preserves_interactive_detached_and_structured_job_identity() { + let temp = tempfile::tempdir().unwrap(); + let helper = structured_process_helper(); + let (sink, _rx) = structured_test_sink("structured-agent", "structured-instance"); + let manager = JobManager::new(1); + *lock_unpoison(&manager.detached_store_root_override) = + Some(temp.path().join("detached-state")); + let jobs = (0..5) + .map(|i| GatedStructuredJob::new(temp.path(), &format!("mixed-{i}"))) + .collect::>(); + for (i, job) in jobs.iter().enumerate() { + match i { + 0 | 3 => { + let mut context = structured_process_context(temp.path(), job.args().len(), false); + context + .structured_execution + .as_mut() + .unwrap() + .execution_source = "run_process_interactive".into(); + manager.enqueue(sink.clone(), PendingJobStart::from_wire( + 1, RunnerPolicy { allow_cwd_anywhere: true, ..RunnerPolicy::default() }, + ShellConfig::default(), SshConfig::default(), temp.path().join("project-registry"), + serde_json::from_value(json!({ + "request_id": format!("request-{}", job.job_id), "client_id":"structured-agent", + "kind":"start_interactive_process_job", "job_id":job.job_id, "cwd":temp.path(), + "command":"", "process":{"executable":helper.path,"args":job.args()}, + "timeout_secs":20,"requested_by":"test","created_at":chrono::Utc::now().timestamp(), + "job_context":context, + })).unwrap(), + )); + } + 1 => enqueue_detached_process_job( + &manager, + sink.clone(), + temp.path(), + &job.job_id, + &helper.path, + job.args(), + None, + 20, + ), + _ => enqueue_gated_structured_job(&manager, &sink, temp.path(), &helper, job), + } + } + wait_for_all_started(&jobs[..1]); + let before = manager.inventory(); + manager.commit_concurrency_limit(4); + manager.start_available_queued(); + wait_for_all_started(&jobs[..4]); + assert!(wait_until(Duration::from_secs(5), || manager + .inventory() + .jobs + .iter() + .filter(|job| job.status == "running") + .count() + == 4)); + manager.clone().commit_concurrency_limit(1); + manager.start_available_queued(); + assert_eq!(active_gated_children(&jobs), 4); + assert!(!jobs[4].started.exists()); + for job in &jobs[..3] { + job.release(); + } + assert!(wait_until(Duration::from_secs(10), || manager + .inventory() + .jobs + .iter() + .filter(|job| job.status == "completed") + .count() + == 3)); + assert!(!jobs[4].started.exists()); + jobs[3].release(); + wait_for_all_started(&jobs[4..]); + jobs[4].release(); + wait_for_job_workers(&manager); + let after = manager.inventory(); + for original in before.jobs { + let current = after + .jobs + .iter() + .find(|job| job.job_id == original.job_id) + .unwrap(); + assert_eq!(current.status, "completed"); + assert_eq!(current.request_id, original.request_id); + assert_eq!(current.context, original.context); + } + for job in &jobs { + assert_gated_job_started_once(job); + } +}