From 083c7877fee11eb00c1575a659f22ed56142fa06 Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Thu, 22 Oct 2026 02:20:41 -0700 Subject: [PATCH 1/2] fix(scheduler): scope the dind image cache on webhook-dispatched jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The webhook/poll path built runtime.CreateConfig without Provider or Repo, so dind.CacheNamespace returned "" and the per-repo image cache was off for every job that path started — which on Linux nodes is all of them (only the WSL/macOS dispatch path set the fields). Every `docker pull` in a job went to the registry: ducks pulled mysql:8.0 78 times in three days, 7-45 s each. Pass the event's provider name and repo, as dispatch.go already does. --- pkg/scheduler/scheduler.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index c0b1a5a..1a52c12 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -2003,6 +2003,12 @@ func (s *Scheduler) handleLocalJob(ctx context.Context, event providers.JobEvent JITConfig: claim.RunnerConfig, Env: claim.Env, Entrypoint: claim.Entrypoint, + // Provider+Repo scope the job's dind image cache. Without + // them dind.CacheNamespace returns "" and every `docker + // pull` in the job goes to the registry: ducks pulled + // mysql:8.0 78 times in 3 days before these were passed. + Provider: providerName(event), + Repo: event.Repo, }) return cerr }) From 260aee2c9599762837c0eade91decd7b0b97290e Mon Sep 17 00:00:00 2001 From: Luther Monson Date: Thu, 22 Oct 2026 02:20:41 -0700 Subject: [PATCH 2/2] fix(cargo): bound git-dependency fetches that stall MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Git dependencies bypass the crate cache and are fetched by libgit2, which has no working stall detection once TLS is up. ephpm job 119658796412 (ducks, 2026-10-22) sat 13 minutes and then 5.5 more on a silent github.com connection fetching the turso git dep before libgit2 gave up with "SSL error: syscall failure" — 19 minutes of a 25 minute job. Set net.git-fetch-with-cli in both the caching and the inert container config, and export GIT_HTTP_LOW_SPEED_LIMIT=1000 / GIT_HTTP_LOW_SPEED_TIME=30 so git abandons a transfer that moves under 1 KB/s for 30 s; Cargo's net.retry then reconnects. Both are skipped for Windows containers, whose images ship no git binary (Cargo fails outright if it cannot run git). --- pkg/proxies/cargo/cache.go | 35 +++++++++++++++++------- pkg/proxies/cargo/cache_test.go | 2 +- pkg/proxies/cargo/cargoproxy.go | 16 ++++++++--- pkg/proxies/cargo/cargoproxy_test.go | 40 ++++++++++++++++++++++++---- 4 files changed, 75 insertions(+), 18 deletions(-) diff --git a/pkg/proxies/cargo/cache.go b/pkg/proxies/cargo/cache.go index 92ea148..d87cd87 100644 --- a/pkg/proxies/cargo/cache.go +++ b/pkg/proxies/cargo/cache.go @@ -318,7 +318,7 @@ func parseCrateDownloadPath(urlPath string) (name, version string, ok bool) { // environment variables are silently ignored (verified against cargo stable: // with the env vars set, cargo still reported "Updating crates.io index"). // Hence a generated file rather than an env var. -func containerConfigTOML(base string) string { +func containerConfigTOML(base, goos string) string { base = strings.TrimRight(base, "/") return `# Generated by ephemerd. Do NOT edit — regenerated on every daemon start. # @@ -336,9 +336,7 @@ replace-with = "ephemerd" [source.ephemerd] registry = "sparse+` + base + indexRoute + `/" -[net] -retry = 3 -` +` + netTOML(goos) } // inertConfigTOML renders the config the proxy installs when it has withdrawn @@ -350,8 +348,8 @@ retry = 3 // destination is a directory Cargo searches — leaving a stale caching config // behind, or a partial one, is what would break builds. // -// [net] retry is kept: it is useful on its own and costs nothing. -func inertConfigTOML() string { +// The [net] table is kept: nothing in it depends on the cache. +func inertConfigTOML(goos string) string { return `# Generated by ephemerd. Do NOT edit — regenerated on every daemon start. # # ephemerd's Cargo pull-through cache is NOT in use right now: the daemon's @@ -364,9 +362,28 @@ func inertConfigTOML() string { # for a dead replacement source (unlike Go's GOPROXY="…|direct"), so when the # cache cannot be trusted the only safe configuration is none at all. -[net] -retry = 3 -` +` + netTOML(goos) +} + +// netTOML renders the [net] table shared by the caching and inert configs. +// +// Git dependencies never touch the cache: Cargo fetches them straight from +// their host. Its built-in libgit2 fetch has no stall detection that works +// once TLS is up — ephpm job 119658796412 (ducks, 2026-10-22) sat 13 minutes, +// then 5½ more, on a silent github.com connection before libgit2 reported +// "SSL error: syscall failure": 19 minutes of a 25 minute job. +// git-fetch-with-cli hands the fetch to the git binary, which honours +// GIT_HTTP_LOW_SPEED_LIMIT/TIME (set by EnvVars) and abandons a stalled +// transfer in seconds, so Cargo's retry gets a fresh connection. +// +// Not on Windows: the Windows runner images ship no git binary, and with +// git-fetch-with-cli set Cargo fails outright when it cannot run git. +func netTOML(goos string) string { + s := "[net]\nretry = 3\n" + if goos != "windows" { + s += "git-fetch-with-cli = true\n" + } + return s } // containerConfigDest is the container path the generated .cargo directory is diff --git a/pkg/proxies/cargo/cache_test.go b/pkg/proxies/cargo/cache_test.go index 9c00de4..cd8257a 100644 --- a/pkg/proxies/cargo/cache_test.go +++ b/pkg/proxies/cargo/cache_test.go @@ -485,7 +485,7 @@ func TestParseCrateDownloadPath(t *testing.T) { } func TestContainerConfigTOML(t *testing.T) { - got := containerConfigTOML("http://10.88.0.1:8083/") + got := containerConfigTOML("http://10.88.0.1:8083/", "linux") // Source replacement is the whole point — without both halves Cargo // silently keeps using crates.io. diff --git a/pkg/proxies/cargo/cargoproxy.go b/pkg/proxies/cargo/cargoproxy.go index 3391946..80e958f 100644 --- a/pkg/proxies/cargo/cargoproxy.go +++ b/pkg/proxies/cargo/cargoproxy.go @@ -332,10 +332,20 @@ func (p *Proxy) advertiseBase() string { // // Only rustup is configured here. Cargo's registry redirect cannot be done // with environment variables (see Mounts). +// +// The GIT_HTTP_LOW_SPEED pair makes git give up on a transfer that moves +// under 1 KB/s for 30 seconds. It is what bounds the git-dependency fetches +// Cargo runs through the git binary (see netTOML); without it git, like +// libgit2, waits on a dead connection indefinitely. Real fetches never run +// that slowly, so actions/checkout and other git users lose nothing. func (p *Proxy) EnvVars() []string { - return []string{ + env := []string{ "RUSTUP_DIST_SERVER=" + p.advertiseBase() + rustupRoute, } + if p.containerOS() != "windows" { + env = append(env, "GIT_HTTP_LOW_SPEED_LIMIT=1000", "GIT_HTTP_LOW_SPEED_TIME=30") + } + return env } // Mounts returns the read-only bind mount that carries the generated Cargo @@ -386,9 +396,9 @@ func (p *Proxy) writeContainerConfigLocked(active bool) error { if err := os.MkdirAll(dir, 0o755); err != nil { return fmt.Errorf("creating cargo conf dir %q: %w", dir, err) } - body := containerConfigTOML(p.advertiseBase()) + body := containerConfigTOML(p.advertiseBase(), p.containerOS()) if !active { - body = inertConfigTOML() + body = inertConfigTOML(p.containerOS()) } target := filepath.Join(dir, "config.toml") tmp, err := os.CreateTemp(dir, ".config-*.toml") diff --git a/pkg/proxies/cargo/cargoproxy_test.go b/pkg/proxies/cargo/cargoproxy_test.go index 987a090..9f2a7cd 100644 --- a/pkg/proxies/cargo/cargoproxy_test.go +++ b/pkg/proxies/cargo/cargoproxy_test.go @@ -695,12 +695,42 @@ func TestEnvVars_PointRustupAtTheProxy(t *testing.T) { p, _ := newTestProxy(t, up, nil) env := p.EnvVars() - if len(env) != 1 { - t.Fatalf("EnvVars() = %v, want exactly the rustup dist server", env) + want := []string{ + "RUSTUP_DIST_SERVER=" + p.advertiseBase() + "/rustup", + "GIT_HTTP_LOW_SPEED_LIMIT=1000", + "GIT_HTTP_LOW_SPEED_TIME=30", + } + if strings.Join(env, "\n") != strings.Join(want, "\n") { + t.Errorf("EnvVars() = %v, want %v", env, want) + } +} + +// TestNetTOML_GitFetchBounded: git dependencies bypass the cache, so the +// only thing that stops a stalled GitHub fetch from eating a whole job is +// handing it to the git CLI (bounded by GIT_HTTP_LOW_SPEED_*). Both configs +// carry it — the inert one is what is live exactly when the network is bad. +// Windows images have no git binary, so there it must stay off. +func TestNetTOML_GitFetchBounded(t *testing.T) { + for _, goos := range []string{"linux", "darwin"} { + for name, cfg := range map[string]string{ + "caching": containerConfigTOML("http://10.88.0.1:8083", goos), + "inert": inertConfigTOML(goos), + } { + if !strings.Contains(cfg, "git-fetch-with-cli = true") { + t.Errorf("%s/%s config lacks git-fetch-with-cli:\n%s", goos, name, cfg) + } + if !strings.Contains(cfg, "retry = 3") { + t.Errorf("%s/%s config lost net.retry:\n%s", goos, name, cfg) + } + } } - want := "RUSTUP_DIST_SERVER=" + p.advertiseBase() + "/rustup" - if env[0] != want { - t.Errorf("EnvVars()[0] = %q, want %q", env[0], want) + for name, cfg := range map[string]string{ + "caching": containerConfigTOML("http://10.88.0.1:8083", "windows"), + "inert": inertConfigTOML("windows"), + } { + if strings.Contains(cfg, "git-fetch-with-cli") { + t.Errorf("windows/%s config sets git-fetch-with-cli, but Windows images have no git:\n%s", name, cfg) + } } }