diff --git a/pkg/proxies/cargo/cache.go b/pkg/proxies/cargo/cache.go index 92ea148..d87cd87 100644 --- a/pkg/proxies/cargo/cache.go +++ b/pkg/proxies/cargo/cache.go @@ -318,7 +318,7 @@ func parseCrateDownloadPath(urlPath string) (name, version string, ok bool) { // environment variables are silently ignored (verified against cargo stable: // with the env vars set, cargo still reported "Updating crates.io index"). // Hence a generated file rather than an env var. -func containerConfigTOML(base string) string { +func containerConfigTOML(base, goos string) string { base = strings.TrimRight(base, "/") return `# Generated by ephemerd. Do NOT edit — regenerated on every daemon start. # @@ -336,9 +336,7 @@ replace-with = "ephemerd" [source.ephemerd] registry = "sparse+` + base + indexRoute + `/" -[net] -retry = 3 -` +` + netTOML(goos) } // inertConfigTOML renders the config the proxy installs when it has withdrawn @@ -350,8 +348,8 @@ retry = 3 // destination is a directory Cargo searches — leaving a stale caching config // behind, or a partial one, is what would break builds. // -// [net] retry is kept: it is useful on its own and costs nothing. -func inertConfigTOML() string { +// The [net] table is kept: nothing in it depends on the cache. +func inertConfigTOML(goos string) string { return `# Generated by ephemerd. Do NOT edit — regenerated on every daemon start. # # ephemerd's Cargo pull-through cache is NOT in use right now: the daemon's @@ -364,9 +362,28 @@ func inertConfigTOML() string { # for a dead replacement source (unlike Go's GOPROXY="…|direct"), so when the # cache cannot be trusted the only safe configuration is none at all. -[net] -retry = 3 -` +` + netTOML(goos) +} + +// netTOML renders the [net] table shared by the caching and inert configs. +// +// Git dependencies never touch the cache: Cargo fetches them straight from +// their host. Its built-in libgit2 fetch has no stall detection that works +// once TLS is up — ephpm job 119658796412 (ducks, 2026-10-22) sat 13 minutes, +// then 5½ more, on a silent github.com connection before libgit2 reported +// "SSL error: syscall failure": 19 minutes of a 25 minute job. +// git-fetch-with-cli hands the fetch to the git binary, which honours +// GIT_HTTP_LOW_SPEED_LIMIT/TIME (set by EnvVars) and abandons a stalled +// transfer in seconds, so Cargo's retry gets a fresh connection. +// +// Not on Windows: the Windows runner images ship no git binary, and with +// git-fetch-with-cli set Cargo fails outright when it cannot run git. +func netTOML(goos string) string { + s := "[net]\nretry = 3\n" + if goos != "windows" { + s += "git-fetch-with-cli = true\n" + } + return s } // containerConfigDest is the container path the generated .cargo directory is diff --git a/pkg/proxies/cargo/cache_test.go b/pkg/proxies/cargo/cache_test.go index 9c00de4..cd8257a 100644 --- a/pkg/proxies/cargo/cache_test.go +++ b/pkg/proxies/cargo/cache_test.go @@ -485,7 +485,7 @@ func TestParseCrateDownloadPath(t *testing.T) { } func TestContainerConfigTOML(t *testing.T) { - got := containerConfigTOML("http://10.88.0.1:8083/") + got := containerConfigTOML("http://10.88.0.1:8083/", "linux") // Source replacement is the whole point — without both halves Cargo // silently keeps using crates.io. diff --git a/pkg/proxies/cargo/cargoproxy.go b/pkg/proxies/cargo/cargoproxy.go index 3391946..80e958f 100644 --- a/pkg/proxies/cargo/cargoproxy.go +++ b/pkg/proxies/cargo/cargoproxy.go @@ -332,10 +332,20 @@ func (p *Proxy) advertiseBase() string { // // Only rustup is configured here. Cargo's registry redirect cannot be done // with environment variables (see Mounts). +// +// The GIT_HTTP_LOW_SPEED pair makes git give up on a transfer that moves +// under 1 KB/s for 30 seconds. It is what bounds the git-dependency fetches +// Cargo runs through the git binary (see netTOML); without it git, like +// libgit2, waits on a dead connection indefinitely. Real fetches never run +// that slowly, so actions/checkout and other git users lose nothing. func (p *Proxy) EnvVars() []string { - return []string{ + env := []string{ "RUSTUP_DIST_SERVER=" + p.advertiseBase() + rustupRoute, } + if p.containerOS() != "windows" { + env = append(env, "GIT_HTTP_LOW_SPEED_LIMIT=1000", "GIT_HTTP_LOW_SPEED_TIME=30") + } + return env } // Mounts returns the read-only bind mount that carries the generated Cargo @@ -386,9 +396,9 @@ func (p *Proxy) writeContainerConfigLocked(active bool) error { if err := os.MkdirAll(dir, 0o755); err != nil { return fmt.Errorf("creating cargo conf dir %q: %w", dir, err) } - body := containerConfigTOML(p.advertiseBase()) + body := containerConfigTOML(p.advertiseBase(), p.containerOS()) if !active { - body = inertConfigTOML() + body = inertConfigTOML(p.containerOS()) } target := filepath.Join(dir, "config.toml") tmp, err := os.CreateTemp(dir, ".config-*.toml") diff --git a/pkg/proxies/cargo/cargoproxy_test.go b/pkg/proxies/cargo/cargoproxy_test.go index 987a090..9f2a7cd 100644 --- a/pkg/proxies/cargo/cargoproxy_test.go +++ b/pkg/proxies/cargo/cargoproxy_test.go @@ -695,12 +695,42 @@ func TestEnvVars_PointRustupAtTheProxy(t *testing.T) { p, _ := newTestProxy(t, up, nil) env := p.EnvVars() - if len(env) != 1 { - t.Fatalf("EnvVars() = %v, want exactly the rustup dist server", env) + want := []string{ + "RUSTUP_DIST_SERVER=" + p.advertiseBase() + "/rustup", + "GIT_HTTP_LOW_SPEED_LIMIT=1000", + "GIT_HTTP_LOW_SPEED_TIME=30", + } + if strings.Join(env, "\n") != strings.Join(want, "\n") { + t.Errorf("EnvVars() = %v, want %v", env, want) + } +} + +// TestNetTOML_GitFetchBounded: git dependencies bypass the cache, so the +// only thing that stops a stalled GitHub fetch from eating a whole job is +// handing it to the git CLI (bounded by GIT_HTTP_LOW_SPEED_*). Both configs +// carry it — the inert one is what is live exactly when the network is bad. +// Windows images have no git binary, so there it must stay off. +func TestNetTOML_GitFetchBounded(t *testing.T) { + for _, goos := range []string{"linux", "darwin"} { + for name, cfg := range map[string]string{ + "caching": containerConfigTOML("http://10.88.0.1:8083", goos), + "inert": inertConfigTOML(goos), + } { + if !strings.Contains(cfg, "git-fetch-with-cli = true") { + t.Errorf("%s/%s config lacks git-fetch-with-cli:\n%s", goos, name, cfg) + } + if !strings.Contains(cfg, "retry = 3") { + t.Errorf("%s/%s config lost net.retry:\n%s", goos, name, cfg) + } + } } - want := "RUSTUP_DIST_SERVER=" + p.advertiseBase() + "/rustup" - if env[0] != want { - t.Errorf("EnvVars()[0] = %q, want %q", env[0], want) + for name, cfg := range map[string]string{ + "caching": containerConfigTOML("http://10.88.0.1:8083", "windows"), + "inert": inertConfigTOML("windows"), + } { + if strings.Contains(cfg, "git-fetch-with-cli") { + t.Errorf("windows/%s config sets git-fetch-with-cli, but Windows images have no git:\n%s", name, cfg) + } } } diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index c0b1a5a..1a52c12 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -2003,6 +2003,12 @@ func (s *Scheduler) handleLocalJob(ctx context.Context, event providers.JobEvent JITConfig: claim.RunnerConfig, Env: claim.Env, Entrypoint: claim.Entrypoint, + // Provider+Repo scope the job's dind image cache. Without + // them dind.CacheNamespace returns "" and every `docker + // pull` in the job goes to the registry: ducks pulled + // mysql:8.0 78 times in 3 days before these were passed. + Provider: providerName(event), + Repo: event.Repo, }) return cerr })