feat: LAN GitHub Actions cache backed by a writable registry (off by default)

cmd/ephemerd/actionscache.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"fmt"
+	"log/slog"
+	runtime_ "runtime"
+
+	"github.com/ephpm/ephemerd/pkg/config"
+	"github.com/ephpm/ephemerd/pkg/networking"
+	"github.com/ephpm/ephemerd/pkg/providers"
+	"github.com/ephpm/ephemerd/pkg/proxies/actionscache"
+	"github.com/ephpm/ephemerd/pkg/scheduler"
+)
+
+// actionsCachePorts returns the gateway port [actions_cache] binds, so the
+// container firewall allows it. Linux only, like the cache itself.
+func actionsCachePorts(cfg *config.Config) []int {
+	if !cfg.ActionsCache.Enabled || runtime_.GOOS != "linux" {
+		return nil
+	}
+	return []int{cfg.ActionsCache.ResolvedPort()}
+}
+
+// startActionsCache starts the LAN Actions cache when [actions_cache] is
+// enabled, returning the scheduler's handle on it (nil when off) and a
+// teardown func.
+//
+// FAIL-OPEN: a cache that cannot start is logged and left out — jobs keep
+// GitHub's cache. One that starts with its registry down is kept: it turns
+// itself on for new jobs as soon as the registry answers (the scheduler asks
+// Healthy() per job).
+func startActionsCache(cfg *config.Config, net *networking.Manager, log *slog.Logger) (scheduler.ActionsCache, func()) {
+	noop := func() {}
+	ac := cfg.ActionsCache
+	if !ac.Enabled {
+		return nil, noop
+	}
+	if runtime_.GOOS != "linux" {
+		log.Warn("actions_cache is Linux-only; leaving it off on this host", "goos", runtime_.GOOS)
+		return nil, noop
+	}
+	addr := fmt.Sprintf("%s:%d", net.GatewayIP(), ac.ResolvedPort())
+	p, err := actionscache.New(actionscache.Config{
+		ListenAddr:         addr,
+		AdvertiseBase:      "http://" + addr,
+		Registry:           ac.Registry,
+		RegistryUser:       ac.Username,
+		RegistryPassword:   ac.ResolvedPassword(),
+		MaxBytesPerRepo:    ac.MaxBytesPerRepo(),
+		MaxEntryBytes:      ac.MaxEntryBytes(),
+		UpstreamResultsURL: ac.UpstreamResultsURL,
+		JobForIP:           net.JobForIP,
+		Log:                log,
+	})
+	if err != nil {
+		log.Warn("actions cache misconfigured; jobs use GitHub's cache", "error", err)
+		return nil, noop
+	}
+	if err := p.Start(); err != nil {
+		log.Warn("failed to start actions cache; jobs use GitHub's cache", "error", err)
+		return nil, noop
+	}
+	return actionsCacheAdapter{p}, func() {
+		if err := p.Stop(); err != nil {
+			log.Warn("error stopping actions cache", "error", err)
+		}
+	}
+}
+
+// actionsCacheAdapter carries the provider's scope type into the proxy's.
+type actionsCacheAdapter struct{ *actionscache.Proxy }
+
+func (a actionsCacheAdapter) Bind(runner string, s providers.CacheScope) {
+	a.Proxy.Bind(runner, actionscache.Scope{Repo: s.Repo, Write: s.Write, Reads: s.Reads})
+}

cmd/ephemerd/main.go

@@ -478,6 +478,8 @@ func serve(ctx context.Context, configFile, imagesDirFlag string, containerdTCPP
 	}
 	// Language package caches (npm, pip, pub) — see pkgproxies.go.
 	gatewayPorts = append(gatewayPorts, pkgProxyPorts(cfg)...)
+	// LAN Actions cache — see actionscache.go.
+	gatewayPorts = append(gatewayPorts, actionsCachePorts(cfg)...)
 
 	// Initialize container networking, retrying through the cold-boot window.
 	//
@@ -586,6 +588,12 @@ func serve(ctx context.Context, configFile, imagesDirFlag string, containerdTCPP
 	defer stopPkgProxies()
 	cacheProxies = append(cacheProxies, pkgProxies...)
 
+	// The LAN Actions cache is per-job, not a CacheProxy: its env var is set
+	// only for jobs the scheduler turns it on for, so it is not collected
+	// into cacheProxyEnvVars below.
+	actionsCache, stopActionsCache := startActionsCache(cfg, net, log)
+	defer stopActionsCache()
+
 	// Collect env vars and mounts from all cache proxies for injection into
 	// containers. Only proxies that actually STARTED are in cacheProxies, so
 	// a failed proxy is never advertised to a job — that is the outer
@@ -873,6 +881,7 @@ func serve(ctx context.Context, configFile, imagesDirFlag string, containerdTCPP
 		RunnerImageForRepo:    cfg.Runner.ImageForRepoOS,
 		Retry:                 retryCfg,
 		OrphanSweep:           orphanCfg,
+		ActionsCache:          actionsCache,
 		Log:                   log,
 	})
 

pkg/config/actionscache.go

@@ -0,0 +1,113 @@
+package config
+
+import (
+	"errors"
+	"os"
+	"strings"
+)
+
+// ActionsCacheConfig configures [actions_cache]: GitHub Actions' cache
+// service (what actions/cache and Swatinem/rust-cache talk to) served from
+// the LAN instead of GitHub, with entries stored in a writable OCI registry
+// (registry:2 in normal, non-proxy mode — the pull-through mirror is
+// read-only and cannot hold them).
+//
+// Linux hosts only. Off by default; with it off nothing about a job changes.
+// With it on, a job only uses the LAN cache when the registry answers its
+// health probe at job start AND the job's runner could be patched; otherwise
+// it uses GitHub's cache exactly as before.
+//
+// Which entries a job may read and write is decided by ephemerd from
+// GitHub's record of the job's run — branch pushes save under their branch,
+// pull requests under their PR, and other events are read-only. See
+// pkg/proxies/actionscache.
+type ActionsCacheConfig struct {
+	// Enabled turns the cache on. Default false.
+	Enabled bool `toml:"enabled"`
+	// Port is the listen port on the bridge gateway. Default 8089.
+	Port int `toml:"port"`
+
+	// Registry is the writable registry's base URL, e.g.
+	// "http://192.168.16.224:5001". Required when enabled.
+	Registry string `toml:"registry"`
+	// Username / Password authenticate to the registry (htpasswd). Jobs
+	// never see them: only ephemerd talks to the registry.
+	Username string `toml:"username"`
+	Password string `toml:"password"`
+	// PasswordEnv names an environment variable to read the password from
+	// instead of writing it in this file. Wins over Password when set.
+	PasswordEnv string `toml:"password_env"`
+
+	// MaxSizeGBPerRepo caps each repository's total cache; the least
+	// recently used entries are deleted past it. Default 100. 0 keeps the
+	// default; a negative value disables eviction.
+	MaxSizeGBPerRepo int `toml:"max_size_gb_per_repo"`
+	// MaxEntryGB refuses a single entry larger than this. Default 10.
+	MaxEntryGB int `toml:"max_entry_gb"`
+
+	// UpstreamResultsURL is where requests that are not the cache (artifact
+	// uploads and downloads) are relayed. Default
+	// "https://results-receiver.actions.githubusercontent.com".
+	UpstreamResultsURL string `toml:"upstream_results_url"`
+}
+
+// DefaultActionsCachePort is the gateway port [actions_cache] binds when
+// port is unset — the next free one after the package caches (8082-8088).
+const DefaultActionsCachePort = 8089
+
+// ResolvedPort returns Port or the default.
+func (c ActionsCacheConfig) ResolvedPort() int {
+	if c.Port > 0 {
+		return c.Port
+	}
+	return DefaultActionsCachePort
+}
+
+// ResolvedPassword returns the registry password, from PasswordEnv when set.
+func (c ActionsCacheConfig) ResolvedPassword() string {
+	if c.PasswordEnv != "" {
+		return os.Getenv(c.PasswordEnv)
+	}
+	return c.Password
+}
+
+// MaxBytesPerRepo returns the eviction cap in bytes (0 = no eviction).
+func (c ActionsCacheConfig) MaxBytesPerRepo() int64 {
+	switch {
+	case c.MaxSizeGBPerRepo < 0:
+		return 0
+	case c.MaxSizeGBPerRepo == 0:
+		return 100 << 30
+	default:
+		return int64(c.MaxSizeGBPerRepo) << 30
+	}
+}
+
+// MaxEntryBytes returns the per-entry limit in bytes.
+func (c ActionsCacheConfig) MaxEntryBytes() int64 {
+	if c.MaxEntryGB <= 0 {
+		return 10 << 30
+	}
+	return int64(c.MaxEntryGB) << 30
+}
+
+func (c ActionsCacheConfig) validate() error {
+	if !c.Enabled {
+		return nil
+	}
+	if strings.TrimSpace(c.Registry) == "" {
+		return errors.New("actions_cache.registry is required when actions_cache is enabled: the writable registry entries are stored in")
+	}
+	if err := validateMirrorURL("actions_cache.registry", c.Registry); err != nil {
+		return err
+	}
+	if c.UpstreamResultsURL != "" {
+		if err := validateMirrorURL("actions_cache.upstream_results_url", c.UpstreamResultsURL); err != nil {
+			return err
+		}
+	}
+	if c.PasswordEnv != "" && os.Getenv(c.PasswordEnv) == "" {
+		return errors.New("actions_cache.password_env names " + c.PasswordEnv + ", which is not set in ephemerd's environment")
+	}
+	return nil
+}

pkg/config/actionscache_test.go

@@ -0,0 +1,33 @@
+package config
+
+import "testing"
+
+func TestActionsCacheConfig(t *testing.T) {
+	var off ActionsCacheConfig
+	if err := off.validate(); err != nil {
+		t.Fatalf("disabled block failed validation: %v", err)
+	}
+	if off.ResolvedPort() != DefaultActionsCachePort || off.MaxBytesPerRepo() != 100<<30 || off.MaxEntryBytes() != 10<<30 {
+		t.Fatal("defaults are wrong")
+	}
+	if err := (ActionsCacheConfig{Enabled: true}).validate(); err == nil {
+		t.Fatal("enabled without a registry passed validation")
+	}
+	if err := (ActionsCacheConfig{Enabled: true, Registry: "cache.lan:5001"}).validate(); err == nil {
+		t.Fatal("a scheme-less registry passed validation")
+	}
+	if err := (ActionsCacheConfig{Enabled: true, Registry: "http://cache.lan:5001"}).validate(); err != nil {
+		t.Fatalf("valid block failed: %v", err)
+	}
+	t.Setenv("EPHEMERD_TEST_AC_PASS", "hunter2")
+	c := ActionsCacheConfig{Enabled: true, Registry: "http://c:5001", Password: "file", PasswordEnv: "EPHEMERD_TEST_AC_PASS"}
+	if c.ResolvedPassword() != "hunter2" {
+		t.Fatal("password_env did not win over password")
+	}
+	if err := (ActionsCacheConfig{Enabled: true, Registry: "http://c:5001", PasswordEnv: "EPHEMERD_TEST_AC_UNSET"}).validate(); err == nil {
+		t.Fatal("password_env naming an unset variable passed validation")
+	}
+	if (ActionsCacheConfig{MaxSizeGBPerRepo: -1}).MaxBytesPerRepo() != 0 {
+		t.Fatal("negative size cap should disable eviction")
+	}
+}

pkg/config/config.go

@@ -45,11 +45,14 @@ type Config struct {
 	// RegistryMirror routes container image pulls through a LAN pull-through
 	// cache instead of the origin registry. See RegistryMirrorConfig.
 	RegistryMirror RegistryMirrorConfig `toml:"registry_mirror"`
-	Runtime        RuntimeConfig        `toml:"runtime"`
-	Runner         RunnerConfig         `toml:"runner"`
-	Metrics        MetricsConfig        `toml:"metrics"`
-	Dispatch       DispatchConfig       `toml:"dispatch"`
-	Log            LogConfig            `toml:"log"`
+	// ActionsCache serves GitHub Actions' cache from the LAN, backed by a
+	// writable registry. See ActionsCacheConfig.
+	ActionsCache ActionsCacheConfig `toml:"actions_cache"`
+	Runtime      RuntimeConfig      `toml:"runtime"`
+	Runner       RunnerConfig       `toml:"runner"`
+	Metrics      MetricsConfig      `toml:"metrics"`
+	Dispatch     DispatchConfig     `toml:"dispatch"`
+	Log          LogConfig          `toml:"log"`
 }
 
 // DispatchConfig configures the host<->VM dispatch gRPC channel. On Windows
@@ -1984,6 +1987,10 @@ func (c *Config) validate() error {
 		return err
 	}
 
+	if err := c.ActionsCache.validate(); err != nil {
+		return err
+	}
+
 	if err := c.Runner.Linux.validate(); err != nil {
 		return err
 	}

pkg/dind/containers.go

@@ -490,7 +490,7 @@ func (s *Server) handleContainerCreate(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if len(req.Env) > 0 {
-		opts = append(opts, oci.WithEnv(req.Env))
+		opts = append(opts, oci.WithEnv(translateResultsEnv(req.Env, s.resultsURL)))
 	}
 
 	if req.WorkingDir != "" {

pkg/dind/dind.go

@@ -48,6 +48,7 @@ type Server struct {
 	jobID          string
 	jobNamespace   string    // per-job containerd namespace for isolation
 	cacheNamespace string    // per-(provider,repo) shared image cache namespace; empty disables caching
+	resultsURL     string    // ACTIONS_RESULTS_URL for container steps; see resultsenv.go
 	transport      Transport // how the API is exposed to the job container; see listen.go
 	dockerDir      string    // <DataDir>/jobs/<JobID>/docker — per-job scratch, exists on every transport
 	sockPath       string    // host-side unix socket path; empty on the TCP transport
@@ -198,6 +199,11 @@ type Config struct {
 	// See listen.go.
 	Transport Transport
 
+	// ResultsURL is the ACTIONS_RESULTS_URL the job's steps must see: the
+	// LAN Actions cache when it is on for this job, "" otherwise. See
+	// resultsenv.go for why dind has to restore it at all.
+	ResultsURL string
+
 	Log *slog.Logger
 }
 
@@ -240,6 +246,7 @@ func New(cfg Config) (*Server, error) {
 		allowPrivileged: cfg.AllowPrivileged,
 		stager:          newBindStager(cfg.DataDir, cfg.JobID, cfg.Log),
 		mirror:          cfg.RegistryMirror,
+		resultsURL:      cfg.ResultsURL,
 		log:             cfg.Log.With("component", "dind", "job_id", cfg.JobID),
 		images:          make(map[string]*imageEntry),
 		containers:      make(map[string]*containerEntry),

pkg/dind/exec.go

@@ -125,6 +125,7 @@ func (s *Server) handleExecCreate(w http.ResponseWriter, r *http.Request, contai
 		env = append(env, spec.Process.Env...)
 	}
 	env = append(env, req.Env...)
+	env = translateResultsEnv(env, s.resultsURL)
 
 	exec := &execEntry{
 		ID:          execID,

pkg/dind/resultsenv.go

@@ -0,0 +1,49 @@
+package dind
+
+import "strings"
+
+// Names the runner exports the results service address under: the real one,
+// and the one a runner patched for the LAN Actions cache exports instead
+// (pkg/runner.PatchResultsURL). Duplicated as literals so dind does not
+// depend on the runner package.
+const (
+	resultsURLVar     = "ACTIONS_RESULTS_URL="
+	resultsRenamedVar = "ACTIONS_RESULTS_ORL="
+)
+
+// translateResultsEnv fixes ACTIONS_RESULTS_URL for a `container:` job.
+//
+// Host steps inherit the runner process's environment, which carries the
+// ACTIONS_RESULTS_URL ephemerd set for the LAN cache. Container steps do
+// not: the runner hands them only the step environment, through
+// `docker exec -e` — and a patched runner exports GitHub's address there as
+// ACTIONS_RESULTS_ORL. Left alone, a container step would see no
+// ACTIONS_RESULTS_URL at all, and upload-artifact would fail the job.
+//
+// So whenever the renamed variable is present, ACTIONS_RESULTS_URL is set to
+// the LAN cache (resultsURL) when this job has one, or restored to GitHub's
+// address when it does not. Env without the renamed variable — every job not
+// using the LAN cache — is returned untouched.
+func translateResultsEnv(env []string, resultsURL string) []string {
+	original, found := "", false
+	for _, kv := range env {
+		if v, ok := strings.CutPrefix(kv, resultsRenamedVar); ok {
+			original, found = v, true
+		}
+	}
+	if !found {
+		return env
+	}
+	want := resultsURL
+	if want == "" {
+		want = original
+	}
+	out := make([]string, 0, len(env)+1)
+	for _, kv := range env {
+		if strings.HasPrefix(kv, resultsURLVar) {
+			continue
+		}
+		out = append(out, kv)
+	}
+	return append(out, resultsURLVar+want)
+}