From 2832c575539dc9a497cbc46ab5a7faee72fd63e0 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Thu, 3 Sep 2026 12:25:54 -0500 Subject: [PATCH 01/19] ci: guard Windows DNS against harden-runner's post-step race; allowlist OS endpoints Windows jobs have been finishing every step, including "Post Harden Runner" and "Complete job", and then never reporting a conclusion until GitHub cancels them 30-40 minutes later (0/day before harden-runner landed on the Windows legs on Aug 31; 5, 9, 23/day on Sept 1-3). Root cause: harden-runner's Windows post step waits at most 10 s for its agent, then kills it, and in the losing case the agent is still restoring the runner's DNS settings, leaving the adapter pointed at a DNS proxy that no longer exists. - Add .github/actions/windows-dns-guard: a scheduled-task watchdog that resets DNS only once harden-runner's post step has begun, the agent is gone, and 127.0.0.1 is still configured. Wired into every Windows leg. - Allowlist the Windows/macOS operating-system endpoints StepSecurity showed blocked on every job (NCSI probes, WNS, update/settings/telemetry, time sync, Sectigo OCSP/CRL, Apple update/CDN hosts); harden-runner stays in block mode everywhere. - Drop the GOPROXY "|direct" fallback in block-mode workflows: it only fans out to blocked vanity-import hosts. - Cancel superseded pull_request runs via a concurrency group. - Skip SARIF uploads on merge_group runs, whose synthetic ref cannot be scanned and was evicting PRs from the merge queue. Fix log: docs/fixes/2026-09-03-harden-runner-windows-dns-restore-race.md Co-Authored-By: Claude Fable 5.1 --- .github/actions/windows-dns-guard/action.yml | 42 +++ .../actions/windows-dns-guard/dns-guard.ps1 | 71 +++++ .github/workflows/codeql.yml | 7 +- .github/workflows/native-ci.yml | 7 +- .github/workflows/pre-commit.yml | 7 +- .github/workflows/setup-go-cache-warmup.yml | 46 +++- .github/workflows/test.yml | 254 +++++++++++++++++- ...-harden-runner-windows-dns-restore-race.md | 162 +++++++++++ 8 files changed, 577 insertions(+), 19 deletions(-) create mode 100644 .github/actions/windows-dns-guard/action.yml create mode 100644 .github/actions/windows-dns-guard/dns-guard.ps1 create mode 100644 docs/fixes/2026-09-03-harden-runner-windows-dns-restore-race.md diff --git a/.github/actions/windows-dns-guard/action.yml b/.github/actions/windows-dns-guard/action.yml new file mode 100644 index 00000000000..9dd179b4a75 --- /dev/null +++ b/.github/actions/windows-dns-guard/action.yml @@ -0,0 +1,42 @@ +name: 'Windows DNS guard (harden-runner post-step race)' +description: > + Registers a one-shot Windows scheduled task that repairs the runner's DNS + client settings if step-security/harden-runner's Windows post step kills + its agent before the agent has finished restoring them. Without this, the + job's steps all succeed but the runner is left pointing at a DNS proxy + that no longer exists, cannot reach the Actions service to report + completion, and is cancelled by GitHub 30-40 minutes later. See + docs/fixes/2026-09-03-harden-runner-windows-dns-restore-race.md. + + The task runs under the Task Scheduler service, outside the step's and + runner's process trees, so it survives step teardown and the runner's + "Cleaning up orphan processes" pass. It only acts once harden-runner's own + post step has started (C:\agent\post_event.json exists), the agent process + is gone, and an interface still lists 127.0.0.1 as its DNS server -- so it + can never fail open while egress enforcement is live. No-op (exits) when + harden-runner restores DNS itself, which is the common case. + + Windows-only; a no-op on other runners. Safe to place anywhere after the + Harden Runner step (it needs the checkout for its script file). + +runs: + using: composite + steps: + - name: Register DNS guard task + if: runner.os == 'Windows' + shell: powershell + env: + GUARD_SCRIPT: ${{ github.action_path }}/dns-guard.ps1 + run: | + $dest = Join-Path $env:RUNNER_TEMP 'atmos-dns-guard.ps1' + Copy-Item -LiteralPath $env:GUARD_SCRIPT -Destination $dest -Force + $log = Join-Path $env:RUNNER_TEMP 'atmos-dns-guard.log' + $action = New-ScheduledTaskAction -Execute 'powershell.exe' ` + -Argument "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$dest`" -LogPath `"$log`"" + $principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest + $settings = New-ScheduledTaskSettingsSet -ExecutionTimeLimit (New-TimeSpan -Hours 2) ` + -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -MultipleInstances IgnoreNew + Register-ScheduledTask -TaskName 'atmos-dns-guard' -Action $action -Principal $principal ` + -Settings $settings -Force | Out-Null + Start-ScheduledTask -TaskName 'atmos-dns-guard' + Write-Host "DNS guard task registered (script: $dest, log: $log)" diff --git a/.github/actions/windows-dns-guard/dns-guard.ps1 b/.github/actions/windows-dns-guard/dns-guard.ps1 new file mode 100644 index 00000000000..6efb96ad0df --- /dev/null +++ b/.github/actions/windows-dns-guard/dns-guard.ps1 @@ -0,0 +1,71 @@ +# Watchdog for step-security/harden-runner's Windows post-step race. +# +# harden-runner's Windows agent enforces egress with a DNS proxy on +# 127.0.0.1:53 and repoints every adapter's DNS server to it. Its post step +# waits at most 10 seconds for the agent to finish, then kills it. When the +# agent is still inside "restoring system DNS" at that moment, the adapter is +# left pointing at a proxy that no longer exists; the runner can no longer +# resolve the Actions service and never reports job completion. +# +# This script polls once a second and repairs DNS only when all three hold: +# 1. C:\agent\post_event.json exists (harden-runner's post step has begun), +# 2. the agent process from C:\agent\agent.pid is gone (or the pid file is), +# 3. an IPv4 interface still lists 127.0.0.1 as a DNS server. +# Condition 1 guarantees we never undo enforcement while it is live: an agent +# crash mid-job keeps the job fail-closed exactly as it is today. +param( + [string]$AgentDir = 'C:\agent', + [string]$LogPath = "$env:TEMP\atmos-dns-guard.log", + [int]$MaxSeconds = 7200 +) + +function Write-Log([string]$Message) { + $line = "{0} {1}" -f (Get-Date -Format 'o'), $Message + Add-Content -LiteralPath $LogPath -Value $line -ErrorAction SilentlyContinue +} + +function Test-AgentAlive { + $pidFile = Join-Path $AgentDir 'agent.pid' + if (-not (Test-Path -LiteralPath $pidFile)) { return $false } + $raw = (Get-Content -LiteralPath $pidFile -ErrorAction SilentlyContinue | Select-Object -First 1) + $agentPid = 0 + if (-not [int]::TryParse(($raw -as [string]).Trim(), [ref]$agentPid)) { return $false } + return $null -ne (Get-Process -Id $agentPid -ErrorAction SilentlyContinue) +} + +function Get-LoopbackDnsInterfaces { + Get-DnsClientServerAddress -AddressFamily IPv4 -ErrorAction SilentlyContinue | + Where-Object { $_.ServerAddresses -contains '127.0.0.1' } +} + +Write-Log "dns guard started (agentDir=$AgentDir)" +$postEvent = Join-Path $AgentDir 'post_event.json' +$deadline = (Get-Date).AddSeconds($MaxSeconds) + +while ((Get-Date) -lt $deadline) { + Start-Sleep -Seconds 1 + if (-not (Test-Path -LiteralPath $postEvent)) { continue } + if (Test-AgentAlive) { continue } + + # Give harden-runner's own graceful path a moment: the agent normally + # restores DNS just before it exits. + Start-Sleep -Seconds 1 + $bad = @(Get-LoopbackDnsInterfaces) + if ($bad.Count -eq 0) { + Write-Log 'post step finished and DNS is healthy; nothing to do' + break + } + foreach ($iface in $bad) { + Write-Log ("harden-runner agent exited with DNS still on 127.0.0.1 for interface '{0}' (index {1}); resetting" -f $iface.InterfaceAlias, $iface.InterfaceIndex) + try { + Set-DnsClientServerAddress -InterfaceIndex $iface.InterfaceIndex -ResetServerAddresses -ErrorAction Stop + } catch { + Write-Log ("reset failed for interface index {0}: {1}" -f $iface.InterfaceIndex, $_.Exception.Message) + } + } + Clear-DnsClientCache -ErrorAction SilentlyContinue + $check = Resolve-DnsName -Name 'pipelines.actions.githubusercontent.com' -Type A -ErrorAction SilentlyContinue + Write-Log ("DNS reset done; pipelines.actions.githubusercontent.com resolves: {0}" -f ($null -ne $check)) + break +} +Write-Log 'dns guard exiting' diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 642fc0f75b9..82676713f71 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -15,9 +15,10 @@ on: - cron: "27 19 * * 2" env: - # Use pipe fallback so transient proxy.golang.org 5xx errors during - # `go mod download` fall back to direct module fetches. - GOPROXY: "https://proxy.golang.org|direct" + # No `|direct` fallback: under harden-runner's block mode a direct fetch + # only fans out to blocked vanity-import hosts and burns minutes; see the + # GOPROXY comment in test.yml. + GOPROXY: "https://proxy.golang.org" # Least-privilege default; every job below declares its own narrower # job-level permissions that override this baseline. diff --git a/.github/workflows/native-ci.yml b/.github/workflows/native-ci.yml index d9956205945..c0f38a9be9d 100644 --- a/.github/workflows/native-ci.yml +++ b/.github/workflows/native-ci.yml @@ -36,9 +36,10 @@ env: ATMOS_VERSION_CHECK_ENABLED: "false" NATIVE_CI_TRIVY_VERSION: "0.70.0" NATIVE_CI_KICS_VERSION: "2.1.20" - # Use pipe fallback so transient proxy.golang.org 5xx errors during - # `go mod download` fall back to direct module fetches. - GOPROXY: "https://proxy.golang.org|direct" + # No `|direct` fallback: under harden-runner's block mode a direct fetch + # only fans out to blocked vanity-import hosts and burns minutes; see the + # GOPROXY comment in test.yml. + GOPROXY: "https://proxy.golang.org" jobs: workflow-groups: diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index fa54308b476..54368870a02 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -18,9 +18,10 @@ env: # golangci-lint: Already runs in codeql.yml lint-golangci job # lintroller: Already runs in codeql.yml lint-golangci job (via custom-gcl binary) SKIP: go-build-mod,golangci-lint,lintroller - # Use pipe fallback so transient proxy.golang.org 5xx errors during - # `go mod download` fall back to direct module fetches. - GOPROXY: "https://proxy.golang.org|direct" + # No `|direct` fallback: under harden-runner's block mode a direct fetch + # only fans out to blocked vanity-import hosts and burns minutes; see the + # GOPROXY comment in test.yml. + GOPROXY: "https://proxy.golang.org" jobs: pre-commit: diff --git a/.github/workflows/setup-go-cache-warmup.yml b/.github/workflows/setup-go-cache-warmup.yml index c7c4b7ee1a5..2d7b545f07e 100644 --- a/.github/workflows/setup-go-cache-warmup.yml +++ b/.github/workflows/setup-go-cache-warmup.yml @@ -14,9 +14,10 @@ permissions: env: ATMOS_BOOTSTRAP_VERSION: "1.223.0" - # Use pipe fallback so transient proxy.golang.org 5xx errors during - # `go mod download` fall back to direct module fetches. - GOPROXY: "https://proxy.golang.org|direct" + # No `|direct` fallback: under harden-runner's block mode a direct fetch + # only fans out to blocked vanity-import hosts and burns minutes; see the + # GOPROXY comment in test.yml. + GOPROXY: "https://proxy.golang.org" jobs: cache-warmup: @@ -42,12 +43,45 @@ jobs: uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 with: egress-policy: block + # Operating-system background services on the hosted Windows and + # macOS images are listed after the job-specific endpoints; see the + # `test` job's allowed-endpoints comment in test.yml for the + # rationale and keep the two lists in sync. allowed-endpoints: > github.com:443 api.github.com:443 release-assets.githubusercontent.com:443 proxy.golang.org:443 + sum.golang.org:443 storage.googleapis.com:443 + google.golang.org:443 + modernc.org:443 + www.msftconnecttest.com:443 + ipv6.msftconnecttest.com:443 + www.msftncsi.com:443 + ipv6.msftncsi.com:443 + time.windows.com:443 + client.wns.windows.com:443 + settings-win.data.microsoft.com:443 + *.events.data.microsoft.com:443 + fe2cr.update.microsoft.com:443 + ecs.office.com:443 + pti.store.microsoft.com:443 + login.live.com:443 + go.microsoft.com:443 + www.microsoft.com:443 + ocsp.sectigo.com:80 + ocsp.sectigo.com:443 + crl.sectigo.com:80 + crl.sectigo.com:443 + *.apple.com:443 + *.aaplimg.com:443 + *.mzstatic.com:443 + *.icloud.com:443 + *.apple-dns.net:443 + ocsp.usertrust.com:80 + ocsp.digicert.com:80 + 0.pool.ntp.org:123 - name: Add GNU tar to PATH (significantly faster than windows tar) if: matrix.target == 'windows' @@ -59,6 +93,12 @@ jobs: with: persist-credentials: false + # See test.yml: repairs DNS if harden-runner's Windows post step kills + # its agent before it has restored the runner's DNS settings. + - name: Guard DNS against the harden-runner post-step race + if: matrix.target == 'windows' + uses: ./.github/actions/windows-dns-guard + - name: Set up Go uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2f573c90367..0aebbf62a13 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,6 +22,16 @@ on: workflow_dispatch: +# A newer push to the same PR supersedes the previous run: cancel it instead +# of letting both compete for runners (30 acceptance shards each, and the org +# macOS concurrency cap is small enough that a second run's macOS legs queue +# for up to ~30 minutes). merge_group, push, and workflow_dispatch runs are +# keyed by SHA and are never cancelled -- a merge-queue entry or a main-branch +# run must always finish. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # Grant `packages: read` so jobs that pull OCI images from ghcr.io # (e.g. vendor pulls in mock/acceptance tests) can authenticate with # the auto-generated GITHUB_TOKEN. The default PR-event scope is @@ -52,11 +62,16 @@ env: # OS (see tests/cli_test.go's testShard/testCaseShard and the `test` job's # matrix.shard list below, which must stay in sync with this count). TEST_SHARD_COUNT: "10" - # Use pipe fallback so transient proxy.golang.org errors (5xx, HTTP/2 stream - # resets) during `go mod download` fall back to direct module fetches. The - # default comma-separated GOPROXY list only falls through on "not found" - # responses, not network errors. See .github/workflows/native-ci.yml. - GOPROXY: "https://proxy.golang.org|direct" + # No `|direct` fallback: every job here runs under harden-runner's block + # mode, where a direct fetch fans out to ~25 vanity-import hosts (k8s.io, + # go.uber.org, gopkg.in, helm.sh, ...) that are not allowlisted, so the + # fallback can only ever burn minutes on blocked lookups (observed: a 9+ + # minute "Get dependencies" step after one proxy.golang.org hiccup). The + # transient 5xx / HTTP/2 stream resets the fallback used to paper over are + # handled by `atmos build deps`'s retry policy (.atmos.d/build.yaml) + # instead. If the retry policy ever proves insufficient, allowlist the + # vanity hosts rather than restoring the fallback. + GOPROXY: "https://proxy.golang.org" jobs: # ensure the code builds... @@ -97,6 +112,18 @@ jobs: uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 with: egress-policy: block + # Operating-system background services on the hosted Windows and + # macOS images (connectivity probes, push notifications, update and + # settings/telemetry checks, time sync, certificate revocation for + # GitHub's Sectigo chain, Apple software-update and CDN hosts) are + # listed below their job-specific endpoints. StepSecurity Insights + # showed them blocked on every Windows/macOS job -- they are not + # implicitly allowed -- and each blocked lookup is retried by the OS + # for the whole job (NCSI probes every few seconds; schannel + # revocation checks stall TLS to github.com). Allowing them removes + # that churn without widening what our own steps can reach. Source: + # StepSecurity Insights -> Network Events; keep in sync with the + # other Windows/macOS legs in this file. allowed-endpoints: > api.github.com:443 github.com:443 @@ -111,6 +138,32 @@ jobs: storage.googleapis.com:443 google.golang.org:443 modernc.org:443 + www.msftconnecttest.com:443 + ipv6.msftconnecttest.com:443 + www.msftncsi.com:443 + ipv6.msftncsi.com:443 + time.windows.com:443 + client.wns.windows.com:443 + settings-win.data.microsoft.com:443 + *.events.data.microsoft.com:443 + fe2cr.update.microsoft.com:443 + ecs.office.com:443 + pti.store.microsoft.com:443 + login.live.com:443 + go.microsoft.com:443 + www.microsoft.com:443 + ocsp.sectigo.com:80 + ocsp.sectigo.com:443 + crl.sectigo.com:80 + crl.sectigo.com:443 + *.apple.com:443 + *.aaplimg.com:443 + *.mzstatic.com:443 + *.icloud.com:443 + *.apple-dns.net:443 + ocsp.usertrust.com:80 + ocsp.digicert.com:80 + 0.pool.ntp.org:123 - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2.3.0 if: matrix.target == 'linux' @@ -130,6 +183,16 @@ jobs: with: persist-credentials: false + # Repairs the runner's DNS if harden-runner's Windows post step kills its + # agent mid-"restoring system DNS" (a race against its 10-second wait); + # without it the job's steps all pass but the job never reports + # completion and GitHub cancels it 30-40 minutes later. Windows-only, + # no-op elsewhere. See the action's description and + # docs/fixes/2026-09-03-harden-runner-windows-dns-restore-race.md. + - name: Guard DNS against the harden-runner post-step race + if: matrix.target == 'windows' && ! github.event.pull_request.draft + uses: ./.github/actions/windows-dns-guard + - name: Set up Go if: ${{ ! ( matrix.target == 'windows' && github.event.pull_request.draft ) }} # setup-go v6 requires runner v2.327.1+ and can affect toolchain handling. @@ -278,6 +341,18 @@ jobs: uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 with: egress-policy: block + # Operating-system background services on the hosted Windows and + # macOS images (connectivity probes, push notifications, update and + # settings/telemetry checks, time sync, certificate revocation for + # GitHub's Sectigo chain, Apple software-update and CDN hosts) are + # listed below their job-specific endpoints. StepSecurity Insights + # showed them blocked on every Windows/macOS job -- they are not + # implicitly allowed -- and each blocked lookup is retried by the OS + # for the whole job (NCSI probes every few seconds; schannel + # revocation checks stall TLS to github.com). Allowing them removes + # that churn without widening what our own steps can reach. Source: + # StepSecurity Insights -> Network Events; keep in sync with the + # other Windows/macOS legs in this file. allowed-endpoints: > api.github.com:443 checkpoint-api.hashicorp.com:443 @@ -295,6 +370,32 @@ jobs: storage.googleapis.com:443 google.golang.org:443 modernc.org:443 + www.msftconnecttest.com:443 + ipv6.msftconnecttest.com:443 + www.msftncsi.com:443 + ipv6.msftncsi.com:443 + time.windows.com:443 + client.wns.windows.com:443 + settings-win.data.microsoft.com:443 + *.events.data.microsoft.com:443 + fe2cr.update.microsoft.com:443 + ecs.office.com:443 + pti.store.microsoft.com:443 + login.live.com:443 + go.microsoft.com:443 + www.microsoft.com:443 + ocsp.sectigo.com:80 + ocsp.sectigo.com:443 + crl.sectigo.com:80 + crl.sectigo.com:443 + *.apple.com:443 + *.aaplimg.com:443 + *.mzstatic.com:443 + *.icloud.com:443 + *.apple-dns.net:443 + ocsp.usertrust.com:80 + ocsp.digicert.com:80 + 0.pool.ntp.org:123 - name: Check out code into the Go module directory if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} @@ -302,6 +403,16 @@ jobs: with: persist-credentials: false + # Repairs the runner's DNS if harden-runner's Windows post step kills its + # agent mid-"restoring system DNS" (a race against its 10-second wait); + # without it the job's steps all pass but the job never reports + # completion and GitHub cancels it 30-40 minutes later. Windows-only, + # no-op elsewhere. See the action's description and + # docs/fixes/2026-09-03-harden-runner-windows-dns-restore-race.md. + - name: Guard DNS against the harden-runner post-step race + if: matrix.flavor.target == 'windows' && ! github.event.pull_request.draft + uses: ./.github/actions/windows-dns-guard + - name: Add GNU tar to PATH (significantly faster than windows tar) if: matrix.flavor.target == 'windows' && ! github.event.pull_request.draft shell: pwsh @@ -468,6 +579,18 @@ jobs: # single run (MCP pagination limitation), so carries more residual risk # of an untested-path endpoint than other workflows in this rollout. egress-policy: block + # Operating-system background services on the hosted Windows and + # macOS images (connectivity probes, push notifications, update and + # settings/telemetry checks, time sync, certificate revocation for + # GitHub's Sectigo chain, Apple software-update and CDN hosts) are + # listed below their job-specific endpoints. StepSecurity Insights + # showed them blocked on every Windows/macOS job -- they are not + # implicitly allowed -- and each blocked lookup is retried by the OS + # for the whole job (NCSI probes every few seconds; schannel + # revocation checks stall TLS to github.com). Allowing them removes + # that churn without widening what our own steps can reach. Source: + # StepSecurity Insights -> Network Events; keep in sync with the + # other Windows/macOS legs in this file. allowed-endpoints: > accounts.google.com:443 api.bitbucket.org:443 @@ -497,6 +620,35 @@ jobs: sts.us-east-1.amazonaws.com:443 tuf-repo-cdn.sigstore.dev:443 us.i.posthog.com:443 + sum.golang.org:443 + google.golang.org:443 + modernc.org:443 + www.msftconnecttest.com:443 + ipv6.msftconnecttest.com:443 + www.msftncsi.com:443 + ipv6.msftncsi.com:443 + time.windows.com:443 + client.wns.windows.com:443 + settings-win.data.microsoft.com:443 + *.events.data.microsoft.com:443 + fe2cr.update.microsoft.com:443 + ecs.office.com:443 + pti.store.microsoft.com:443 + login.live.com:443 + go.microsoft.com:443 + www.microsoft.com:443 + ocsp.sectigo.com:80 + ocsp.sectigo.com:443 + crl.sectigo.com:80 + crl.sectigo.com:443 + *.apple.com:443 + *.aaplimg.com:443 + *.mzstatic.com:443 + *.icloud.com:443 + *.apple-dns.net:443 + ocsp.usertrust.com:80 + ocsp.digicert.com:80 + 0.pool.ntp.org:123 - name: Check out code into the Go module directory if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} @@ -504,6 +656,16 @@ jobs: with: persist-credentials: false + # Repairs the runner's DNS if harden-runner's Windows post step kills its + # agent mid-"restoring system DNS" (a race against its 10-second wait); + # without it the job's steps all pass but the job never reports + # completion and GitHub cancels it 30-40 minutes later. Windows-only, + # no-op elsewhere. See the action's description and + # docs/fixes/2026-09-03-harden-runner-windows-dns-restore-race.md. + - name: Guard DNS against the harden-runner post-step race + if: matrix.flavor.target == 'windows' && ! github.event.pull_request.draft + uses: ./.github/actions/windows-dns-guard + - name: Add GNU tar to PATH (significantly faster than windows tar) if: matrix.flavor.target == 'windows' && ! github.event.pull_request.draft shell: pwsh @@ -912,9 +1074,15 @@ jobs: # DL3008 Pin versions in apt-get install ignore: DL3008 + # Code scanning cannot attach results to a merge queue's synthetic + # `refs/heads/gh-readonly-queue/...` ref: the upload fails with "ref ... + # not found", which (with wait-for-processing) fails this step, skips the + # Trivy scan below, and then fails the second upload on the missing SARIF + # file -- evicting the PR from the queue for a result nobody consumes. + # The same commit was already scanned on the PR run. - name: Upload SARIF file uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 - if: always() + if: always() && github.event_name != 'merge_group' with: # Path to SARIF file relative to the root of the repository sarif_file: hadolint.sarif @@ -935,7 +1103,7 @@ jobs: output: trivy-config.sarif - name: Upload Trivy config scan results - if: always() + if: always() && github.event_name != 'merge_group' uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 with: sarif_file: trivy-config.sarif @@ -1133,6 +1301,10 @@ jobs: uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 with: egress-policy: block + # Operating-system background services on the hosted macOS image + # (software update, CDN, certificate revocation, time sync) are + # listed below the job-specific endpoints; see the `test` job's + # allowed-endpoints comment for the rationale. allowed-endpoints: > auth.docker.io:443 charts.bitnami.com:443 @@ -1156,6 +1328,14 @@ jobs: security.ubuntu.com:80 tuf-repo-cdn.sigstore.dev:443 us.i.posthog.com:443 + *.apple.com:443 + *.aaplimg.com:443 + *.mzstatic.com:443 + *.icloud.com:443 + *.apple-dns.net:443 + ocsp.usertrust.com:80 + ocsp.digicert.com:80 + 0.pool.ntp.org:123 - name: Check out code into the Go module directory uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 @@ -1310,12 +1490,24 @@ jobs: uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 with: egress-policy: block + # Operating-system background services on the hosted macOS image + # (software update, CDN, certificate revocation, time sync) are + # listed below the job-specific endpoints; see the `test` job's + # allowed-endpoints comment for the rationale. allowed-endpoints: > api.github.com:443 github.com:443 proxy.golang.org:443 release-assets.githubusercontent.com:443 storage.googleapis.com:443 + *.apple.com:443 + *.aaplimg.com:443 + *.mzstatic.com:443 + *.icloud.com:443 + *.apple-dns.net:443 + ocsp.usertrust.com:80 + ocsp.digicert.com:80 + 0.pool.ntp.org:123 - name: Check out code into the Go module directory uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 @@ -1406,6 +1598,18 @@ jobs: uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 with: egress-policy: block + # Operating-system background services on the hosted Windows and + # macOS images (connectivity probes, push notifications, update and + # settings/telemetry checks, time sync, certificate revocation for + # GitHub's Sectigo chain, Apple software-update and CDN hosts) are + # listed below their job-specific endpoints. StepSecurity Insights + # showed them blocked on every Windows/macOS job -- they are not + # implicitly allowed -- and each blocked lookup is retried by the OS + # for the whole job (NCSI probes every few seconds; schannel + # revocation checks stall TLS to github.com). Allowing them removes + # that churn without widening what our own steps can reach. Source: + # StepSecurity Insights -> Network Events; keep in sync with the + # other Windows/macOS legs in this file. allowed-endpoints: > api.github.com:443 checkpoint-api.hashicorp.com:443 @@ -1419,6 +1623,32 @@ jobs: releases.hashicorp.com:443 tuf-repo-cdn.sigstore.dev:443 us.i.posthog.com:443 + www.msftconnecttest.com:443 + ipv6.msftconnecttest.com:443 + www.msftncsi.com:443 + ipv6.msftncsi.com:443 + time.windows.com:443 + client.wns.windows.com:443 + settings-win.data.microsoft.com:443 + *.events.data.microsoft.com:443 + fe2cr.update.microsoft.com:443 + ecs.office.com:443 + pti.store.microsoft.com:443 + login.live.com:443 + go.microsoft.com:443 + www.microsoft.com:443 + ocsp.sectigo.com:80 + ocsp.sectigo.com:443 + crl.sectigo.com:80 + crl.sectigo.com:443 + *.apple.com:443 + *.aaplimg.com:443 + *.mzstatic.com:443 + *.icloud.com:443 + *.apple-dns.net:443 + ocsp.usertrust.com:80 + ocsp.digicert.com:80 + 0.pool.ntp.org:123 - name: Check out code into the Go module directory if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} @@ -1426,6 +1656,16 @@ jobs: with: persist-credentials: false + # Repairs the runner's DNS if harden-runner's Windows post step kills its + # agent mid-"restoring system DNS" (a race against its 10-second wait); + # without it the job's steps all pass but the job never reports + # completion and GitHub cancels it 30-40 minutes later. Windows-only, + # no-op elsewhere. See the action's description and + # docs/fixes/2026-09-03-harden-runner-windows-dns-restore-race.md. + - name: Guard DNS against the harden-runner post-step race + if: matrix.flavor.target == 'windows' && ! github.event.pull_request.draft + uses: ./.github/actions/windows-dns-guard + - name: Add GNU tar to flavor.target (significantly faster than windows tar) if: matrix.flavor.target == 'windows' && ! github.event.pull_request.draft shell: pwsh diff --git a/docs/fixes/2026-09-03-harden-runner-windows-dns-restore-race.md b/docs/fixes/2026-09-03-harden-runner-windows-dns-restore-race.md new file mode 100644 index 00000000000..c45e5868800 --- /dev/null +++ b/docs/fixes/2026-09-03-harden-runner-windows-dns-restore-race.md @@ -0,0 +1,162 @@ +# Fix: Windows CI jobs left stuck after "Complete job" by harden-runner's DNS-restore race + +**Date:** 2026-09-03 + +## Summary + +Since harden-runner landed on the Windows and macOS legs of `test.yml` (2026-08-31, #2958) and went to +block mode (2026-09-02, #3027), Windows jobs have been finishing every step, including `Post Harden Runner` +and `Complete job`, and then never reporting a conclusion until GitHub cancels them 30–40 minutes later. +`Build (windows)` gates every other job, so one stuck build stalls the whole run, including merge-queue +entries. Root cause is a race in harden-runner's Windows post step: it waits at most 10 s for its agent, then +kills it, and in the losing case the agent is still restoring the runner's DNS settings, leaving the adapter +pointed at a DNS proxy that no longer exists. This change adds a scheduled-task watchdog that repairs DNS +only in that exact state, allowlists the Windows/macOS operating-system endpoints that StepSecurity showed +blocked on every job, removes the `|direct` Go proxy fallback that only fans out to blocked hosts under block +mode, cancels superseded PR runs, and stops the Trivy SARIF upload from evicting PRs from the merge queue. +harden-runner stays in block mode on every leg. + +## Context + +Measured from `test.yml` run data (Aug 24 – Sept 3), raw job logs (which include the harden-runner agent's +own log), StepSecurity network-event telemetry, and harden-runner's source. + +**Mechanism.** harden-runner's Windows agent (v1.0.7-win, hard-coded in `src/install-agent.ts`) enforces +egress with a DNS proxy on `127.0.0.1:53` and repoints every adapter to it with `Set-DnsClientServerAddress`. +Its post step (`src/cleanup.ts`, `handleWindowsCleanup`) writes `C:\agent\post_event.json`, polls for +`done.json` for at most 10 × 1 s, logs `timed out`, sends SIGINT and sees the agent gone within a second. +In every stuck job the agent picked the post event up ~5 s late and had just logged +`[handlePostHardenRunnerEvent] restoring system DNS` (a step that takes ~2.4 s when it completes) when it was +killed. The runner can still upload logs to hosts it already resolved, but the final job-completion call to +the Actions service needs a fresh lookup and never succeeds. + +| Job | Kind | 2nd `timed out` in post step | `system DNS settings restored` | `DNS proxy stopped` | +|---|---|---|---|---| +| 100694525046 Build (windows) | stuck | yes | no | no | +| 100532566143 shard 2 | stuck | yes | no (restore began 5.6 s after post step) | no | +| 100127533462 shard (Sept 2, audit mode) | stuck | yes | no (restore began 4.8 s after) | no | +| 100688004653 Build (windows) | healthy | no | yes (2.9 s → 5.3 s after post step) | yes | +| 100532566215 shard 3 | healthy | no | yes (2.2 s → 4.5 s after) | yes | +| 100696221376, 100696221470 shards | healthy | no | yes | yes | + +**Frequency.** Counting only cancelled jobs whose job-level completion landed ≥ 5 min after their last step +(true zombies; ordinary cancellations land within seconds): + +| Day | true stuck Windows jobs | note | +|---|---|---| +| Aug 25–28 | 0 | 12 "cancelled" jobs all completed ≤ 5 s after their last step (ordinary cancels) | +| Aug 31 (harden-runner `audit` on Windows/macOS legs) | 0 | 11 instant cancels (manual cancel + rerun) | +| Sept 1 (audit) | 5 | | +| Sept 2 (`block` from 21:08 UTC) | 9 | | +| Sept 3 | 23 | ≈ 1.5 % of Windows jobs, ≈ 27 % of runs (18 Windows jobs per run) | + +It happens in audit mode too (the agent and proxy run in both), so audit is not a fix, and harden-runner +v2.21.1 does not change the Windows agent. + +**Blocked OS endpoints.** StepSecurity Insights shows the same operating-system destinations blocked on +every Windows/macOS job, healthy or stuck, from OS services rather than our steps: NCSI connectivity probes +(`www.msftconnecttest.com`, `www.msftncsi.com`, IPv6 variants, retried every few seconds), WNS, +settings/events telemetry, Windows Update, `time.windows.com`, Store/Live/`go.microsoft.com`, and Sectigo +OCSP/CRL (GitHub's certificate chain, hit from `git-remote-https.exe`); on macOS, Apple software-update, CDN, +iCloud and revocation hosts plus NTP. None are implicitly allowed (StepSecurity's docs: only their own agent +endpoints and dot-less hostnames are), and `allowed-endpoints` supports domain wildcards. + +**Go proxy fallback.** With `GOPROXY=https://proxy.golang.org|direct`, one proxy hiccup on a harden-runner PR +run fanned `go mod download` out to ~25 vanity-import hosts (`k8s.io`, `go.uber.org`, `gopkg.in`, `helm.sh`, +…), all blocked, and `Get dependencies` took 9+ minutes of retries. Under block mode the fallback can only +ever cost time. + +**Merge queue.** `[lint] Dockerfile` failed on 16 `merge_group` runs: `codeql-action/upload-sarif` fails with +`ref 'refs/heads/gh-readonly-queue/…' not found`, which skips the Trivy scan and fails the second upload on +the missing SARIF file, evicting the PR from the queue. + +## Changes + +- `.github/actions/windows-dns-guard/` (new): composite action that registers a one-shot scheduled task + (`atmos-dns-guard`, SYSTEM, outside the runner's process tree so it survives step teardown and the + runner's orphan-process cleanup) running `dns-guard.ps1`. The script polls once a second and resets DNS + (`Set-DnsClientServerAddress -ResetServerAddresses` + `Clear-DnsClientCache`) only when + `C:\agent\post_event.json` exists, the agent from `C:\agent\agent.pid` is gone, and an IPv4 interface still + lists `127.0.0.1`. The first condition keeps an agent crash mid-job fail-closed exactly as today. It logs to + `$RUNNER_TEMP\atmos-dns-guard.log` and exits after acting or when DNS is already healthy. +- `.github/workflows/test.yml`: + - Guard step on every Windows leg (`build`, `terraform-registry-cache`, `test` × 10 shards, `mock`), + right after checkout, gated the same way as the other Windows steps (skipped on draft PRs). + - Windows and macOS OS endpoints added to the `allowed-endpoints` of `build`, `terraform-registry-cache`, + `test`, `mock`, `k3s` (macOS leg) and `kubernetes-e2e` (macOS leg), with a comment naming the source and + the trade-off. `test` also gains `sum.golang.org`, `google.golang.org` and `modernc.org` for parity with + `build` (its Linux/macOS legs run `atmos build deps`). Firefox telemetry stays blocked. + - `GOPROXY` drops `|direct`; `atmos build deps`'s retry policy covers the transient proxy errors it existed + for. + - Workflow-level `concurrency` cancels a PR's superseded run on a new push; `merge_group`, `push` and + `workflow_dispatch` are keyed by SHA and never cancelled. + - Both `codeql-action/upload-sarif` steps in the `docker` job skip on `merge_group`. +- `.github/workflows/setup-go-cache-warmup.yml`: guard step, the same OS endpoints and Go hosts, `GOPROXY`. +- `.github/workflows/native-ci.yml`, `codeql.yml`, `pre-commit.yml`: `GOPROXY` drops `|direct` (all run under + block mode). + +## Validation + +- `python3 -c 'import yaml; yaml.safe_load(...)'` on every edited workflow and the new action: parses. +- `actionlint` on the five edited workflows: no new findings (one pre-existing SC2086 note in + `pre-commit.yml`, untouched by this change). +- The watchdog's three preconditions were checked against harden-runner's source: `agent.pid` and + `post_event.json` paths and the post step's "wait 10 s, then kill, then delete the pid file" sequence are + as read from `src/install-agent.ts` and `src/cleanup.ts` at `main`. +- Not validated locally: the scheduled task itself needs a Windows runner. Validation is the first CI run on + this branch (guard registers on every Windows job; `$RUNNER_TEMP\atmos-dns-guard.log` is not uploaded, so + the observable signal is the job-level outcome) and the 48-hour measurement below. + +Post-merge measurement (run from any checkout): + +```bash +# True stuck Windows jobs: cancelled, every step succeeded, completion landed >= 5 min after the last step. +gh run list -R cloudposse/atmos -w test.yml --created ">=2026-09-04" -L 300 \ + --json databaseId,conclusion --jq '.[]|select(.conclusion!="success")|.databaseId' | +while read id; do + gh api "repos/cloudposse/atmos/actions/runs/$id/jobs?per_page=100" --paginate --jq ' + .jobs[] | select(.conclusion=="cancelled") | + select(all(.steps[]; .conclusion=="success" or .conclusion=="skipped")) | + select(((.completed_at|fromdate) - (.steps[-1].completed_at|fromdate)) >= 300) | + [.run_id, .name] | @tsv' +done + +# Did harden-runner restore DNS on its own in a given Windows job? (0 = the guard had to act) +gh api repos/cloudposse/atmos/actions/jobs//logs | grep -cE 'system DNS settings restored' +``` + +Success criteria: true stuck Windows jobs 0/day (23/day on Sept 3); no `infra`-labelled blocked calls on +Windows/macOS in StepSecurity Insights; no `Could not resolve host` on Windows checkout; no Trivy failures on +merge-queue runs. + +## Follow-ups + +- Upstream report to StepSecurity (harden-runner) — draft below; to be filed once approved. +- Phase 2 of the CI-stability plan (Windows Defender exclusions; restore-only toolchain cache and shipping + toolchains in the build artifact; the Actions cache is a 10 GB LRU being churned by ~5 GB per run, so + nothing saved from `main` survives) and Phase 3 (auto-rerun of infra-cancelled PR runs) are tracked in + the follow-up issue linked from the PR. + +### Upstream issue draft (step-security/harden-runner) + +**Title:** Windows: post step's 10 s `done.json` wait races the agent's DNS restore; job never reports +completion + +On GitHub-hosted `windows-latest` with harden-runner v2.21.0 (Windows agent v1.0.7-win), in both `audit` and +`block` mode, about 1.5 % of our jobs finish every step — including `Post Harden Runner` and `Complete job` — +and then never report a conclusion; GitHub cancels them 30–40 minutes later, well past `timeout-minutes`. + +From the agent log appended to the post step output: healthy jobs end with +`[handlePostHardenRunnerEvent] restoring system DNS` → `system DNS settings restored` → `DNS proxy stopped`. +Stuck jobs log a second `timed out` from `handleWindowsCleanup` (the 10 × 1 s `done.json` wait), then +`stopping windows agent process` / `agent process stopped gracefully`, and the agent log stops at +`restoring system DNS` (or before it). The agent picked up `post_event.json` ~5 s after the post step began; +the restore takes ~2.4 s, so it loses the 10 s race a few percent of the time. The adapter is left on +`127.0.0.1` with no proxy behind it, and the runner cannot resolve the Actions service to complete the job. + +Examples (cloudposse/atmos, public): stuck jobs 100694525046, 100532566143, 100127533462; healthy for +comparison 100688004653, 100532566215. + +Requests: (1) wait for the DNS restore to finish (or restore DNS from the action as a fallback after killing +the agent); (2) consider implicitly allowing Windows/macOS OS endpoints in block mode (NCSI probes, WNS, +update/settings/telemetry, time sync, OCSP/CRL, Apple update/CDN) — they are blocked on every job today. From 97e87d1149577bd7fa88b6f51c8658d7425abc1b Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Thu, 3 Sep 2026 12:35:29 -0500 Subject: [PATCH 02/19] ci: allow the whole NTP pool (*.pool.ntp.org), not just 0.pool.ntp.org The pool advertises four rotating names (0-3.pool.ntp.org); a single one was only what happened to be sampled. Push deferred until the first CI run of the guard finishes so its Windows data isn't cancelled by the new concurrency group. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/setup-go-cache-warmup.yml | 2 +- .github/workflows/test.yml | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/setup-go-cache-warmup.yml b/.github/workflows/setup-go-cache-warmup.yml index 2d7b545f07e..fedbdfb6a77 100644 --- a/.github/workflows/setup-go-cache-warmup.yml +++ b/.github/workflows/setup-go-cache-warmup.yml @@ -81,7 +81,7 @@ jobs: *.apple-dns.net:443 ocsp.usertrust.com:80 ocsp.digicert.com:80 - 0.pool.ntp.org:123 + *.pool.ntp.org:123 - name: Add GNU tar to PATH (significantly faster than windows tar) if: matrix.target == 'windows' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0aebbf62a13..90acdb36269 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -163,7 +163,7 @@ jobs: *.apple-dns.net:443 ocsp.usertrust.com:80 ocsp.digicert.com:80 - 0.pool.ntp.org:123 + *.pool.ntp.org:123 - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2.3.0 if: matrix.target == 'linux' @@ -395,7 +395,7 @@ jobs: *.apple-dns.net:443 ocsp.usertrust.com:80 ocsp.digicert.com:80 - 0.pool.ntp.org:123 + *.pool.ntp.org:123 - name: Check out code into the Go module directory if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} @@ -648,7 +648,7 @@ jobs: *.apple-dns.net:443 ocsp.usertrust.com:80 ocsp.digicert.com:80 - 0.pool.ntp.org:123 + *.pool.ntp.org:123 - name: Check out code into the Go module directory if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} @@ -1335,7 +1335,7 @@ jobs: *.apple-dns.net:443 ocsp.usertrust.com:80 ocsp.digicert.com:80 - 0.pool.ntp.org:123 + *.pool.ntp.org:123 - name: Check out code into the Go module directory uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 @@ -1507,7 +1507,7 @@ jobs: *.apple-dns.net:443 ocsp.usertrust.com:80 ocsp.digicert.com:80 - 0.pool.ntp.org:123 + *.pool.ntp.org:123 - name: Check out code into the Go module directory uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 @@ -1648,7 +1648,7 @@ jobs: *.apple-dns.net:443 ocsp.usertrust.com:80 ocsp.digicert.com:80 - 0.pool.ntp.org:123 + *.pool.ntp.org:123 - name: Check out code into the Go module directory if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} From aeadac9b49f3addcbace2302d7917286ee75c437 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Thu, 3 Sep 2026 12:46:33 -0500 Subject: [PATCH 03/19] ci: address review on the Windows DNS guard - Skip Harden Runner on draft-PR Windows jobs: every other Windows step (including checkout) is already skipped there, so its post step would race the agent's DNS restore with no guard available. - Run the guard's scheduled task as the runner's own Administrator account (S4U) instead of SYSTEM; it grants nothing the job does not already have. - Wrap the plain go mod download steps in pre-commit.yml and codeql.yml with go-mod-download-retry now that GOPROXY has no direct fallback. - Measurement query: filter to Windows jobs and tolerate fractional-second timestamps. Co-Authored-By: Claude Fable 5.1 --- .github/actions/windows-dns-guard/action.yml | 12 +++++++++--- .github/workflows/codeql.yml | 4 +++- .github/workflows/pre-commit.yml | 8 +++++--- .github/workflows/test.yml | 15 ++++++++++++++- ...9-03-harden-runner-windows-dns-restore-race.md | 10 +++++++--- 5 files changed, 38 insertions(+), 11 deletions(-) diff --git a/.github/actions/windows-dns-guard/action.yml b/.github/actions/windows-dns-guard/action.yml index 9dd179b4a75..5d33d91af9e 100644 --- a/.github/actions/windows-dns-guard/action.yml +++ b/.github/actions/windows-dns-guard/action.yml @@ -1,6 +1,7 @@ name: 'Windows DNS guard (harden-runner post-step race)' description: > - Registers a one-shot Windows scheduled task that repairs the runner's DNS + Registers a one-shot Windows scheduled task (running as the runner's own + Administrator account) that repairs the runner's DNS client settings if step-security/harden-runner's Windows post step kills its agent before the agent has finished restoring them. Without this, the job's steps all succeed but the runner is left pointing at a DNS proxy @@ -9,7 +10,7 @@ description: > docs/fixes/2026-09-03-harden-runner-windows-dns-restore-race.md. The task runs under the Task Scheduler service, outside the step's and - runner's process trees, so it survives step teardown and the runner's + runner's process trees (but with no more rights than the job already has), so it survives step teardown and the runner's "Cleaning up orphan processes" pass. It only acts once harden-runner's own post step has started (C:\agent\post_event.json exists), the agent process is gone, and an interface still lists 127.0.0.1 as its DNS server -- so it @@ -33,7 +34,12 @@ runs: $log = Join-Path $env:RUNNER_TEMP 'atmos-dns-guard.log' $action = New-ScheduledTaskAction -Execute 'powershell.exe' ` -Argument "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$dest`" -LogPath `"$log`"" - $principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest + # Run as the runner's own (already Administrator) account rather than + # SYSTEM: the script comes from the checkout, and the job's other steps + # already execute that checkout with the same rights, so this grants + # nothing extra. S4U lets the task start whether or not the session is + # interactive. + $principal = New-ScheduledTaskPrincipal -UserId "$env:USERDOMAIN\$env:USERNAME" -LogonType S4U -RunLevel Highest $settings = New-ScheduledTaskSettingsSet -ExecutionTimeLimit (New-TimeSpan -Hours 2) ` -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -MultipleInstances IgnoreNew Register-ScheduledTask -TaskName 'atmos-dns-guard' -Action $action -Principal $principal ` diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 82676713f71..0b5ce6f2a19 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -268,8 +268,10 @@ jobs: # Pre-populate module cache to prevent goanalysis_metalinter failures. # Without this, the linter may fail with "could not load export data" errors. # See: https://github.com/golangci/golangci-lint/issues/5437 + # With retry: GOPROXY no longer falls back to direct fetches (they only + # hit blocked vanity hosts under harden-runner's block mode). - name: Download modules - run: go mod download + uses: ./.github/actions/go-mod-download-retry # Install the golangci-lint v2 CLI tool (not the linters themselves). # This tool is needed to run `golangci-lint custom` which builds a custom binary diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 54368870a02..22c97b3c655 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -68,10 +68,12 @@ jobs: # Add Go bin to PATH echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + # Pre-download dependencies to prevent go mod tidy from trying to fetch + # internal packages. With retry: GOPROXY no longer falls back to direct + # fetches (they only hit blocked vanity hosts under harden-runner's block + # mode), so transient proxy errors are handled by retrying instead. - name: Download Go module dependencies - run: | - # Pre-download dependencies to prevent go mod tidy from trying to fetch internal packages - go mod download + uses: ./.github/actions/go-mod-download-retry - name: Set up Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 90acdb36269..01ce7561a4a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -107,8 +107,12 @@ jobs: # (what this repo uses) only covers GitHub-hosted runners — self-hosted # runners including RunsOn need a StepSecurity Enterprise license plus # RunsOn-side config we're not doing here. + # Draft PRs skip every Windows step below (no checkout, nothing to + # protect), so skip Harden Runner there too: its post step would still + # race the agent's DNS restore, and the DNS guard (a local action) needs + # the checkout. - name: Harden Runner - if: matrix.target != 'linux' + if: matrix.target != 'linux' && ! ( matrix.target == 'windows' && github.event.pull_request.draft ) uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 with: egress-policy: block @@ -337,7 +341,10 @@ jobs: timeout-minutes: ${{ matrix.flavor.target == 'windows' && 45 || 20 }} runs-on: ${{ matrix.flavor.os }} steps: + # Draft PRs skip every Windows step in this job (no checkout), so skip + # Harden Runner for them as well; see the `build` job's note. - name: Harden Runner + if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 with: egress-policy: block @@ -569,7 +576,10 @@ jobs: timeout-minutes: ${{ matrix.jobTimeoutOverride || matrix.flavor.jobTimeout }} runs-on: ${{ matrix.flavor.os }} steps: + # Draft PRs skip every Windows step in this job (no checkout), so skip + # Harden Runner for them as well; see the `build` job's note. - name: Harden Runner + if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 with: # Union of core, tool-driven endpoints across the linux/windows/macos @@ -1594,7 +1604,10 @@ jobs: timeout-minutes: 20 steps: + # Draft PRs skip every Windows step in this job (no checkout), so skip + # Harden Runner for them as well; see the `build` job's note. - name: Harden Runner + if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} uses: step-security/harden-runner@05e31511f85b41b11d1cf0ef85d0992719546e2c # v2.21.0 with: egress-policy: block diff --git a/docs/fixes/2026-09-03-harden-runner-windows-dns-restore-race.md b/docs/fixes/2026-09-03-harden-runner-windows-dns-restore-race.md index c45e5868800..464dce7fd50 100644 --- a/docs/fixes/2026-09-03-harden-runner-windows-dns-restore-race.md +++ b/docs/fixes/2026-09-03-harden-runner-windows-dns-restore-race.md @@ -73,7 +73,7 @@ the missing SARIF file, evicting the PR from the queue. ## Changes - `.github/actions/windows-dns-guard/` (new): composite action that registers a one-shot scheduled task - (`atmos-dns-guard`, SYSTEM, outside the runner's process tree so it survives step teardown and the + (`atmos-dns-guard`, running as the runner's own Administrator account, outside the runner's process tree so it survives step teardown and the runner's orphan-process cleanup) running `dns-guard.ps1`. The script polls once a second and resets DNS (`Set-DnsClientServerAddress -ResetServerAddresses` + `Clear-DnsClientCache`) only when `C:\agent\post_event.json` exists, the agent from `C:\agent\agent.pid` is gone, and an IPv4 interface still @@ -81,7 +81,9 @@ the missing SARIF file, evicting the PR from the queue. `$RUNNER_TEMP\atmos-dns-guard.log` and exits after acting or when DNS is already healthy. - `.github/workflows/test.yml`: - Guard step on every Windows leg (`build`, `terraform-registry-cache`, `test` × 10 shards, `mock`), - right after checkout, gated the same way as the other Windows steps (skipped on draft PRs). + right after checkout, gated the same way as the other Windows steps. Draft PRs skip every Windows step + including checkout, so Harden Runner is now skipped for them too rather than left to race without the + guard. - Windows and macOS OS endpoints added to the `allowed-endpoints` of `build`, `terraform-registry-cache`, `test`, `mock`, `k3s` (macOS leg) and `kubernetes-e2e` (macOS leg), with a comment naming the source and the trade-off. `test` also gains `sum.golang.org`, `google.golang.org` and `modernc.org` for parity with @@ -115,9 +117,11 @@ gh run list -R cloudposse/atmos -w test.yml --created ">=2026-09-04" -L 300 \ --json databaseId,conclusion --jq '.[]|select(.conclusion!="success")|.databaseId' | while read id; do gh api "repos/cloudposse/atmos/actions/runs/$id/jobs?per_page=100" --paginate --jq ' + def ts: sub("\\.[0-9]+Z$"; "Z") | fromdate; .jobs[] | select(.conclusion=="cancelled") | + select(any(.labels[]?; test("windows"; "i")) or (.name|test("windows"; "i"))) | select(all(.steps[]; .conclusion=="success" or .conclusion=="skipped")) | - select(((.completed_at|fromdate) - (.steps[-1].completed_at|fromdate)) >= 300) | + select(((.completed_at|ts) - (.steps[-1].completed_at|ts)) >= 300) | [.run_id, .name] | @tsv' done From 7fa797daae850cc147133ad9d066e5c411dc2133 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Thu, 3 Sep 2026 12:57:41 -0500 Subject: [PATCH 04/19] ci: Windows Defender exclusions and restore-only toolchain cache on shards Phase 2 (part 1) of the CI-stability plan, stacked on the Phase 1 branch. Windows Defender: on the Windows shards "Set up Go" averages 5.2 min (max 10.5) restoring a 1.9 GB go-build+mod cache; the download is ~20 s, the rest is tar/zstd extraction with Defender's real-time scanner inspecting every extracted file, and the same scanner holding handles on fresh test files surfaces as testing.TempDir cleanup failures ("unlinkat ... being used by another process"). Add a continue-on-error pwsh step to every Windows leg of test.yml (build, terraform-registry-cache, test, mock) and to setup-go-cache-warmup.yml that excludes D:\a, C:\hostedtoolcache\windows, the Go caches, and the temp dirs, plus go.exe as a process. These are ephemeral VMs and the step does not touch harden-runner's egress policy. Restore-only toolchain cache: the repo's Actions cache is 18.9 GB across 17 refs/pull/N/merge-scoped entries against a 10 GB LRU quota, so the static atmos-toolchain---v2 key never hits ("Cache not found for input keys" on every shard), then all 10 shards race to save it ("Unable to reserve cache with key ...") and "Post Cache Atmos toolchain" costs 43 s avg / 3 min max per shard for nothing. Add an opt-in restore-only input (default 'false') to actions/cache/action.yml that switches to actions/cache/restore at the same pinned SHA, keep cache-hit and key outputs working, document it in the README, and set it on the test job's "Cache Atmos toolchain" step for all three OSes. terraform-registry-cache stays the single writer per OS and is unchanged. Fix log: docs/fixes/2026-09-03-windows-defender-exclusions-and-restore-only-toolchain-cache.md Co-Authored-By: Claude Fable 5.1 --- .github/workflows/setup-go-cache-warmup.yml | 26 ++++ .github/workflows/test.yml | 117 ++++++++++++++++++ actions/cache/README.md | 28 ++++- actions/cache/action.yml | 27 +++- ...usions-and-restore-only-toolchain-cache.md | 93 ++++++++++++++ 5 files changed, 287 insertions(+), 4 deletions(-) create mode 100644 docs/fixes/2026-09-03-windows-defender-exclusions-and-restore-only-toolchain-cache.md diff --git a/.github/workflows/setup-go-cache-warmup.yml b/.github/workflows/setup-go-cache-warmup.yml index fedbdfb6a77..0696ea8341d 100644 --- a/.github/workflows/setup-go-cache-warmup.yml +++ b/.github/workflows/setup-go-cache-warmup.yml @@ -99,6 +99,32 @@ jobs: if: matrix.target == 'windows' uses: ./.github/actions/windows-dns-guard + # Windows Defender exclusions. Measured on the Windows shards (Aug 24 - + # Sept 3): "Set up Go" averages 5.2 min (max 10.5) restoring a 1.9 GB + # go-build+mod cache; the download is ~20 s at 100 MB/s and the rest is + # tar/zstd extraction with Defender's real-time scanner inspecting every + # extracted file. The same scanner holds handles on freshly written test + # files, which surfaces as testing.TempDir cleanup failures + # ("unlinkat ... being used by another process"). These runners are + # ephemeral VMs discarded at job end, so there is nothing to protect from + # files the job itself just wrote. Landing this right after the + # harden-runner block-mode PR is fine: it does not touch the egress + # policy at all, it only stops the AV from re-scanning the workspace, the + # hosted tool cache, the Go caches, and the temp dirs. continue-on-error + # keeps a future locked-down image from failing the job. + - name: Exclude the workspace and Go caches from Windows Defender + if: matrix.target == 'windows' + shell: pwsh + continue-on-error: true + run: | + Add-MpPreference -ExclusionPath 'D:\a' + Add-MpPreference -ExclusionPath 'C:\hostedtoolcache\windows' + Add-MpPreference -ExclusionPath "$env:USERPROFILE\go" + Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\go-build" + Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\Temp" + Add-MpPreference -ExclusionPath "$env:RUNNER_TEMP" + Add-MpPreference -ExclusionProcess 'go.exe' + - name: Set up Go uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 01ce7561a4a..cf822de191e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -197,6 +197,32 @@ jobs: if: matrix.target == 'windows' && ! github.event.pull_request.draft uses: ./.github/actions/windows-dns-guard + # Windows Defender exclusions. Measured on the Windows shards (Aug 24 - + # Sept 3): "Set up Go" averages 5.2 min (max 10.5) restoring a 1.9 GB + # go-build+mod cache; the download is ~20 s at 100 MB/s and the rest is + # tar/zstd extraction with Defender's real-time scanner inspecting every + # extracted file. The same scanner holds handles on freshly written test + # files, which surfaces as testing.TempDir cleanup failures + # ("unlinkat ... being used by another process"). These runners are + # ephemeral VMs discarded at job end, so there is nothing to protect from + # files the job itself just wrote. Landing this right after the + # harden-runner block-mode PR is fine: it does not touch the egress + # policy at all, it only stops the AV from re-scanning the workspace, the + # hosted tool cache, the Go caches, and the temp dirs. continue-on-error + # keeps a future locked-down image from failing the job. + - name: Exclude the workspace and Go caches from Windows Defender + if: matrix.target == 'windows' && ! github.event.pull_request.draft + shell: pwsh + continue-on-error: true + run: | + Add-MpPreference -ExclusionPath 'D:\a' + Add-MpPreference -ExclusionPath 'C:\hostedtoolcache\windows' + Add-MpPreference -ExclusionPath "$env:USERPROFILE\go" + Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\go-build" + Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\Temp" + Add-MpPreference -ExclusionPath "$env:RUNNER_TEMP" + Add-MpPreference -ExclusionProcess 'go.exe' + - name: Set up Go if: ${{ ! ( matrix.target == 'windows' && github.event.pull_request.draft ) }} # setup-go v6 requires runner v2.327.1+ and can affect toolchain handling. @@ -425,6 +451,32 @@ jobs: shell: pwsh run: echo "C:\Program Files\Git\usr\bin" >> $Env:GITHUB_PATH + # Windows Defender exclusions. Measured on the Windows shards (Aug 24 - + # Sept 3): "Set up Go" averages 5.2 min (max 10.5) restoring a 1.9 GB + # go-build+mod cache; the download is ~20 s at 100 MB/s and the rest is + # tar/zstd extraction with Defender's real-time scanner inspecting every + # extracted file. The same scanner holds handles on freshly written test + # files, which surfaces as testing.TempDir cleanup failures + # ("unlinkat ... being used by another process"). These runners are + # ephemeral VMs discarded at job end, so there is nothing to protect from + # files the job itself just wrote. Landing this right after the + # harden-runner block-mode PR is fine: it does not touch the egress + # policy at all, it only stops the AV from re-scanning the workspace, the + # hosted tool cache, the Go caches, and the temp dirs. continue-on-error + # keeps a future locked-down image from failing the job. + - name: Exclude the workspace and Go caches from Windows Defender + if: matrix.flavor.target == 'windows' && ! github.event.pull_request.draft + shell: pwsh + continue-on-error: true + run: | + Add-MpPreference -ExclusionPath 'D:\a' + Add-MpPreference -ExclusionPath 'C:\hostedtoolcache\windows' + Add-MpPreference -ExclusionPath "$env:USERPROFILE\go" + Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\go-build" + Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\Temp" + Add-MpPreference -ExclusionPath "$env:RUNNER_TEMP" + Add-MpPreference -ExclusionProcess 'go.exe' + - name: Set up Atmos (install build artifact for ${{ matrix.flavor.target }}) if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} uses: ./.github/actions/setup-atmos-install @@ -681,6 +733,32 @@ jobs: shell: pwsh run: echo "C:\Program Files\Git\usr\bin" >> $Env:GITHUB_PATH + # Windows Defender exclusions. Measured on the Windows shards (Aug 24 - + # Sept 3): "Set up Go" averages 5.2 min (max 10.5) restoring a 1.9 GB + # go-build+mod cache; the download is ~20 s at 100 MB/s and the rest is + # tar/zstd extraction with Defender's real-time scanner inspecting every + # extracted file. The same scanner holds handles on freshly written test + # files, which surfaces as testing.TempDir cleanup failures + # ("unlinkat ... being used by another process"). These runners are + # ephemeral VMs discarded at job end, so there is nothing to protect from + # files the job itself just wrote. Landing this right after the + # harden-runner block-mode PR is fine: it does not touch the egress + # policy at all, it only stops the AV from re-scanning the workspace, the + # hosted tool cache, the Go caches, and the temp dirs. continue-on-error + # keeps a future locked-down image from failing the job. + - name: Exclude the workspace and Go caches from Windows Defender + if: matrix.flavor.target == 'windows' && ! github.event.pull_request.draft + shell: pwsh + continue-on-error: true + run: | + Add-MpPreference -ExclusionPath 'D:\a' + Add-MpPreference -ExclusionPath 'C:\hostedtoolcache\windows' + Add-MpPreference -ExclusionPath "$env:USERPROFILE\go" + Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\go-build" + Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\Temp" + Add-MpPreference -ExclusionPath "$env:RUNNER_TEMP" + Add-MpPreference -ExclusionProcess 'go.exe' + - name: Set up Atmos (install build artifact for ${{ matrix.flavor.target }}) if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} uses: ./.github/actions/setup-atmos-install @@ -695,10 +773,23 @@ jobs: # defaults and Terraform's plugin cache is not safe for shared concurrent # use. This cache step restores/saves toolchain bits only; it must stay a # pure accelerator. + # + # Restore-only on the shards. The repo's Actions cache holds 18.9 GB in + # 17 entries against a 10 GB LRU quota, every entry scoped to a + # refs/pull/N/merge ref and minutes old, so the static key + # atmos-toolchain---v2 never hits from main (every shard logs + # "Cache not found for input keys"). All 10 shards then raced to save + # the same key and logged "Unable to reserve cache with key ..., another + # job may be creating this cache", with "Post Cache Atmos toolchain" + # costing 43 s avg / 3 min max per shard for nothing. The + # terraform-registry-cache job keeps the plain (restore+save) step and is + # the single writer of this key per OS. - name: Cache Atmos toolchain if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} continue-on-error: true uses: ./actions/cache + with: + restore-only: 'true' - name: Install Terraform, OpenTofu, Packer, Helm, and Helmfile if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} @@ -1684,6 +1775,32 @@ jobs: shell: pwsh run: echo "C:\Program Files\Git\usr\bin" >> $Env:GITHUB_PATH + # Windows Defender exclusions. Measured on the Windows shards (Aug 24 - + # Sept 3): "Set up Go" averages 5.2 min (max 10.5) restoring a 1.9 GB + # go-build+mod cache; the download is ~20 s at 100 MB/s and the rest is + # tar/zstd extraction with Defender's real-time scanner inspecting every + # extracted file. The same scanner holds handles on freshly written test + # files, which surfaces as testing.TempDir cleanup failures + # ("unlinkat ... being used by another process"). These runners are + # ephemeral VMs discarded at job end, so there is nothing to protect from + # files the job itself just wrote. Landing this right after the + # harden-runner block-mode PR is fine: it does not touch the egress + # policy at all, it only stops the AV from re-scanning the workspace, the + # hosted tool cache, the Go caches, and the temp dirs. continue-on-error + # keeps a future locked-down image from failing the job. + - name: Exclude the workspace and Go caches from Windows Defender + if: matrix.flavor.target == 'windows' && ! github.event.pull_request.draft + shell: pwsh + continue-on-error: true + run: | + Add-MpPreference -ExclusionPath 'D:\a' + Add-MpPreference -ExclusionPath 'C:\hostedtoolcache\windows' + Add-MpPreference -ExclusionPath "$env:USERPROFILE\go" + Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\go-build" + Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\Temp" + Add-MpPreference -ExclusionPath "$env:RUNNER_TEMP" + Add-MpPreference -ExclusionProcess 'go.exe' + - name: Set up Atmos (install build artifact for ${{ matrix.flavor.target }}) if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} uses: ./.github/actions/setup-atmos-install diff --git a/actions/cache/README.md b/actions/cache/README.md index 064cc67a036..d53a4d1297a 100644 --- a/actions/cache/README.md +++ b/actions/cache/README.md @@ -38,11 +38,35 @@ ci: - 'atmos-toolchain-{{.OS}}-{{.Arch}}-' ``` +### Inputs + +| Input | Default | Description | +| --- | --- | --- | +| `restore-only` | `'false'` | When `'true'`, restore the cache but never save it (uses `actions/cache/restore` instead of `actions/cache`, so there is no post step). | + +#### Many parallel consumers, one writer + +If several jobs (for example, a matrix of test shards) restore the same key, +let exactly one job save it and mark the rest `restore-only`. Otherwise every +shard that misses tries to save the same key at the end of the job: all but +one fail with `Unable to reserve cache with key ..., another job may be +creating this cache`, and each still pays the tar + zstd + upload cost first. + +```yaml +# The single writer (saves on a miss): +- uses: cloudposse/atmos/actions/cache@v1 + +# Every parallel consumer (restores only, no post step): +- uses: cloudposse/atmos/actions/cache@v1 + with: + restore-only: 'true' +``` + ### Outputs | Output | Description | | --- | --- | -| `cache-hit` | `true` when `actions/cache` found an exact key match. | +| `cache-hit` | `true` when `actions/cache` (or `actions/cache/restore`) found an exact key match. | | `key` | The resolved cache key. | ## How it compares @@ -60,4 +84,4 @@ If you need Atmos to *own* restore/save (rather than `actions/cache`), use the This action ships inside the Atmos repository, so the ref is an Atmos release: pin to `@v1` (moving major tag), `@vX.Y.Z`, or a commit SHA. It internally pins -`actions/cache` to a SHA (`v5.0.5`). +`actions/cache` and `actions/cache/restore` to the same SHA (`v5.0.5`). diff --git a/actions/cache/action.yml b/actions/cache/action.yml index b06a52cc46b..5f7f3801802 100644 --- a/actions/cache/action.yml +++ b/actions/cache/action.yml @@ -14,10 +14,23 @@ branding: icon: 'archive' color: 'blue' +inputs: + restore-only: + description: >- + When 'true', only restore the cache (actions/cache/restore) and never + save it in the post step. Use this on jobs that fan out into many + parallel consumers of one key (for example test shards) and leave a + single writer job to save. Saves otherwise race for the same key + ("Unable to reserve cache ... another job may be creating this cache") + and each one still pays the tar+zstd upload cost. + required: false + default: 'false' + outputs: cache-hit: - description: 'Whether an exact key match was found (from actions/cache).' - value: ${{ steps.cache.outputs.cache-hit }} + description: 'Whether an exact key match was found (from actions/cache or actions/cache/restore).' + # Exactly one of the two cache steps runs; a skipped step's output is empty. + value: ${{ steps.cache.outputs.cache-hit || steps.cache-restore.outputs.cache-hit }} key: description: 'The resolved cache key.' value: ${{ steps.meta.outputs.key }} @@ -43,8 +56,18 @@ runs: exit 1 fi - id: cache + if: inputs.restore-only != 'true' uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: key: ${{ steps.meta.outputs.key }} path: ${{ steps.meta.outputs.path }} restore-keys: ${{ steps.meta.outputs.restore-keys }} + # actions/cache/restore ships in the same repository and tag as + # actions/cache, so the same SHA pins both. + - id: cache-restore + if: inputs.restore-only == 'true' + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + key: ${{ steps.meta.outputs.key }} + path: ${{ steps.meta.outputs.path }} + restore-keys: ${{ steps.meta.outputs.restore-keys }} diff --git a/docs/fixes/2026-09-03-windows-defender-exclusions-and-restore-only-toolchain-cache.md b/docs/fixes/2026-09-03-windows-defender-exclusions-and-restore-only-toolchain-cache.md new file mode 100644 index 00000000000..da76813b4c1 --- /dev/null +++ b/docs/fixes/2026-09-03-windows-defender-exclusions-and-restore-only-toolchain-cache.md @@ -0,0 +1,93 @@ +# Fix: Windows Defender exclusions on CI Windows legs and restore-only toolchain cache on the acceptance shards + +**Date:** 2026-09-03 + +## Summary + +Two CI-only changes from Phase 2 of the CI-stability plan (see +`docs/fixes/2026-09-03-harden-runner-windows-dns-restore-race.md` for Phase 1). First, every Windows leg of +`test.yml` (and the nightly Go cache warmup) now excludes the workspace, the hosted tool cache, the Go +caches, and the temp directories from Windows Defender's real-time scanner before `Set up Go` runs. Second, +the `test` job's ten shards per OS restore the Atmos toolchain cache without ever saving it: the +`actions/cache` composite action gained a `restore-only` input that switches it to `actions/cache/restore`, +and `terraform-registry-cache` stays the single writer of the key per OS. No user-visible behavior changes; +the new action input is opt-in and defaults to the previous behavior. + +## Context + +Measured on `test.yml` runs from Aug 24 to Sept 3 (80 successful Windows shards, step-level timelines, raw +job logs, and `gh api repos/cloudposse/atmos/actions/cache/usage`). + +**Windows shard time.** Median 14.1 min, p90 16.6, max 36. `Set up Go` alone averages 5.2 min (max 10.5) +restoring a 1.9 GB go-build + mod cache. The download is about 20 s at 100 MB/s; the remainder is tar/zstd +extraction with Defender's real-time scanner inspecting every extracted file. The same scanner holding +handles on freshly written files is what surfaces as `testing.TempDir` cleanup failures +(`unlinkat ... being used by another process`) in the acceptance tests. The existing +`transientErrorDetector` in `internal/ci/acceptance/command.go` only covers the Go-toolchain variant of +that error. + +**Toolchain cache.** The repository's Actions cache holds 18.9 GB across 17 entries against a 10 GB LRU +quota. Every entry is scoped to a `refs/pull/N/merge` ref and is minutes old (14 `setup-go-*` entries at +1.6 to 1.9 GB, 8 `atmos-toolchain-*` entries at 350 to 470 MB). Each run writes about 5 GB, so nothing +saved from `main` survives and the static key `atmos-toolchain---v2` never hits: every shard +logs `Cache not found for input keys`, then all 10 shards race to save the same key and log +`Unable to reserve cache with key ..., another job may be creating this cache`. `Post Cache Atmos +toolchain` costs 43 s on average and up to 3 min per shard for nothing. + +**Why it is safe to land right after the harden-runner block-mode PR.** The Defender step does not touch +harden-runner's egress policy. It only stops the antivirus from re-scanning files the job itself just wrote +on an ephemeral VM that is discarded when the job ends. The step is `continue-on-error: true` so a future +locked-down image cannot fail the job. + +## Changes + +- `.github/workflows/test.yml` + - New step `Exclude the workspace and Go caches from Windows Defender` (`shell: pwsh`, + `continue-on-error: true`, same draft-PR gating as the neighbouring `Add GNU tar to PATH` step) on the + Windows legs of `build` (after the `Guard DNS against the harden-runner post-step race` step, before + `Set up Go`), `terraform-registry-cache`, `test`, and `mock` (each immediately after `Add GNU tar to + PATH`). It runs `Add-MpPreference -ExclusionPath` for `D:\a`, `C:\hostedtoolcache\windows`, + `$env:USERPROFILE\go`, `$env:LOCALAPPDATA\go-build`, `$env:LOCALAPPDATA\Temp`, `$env:RUNNER_TEMP`, and + `Add-MpPreference -ExclusionProcess 'go.exe'`. + - The `test` job's `Cache Atmos toolchain` step (all three OSes) now passes `restore-only: 'true'`, with a + comment carrying the cache data above and naming `terraform-registry-cache` as the single writer. The + `terraform-registry-cache` job's cache step is unchanged. +- `.github/workflows/setup-go-cache-warmup.yml`: the same Defender step on the Windows leg, after the DNS + guard and before `Set up Go`. +- `actions/cache/action.yml`: new optional input `restore-only` (string, default `'false'`). When `'true'` + the action runs `actions/cache/restore` (same repository and tag as `actions/cache`, so the same pinned + SHA `27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5`) with identical `key`, `path`, and + `restore-keys`; otherwise `actions/cache` runs as before. Exactly one of the two steps runs, so the + `cache-hit` output is `steps.cache.outputs.cache-hit || steps.cache-restore.outputs.cache-hit`; the `key` + output is unchanged. +- `actions/cache/README.md`: documents the input and the "many parallel consumers, one writer" pattern. + +Not changed on purpose (later PRs): `atmos.yaml` `ci.cache.key`, `setup-go` caching, and the Go cache +warmup's writer role. + +## Validation + +- `python3 -c 'import yaml; yaml.safe_load(open(f))'` on `.github/workflows/test.yml`, + `.github/workflows/setup-go-cache-warmup.yml`, and `actions/cache/action.yml`: all parse; the action + parses with `inputs: [restore-only]` and steps `meta`, `validate`, `cache`, `cache-restore`. +- `actionlint .github/workflows/test.yml .github/workflows/setup-go-cache-warmup.yml`: clean (exit 0). + actionlint does not lint composite actions, so `actions/cache/action.yml` is covered by the YAML parse + and the Go regression test below only. +- `go test github.com/cloudposse/atmos/cmd -run TestAtmosCacheActionValidatesMetadataBeforeActionsCache`: + passes (it asserts the metadata validation step still precedes the cache steps). +- `git diff` reviewed for accidental changes: only the four files above. +- Not validated here: the Windows timing effect itself. `Set up Go` extraction time and + `Post Cache Atmos toolchain` duration on the shards are measured from the PR's own CI runs with the + "Windows shard step durations" query in the Phase 1 fix log; the expected outcomes are `Set up Go` from + 5 to 10 min down to about 1 min, `Post Cache Atmos toolchain` absent from the 30 shard jobs, and no + `Unable to reserve cache` lines in shard logs. + +## Follow-ups + +- Two further Phase 2 changes are separate PRs already in flight and have no GitHub issue numbers yet (the + repository rule against opening unprompted issues applies; the numbers are added here when the PRs open): + the single-writer hashed toolchain key (`atmos-toolchain-{{.OS}}-{{.Arch}}-{{ hashFiles ".tool-versions" }}` + in `atmos.yaml` `ci.cache.key`, once `.tool-versions` lands), and shipping the toolchains inside the + existing `build-artifacts-` artifact so the shards stop installing them at all. +- Phase 2c (measure the cache-free Windows shard) and the `setup-go` single-writer change are also still + open; same tracking note. From 951158dcf2789eb46f73e1697b3ad0355d179cdc Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Thu, 3 Sep 2026 13:14:49 -0500 Subject: [PATCH 05/19] ci: install the toolchain once per OS and ship it in a build artifact Every acceptance shard, terraform-registry-cache leg and mock job ran the five `atmos toolchain install --default ...` commands (~400 MB of release downloads, ~2 min, per job) because the toolchain cache never hits: the Go caches churn the 10 GB Actions cache first. Install the tools once in the build job, ship `toolchain/bin` as a `toolchain-` artifact (1-day retention), and have the consumers unpack it into their toolchain cache before installing. The consumers install from a job-local tool-versions file instead of per tool: only `installFromToolVersions` takes the "already installed" skip path, while the per-tool `--default owner/repo@version` form always re-resolves, re-verifies and re-extracts (it hung offline even with the archive cached). Both forms live in a new composite action so the tool list stays identical between the build job and its consumers; a missing artifact degrades to the network install. Co-Authored-By: Claude Fable 5.1 --- .github/actions/ci-toolchain/action.yml | 153 ++++++++++++++++++ .github/workflows/test.yml | 134 +++++++++++---- ...-ci-toolchain-shipped-in-build-artifact.md | 128 +++++++++++++++ 3 files changed, 388 insertions(+), 27 deletions(-) create mode 100644 .github/actions/ci-toolchain/action.yml create mode 100644 docs/fixes/2026-09-03-ci-toolchain-shipped-in-build-artifact.md diff --git a/.github/actions/ci-toolchain/action.yml b/.github/actions/ci-toolchain/action.yml new file mode 100644 index 00000000000..835f504d4a1 --- /dev/null +++ b/.github/actions/ci-toolchain/action.yml @@ -0,0 +1,153 @@ +name: 'Set up CI toolchain' +description: > + Installs the external tools the CI jobs need (Terraform, OpenTofu, Packer, + Helm, Helmfile) with `atmos toolchain install`, optionally seeding the Atmos + toolchain cache from a `toolchain.tar` artifact produced earlier in the same + run, and exports their directories to PATH via `atmos toolchain env`. + + The pins are written to a job-local tool-versions file, and both the install + and the PATH export read that file (`--tool-versions`), for two reasons: + + 1. `atmos toolchain install` only takes its "already installed, skip" path + when installing from a tool-versions file. The per-tool + `atmos toolchain install --default owner/repo@version` form always + re-resolves the registry, re-downloads or re-verifies the release + signature, and re-extracts, even when the exact version is already on + disk, so it can never be a no-op and always needs the network. With a + restored artifact, the from-file form skips every tool without a single + network request; without one (or for a tool the artifact lacks) it + installs exactly what is missing, so the network install remains the + safety net. + 2. The repository's own `.tool-versions` is left untouched: nothing in CI + rewrites its defaults any more. + + `hashicorp/terraform` is always pinned to the version in the repository's + `.tool-versions`; every other tool comes from the `tool-versions` input. + + Requires the `atmos` binary to be on PATH (install it before this step). + +inputs: + tool-versions: + description: > + Tools to install, one per line, in `.tool-versions` format + (`owner/repo version`). `hashicorp/terraform` is added automatically + from the repository's `.tool-versions`. Blank lines are ignored. + required: true + artifact: + description: > + Name of a workflow artifact containing a `toolchain.tar` (as packaged by + the `build` job) to unpack into the Atmos toolchain cache before + installing. Tools present in it are skipped by the install step. Leave + empty to install everything from the network (what the `build` job does + to produce the artifact in the first place). A missing or failed + download is a warning, not an error: the install step then falls back + to the network for whatever is absent. + required: false + default: '' + github-token: + description: 'GitHub token for the toolchain registry and release-asset lookups (raises the unauthenticated API rate limit)' + required: true + +outputs: + tool-versions-file: + description: 'Absolute path of the generated tool-versions file' + value: ${{ steps.pins.outputs.file }} + cache-root: + description: 'Absolute path of the Atmos cache root (the parent of the `toolchain` directory), in the OS-native form' + value: ${{ steps.root.outputs.path }} + +runs: + using: composite + steps: + - name: Write the CI tool pins + id: pins + shell: bash + env: + CI_TOOL_VERSIONS: ${{ inputs.tool-versions }} + run: | + file="${RUNNER_TEMP}/atmos-ci.tool-versions" + # Terraform is pinned once, in the repository's .tool-versions; the + # first version on that line is the default. + terraform_version="$(sed -n 's#^hashicorp/terraform[[:space:]]\{1,\}\([^[:space:]]*\).*#\1#p' .tool-versions | head -n 1)" + if [ -z "${terraform_version}" ]; then + echo "::error::.tool-versions has no hashicorp/terraform entry to pin Terraform from" + exit 1 + fi + { + echo "hashicorp/terraform ${terraform_version}" + printf '%s\n' "${CI_TOOL_VERSIONS}" | sed '/^[[:space:]]*$/d' + } > "${file}" + echo "Tool pins (${file}):" + cat "${file}" + echo "file=${file}" >> "$GITHUB_OUTPUT" + + - name: Resolve the Atmos cache root + id: root + shell: bash + run: | + # The toolchain lives at /toolchain (see pkg/toolchain + # GetInstallPath). Ask atmos for the root instead of hardcoding the + # per-OS XDG default; --format=env prints one KEY=VALUE per line. + root="$(atmos ci cache paths --format=env 2>/dev/null | sed -n 's/^ATMOS_CI_CACHE_PATHS=//p')" + if [ -z "${root}" ]; then + echo "::error::atmos ci cache paths did not report a cache root" + exit 1 + fi + echo "Atmos cache root: ${root}" + echo "path=${root}" >> "$GITHUB_OUTPUT" + + - name: Download the toolchain artifact + id: download + if: inputs.artifact != '' + # A pure accelerator: if the artifact is missing (the build leg that + # produces it was skipped) or the download keeps failing, fall through + # to the network install below instead of failing the job. + continue-on-error: true + uses: ./.github/actions/download-artifact-retry + with: + name: ${{ inputs.artifact }} + path: ${{ runner.temp }}/atmos-ci-toolchain + + - name: Unpack the toolchain into the Atmos cache root + if: inputs.artifact != '' && steps.download.outcome == 'success' + shell: bash + env: + ATMOS_CACHE_ROOT: ${{ steps.root.outputs.path }} + TOOLCHAIN_TAR: ${{ runner.temp }}/atmos-ci-toolchain/toolchain.tar + run: | + root="${ATMOS_CACHE_ROOT}" + tarball="${TOOLCHAIN_TAR}" + # Git Bash on Windows: give tar POSIX paths. A native `D:\...` path + # would be read by GNU tar as a `host:file` remote archive. + if command -v cygpath >/dev/null 2>&1; then + root="$(cygpath -u "${root}")" + tarball="$(cygpath -u "${tarball}")" + fi + if [ ! -f "${tarball}" ]; then + echo "::warning::toolchain artifact contained no toolchain.tar; installing from the network instead" + exit 0 + fi + mkdir -p "${root}" + tar -C "${root}" -xf "${tarball}" + echo "Restored toolchain into ${root}/toolchain:" + find "${root}/toolchain/bin" -mindepth 3 -maxdepth 3 -type d | sort + + - name: Report a missing toolchain artifact + if: inputs.artifact != '' && steps.download.outcome != 'success' + shell: bash + env: + ARTIFACT_NAME: ${{ inputs.artifact }} + run: echo "::warning::could not download the ${ARTIFACT_NAME} artifact; installing the toolchain from the network instead" + + - name: Install any tool the cache is missing + shell: bash + env: + GITHUB_TOKEN: ${{ inputs.github-token }} + TOOL_VERSIONS_FILE: ${{ steps.pins.outputs.file }} + run: atmos toolchain install --tool-versions "${TOOL_VERSIONS_FILE}" + + - name: Export the toolchain directories to PATH + shell: bash + env: + TOOL_VERSIONS_FILE: ${{ steps.pins.outputs.file }} + run: atmos toolchain env --tool-versions "${TOOL_VERSIONS_FILE}" --format=github diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 01ce7561a4a..34d23a9a8d8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -130,6 +130,7 @@ jobs: # other Windows/macOS legs in this file. allowed-endpoints: > api.github.com:443 + get.helm.sh:443 github.com:443 raw.githubusercontent.com:443 rekor.sigstore.dev:443 @@ -257,6 +258,72 @@ jobs: shell: bash run: go tool mage acceptance:verify "${{ matrix.target }}" "$TEST_SHARD_COUNT" + # The acceptance shards, the terraform-registry-cache job, and the mock + # jobs all need the same external tools (Terraform, OpenTofu, Packer, + # Helm, Helmfile). Install them once per OS here and ship the installed + # tree as a `toolchain-` artifact; those jobs unpack it into + # their own toolchain cache before `atmos toolchain install`, which then + # skips every tool. That replaces 30+ per-shard installs (~400 MB of + # release downloads and ~2 min each, plus a per-shard dependency on + # releases.hashicorp.com/GitHub/get.helm.sh) and the "Cache Atmos + # toolchain" step that never hits because the Go caches churn the 10 GB + # Actions cache. Uses the atmos just built, not the bootstrap release, so + # the on-disk layout is the one the consumers' binary expects. See + # docs/fixes/2026-09-03-ci-toolchain-shipped-in-build-artifact.md. + # macos-intel is skipped: its only consumer (the k3s macOS job) installs + # no toolchain. + - name: Put the freshly built atmos on PATH for the toolchain install + if: ${{ matrix.target != 'macos-intel' && ! ( matrix.target == 'windows' && github.event.pull_request.draft ) }} + shell: bash + run: echo "${GITHUB_WORKSPACE}/build" >> "$GITHUB_PATH" + + - name: Install the CI toolchain (Terraform, OpenTofu, Packer, Helm, Helmfile) + id: toolchain + if: ${{ matrix.target != 'macos-intel' && ! ( matrix.target == 'windows' && github.event.pull_request.draft ) }} + uses: ./.github/actions/ci-toolchain + with: + tool-versions: | + opentofu/opentofu ${{ env.OPEN_TOFU_VERSION }} + hashicorp/packer ${{ env.PACKER_VERSION }} + helm/helm ${{ env.HELM_VERSION }} + helmfile/helmfile ${{ env.HELMFILE_VERSION }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Package the CI toolchain + if: ${{ matrix.target != 'macos-intel' && ! ( matrix.target == 'windows' && github.event.pull_request.draft ) }} + shell: bash + env: + ATMOS_CACHE_ROOT: ${{ steps.toolchain.outputs.cache-root }} + run: | + root="${ATMOS_CACHE_ROOT}" + out="${RUNNER_TEMP}/toolchain.tar" + # Git Bash on Windows: give tar POSIX paths. A native `D:\...` path + # would be read by GNU tar as a `host:file` remote archive. + if command -v cygpath >/dev/null 2>&1; then + root="$(cygpath -u "${root}")" + out="$(cygpath -u "${out}")" + fi + cd "${root}" + # Ship the installed tree (bin////...) plus the + # registry index and lockfile. The downloaded release archives that + # sit next to them are left out: once a tool is on disk, + # `atmos toolchain install` never looks at them again. + members=(toolchain/bin) + for f in toolchain/toolchain.lock.yaml toolchain/aqua-registry-index.json; do + [ -f "${f}" ] && members+=("${f}") + done + tar -cf "${out}" "${members[@]}" + ls -la "${out}" + + - name: Upload the CI toolchain artifact + if: ${{ matrix.target != 'macos-intel' && ! ( matrix.target == 'windows' && github.event.pull_request.draft ) }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: toolchain-${{ matrix.target }} + path: ${{ runner.temp }}/toolchain.tar + if-no-files-found: error + retention-days: 1 + - name: Upload build artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: ${{ ! ( matrix.target == 'windows' && github.event.pull_request.draft ) }} @@ -438,17 +505,22 @@ jobs: continue-on-error: true uses: ./actions/cache - - name: Install Terraform, OpenTofu, Packer, Helm, and Helmfile + # Seed the toolchain cache from the tarball the `build` job packaged for + # this OS (its "Package the CI toolchain" step), so every tool is already + # on disk and `atmos toolchain install` skips it without touching the + # network; anything the tarball lacks is installed from the network as + # before. The pins here must match the build job's. + - name: Set up Terraform, OpenTofu, Packer, Helm, and Helmfile from the build artifact if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - atmos toolchain install --default hashicorp/terraform - atmos toolchain install --default opentofu/opentofu@${{ env.OPEN_TOFU_VERSION }} - atmos toolchain install --default hashicorp/packer@${{ env.PACKER_VERSION }} - atmos toolchain install --default helm/helm@${{ env.HELM_VERSION }} - atmos toolchain install --default helmfile/helmfile@${{ env.HELMFILE_VERSION }} - atmos toolchain env --format=github + uses: ./.github/actions/ci-toolchain + with: + artifact: toolchain-${{ matrix.flavor.target }} + tool-versions: | + opentofu/opentofu ${{ env.OPEN_TOFU_VERSION }} + hashicorp/packer ${{ env.PACKER_VERSION }} + helm/helm ${{ env.HELM_VERSION }} + helmfile/helmfile ${{ env.HELMFILE_VERSION }} + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Verify Terraform, OpenTofu, Packer, Helm, and Helmfile if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} @@ -700,17 +772,22 @@ jobs: continue-on-error: true uses: ./actions/cache - - name: Install Terraform, OpenTofu, Packer, Helm, and Helmfile + # Seed the toolchain cache from the tarball the `build` job packaged for + # this OS (its "Package the CI toolchain" step), so every tool is already + # on disk and `atmos toolchain install` skips it without touching the + # network; anything the tarball lacks is installed from the network as + # before. The pins here must match the build job's. + - name: Set up Terraform, OpenTofu, Packer, Helm, and Helmfile from the build artifact if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - atmos toolchain install --default hashicorp/terraform - atmos toolchain install --default opentofu/opentofu@${{ env.OPEN_TOFU_VERSION }} - atmos toolchain install --default hashicorp/packer@${{ env.PACKER_VERSION }} - atmos toolchain install --default helm/helm@${{ env.HELM_VERSION }} - atmos toolchain install --default helmfile/helmfile@${{ env.HELMFILE_VERSION }} - atmos toolchain env --format=github + uses: ./.github/actions/ci-toolchain + with: + artifact: toolchain-${{ matrix.flavor.target }} + tool-versions: | + opentofu/opentofu ${{ env.OPEN_TOFU_VERSION }} + hashicorp/packer ${{ env.PACKER_VERSION }} + helm/helm ${{ env.HELM_VERSION }} + helmfile/helmfile ${{ env.HELMFILE_VERSION }} + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Verify Terraform, OpenTofu, Packer, Helm, and Helmfile if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} @@ -1692,14 +1769,17 @@ jobs: path: ${{ github.workspace }} add-to-path: 'true' - - name: Install Terraform and OpenTofu with Atmos toolchain + # Seed the toolchain cache from the tarball the `build` job packaged for + # this OS, so both tools are already on disk and `atmos toolchain install` + # skips them (see the `test` job's step of the same shape). + - name: Set up Terraform and OpenTofu from the build artifact if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - atmos toolchain install hashicorp/terraform - atmos toolchain install --default opentofu/opentofu@${{ env.OPEN_TOFU_VERSION }} - atmos toolchain env --format=github + uses: ./.github/actions/ci-toolchain + with: + artifact: toolchain-${{ matrix.flavor.target }} + tool-versions: | + opentofu/opentofu ${{ env.OPEN_TOFU_VERSION }} + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Verify OpenTofu if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} diff --git a/docs/fixes/2026-09-03-ci-toolchain-shipped-in-build-artifact.md b/docs/fixes/2026-09-03-ci-toolchain-shipped-in-build-artifact.md new file mode 100644 index 00000000000..b8978b83345 --- /dev/null +++ b/docs/fixes/2026-09-03-ci-toolchain-shipped-in-build-artifact.md @@ -0,0 +1,128 @@ +# Fix: CI installs the toolchain once per OS and ships it in a build artifact + +**Date:** 2026-09-03 + +## Summary + +Every one of the 30 acceptance shards in `test.yml` (`test` job, 10 shards × linux/windows/macos), plus the +three `terraform-registry-cache` legs and the `mock` jobs, ran `atmos toolchain install --default ...` for +Terraform, OpenTofu, Packer, Helm and Helmfile: ~400 MB of release downloads and ~2 minutes per job, with +a per-job dependency on releases.hashicorp.com, GitHub releases, get.helm.sh and Sigstore. The +"Cache Atmos toolchain" step meant to short-circuit that never hit, because the ~5 GB of Go caches written +per run churn the repository's 10 GB Actions cache before any toolchain entry survives. This change installs +the five tools once per OS in the `build` job, packages the installed tree as a `toolchain-` +artifact (1-day retention, no cache quota), and has the consuming jobs unpack it into their own Atmos +toolchain cache before `atmos toolchain install`, which then skips every tool without a network request. +The network install stays as the safety net for anything the artifact lacks. + +## Context + +Measured from run logs: every shard logged `Cache not found for input keys: atmos-toolchain-...`, and the +Actions cache listing showed only PR-scoped Go cache entries minutes old. The plan +(Findings §4, Phase 2b) preferred shipping the toolchain in the build artifact over a hashed cache key, +because the cache quota problem is structural (Go caches) and would need the Go footprint to shrink first. + +What the code says about the mechanics (read, then verified locally, see Validation): + +- `atmos toolchain install` puts tools under `/toolchain/bin////` + (`pkg/toolchain/setup.go` `GetInstallPath`, `pkg/toolchain/installer/installer.go` `New`), where the + cache root is the Atmos XDG cache dir (`~/.cache/atmos` on Linux/macOS, `%LOCALAPPDATA%\cache\atmos` + on Windows). `atmos ci cache paths --format=env` prints that root (`ATMOS_CI_CACHE_PATHS=...`), so the + workflow never hardcodes a per-OS path. The tree contains no symlinks and no absolute paths, so it is + relocatable between machines with different home directories. +- The per-tool form the workflow used, `atmos toolchain install --default owner/repo@version`, never takes + the "already installed" path: `RunInstall` (`pkg/toolchain/install.go`) always calls + `InstallSingleTool`, which re-resolves the registry, re-downloads or re-verifies the release signature + (checksums from releases.hashicorp.com, cosign against Sigstore for OpenTofu) and re-extracts. Only the + from-file path (`installFromToolVersions` → `installOrSkipTool`) checks `FindBinaryPath` first and skips. + With the network blocked, the per-tool form hung for the full 2-minute command timeout even with the + download archive already cached; the from-file form skipped every tool in under a second. +- `actions/upload-artifact` zips lose executable bits, so the tree is shipped as a tar, not as loose files. +- `.tool-versions` handling is being consolidated separately (#3022); the pins still come from the + workflow `env` (`OPEN_TOFU_VERSION`, `PACKER_VERSION`, `HELM_VERSION`, `HELMFILE_VERSION`), and Terraform + from the repository's `.tool-versions`, exactly as before. + +## Changes + +- `.github/actions/ci-toolchain/action.yml` (new composite action, used by all four jobs so the tool list + and the install command are identical everywhere): + 1. writes a job-local tool-versions file from the `tool-versions` input plus the repository's + `hashicorp/terraform` pin; + 2. resolves the Atmos cache root with `atmos ci cache paths --format=env`; + 3. when `artifact` is set, downloads it (`download-artifact-retry`, `continue-on-error`) and untars + `toolchain.tar` into the cache root (`cygpath` on Windows so GNU tar does not read `D:\...` as a + remote `host:file`); a missing artifact is a `::warning::`, not a failure; + 4. `atmos toolchain install --tool-versions ` (skips what is on disk, installs the rest); + 5. `atmos toolchain env --tool-versions --format=github`. +- `.github/workflows/test.yml`: + - `build` job: after "Verify acceptance shard plan", puts `./build` on PATH so the install runs with the + atmos just built (same on-disk layout the consumers' binary expects), installs the five tools via the + action, tars `toolchain/bin` plus `toolchain.lock.yaml` and `aqua-registry-index.json` (the downloaded + release archives next to them are left out), and uploads it as `toolchain-` with + `retention-days: 1`. Skipped on `macos-intel`, whose only consumer (the k3s macOS job) installs no + toolchain. `get.helm.sh:443` added to the build job's harden-runner allowlist (Helm's release host). + - `test` and `terraform-registry-cache` jobs: the six-line "Install Terraform, OpenTofu, Packer, Helm, + and Helmfile" `run:` step is replaced by the action with `artifact: toolchain-`. The + "Cache Atmos toolchain" step is untouched (a sibling PR makes it restore-only). + - `mock` job: same replacement for its Terraform + OpenTofu install. + - Consumers no longer rewrite the repository's `.tool-versions` (the old `--default` calls did, on every + job); the acceptance harness finds tools through PATH (`tests/testhelpers/toolchain.go` + `ProvisionToolchain`) and gives every test case its own `XDG_CACHE_HOME`, so nothing reads that file. + +Expected artifact size: the local tar of Terraform + OpenTofu + Helm + the cosign verifier that OpenTofu's +signature check bootstraps is 439 MB uncompressed; with Packer and Helmfile, roughly 600 MB per OS, which +`upload-artifact` compresses to an estimated 250–300 MB. Three of them per run, kept for one day. The +existing `build-artifacts-` is unchanged (it is downloaded into `/usr/local/bin` by a dozen jobs +that need no toolchain, which is why the toolchain is a separate artifact). + +## Validation + +- `actionlint .github/workflows/test.yml`: clean. `yq` parses both files (actionlint does not lint + composite actions; run on `action.yml` it reports the expected "not a workflow" errors only). +- Local mechanics on macOS with a build of this branch (`go build -o /tmp/atmos-ci .`), scratch caches under + `/tmp`, and the network blocked for the offline steps with `HTTPS_PROXY=http://127.0.0.1:9`: + + ```text + $ ATMOS_XDG_CACHE_HOME=/tmp/tc atmos ci cache paths --format=json --path toolchain + { "key": "atmos-toolchain-darwin-arm64-v2", "paths": ["/tmp/tc/atmos/toolchain"], ... } + + $ ATMOS_XDG_CACHE_HOME=/tmp/tc atmos toolchain install --default hashicorp/terraform + ✓ Installed hashicorp/terraform@1.15.8 to /tmp/tc/atmos/toolchain/bin/hashicorp/terraform/1.15.8/terraform (106mb) + $ find /tmp/tc -type l; grep -rl /tmp/tc /tmp/tc # no symlinks, no embedded absolute paths + + # per-tool form, archive cached, network blocked: hangs (killed at the 2 min command timeout) + $ ATMOS_XDG_CACHE_HOME=/tmp/tc2 HTTPS_PROXY=http://127.0.0.1:9 atmos toolchain install --default hashicorp/terraform + Command timed out after 2m 0s + + # from-file form against a tar restored into an EMPTY cache root, network blocked + $ tar -C /tmp/tc/atmos -cf /tmp/tc-artifact/toolchain.tar toolchain/bin toolchain/toolchain.lock.yaml toolchain/aqua-registry-index.json + $ tar -C /tmp/tc3/atmos -xf /tmp/tc-artifact/toolchain.tar + $ ATMOS_XDG_CACHE_HOME=/tmp/tc3 HTTPS_PROXY=http://127.0.0.1:9 atmos toolchain install --tool-versions /tmp/ci.tool-versions + ✓ Skipped opentofu/opentofu@1.12.5 (already installed) + ✓ Skipped helm/helm@v3.19.2 (already installed) + ✓ Skipped hashicorp/terraform@1.15.8 (already installed) + ✓ Installed 0 tools, skipped 3 + $ ATMOS_XDG_CACHE_HOME=/tmp/tc3 HTTPS_PROXY=http://127.0.0.1:9 atmos toolchain env --tool-versions /tmp/ci.tool-versions --format=github + /tmp/tc3/atmos/toolchain/bin/hashicorp/terraform/1.15.8 + /tmp/tc3/atmos/toolchain/bin/helm/helm/v3.19.2 + /tmp/tc3/atmos/toolchain/bin/opentofu/opentofu/1.12.5 + $ /tmp/tc3/atmos/toolchain/bin/hashicorp/terraform/1.15.8/terraform version | head -1 # Terraform v1.15.8 + $ /tmp/tc3/atmos/toolchain/bin/opentofu/opentofu/1.12.5/tofu version | head -1 # OpenTofu v1.12.5 + $ /tmp/tc3/atmos/toolchain/bin/helm/helm/v3.19.2/helm version --short # v3.19.2+g8766e71 + ``` + +- The exact `run:` snippets of the action and of the build job's "Package the CI toolchain" step, extracted + with `yq` and executed with `RUNNER_TEMP`, `GITHUB_OUTPUT` and `GITHUB_PATH` pointed at scratch files and + `ATMOS_XDG_CACHE_HOME` at an empty root: pins file written (Terraform pinned from `.tool-versions`, blank + input lines dropped), root resolved, tar unpacked, install skipped all three tools offline, PATH appended + to `$GITHUB_PATH`, tar re-packaged with the expected members, missing-tarball path exits 0 with a + `::warning::`, and the repository's `.tool-versions` stayed clean. +- Not validated: Windows (no local Windows machine; the `cygpath` conversions and GNU tar under Git Bash + are reasoned from the existing "Add GNU tar to PATH" steps), the actual artifact size and transfer time + on the hosted runners, and a run of the workflow itself. The first CI run of this PR is the real test; + a failed artifact download degrades to today's behaviour by design. +- No Go code or test fixtures were changed. + +## Follow-ups + +None. From 69da94ff8b5ba586c842a54178f115ff6eec7236 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Thu, 3 Sep 2026 13:30:39 -0500 Subject: [PATCH 06/19] docs: fix indentation in the CI toolchain fix log Co-Authored-By: Claude Fable 5.1 --- .../2026-09-03-ci-toolchain-shipped-in-build-artifact.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/fixes/2026-09-03-ci-toolchain-shipped-in-build-artifact.md b/docs/fixes/2026-09-03-ci-toolchain-shipped-in-build-artifact.md index b8978b83345..2f199956b29 100644 --- a/docs/fixes/2026-09-03-ci-toolchain-shipped-in-build-artifact.md +++ b/docs/fixes/2026-09-03-ci-toolchain-shipped-in-build-artifact.md @@ -47,11 +47,11 @@ What the code says about the mechanics (read, then verified locally, see Validat - `.github/actions/ci-toolchain/action.yml` (new composite action, used by all four jobs so the tool list and the install command are identical everywhere): 1. writes a job-local tool-versions file from the `tool-versions` input plus the repository's - `hashicorp/terraform` pin; + `hashicorp/terraform` pin; 2. resolves the Atmos cache root with `atmos ci cache paths --format=env`; 3. when `artifact` is set, downloads it (`download-artifact-retry`, `continue-on-error`) and untars - `toolchain.tar` into the cache root (`cygpath` on Windows so GNU tar does not read `D:\...` as a - remote `host:file`); a missing artifact is a `::warning::`, not a failure; + `toolchain.tar` into the cache root (`cygpath` on Windows so GNU tar does not read `D:\...` as a + remote `host:file`); a missing artifact is a `::warning::`, not a failure; 4. `atmos toolchain install --tool-versions ` (skips what is on disk, installs the rest); 5. `atmos toolchain env --tool-versions --format=github`. - `.github/workflows/test.yml`: From b343149b61ce0d6050dd5c93c8306242c6b1d086 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Thu, 3 Sep 2026 17:42:27 -0500 Subject: [PATCH 07/19] fix(ci): drop the go.exe Defender process exclusion; it's spoofable Add-MpPreference -ExclusionProcess 'go.exe' matches on image name only, not path, so it would exempt any binary named go.exe from real-time scanning regardless of where it actually lives - including one planted by a compromised dependency during go build/go generate. The path exclusions already in this step cover the actual scanning cost (file I/O in the Go caches and workspace), so removing the process exclusion gives up nothing measured. (Skipped a second CodeRabbit finding on this PR: it asked test/ terraform-registry-cache to needs: a "cache-writer" job that doesn't exist in this file - terraform-registry-cache itself is the writer, and test is deliberately parallel to it so 30 shards don't wait on or race the single writer job. Making test depend on it would serialize two long-running job groups that currently run concurrently.) Co-Authored-By: Claude Sonnet 5 --- .github/workflows/setup-go-cache-warmup.yml | 1 - .github/workflows/test.yml | 4 ---- ...fender-exclusions-and-restore-only-toolchain-cache.md | 9 +++++++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/setup-go-cache-warmup.yml b/.github/workflows/setup-go-cache-warmup.yml index 0696ea8341d..4948699a939 100644 --- a/.github/workflows/setup-go-cache-warmup.yml +++ b/.github/workflows/setup-go-cache-warmup.yml @@ -123,7 +123,6 @@ jobs: Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\go-build" Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\Temp" Add-MpPreference -ExclusionPath "$env:RUNNER_TEMP" - Add-MpPreference -ExclusionProcess 'go.exe' - name: Set up Go uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cf822de191e..d9ad8312c12 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -221,7 +221,6 @@ jobs: Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\go-build" Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\Temp" Add-MpPreference -ExclusionPath "$env:RUNNER_TEMP" - Add-MpPreference -ExclusionProcess 'go.exe' - name: Set up Go if: ${{ ! ( matrix.target == 'windows' && github.event.pull_request.draft ) }} @@ -475,7 +474,6 @@ jobs: Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\go-build" Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\Temp" Add-MpPreference -ExclusionPath "$env:RUNNER_TEMP" - Add-MpPreference -ExclusionProcess 'go.exe' - name: Set up Atmos (install build artifact for ${{ matrix.flavor.target }}) if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} @@ -757,7 +755,6 @@ jobs: Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\go-build" Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\Temp" Add-MpPreference -ExclusionPath "$env:RUNNER_TEMP" - Add-MpPreference -ExclusionProcess 'go.exe' - name: Set up Atmos (install build artifact for ${{ matrix.flavor.target }}) if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} @@ -1799,7 +1796,6 @@ jobs: Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\go-build" Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\Temp" Add-MpPreference -ExclusionPath "$env:RUNNER_TEMP" - Add-MpPreference -ExclusionProcess 'go.exe' - name: Set up Atmos (install build artifact for ${{ matrix.flavor.target }}) if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} diff --git a/docs/fixes/2026-09-03-windows-defender-exclusions-and-restore-only-toolchain-cache.md b/docs/fixes/2026-09-03-windows-defender-exclusions-and-restore-only-toolchain-cache.md index da76813b4c1..b7089342844 100644 --- a/docs/fixes/2026-09-03-windows-defender-exclusions-and-restore-only-toolchain-cache.md +++ b/docs/fixes/2026-09-03-windows-defender-exclusions-and-restore-only-toolchain-cache.md @@ -47,8 +47,13 @@ locked-down image cannot fail the job. Windows legs of `build` (after the `Guard DNS against the harden-runner post-step race` step, before `Set up Go`), `terraform-registry-cache`, `test`, and `mock` (each immediately after `Add GNU tar to PATH`). It runs `Add-MpPreference -ExclusionPath` for `D:\a`, `C:\hostedtoolcache\windows`, - `$env:USERPROFILE\go`, `$env:LOCALAPPDATA\go-build`, `$env:LOCALAPPDATA\Temp`, `$env:RUNNER_TEMP`, and - `Add-MpPreference -ExclusionProcess 'go.exe'`. + `$env:USERPROFILE\go`, `$env:LOCALAPPDATA\go-build`, `$env:LOCALAPPDATA\Temp`, and `$env:RUNNER_TEMP` - + path-scoped exclusions only. An earlier draft also added + `Add-MpPreference -ExclusionProcess 'go.exe'`, a name-only process exclusion that would have skipped + real-time scanning for *any* binary named `go.exe` regardless of where it actually lives - including one + planted by a compromised dependency during `go build`/`go generate`. Dropped it: the path exclusions + above already cover the actual scanning cost this fix targets (file I/O in the Go caches and workspace), + so nothing measured is given up. - The `test` job's `Cache Atmos toolchain` step (all three OSes) now passes `restore-only: 'true'`, with a comment carrying the cache data above and naming `terraform-registry-cache` as the single writer. The `terraform-registry-cache` job's cache step is unchanged. From 55e78cbe5fa7b721687593d52ba1fbfa09802828 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Thu, 3 Sep 2026 18:15:33 -0500 Subject: [PATCH 08/19] docs(fixes): correct the mock job's tool list in the summary The mock job only requests opentofu/opentofu (plus the terraform ci-toolchain always adds) - not packer/helm/helmfile like the test and terraform-registry-cache jobs. The Changes section already said this correctly; only the Summary paragraph overstated it. Co-Authored-By: Claude Sonnet 5 --- .../2026-09-03-ci-toolchain-shipped-in-build-artifact.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/fixes/2026-09-03-ci-toolchain-shipped-in-build-artifact.md b/docs/fixes/2026-09-03-ci-toolchain-shipped-in-build-artifact.md index 2f199956b29..5487f9b2a33 100644 --- a/docs/fixes/2026-09-03-ci-toolchain-shipped-in-build-artifact.md +++ b/docs/fixes/2026-09-03-ci-toolchain-shipped-in-build-artifact.md @@ -4,10 +4,11 @@ ## Summary -Every one of the 30 acceptance shards in `test.yml` (`test` job, 10 shards × linux/windows/macos), plus the -three `terraform-registry-cache` legs and the `mock` jobs, ran `atmos toolchain install --default ...` for -Terraform, OpenTofu, Packer, Helm and Helmfile: ~400 MB of release downloads and ~2 minutes per job, with -a per-job dependency on releases.hashicorp.com, GitHub releases, get.helm.sh and Sigstore. The +Every one of the 30 acceptance shards in `test.yml` (`test` job, 10 shards × linux/windows/macos) and the +three `terraform-registry-cache` legs ran `atmos toolchain install --default ...` for Terraform, OpenTofu, +Packer, Helm and Helmfile; the `mock` jobs did the same for just Terraform and OpenTofu, the only two they +need. ~400 MB of release downloads and ~2 minutes per job, with a per-job dependency on +releases.hashicorp.com, GitHub releases, get.helm.sh and Sigstore. The "Cache Atmos toolchain" step meant to short-circuit that never hit, because the ~5 GB of Go caches written per run churn the repository's 10 GB Actions cache before any toolchain entry survives. This change installs the five tools once per OS in the `build` job, packages the installed tree as a `toolchain-` From 145a8c42a70f6b16831a9b51d63c0653a25a869a Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 4 Sep 2026 08:10:44 -0500 Subject: [PATCH 09/19] temp: diagnostic - dump live Windows Defender state (to be reverted) --- .github/workflows/setup-go-cache-warmup.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/setup-go-cache-warmup.yml b/.github/workflows/setup-go-cache-warmup.yml index 12c6442c0f4..2a216add511 100644 --- a/.github/workflows/setup-go-cache-warmup.yml +++ b/.github/workflows/setup-go-cache-warmup.yml @@ -112,6 +112,16 @@ jobs: # policy at all, it only stops the AV from re-scanning the workspace, the # hosted tool cache, the Go caches, and the temp dirs. continue-on-error # keeps a future locked-down image from failing the job. + - name: TEMP DIAGNOSTIC - Windows Defender live state + if: matrix.target == 'windows' + shell: pwsh + continue-on-error: true + run: | + Write-Host "--- Get-MpComputerStatus ---" + Get-MpComputerStatus | Format-List AntivirusEnabled, RealTimeProtectionEnabled, AMServiceEnabled, AntispywareEnabled, OnAccessProtectionEnabled, IsTamperProtected + Write-Host "--- Get-MpPreference (pre-exclusion) ---" + Get-MpPreference | Format-List DisableRealtimeMonitoring, DisableBehaviorMonitoring, DisableIOAVProtection, ExclusionPath + - name: Exclude the workspace and Go caches from Windows Defender if: matrix.target == 'windows' shell: pwsh @@ -124,6 +134,13 @@ jobs: Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\Temp" Add-MpPreference -ExclusionPath "$env:RUNNER_TEMP" + - name: TEMP DIAGNOSTIC - Windows Defender state after our exclusions + if: matrix.target == 'windows' + shell: pwsh + continue-on-error: true + run: | + Get-MpPreference | Format-List DisableRealtimeMonitoring, ExclusionPath + - name: Set up Go uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 with: From bc97d8c98a5da35385a483b42fe920596a2674de Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 4 Sep 2026 08:31:57 -0500 Subject: [PATCH 10/19] Revert "temp: diagnostic - dump live Windows Defender state (to be reverted)" This reverts commit 145a8c42a70f6b16831a9b51d63c0653a25a869a. --- .github/workflows/setup-go-cache-warmup.yml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/.github/workflows/setup-go-cache-warmup.yml b/.github/workflows/setup-go-cache-warmup.yml index 2a216add511..12c6442c0f4 100644 --- a/.github/workflows/setup-go-cache-warmup.yml +++ b/.github/workflows/setup-go-cache-warmup.yml @@ -112,16 +112,6 @@ jobs: # policy at all, it only stops the AV from re-scanning the workspace, the # hosted tool cache, the Go caches, and the temp dirs. continue-on-error # keeps a future locked-down image from failing the job. - - name: TEMP DIAGNOSTIC - Windows Defender live state - if: matrix.target == 'windows' - shell: pwsh - continue-on-error: true - run: | - Write-Host "--- Get-MpComputerStatus ---" - Get-MpComputerStatus | Format-List AntivirusEnabled, RealTimeProtectionEnabled, AMServiceEnabled, AntispywareEnabled, OnAccessProtectionEnabled, IsTamperProtected - Write-Host "--- Get-MpPreference (pre-exclusion) ---" - Get-MpPreference | Format-List DisableRealtimeMonitoring, DisableBehaviorMonitoring, DisableIOAVProtection, ExclusionPath - - name: Exclude the workspace and Go caches from Windows Defender if: matrix.target == 'windows' shell: pwsh @@ -134,13 +124,6 @@ jobs: Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\Temp" Add-MpPreference -ExclusionPath "$env:RUNNER_TEMP" - - name: TEMP DIAGNOSTIC - Windows Defender state after our exclusions - if: matrix.target == 'windows' - shell: pwsh - continue-on-error: true - run: | - Get-MpPreference | Format-List DisableRealtimeMonitoring, ExclusionPath - - name: Set up Go uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 with: From faceee0e2b4599efd009f882826af06dcabb050a Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 4 Sep 2026 08:44:18 -0500 Subject: [PATCH 11/19] ci: drop Windows Defender exclusions, they're a no-op GitHub's own actions/runner-images build script already disables real-time monitoring and excludes C:\ and D:\ entirely on windows-latest images, before any workflow step runs. Confirmed live via a temporary diagnostic step (Get-MpComputerStatus/Get-MpPreference) on an actual windows-latest runner: RealTimeProtectionEnabled was already False and ExclusionPath already {C:\, D:\} before our own Add-MpPreference calls, which only added redundant subpaths already covered by the existing C:\ exclusion. Since Defender was never doing real-time scanning to begin with, it wasn't the cause of the measured Windows slowdown this step was meant to fix - that root cause is still open. Splits the fix-log doc accordingly: the restore-only-toolchain-cache half (unrelated, unaffected) moves to its own file; a new doc records the Defender investigation and reversal so it isn't re-attempted blind later. --- .github/workflows/setup-go-cache-warmup.yml | 25 ----- .github/workflows/test.yml | 100 ------------------ ...-restore-only-toolchain-cache-on-shards.md | 66 ++++++++++++ ...usions-and-restore-only-toolchain-cache.md | 98 ----------------- ...-windows-defender-exclusions-are-a-noop.md | 65 ++++++++++++ 5 files changed, 131 insertions(+), 223 deletions(-) create mode 100644 docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md delete mode 100644 docs/fixes/2026-09-03-windows-defender-exclusions-and-restore-only-toolchain-cache.md create mode 100644 docs/fixes/2026-09-04-windows-defender-exclusions-are-a-noop.md diff --git a/.github/workflows/setup-go-cache-warmup.yml b/.github/workflows/setup-go-cache-warmup.yml index 12c6442c0f4..a775ec39f9a 100644 --- a/.github/workflows/setup-go-cache-warmup.yml +++ b/.github/workflows/setup-go-cache-warmup.yml @@ -99,31 +99,6 @@ jobs: if: matrix.target == 'windows' uses: ./.github/actions/windows-dns-guard - # Windows Defender exclusions. Measured on the Windows shards (Aug 24 - - # Sept 3): "Set up Go" averages 5.2 min (max 10.5) restoring a 1.9 GB - # go-build+mod cache; the download is ~20 s at 100 MB/s and the rest is - # tar/zstd extraction with Defender's real-time scanner inspecting every - # extracted file. The same scanner holds handles on freshly written test - # files, which surfaces as testing.TempDir cleanup failures - # ("unlinkat ... being used by another process"). These runners are - # ephemeral VMs discarded at job end, so there is nothing to protect from - # files the job itself just wrote. Landing this right after the - # harden-runner block-mode PR is fine: it does not touch the egress - # policy at all, it only stops the AV from re-scanning the workspace, the - # hosted tool cache, the Go caches, and the temp dirs. continue-on-error - # keeps a future locked-down image from failing the job. - - name: Exclude the workspace and Go caches from Windows Defender - if: matrix.target == 'windows' - shell: pwsh - continue-on-error: true - run: | - Add-MpPreference -ExclusionPath 'D:\a' - Add-MpPreference -ExclusionPath 'C:\hostedtoolcache\windows' - Add-MpPreference -ExclusionPath "$env:USERPROFILE\go" - Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\go-build" - Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\Temp" - Add-MpPreference -ExclusionPath "$env:RUNNER_TEMP" - - name: Set up Go uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 59645203dda..abbca4a3c07 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -197,31 +197,6 @@ jobs: if: matrix.target == 'windows' && ! github.event.pull_request.draft uses: ./.github/actions/windows-dns-guard - # Windows Defender exclusions. Measured on the Windows shards (Aug 24 - - # Sept 3): "Set up Go" averages 5.2 min (max 10.5) restoring a 1.9 GB - # go-build+mod cache; the download is ~20 s at 100 MB/s and the rest is - # tar/zstd extraction with Defender's real-time scanner inspecting every - # extracted file. The same scanner holds handles on freshly written test - # files, which surfaces as testing.TempDir cleanup failures - # ("unlinkat ... being used by another process"). These runners are - # ephemeral VMs discarded at job end, so there is nothing to protect from - # files the job itself just wrote. Landing this right after the - # harden-runner block-mode PR is fine: it does not touch the egress - # policy at all, it only stops the AV from re-scanning the workspace, the - # hosted tool cache, the Go caches, and the temp dirs. continue-on-error - # keeps a future locked-down image from failing the job. - - name: Exclude the workspace and Go caches from Windows Defender - if: matrix.target == 'windows' && ! github.event.pull_request.draft - shell: pwsh - continue-on-error: true - run: | - Add-MpPreference -ExclusionPath 'D:\a' - Add-MpPreference -ExclusionPath 'C:\hostedtoolcache\windows' - Add-MpPreference -ExclusionPath "$env:USERPROFILE\go" - Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\go-build" - Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\Temp" - Add-MpPreference -ExclusionPath "$env:RUNNER_TEMP" - - name: Set up Go if: ${{ ! ( matrix.target == 'windows' && github.event.pull_request.draft ) }} # setup-go v6 requires runner v2.327.1+ and can affect toolchain handling. @@ -451,31 +426,6 @@ jobs: shell: pwsh run: echo "C:\Program Files\Git\usr\bin" >> $Env:GITHUB_PATH - # Windows Defender exclusions. Measured on the Windows shards (Aug 24 - - # Sept 3): "Set up Go" averages 5.2 min (max 10.5) restoring a 1.9 GB - # go-build+mod cache; the download is ~20 s at 100 MB/s and the rest is - # tar/zstd extraction with Defender's real-time scanner inspecting every - # extracted file. The same scanner holds handles on freshly written test - # files, which surfaces as testing.TempDir cleanup failures - # ("unlinkat ... being used by another process"). These runners are - # ephemeral VMs discarded at job end, so there is nothing to protect from - # files the job itself just wrote. Landing this right after the - # harden-runner block-mode PR is fine: it does not touch the egress - # policy at all, it only stops the AV from re-scanning the workspace, the - # hosted tool cache, the Go caches, and the temp dirs. continue-on-error - # keeps a future locked-down image from failing the job. - - name: Exclude the workspace and Go caches from Windows Defender - if: matrix.flavor.target == 'windows' && ! github.event.pull_request.draft - shell: pwsh - continue-on-error: true - run: | - Add-MpPreference -ExclusionPath 'D:\a' - Add-MpPreference -ExclusionPath 'C:\hostedtoolcache\windows' - Add-MpPreference -ExclusionPath "$env:USERPROFILE\go" - Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\go-build" - Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\Temp" - Add-MpPreference -ExclusionPath "$env:RUNNER_TEMP" - - name: Set up Atmos (install build artifact for ${{ matrix.flavor.target }}) if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} uses: ./.github/actions/setup-atmos-install @@ -732,31 +682,6 @@ jobs: shell: pwsh run: echo "C:\Program Files\Git\usr\bin" >> $Env:GITHUB_PATH - # Windows Defender exclusions. Measured on the Windows shards (Aug 24 - - # Sept 3): "Set up Go" averages 5.2 min (max 10.5) restoring a 1.9 GB - # go-build+mod cache; the download is ~20 s at 100 MB/s and the rest is - # tar/zstd extraction with Defender's real-time scanner inspecting every - # extracted file. The same scanner holds handles on freshly written test - # files, which surfaces as testing.TempDir cleanup failures - # ("unlinkat ... being used by another process"). These runners are - # ephemeral VMs discarded at job end, so there is nothing to protect from - # files the job itself just wrote. Landing this right after the - # harden-runner block-mode PR is fine: it does not touch the egress - # policy at all, it only stops the AV from re-scanning the workspace, the - # hosted tool cache, the Go caches, and the temp dirs. continue-on-error - # keeps a future locked-down image from failing the job. - - name: Exclude the workspace and Go caches from Windows Defender - if: matrix.flavor.target == 'windows' && ! github.event.pull_request.draft - shell: pwsh - continue-on-error: true - run: | - Add-MpPreference -ExclusionPath 'D:\a' - Add-MpPreference -ExclusionPath 'C:\hostedtoolcache\windows' - Add-MpPreference -ExclusionPath "$env:USERPROFILE\go" - Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\go-build" - Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\Temp" - Add-MpPreference -ExclusionPath "$env:RUNNER_TEMP" - - name: Set up Atmos (install build artifact for ${{ matrix.flavor.target }}) if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} uses: ./.github/actions/setup-atmos-install @@ -1797,31 +1722,6 @@ jobs: shell: pwsh run: echo "C:\Program Files\Git\usr\bin" >> $Env:GITHUB_PATH - # Windows Defender exclusions. Measured on the Windows shards (Aug 24 - - # Sept 3): "Set up Go" averages 5.2 min (max 10.5) restoring a 1.9 GB - # go-build+mod cache; the download is ~20 s at 100 MB/s and the rest is - # tar/zstd extraction with Defender's real-time scanner inspecting every - # extracted file. The same scanner holds handles on freshly written test - # files, which surfaces as testing.TempDir cleanup failures - # ("unlinkat ... being used by another process"). These runners are - # ephemeral VMs discarded at job end, so there is nothing to protect from - # files the job itself just wrote. Landing this right after the - # harden-runner block-mode PR is fine: it does not touch the egress - # policy at all, it only stops the AV from re-scanning the workspace, the - # hosted tool cache, the Go caches, and the temp dirs. continue-on-error - # keeps a future locked-down image from failing the job. - - name: Exclude the workspace and Go caches from Windows Defender - if: matrix.flavor.target == 'windows' && ! github.event.pull_request.draft - shell: pwsh - continue-on-error: true - run: | - Add-MpPreference -ExclusionPath 'D:\a' - Add-MpPreference -ExclusionPath 'C:\hostedtoolcache\windows' - Add-MpPreference -ExclusionPath "$env:USERPROFILE\go" - Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\go-build" - Add-MpPreference -ExclusionPath "$env:LOCALAPPDATA\Temp" - Add-MpPreference -ExclusionPath "$env:RUNNER_TEMP" - - name: Set up Atmos (install build artifact for ${{ matrix.flavor.target }}) if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} uses: ./.github/actions/setup-atmos-install diff --git a/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md b/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md new file mode 100644 index 00000000000..88ca4d96e9d --- /dev/null +++ b/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md @@ -0,0 +1,66 @@ +# Fix: restore-only toolchain cache on the acceptance shards + +**Date:** 2026-09-03 (Defender-exclusion half removed 2026-09-04; see +`docs/fixes/2026-09-04-windows-defender-exclusions-are-a-noop.md`) + +## Summary + +Part of Phase 2 of the CI-stability plan (see +`docs/fixes/2026-09-03-harden-runner-windows-dns-restore-race.md` for Phase 1). The `test` job's ten shards +per OS restore the Atmos toolchain cache without ever saving it: the `actions/cache` composite action gained +a `restore-only` input that switches it to `actions/cache/restore`, and `terraform-registry-cache` stays the +single writer of the key per OS. No user-visible behavior changes; the new action input is opt-in and +defaults to the previous behavior. + +## Context + +Measured on `test.yml` runs from Aug 24 to Sept 3 (80 successful Windows shards, step-level timelines, raw +job logs, and `gh api repos/cloudposse/atmos/actions/cache/usage`). + +**Toolchain cache.** The repository's Actions cache holds 18.9 GB across 17 entries against a 10 GB LRU +quota. Every entry is scoped to a `refs/pull/N/merge` ref and is minutes old (14 `setup-go-*` entries at +1.6 to 1.9 GB, 8 `atmos-toolchain-*` entries at 350 to 470 MB). Each run writes about 5 GB, so nothing +saved from `main` survives and the static key `atmos-toolchain---v2` never hits: every shard +logs `Cache not found for input keys`, then all 10 shards race to save the same key and log +`Unable to reserve cache with key ..., another job may be creating this cache`. `Post Cache Atmos +toolchain` costs 43 s on average and up to 3 min per shard for nothing. + +## Changes + +- `.github/workflows/test.yml`: the `test` job's `Cache Atmos toolchain` step (all three OSes) now passes + `restore-only: 'true'`, with a comment carrying the cache data above and naming + `terraform-registry-cache` as the single writer. The `terraform-registry-cache` job's cache step is + unchanged. +- `actions/cache/action.yml`: new optional input `restore-only` (string, default `'false'`). When `'true'` + the action runs `actions/cache/restore` (same repository and tag as `actions/cache`, so the same pinned + SHA `27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5`) with identical `key`, `path`, and + `restore-keys`; otherwise `actions/cache` runs as before. Exactly one of the two steps runs, so the + `cache-hit` output is `steps.cache.outputs.cache-hit || steps.cache-restore.outputs.cache-hit`; the `key` + output is unchanged. +- `actions/cache/README.md`: documents the input and the "many parallel consumers, one writer" pattern. + +Not changed on purpose (later PRs): `atmos.yaml` `ci.cache.key`, `setup-go` caching, and the Go cache +warmup's writer role. + +## Validation + +- `python3 -c 'import yaml; yaml.safe_load(open(f))'` on `.github/workflows/test.yml` and + `actions/cache/action.yml`: both parse; the action parses with `inputs: [restore-only]` and steps `meta`, + `validate`, `cache`, `cache-restore`. +- `actionlint .github/workflows/test.yml`: clean (exit 0). actionlint does not lint composite actions, so + `actions/cache/action.yml` is covered by the YAML parse and the Go regression test below only. +- `go test github.com/cloudposse/atmos/cmd -run TestAtmosCacheActionValidatesMetadataBeforeActionsCache`: + passes (it asserts the metadata validation step still precedes the cache steps). +- Not validated here: the Windows timing effect itself. `Post Cache Atmos toolchain` duration on the shards + is measured from the PR's own CI runs; the expected outcome is `Post Cache Atmos toolchain` absent from + the 30 shard jobs and no `Unable to reserve cache` lines in shard logs. + +## Follow-ups + +- Two further Phase 2 changes are separate PRs already in flight and have no GitHub issue numbers yet (the + repository rule against opening unprompted issues applies; the numbers are added here when the PRs open): + the single-writer hashed toolchain key (`atmos-toolchain-{{.OS}}-{{.Arch}}-{{ hashFiles ".tool-versions" }}` + in `atmos.yaml` `ci.cache.key`, once `.tool-versions` lands), and shipping the toolchains inside the + existing `build-artifacts-` artifact so the shards stop installing them at all. +- Phase 2c (measure the cache-free Windows shard) and the `setup-go` single-writer change are also still + open; same tracking note. diff --git a/docs/fixes/2026-09-03-windows-defender-exclusions-and-restore-only-toolchain-cache.md b/docs/fixes/2026-09-03-windows-defender-exclusions-and-restore-only-toolchain-cache.md deleted file mode 100644 index b7089342844..00000000000 --- a/docs/fixes/2026-09-03-windows-defender-exclusions-and-restore-only-toolchain-cache.md +++ /dev/null @@ -1,98 +0,0 @@ -# Fix: Windows Defender exclusions on CI Windows legs and restore-only toolchain cache on the acceptance shards - -**Date:** 2026-09-03 - -## Summary - -Two CI-only changes from Phase 2 of the CI-stability plan (see -`docs/fixes/2026-09-03-harden-runner-windows-dns-restore-race.md` for Phase 1). First, every Windows leg of -`test.yml` (and the nightly Go cache warmup) now excludes the workspace, the hosted tool cache, the Go -caches, and the temp directories from Windows Defender's real-time scanner before `Set up Go` runs. Second, -the `test` job's ten shards per OS restore the Atmos toolchain cache without ever saving it: the -`actions/cache` composite action gained a `restore-only` input that switches it to `actions/cache/restore`, -and `terraform-registry-cache` stays the single writer of the key per OS. No user-visible behavior changes; -the new action input is opt-in and defaults to the previous behavior. - -## Context - -Measured on `test.yml` runs from Aug 24 to Sept 3 (80 successful Windows shards, step-level timelines, raw -job logs, and `gh api repos/cloudposse/atmos/actions/cache/usage`). - -**Windows shard time.** Median 14.1 min, p90 16.6, max 36. `Set up Go` alone averages 5.2 min (max 10.5) -restoring a 1.9 GB go-build + mod cache. The download is about 20 s at 100 MB/s; the remainder is tar/zstd -extraction with Defender's real-time scanner inspecting every extracted file. The same scanner holding -handles on freshly written files is what surfaces as `testing.TempDir` cleanup failures -(`unlinkat ... being used by another process`) in the acceptance tests. The existing -`transientErrorDetector` in `internal/ci/acceptance/command.go` only covers the Go-toolchain variant of -that error. - -**Toolchain cache.** The repository's Actions cache holds 18.9 GB across 17 entries against a 10 GB LRU -quota. Every entry is scoped to a `refs/pull/N/merge` ref and is minutes old (14 `setup-go-*` entries at -1.6 to 1.9 GB, 8 `atmos-toolchain-*` entries at 350 to 470 MB). Each run writes about 5 GB, so nothing -saved from `main` survives and the static key `atmos-toolchain---v2` never hits: every shard -logs `Cache not found for input keys`, then all 10 shards race to save the same key and log -`Unable to reserve cache with key ..., another job may be creating this cache`. `Post Cache Atmos -toolchain` costs 43 s on average and up to 3 min per shard for nothing. - -**Why it is safe to land right after the harden-runner block-mode PR.** The Defender step does not touch -harden-runner's egress policy. It only stops the antivirus from re-scanning files the job itself just wrote -on an ephemeral VM that is discarded when the job ends. The step is `continue-on-error: true` so a future -locked-down image cannot fail the job. - -## Changes - -- `.github/workflows/test.yml` - - New step `Exclude the workspace and Go caches from Windows Defender` (`shell: pwsh`, - `continue-on-error: true`, same draft-PR gating as the neighbouring `Add GNU tar to PATH` step) on the - Windows legs of `build` (after the `Guard DNS against the harden-runner post-step race` step, before - `Set up Go`), `terraform-registry-cache`, `test`, and `mock` (each immediately after `Add GNU tar to - PATH`). It runs `Add-MpPreference -ExclusionPath` for `D:\a`, `C:\hostedtoolcache\windows`, - `$env:USERPROFILE\go`, `$env:LOCALAPPDATA\go-build`, `$env:LOCALAPPDATA\Temp`, and `$env:RUNNER_TEMP` - - path-scoped exclusions only. An earlier draft also added - `Add-MpPreference -ExclusionProcess 'go.exe'`, a name-only process exclusion that would have skipped - real-time scanning for *any* binary named `go.exe` regardless of where it actually lives - including one - planted by a compromised dependency during `go build`/`go generate`. Dropped it: the path exclusions - above already cover the actual scanning cost this fix targets (file I/O in the Go caches and workspace), - so nothing measured is given up. - - The `test` job's `Cache Atmos toolchain` step (all three OSes) now passes `restore-only: 'true'`, with a - comment carrying the cache data above and naming `terraform-registry-cache` as the single writer. The - `terraform-registry-cache` job's cache step is unchanged. -- `.github/workflows/setup-go-cache-warmup.yml`: the same Defender step on the Windows leg, after the DNS - guard and before `Set up Go`. -- `actions/cache/action.yml`: new optional input `restore-only` (string, default `'false'`). When `'true'` - the action runs `actions/cache/restore` (same repository and tag as `actions/cache`, so the same pinned - SHA `27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5`) with identical `key`, `path`, and - `restore-keys`; otherwise `actions/cache` runs as before. Exactly one of the two steps runs, so the - `cache-hit` output is `steps.cache.outputs.cache-hit || steps.cache-restore.outputs.cache-hit`; the `key` - output is unchanged. -- `actions/cache/README.md`: documents the input and the "many parallel consumers, one writer" pattern. - -Not changed on purpose (later PRs): `atmos.yaml` `ci.cache.key`, `setup-go` caching, and the Go cache -warmup's writer role. - -## Validation - -- `python3 -c 'import yaml; yaml.safe_load(open(f))'` on `.github/workflows/test.yml`, - `.github/workflows/setup-go-cache-warmup.yml`, and `actions/cache/action.yml`: all parse; the action - parses with `inputs: [restore-only]` and steps `meta`, `validate`, `cache`, `cache-restore`. -- `actionlint .github/workflows/test.yml .github/workflows/setup-go-cache-warmup.yml`: clean (exit 0). - actionlint does not lint composite actions, so `actions/cache/action.yml` is covered by the YAML parse - and the Go regression test below only. -- `go test github.com/cloudposse/atmos/cmd -run TestAtmosCacheActionValidatesMetadataBeforeActionsCache`: - passes (it asserts the metadata validation step still precedes the cache steps). -- `git diff` reviewed for accidental changes: only the four files above. -- Not validated here: the Windows timing effect itself. `Set up Go` extraction time and - `Post Cache Atmos toolchain` duration on the shards are measured from the PR's own CI runs with the - "Windows shard step durations" query in the Phase 1 fix log; the expected outcomes are `Set up Go` from - 5 to 10 min down to about 1 min, `Post Cache Atmos toolchain` absent from the 30 shard jobs, and no - `Unable to reserve cache` lines in shard logs. - -## Follow-ups - -- Two further Phase 2 changes are separate PRs already in flight and have no GitHub issue numbers yet (the - repository rule against opening unprompted issues applies; the numbers are added here when the PRs open): - the single-writer hashed toolchain key (`atmos-toolchain-{{.OS}}-{{.Arch}}-{{ hashFiles ".tool-versions" }}` - in `atmos.yaml` `ci.cache.key`, once `.tool-versions` lands), and shipping the toolchains inside the - existing `build-artifacts-` artifact so the shards stop installing them at all. -- Phase 2c (measure the cache-free Windows shard) and the `setup-go` single-writer change are also still - open; same tracking note. diff --git a/docs/fixes/2026-09-04-windows-defender-exclusions-are-a-noop.md b/docs/fixes/2026-09-04-windows-defender-exclusions-are-a-noop.md new file mode 100644 index 00000000000..7bd11effb97 --- /dev/null +++ b/docs/fixes/2026-09-04-windows-defender-exclusions-are-a-noop.md @@ -0,0 +1,65 @@ +# Non-fix: Windows Defender exclusions on CI Windows legs were a no-op + +**Date:** 2026-09-04 + +## Summary + +An earlier draft of `docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md` (then titled +"Windows Defender exclusions and restore-only toolchain cache on shards") added +`Add-MpPreference -ExclusionPath` calls for the workspace, hosted tool cache, Go caches, and temp +directories on every Windows leg of `test.yml` and `setup-go-cache-warmup.yml`, on the theory that Windows +Defender's real-time scanner inspecting every extracted file was the cause of `Set up Go` averaging 5.2 min +(max 10.5) restoring a 1.9 GB go-build+mod cache, and of `testing.TempDir` cleanup failures +(`unlinkat ... being used by another process`). + +That theory was wrong. GitHub's own `actions/runner-images` build script +(`images/windows/scripts/build/Configure-WindowsDefender.ps1`, which builds the `windows-latest` image these +jobs actually run on) already disables real-time monitoring and excludes `C:\` and `D:\` - the entire +filesystem - at the image level, before any workflow step runs: + +```powershell +@{DisableRealtimeMonitoring = $true} +@{ScanAvgCPULoadFactor = 5; ExclusionPath = @("D:\", "C:\")} +``` + +Confirmed live, not just from the build script, by adding a temporary diagnostic step to +`setup-go-cache-warmup.yml` and running it on an actual `windows-latest` runner (queried via +`Get-MpComputerStatus`/`Get-MpPreference` immediately before and after the exclusion step): + +``` +--- before our exclusion step --- +RealTimeProtectionEnabled : False +DisableRealtimeMonitoring : True +ExclusionPath : {C:\, D:\} + +--- after our exclusion step --- +DisableRealtimeMonitoring : True +ExclusionPath : {C:\, C:\hostedtoolcache\windows, C:\Users\runneradmin\...\go-build, ...} +``` + +Every path our step added was already a subpath of the pre-existing `C:\` exclusion. Real-time protection +was off, and the entire drive was already excluded, before the step ran; the step changed nothing +observable about Defender's actual scanning behavior. Since Defender was never doing real-time scanning to +begin with, it cannot be the cause of the measured Windows slowdown - that has a different, still-unknown +root cause (Windows itself is roughly 2-3x slower than Linux/macOS runners in this repo's CI regardless of +this change; NTFS semantics, process-spawn overhead, and tar/zstd extraction cost are more likely +candidates than antivirus). + +## Fix + +Removed the `Exclude the workspace and Go caches from Windows Defender` step (and its explanatory comment) +from `build`, `terraform-registry-cache`, `test`, `mock` in `.github/workflows/test.yml`, and from +`.github/workflows/setup-go-cache-warmup.yml`. The restore-only-toolchain-cache change in the same original +PR is unaffected and unrelated - see `docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md`. + +## Validation + +- `atmos ci validate .github/workflows/test.yml .github/workflows/setup-go-cache-warmup.yml`: both valid. +- Diagnostic run: `gh run view 33877782841` (branch + `osterman/ci-windows-defender-restore-only-cache`, `setup-go-cache-warmup.yml`, `Cache warmup (windows)` + job) - live evidence quoted above. + +## Follow-ups + +- The actual cause of Windows CI being ~2-3x slower than Linux/macOS in this repo remains open. Worth a + dedicated investigation rather than another blind antivirus-shaped guess. From 64dbfc5e2da31f0a00d34f0f8e8769645128a361 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 4 Sep 2026 15:27:35 -0500 Subject: [PATCH 12/19] ci: the CI toolchain is one step - atmos toolchain install from .tool-versions Replace the hand-rolled toolchain artifact (tar + cygpath + upload + download + unpack, and a per-job tool list fed from workflow env vars that main had since removed) with what a developer runs: the Atmos Cache action (save on the build job, restore-only on every consumer), `atmos toolchain install` from .tool-versions, and `atmos toolchain env --format=github`. Every job that needs the toolchain is one step; versions live in .tool-versions only. Atmos's own cache is enough now: the Actions cache limit is 50 GB and restore-only consumers no longer race to save one key. Co-Authored-By: Claude Fable 5.1 --- .github/actions/ci-toolchain/action.yml | 160 ++++-------------- .github/workflows/test.yml | 139 +++------------ ...-ci-toolchain-shipped-in-build-artifact.md | 129 -------------- .../fixes/2026-09-04-ci-toolchain-one-step.md | 56 ++++++ 4 files changed, 109 insertions(+), 375 deletions(-) delete mode 100644 docs/fixes/2026-09-03-ci-toolchain-shipped-in-build-artifact.md create mode 100644 docs/fixes/2026-09-04-ci-toolchain-one-step.md diff --git a/.github/actions/ci-toolchain/action.yml b/.github/actions/ci-toolchain/action.yml index 835f504d4a1..2fb0b779690 100644 --- a/.github/actions/ci-toolchain/action.yml +++ b/.github/actions/ci-toolchain/action.yml @@ -1,153 +1,53 @@ name: 'Set up CI toolchain' description: > - Installs the external tools the CI jobs need (Terraform, OpenTofu, Packer, - Helm, Helmfile) with `atmos toolchain install`, optionally seeding the Atmos - toolchain cache from a `toolchain.tar` artifact produced earlier in the same - run, and exports their directories to PATH via `atmos toolchain env`. - - The pins are written to a job-local tool-versions file, and both the install - and the PATH export read that file (`--tool-versions`), for two reasons: - - 1. `atmos toolchain install` only takes its "already installed, skip" path - when installing from a tool-versions file. The per-tool - `atmos toolchain install --default owner/repo@version` form always - re-resolves the registry, re-downloads or re-verifies the release - signature, and re-extracts, even when the exact version is already on - disk, so it can never be a no-op and always needs the network. With a - restored artifact, the from-file form skips every tool without a single - network request; without one (or for a tool the artifact lacks) it - installs exactly what is missing, so the network install remains the - safety net. - 2. The repository's own `.tool-versions` is left untouched: nothing in CI - rewrites its defaults any more. - - `hashicorp/terraform` is always pinned to the version in the repository's - `.tool-versions`; every other tool comes from the `tool-versions` input. + Puts the tools `.tool-versions` pins (Terraform, OpenTofu, Packer, Helm, + Helmfile, ...) on PATH the way a developer does: `atmos toolchain install`, + which reads `.tool-versions` and skips every tool already on disk, then + `atmos toolchain env` to export their directories to PATH. Versions live in + `.tool-versions` only. + + Persistence across jobs and runs is atmos's own `ci.cache` (the Atmos Cache + action, ./actions/cache): the one producer job per OS saves the toolchain + directory at job end (`cache: save`); every consumer restores it and never + saves (`cache: restore-only`), so ten shards do not race to write one key. + On a miss, `atmos toolchain install` installs from the network as it always + did. Requires the `atmos` binary to be on PATH (install it before this step). inputs: - tool-versions: - description: > - Tools to install, one per line, in `.tool-versions` format - (`owner/repo version`). `hashicorp/terraform` is added automatically - from the repository's `.tool-versions`. Blank lines are ignored. + github-token: + description: 'GitHub token for the toolchain registry and release-asset lookups (raises the unauthenticated API rate limit).' required: true - artifact: + cache: description: > - Name of a workflow artifact containing a `toolchain.tar` (as packaged by - the `build` job) to unpack into the Atmos toolchain cache before - installing. Tools present in it are skipped by the install step. Leave - empty to install everything from the network (what the `build` job does - to produce the artifact in the first place). A missing or failed - download is a warning, not an error: the install step then falls back - to the network for whatever is absent. + How this job uses the Atmos toolchain cache: `save` (restore, and save + at job end - one producer job per OS), `restore-only` (consumers), or + `none`. required: false - default: '' - github-token: - description: 'GitHub token for the toolchain registry and release-asset lookups (raises the unauthenticated API rate limit)' - required: true - -outputs: - tool-versions-file: - description: 'Absolute path of the generated tool-versions file' - value: ${{ steps.pins.outputs.file }} - cache-root: - description: 'Absolute path of the Atmos cache root (the parent of the `toolchain` directory), in the OS-native form' - value: ${{ steps.root.outputs.path }} + default: 'restore-only' runs: using: composite steps: - - name: Write the CI tool pins - id: pins - shell: bash - env: - CI_TOOL_VERSIONS: ${{ inputs.tool-versions }} - run: | - file="${RUNNER_TEMP}/atmos-ci.tool-versions" - # Terraform is pinned once, in the repository's .tool-versions; the - # first version on that line is the default. - terraform_version="$(sed -n 's#^hashicorp/terraform[[:space:]]\{1,\}\([^[:space:]]*\).*#\1#p' .tool-versions | head -n 1)" - if [ -z "${terraform_version}" ]; then - echo "::error::.tool-versions has no hashicorp/terraform entry to pin Terraform from" - exit 1 - fi - { - echo "hashicorp/terraform ${terraform_version}" - printf '%s\n' "${CI_TOOL_VERSIONS}" | sed '/^[[:space:]]*$/d' - } > "${file}" - echo "Tool pins (${file}):" - cat "${file}" - echo "file=${file}" >> "$GITHUB_OUTPUT" - - - name: Resolve the Atmos cache root - id: root - shell: bash - run: | - # The toolchain lives at /toolchain (see pkg/toolchain - # GetInstallPath). Ask atmos for the root instead of hardcoding the - # per-OS XDG default; --format=env prints one KEY=VALUE per line. - root="$(atmos ci cache paths --format=env 2>/dev/null | sed -n 's/^ATMOS_CI_CACHE_PATHS=//p')" - if [ -z "${root}" ]; then - echo "::error::atmos ci cache paths did not report a cache root" - exit 1 - fi - echo "Atmos cache root: ${root}" - echo "path=${root}" >> "$GITHUB_OUTPUT" + - name: Restore and save the Atmos toolchain cache + if: inputs.cache == 'save' + continue-on-error: true + uses: ./actions/cache - - name: Download the toolchain artifact - id: download - if: inputs.artifact != '' - # A pure accelerator: if the artifact is missing (the build leg that - # produces it was skipped) or the download keeps failing, fall through - # to the network install below instead of failing the job. + - name: Restore the Atmos toolchain cache + if: inputs.cache == 'restore-only' continue-on-error: true - uses: ./.github/actions/download-artifact-retry + uses: ./actions/cache with: - name: ${{ inputs.artifact }} - path: ${{ runner.temp }}/atmos-ci-toolchain - - - name: Unpack the toolchain into the Atmos cache root - if: inputs.artifact != '' && steps.download.outcome == 'success' - shell: bash - env: - ATMOS_CACHE_ROOT: ${{ steps.root.outputs.path }} - TOOLCHAIN_TAR: ${{ runner.temp }}/atmos-ci-toolchain/toolchain.tar - run: | - root="${ATMOS_CACHE_ROOT}" - tarball="${TOOLCHAIN_TAR}" - # Git Bash on Windows: give tar POSIX paths. A native `D:\...` path - # would be read by GNU tar as a `host:file` remote archive. - if command -v cygpath >/dev/null 2>&1; then - root="$(cygpath -u "${root}")" - tarball="$(cygpath -u "${tarball}")" - fi - if [ ! -f "${tarball}" ]; then - echo "::warning::toolchain artifact contained no toolchain.tar; installing from the network instead" - exit 0 - fi - mkdir -p "${root}" - tar -C "${root}" -xf "${tarball}" - echo "Restored toolchain into ${root}/toolchain:" - find "${root}/toolchain/bin" -mindepth 3 -maxdepth 3 -type d | sort + restore-only: 'true' - - name: Report a missing toolchain artifact - if: inputs.artifact != '' && steps.download.outcome != 'success' - shell: bash - env: - ARTIFACT_NAME: ${{ inputs.artifact }} - run: echo "::warning::could not download the ${ARTIFACT_NAME} artifact; installing the toolchain from the network instead" - - - name: Install any tool the cache is missing + - name: Install the tools in .tool-versions that are not already on disk shell: bash env: GITHUB_TOKEN: ${{ inputs.github-token }} - TOOL_VERSIONS_FILE: ${{ steps.pins.outputs.file }} - run: atmos toolchain install --tool-versions "${TOOL_VERSIONS_FILE}" + run: atmos toolchain install - name: Export the toolchain directories to PATH shell: bash - env: - TOOL_VERSIONS_FILE: ${{ steps.pins.outputs.file }} - run: atmos toolchain env --tool-versions "${TOOL_VERSIONS_FILE}" --format=github + run: atmos toolchain env --format=github diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7889e545f56..fdda3ab7f9d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -266,72 +266,24 @@ jobs: shell: bash run: go tool mage acceptance:verify "${{ matrix.target }}" "$TEST_SHARD_COUNT" - # The acceptance shards, the terraform-registry-cache job, and the mock - # jobs all need the same external tools (Terraform, OpenTofu, Packer, - # Helm, Helmfile). Install them once per OS here and ship the installed - # tree as a `toolchain-` artifact; those jobs unpack it into - # their own toolchain cache before `atmos toolchain install`, which then - # skips every tool. That replaces 30+ per-shard installs (~400 MB of - # release downloads and ~2 min each, plus a per-shard dependency on - # releases.hashicorp.com/GitHub/get.helm.sh) and the "Cache Atmos - # toolchain" step that never hits because the Go caches churn the 10 GB - # Actions cache. Uses the atmos just built, not the bootstrap release, so - # the on-disk layout is the one the consumers' binary expects. See - # docs/fixes/2026-09-03-ci-toolchain-shipped-in-build-artifact.md. - # macos-intel is skipped: its only consumer (the k3s macOS job) installs - # no toolchain. + # Install the toolchain once per OS with the atmos just built (so the + # on-disk layout is the one the consumers' binary expects) and save it + # to atmos's ci.cache; the acceptance shards, terraform-registry-cache + # and mock jobs restore it (never save) and run the same + # `atmos toolchain install`, which then skips every tool. macos-intel is + # skipped: its only consumer (the k3s macOS job) installs no toolchain. - name: Put the freshly built atmos on PATH for the toolchain install if: ${{ matrix.target != 'macos-intel' && ! ( matrix.target == 'windows' && github.event.pull_request.draft ) }} shell: bash run: echo "${GITHUB_WORKSPACE}/build" >> "$GITHUB_PATH" - - name: Install the CI toolchain (Terraform, OpenTofu, Packer, Helm, Helmfile) - id: toolchain + - name: Install the CI toolchain (atmos toolchain install from .tool-versions; saves the cache) if: ${{ matrix.target != 'macos-intel' && ! ( matrix.target == 'windows' && github.event.pull_request.draft ) }} uses: ./.github/actions/ci-toolchain with: - tool-versions: | - opentofu/opentofu ${{ env.OPEN_TOFU_VERSION }} - hashicorp/packer ${{ env.PACKER_VERSION }} - helm/helm ${{ env.HELM_VERSION }} - helmfile/helmfile ${{ env.HELMFILE_VERSION }} + cache: save github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Package the CI toolchain - if: ${{ matrix.target != 'macos-intel' && ! ( matrix.target == 'windows' && github.event.pull_request.draft ) }} - shell: bash - env: - ATMOS_CACHE_ROOT: ${{ steps.toolchain.outputs.cache-root }} - run: | - root="${ATMOS_CACHE_ROOT}" - out="${RUNNER_TEMP}/toolchain.tar" - # Git Bash on Windows: give tar POSIX paths. A native `D:\...` path - # would be read by GNU tar as a `host:file` remote archive. - if command -v cygpath >/dev/null 2>&1; then - root="$(cygpath -u "${root}")" - out="$(cygpath -u "${out}")" - fi - cd "${root}" - # Ship the installed tree (bin////...) plus the - # registry index and lockfile. The downloaded release archives that - # sit next to them are left out: once a tool is on disk, - # `atmos toolchain install` never looks at them again. - members=(toolchain/bin) - for f in toolchain/toolchain.lock.yaml toolchain/aqua-registry-index.json; do - [ -f "${f}" ] && members+=("${f}") - done - tar -cf "${out}" "${members[@]}" - ls -la "${out}" - - - name: Upload the CI toolchain artifact - if: ${{ matrix.target != 'macos-intel' && ! ( matrix.target == 'windows' && github.event.pull_request.draft ) }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: toolchain-${{ matrix.target }} - path: ${{ runner.temp }}/toolchain.tar - if-no-files-found: error - retention-days: 1 - - name: Upload build artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: ${{ ! ( matrix.target == 'windows' && github.event.pull_request.draft ) }} @@ -509,26 +461,16 @@ jobs: path: ${{ github.workspace }} add-to-path: 'true' - - name: Cache Atmos toolchain - if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} - continue-on-error: true - uses: ./actions/cache - - # Seed the toolchain cache from the tarball the `build` job packaged for - # this OS (its "Package the CI toolchain" step), so every tool is already - # on disk and `atmos toolchain install` skips it without touching the - # network; anything the tarball lacks is installed from the network as - # before. The pins here must match the build job's. - - name: Set up Terraform, OpenTofu, Packer, Helm, and Helmfile from the build artifact + # `atmos toolchain install` from .tool-versions, with the toolchain + # directory restored from atmos's own ci.cache (saved by the build job + # for this OS; consumers never save, so shards do not race for the key). + # Do not export ATMOS_XDG_CACHE_HOME or TF_PLUGIN_CACHE_DIR here: many + # tests assert XDG defaults and Terraform's plugin cache is not safe for + # shared concurrent use. + - name: Set up the CI toolchain if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} uses: ./.github/actions/ci-toolchain with: - artifact: toolchain-${{ matrix.flavor.target }} - tool-versions: | - opentofu/opentofu ${{ env.OPEN_TOFU_VERSION }} - hashicorp/packer ${{ env.PACKER_VERSION }} - helm/helm ${{ env.HELM_VERSION }} - helmfile/helmfile ${{ env.HELMFILE_VERSION }} github-token: ${{ secrets.GITHUB_TOKEN }} - name: Verify Terraform, OpenTofu, Packer, Helm, and Helmfile @@ -760,45 +702,16 @@ jobs: path: ${{ github.workspace }} add-to-path: 'true' - # Dogfood `atmos ci cache`: cache the configured Atmos cache root via the - # recommended composite action. Do not export ATMOS_XDG_CACHE_HOME or - # TF_PLUGIN_CACHE_DIR for the full acceptance job: many tests assert XDG - # defaults and Terraform's plugin cache is not safe for shared concurrent - # use. This cache step restores/saves toolchain bits only; it must stay a - # pure accelerator. - # - # Restore-only on the shards. The repo's Actions cache holds 18.9 GB in - # 17 entries against a 10 GB LRU quota, every entry scoped to a - # refs/pull/N/merge ref and minutes old, so the static key - # atmos-toolchain---v2 never hits from main (every shard logs - # "Cache not found for input keys"). All 10 shards then raced to save - # the same key and logged "Unable to reserve cache with key ..., another - # job may be creating this cache", with "Post Cache Atmos toolchain" - # costing 43 s avg / 3 min max per shard for nothing. The - # terraform-registry-cache job keeps the plain (restore+save) step and is - # the single writer of this key per OS. - - name: Cache Atmos toolchain - if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} - continue-on-error: true - uses: ./actions/cache - with: - restore-only: 'true' - - # Seed the toolchain cache from the tarball the `build` job packaged for - # this OS (its "Package the CI toolchain" step), so every tool is already - # on disk and `atmos toolchain install` skips it without touching the - # network; anything the tarball lacks is installed from the network as - # before. The pins here must match the build job's. - - name: Set up Terraform, OpenTofu, Packer, Helm, and Helmfile from the build artifact + # `atmos toolchain install` from .tool-versions, with the toolchain + # directory restored from atmos's own ci.cache (saved by the build job + # for this OS; consumers never save, so shards do not race for the key). + # Do not export ATMOS_XDG_CACHE_HOME or TF_PLUGIN_CACHE_DIR here: many + # tests assert XDG defaults and Terraform's plugin cache is not safe for + # shared concurrent use. + - name: Set up the CI toolchain if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} uses: ./.github/actions/ci-toolchain with: - artifact: toolchain-${{ matrix.flavor.target }} - tool-versions: | - opentofu/opentofu ${{ env.OPEN_TOFU_VERSION }} - hashicorp/packer ${{ env.PACKER_VERSION }} - helm/helm ${{ env.HELM_VERSION }} - helmfile/helmfile ${{ env.HELMFILE_VERSION }} github-token: ${{ secrets.GITHUB_TOKEN }} - name: Verify Terraform, OpenTofu, Packer, Helm, and Helmfile @@ -1909,16 +1822,10 @@ jobs: path: ${{ github.workspace }} add-to-path: 'true' - # Seed the toolchain cache from the tarball the `build` job packaged for - # this OS, so both tools are already on disk and `atmos toolchain install` - # skips them (see the `test` job's step of the same shape). - - name: Set up Terraform and OpenTofu from the build artifact + - name: Set up the CI toolchain if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} uses: ./.github/actions/ci-toolchain with: - artifact: toolchain-${{ matrix.flavor.target }} - tool-versions: | - opentofu/opentofu ${{ env.OPEN_TOFU_VERSION }} github-token: ${{ secrets.GITHUB_TOKEN }} - name: Verify OpenTofu diff --git a/docs/fixes/2026-09-03-ci-toolchain-shipped-in-build-artifact.md b/docs/fixes/2026-09-03-ci-toolchain-shipped-in-build-artifact.md deleted file mode 100644 index 5487f9b2a33..00000000000 --- a/docs/fixes/2026-09-03-ci-toolchain-shipped-in-build-artifact.md +++ /dev/null @@ -1,129 +0,0 @@ -# Fix: CI installs the toolchain once per OS and ships it in a build artifact - -**Date:** 2026-09-03 - -## Summary - -Every one of the 30 acceptance shards in `test.yml` (`test` job, 10 shards × linux/windows/macos) and the -three `terraform-registry-cache` legs ran `atmos toolchain install --default ...` for Terraform, OpenTofu, -Packer, Helm and Helmfile; the `mock` jobs did the same for just Terraform and OpenTofu, the only two they -need. ~400 MB of release downloads and ~2 minutes per job, with a per-job dependency on -releases.hashicorp.com, GitHub releases, get.helm.sh and Sigstore. The -"Cache Atmos toolchain" step meant to short-circuit that never hit, because the ~5 GB of Go caches written -per run churn the repository's 10 GB Actions cache before any toolchain entry survives. This change installs -the five tools once per OS in the `build` job, packages the installed tree as a `toolchain-` -artifact (1-day retention, no cache quota), and has the consuming jobs unpack it into their own Atmos -toolchain cache before `atmos toolchain install`, which then skips every tool without a network request. -The network install stays as the safety net for anything the artifact lacks. - -## Context - -Measured from run logs: every shard logged `Cache not found for input keys: atmos-toolchain-...`, and the -Actions cache listing showed only PR-scoped Go cache entries minutes old. The plan -(Findings §4, Phase 2b) preferred shipping the toolchain in the build artifact over a hashed cache key, -because the cache quota problem is structural (Go caches) and would need the Go footprint to shrink first. - -What the code says about the mechanics (read, then verified locally, see Validation): - -- `atmos toolchain install` puts tools under `/toolchain/bin////` - (`pkg/toolchain/setup.go` `GetInstallPath`, `pkg/toolchain/installer/installer.go` `New`), where the - cache root is the Atmos XDG cache dir (`~/.cache/atmos` on Linux/macOS, `%LOCALAPPDATA%\cache\atmos` - on Windows). `atmos ci cache paths --format=env` prints that root (`ATMOS_CI_CACHE_PATHS=...`), so the - workflow never hardcodes a per-OS path. The tree contains no symlinks and no absolute paths, so it is - relocatable between machines with different home directories. -- The per-tool form the workflow used, `atmos toolchain install --default owner/repo@version`, never takes - the "already installed" path: `RunInstall` (`pkg/toolchain/install.go`) always calls - `InstallSingleTool`, which re-resolves the registry, re-downloads or re-verifies the release signature - (checksums from releases.hashicorp.com, cosign against Sigstore for OpenTofu) and re-extracts. Only the - from-file path (`installFromToolVersions` → `installOrSkipTool`) checks `FindBinaryPath` first and skips. - With the network blocked, the per-tool form hung for the full 2-minute command timeout even with the - download archive already cached; the from-file form skipped every tool in under a second. -- `actions/upload-artifact` zips lose executable bits, so the tree is shipped as a tar, not as loose files. -- `.tool-versions` handling is being consolidated separately (#3022); the pins still come from the - workflow `env` (`OPEN_TOFU_VERSION`, `PACKER_VERSION`, `HELM_VERSION`, `HELMFILE_VERSION`), and Terraform - from the repository's `.tool-versions`, exactly as before. - -## Changes - -- `.github/actions/ci-toolchain/action.yml` (new composite action, used by all four jobs so the tool list - and the install command are identical everywhere): - 1. writes a job-local tool-versions file from the `tool-versions` input plus the repository's - `hashicorp/terraform` pin; - 2. resolves the Atmos cache root with `atmos ci cache paths --format=env`; - 3. when `artifact` is set, downloads it (`download-artifact-retry`, `continue-on-error`) and untars - `toolchain.tar` into the cache root (`cygpath` on Windows so GNU tar does not read `D:\...` as a - remote `host:file`); a missing artifact is a `::warning::`, not a failure; - 4. `atmos toolchain install --tool-versions ` (skips what is on disk, installs the rest); - 5. `atmos toolchain env --tool-versions --format=github`. -- `.github/workflows/test.yml`: - - `build` job: after "Verify acceptance shard plan", puts `./build` on PATH so the install runs with the - atmos just built (same on-disk layout the consumers' binary expects), installs the five tools via the - action, tars `toolchain/bin` plus `toolchain.lock.yaml` and `aqua-registry-index.json` (the downloaded - release archives next to them are left out), and uploads it as `toolchain-` with - `retention-days: 1`. Skipped on `macos-intel`, whose only consumer (the k3s macOS job) installs no - toolchain. `get.helm.sh:443` added to the build job's harden-runner allowlist (Helm's release host). - - `test` and `terraform-registry-cache` jobs: the six-line "Install Terraform, OpenTofu, Packer, Helm, - and Helmfile" `run:` step is replaced by the action with `artifact: toolchain-`. The - "Cache Atmos toolchain" step is untouched (a sibling PR makes it restore-only). - - `mock` job: same replacement for its Terraform + OpenTofu install. - - Consumers no longer rewrite the repository's `.tool-versions` (the old `--default` calls did, on every - job); the acceptance harness finds tools through PATH (`tests/testhelpers/toolchain.go` - `ProvisionToolchain`) and gives every test case its own `XDG_CACHE_HOME`, so nothing reads that file. - -Expected artifact size: the local tar of Terraform + OpenTofu + Helm + the cosign verifier that OpenTofu's -signature check bootstraps is 439 MB uncompressed; with Packer and Helmfile, roughly 600 MB per OS, which -`upload-artifact` compresses to an estimated 250–300 MB. Three of them per run, kept for one day. The -existing `build-artifacts-` is unchanged (it is downloaded into `/usr/local/bin` by a dozen jobs -that need no toolchain, which is why the toolchain is a separate artifact). - -## Validation - -- `actionlint .github/workflows/test.yml`: clean. `yq` parses both files (actionlint does not lint - composite actions; run on `action.yml` it reports the expected "not a workflow" errors only). -- Local mechanics on macOS with a build of this branch (`go build -o /tmp/atmos-ci .`), scratch caches under - `/tmp`, and the network blocked for the offline steps with `HTTPS_PROXY=http://127.0.0.1:9`: - - ```text - $ ATMOS_XDG_CACHE_HOME=/tmp/tc atmos ci cache paths --format=json --path toolchain - { "key": "atmos-toolchain-darwin-arm64-v2", "paths": ["/tmp/tc/atmos/toolchain"], ... } - - $ ATMOS_XDG_CACHE_HOME=/tmp/tc atmos toolchain install --default hashicorp/terraform - ✓ Installed hashicorp/terraform@1.15.8 to /tmp/tc/atmos/toolchain/bin/hashicorp/terraform/1.15.8/terraform (106mb) - $ find /tmp/tc -type l; grep -rl /tmp/tc /tmp/tc # no symlinks, no embedded absolute paths - - # per-tool form, archive cached, network blocked: hangs (killed at the 2 min command timeout) - $ ATMOS_XDG_CACHE_HOME=/tmp/tc2 HTTPS_PROXY=http://127.0.0.1:9 atmos toolchain install --default hashicorp/terraform - Command timed out after 2m 0s - - # from-file form against a tar restored into an EMPTY cache root, network blocked - $ tar -C /tmp/tc/atmos -cf /tmp/tc-artifact/toolchain.tar toolchain/bin toolchain/toolchain.lock.yaml toolchain/aqua-registry-index.json - $ tar -C /tmp/tc3/atmos -xf /tmp/tc-artifact/toolchain.tar - $ ATMOS_XDG_CACHE_HOME=/tmp/tc3 HTTPS_PROXY=http://127.0.0.1:9 atmos toolchain install --tool-versions /tmp/ci.tool-versions - ✓ Skipped opentofu/opentofu@1.12.5 (already installed) - ✓ Skipped helm/helm@v3.19.2 (already installed) - ✓ Skipped hashicorp/terraform@1.15.8 (already installed) - ✓ Installed 0 tools, skipped 3 - $ ATMOS_XDG_CACHE_HOME=/tmp/tc3 HTTPS_PROXY=http://127.0.0.1:9 atmos toolchain env --tool-versions /tmp/ci.tool-versions --format=github - /tmp/tc3/atmos/toolchain/bin/hashicorp/terraform/1.15.8 - /tmp/tc3/atmos/toolchain/bin/helm/helm/v3.19.2 - /tmp/tc3/atmos/toolchain/bin/opentofu/opentofu/1.12.5 - $ /tmp/tc3/atmos/toolchain/bin/hashicorp/terraform/1.15.8/terraform version | head -1 # Terraform v1.15.8 - $ /tmp/tc3/atmos/toolchain/bin/opentofu/opentofu/1.12.5/tofu version | head -1 # OpenTofu v1.12.5 - $ /tmp/tc3/atmos/toolchain/bin/helm/helm/v3.19.2/helm version --short # v3.19.2+g8766e71 - ``` - -- The exact `run:` snippets of the action and of the build job's "Package the CI toolchain" step, extracted - with `yq` and executed with `RUNNER_TEMP`, `GITHUB_OUTPUT` and `GITHUB_PATH` pointed at scratch files and - `ATMOS_XDG_CACHE_HOME` at an empty root: pins file written (Terraform pinned from `.tool-versions`, blank - input lines dropped), root resolved, tar unpacked, install skipped all three tools offline, PATH appended - to `$GITHUB_PATH`, tar re-packaged with the expected members, missing-tarball path exits 0 with a - `::warning::`, and the repository's `.tool-versions` stayed clean. -- Not validated: Windows (no local Windows machine; the `cygpath` conversions and GNU tar under Git Bash - are reasoned from the existing "Add GNU tar to PATH" steps), the actual artifact size and transfer time - on the hosted runners, and a run of the workflow itself. The first CI run of this PR is the real test; - a failed artifact download degrades to today's behaviour by design. -- No Go code or test fixtures were changed. - -## Follow-ups - -None. diff --git a/docs/fixes/2026-09-04-ci-toolchain-one-step.md b/docs/fixes/2026-09-04-ci-toolchain-one-step.md new file mode 100644 index 00000000000..eb04109ce32 --- /dev/null +++ b/docs/fixes/2026-09-04-ci-toolchain-one-step.md @@ -0,0 +1,56 @@ +# Fix: the CI toolchain is one step - `atmos toolchain install` from `.tool-versions` + +**Date:** 2026-09-04 (supersedes the 2026-09-03 "ship the toolchain as a build artifact" design) + +## Summary + +Every acceptance shard, the `terraform-registry-cache` legs and the `mock` jobs need the external +tools the repository pins in `.tool-versions` (Terraform, OpenTofu, Packer, Helm, Helmfile, ...). +Two things had grown around that over time: + +1. **Versions duplicated in the workflow.** `test.yml` carried `OPEN_TOFU_VERSION`, + `PACKER_VERSION`, `HELM_VERSION`, `HELMFILE_VERSION` next to the pins in `.tool-versions`, and the + two drifted (a stale `OPEN_TOFU_VERSION` of 1.12.2 behind the file's 1.12.5 went undetected; + #3022 removed the env vars). The first version of this PR still fed those variables into its + action, so after merging main it would have written empty pins. +2. **Hand-rolled caching.** Because the "Cache Atmos toolchain" step never hit (the ~5 GB of Go caches + written per run churned the 10 GB Actions cache before a toolchain entry survived), the first + version of this PR built its own mechanism: the build job tarred the installed tree with + `cygpath`-aware shell, uploaded it as a `toolchain-` artifact, and every consumer downloaded and + unpacked it before installing. ~150 lines of shell doing what atmos already does. + +## Fix + +`.github/actions/ci-toolchain` is now exactly what a developer runs, plus the cache atmos already +knows how to describe: + +1. `./actions/cache` (the Atmos Cache action, driven by `ci.cache` in `atmos.yaml`): `cache: save` on + the one producer job per OS (the `build` job, with the atmos it just built), `restore-only` (the + default) on every consumer, so ten shards restore one key and none of them races to write it + (`restore-only` comes from #3038, merged into this branch). +2. `atmos toolchain install`: reads `.tool-versions`, installs what is not on disk, skips the rest. + This is the from-file form, the only one that takes the "already installed" path; the per-tool + `--default owner/repo@version` form always re-resolves and re-verifies. +3. `atmos toolchain env --format=github`: exports the tool directories to `PATH`. + +Every job that needs the toolchain is one step: `uses: ./.github/actions/ci-toolchain` with the +token (and `cache: save` on the build job). Versions are pinned in `.tool-versions` only. No tar, no +artifact, no `cygpath`, no per-job tool list. + +Why atmos's cache is enough now: the repository's Actions cache limit was raised from 10 GB to 50 GB +(org setting, 2026-09-04) after measuring that main's entries never survived a run at 10 GB, and +`restore-only` on consumers removes the ten-way save race. A toolchain entry per OS is ~400-600 MB. + +## Validation + +- `actionlint .github/workflows/test.yml`, pre-commit hooks clean. +- The PR's own `Tests` run: the build job saves `atmos-toolchain---v2`; each shard's + `Set up the CI toolchain` step restores it and `atmos toolchain install` logs every tool as already + installed. + +## Follow-ups + +- `HELM_DIFF_VERSION` (a Helm plugin, not a toolchain tool) is still a workflow env var; the + `helm plugin install` step stays as is. +- Jobs that install a single tool inline (`atmos toolchain install opentofu/opentofu` in the floci, + kubernetes-e2e and container-step jobs) could use the same action; left as they are here. From 69b83371b5a3a6fb34e39133a69e6f9df96e74a3 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 4 Sep 2026 15:33:47 -0500 Subject: [PATCH 13/19] ci(actions/cache): a mode input instead of a restore-only boolean restore-and-save (default) or restore-only. A value lets a caller pass its own mode straight through in one step; the boolean made every caller branch on which step to run. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/test.yml | 2 +- actions/cache/README.md | 6 ++--- actions/cache/action.yml | 22 ++++++++++--------- ...-restore-only-toolchain-cache-on-shards.md | 8 +++---- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index abbca4a3c07..4cff951143c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -712,7 +712,7 @@ jobs: continue-on-error: true uses: ./actions/cache with: - restore-only: 'true' + mode: restore-only - name: Install Terraform, OpenTofu, Packer, Helm, and Helmfile if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} diff --git a/actions/cache/README.md b/actions/cache/README.md index d53a4d1297a..849e7e70ad7 100644 --- a/actions/cache/README.md +++ b/actions/cache/README.md @@ -42,12 +42,12 @@ ci: | Input | Default | Description | | --- | --- | --- | -| `restore-only` | `'false'` | When `'true'`, restore the cache but never save it (uses `actions/cache/restore` instead of `actions/cache`, so there is no post step). | +| `mode` | `'restore-and-save'` | `restore-and-save`: restore and save in the post step (`actions/cache`). `restore-only`: restore but never save (`actions/cache/restore`, no post step). | #### Many parallel consumers, one writer If several jobs (for example, a matrix of test shards) restore the same key, -let exactly one job save it and mark the rest `restore-only`. Otherwise every +let exactly one job save it and put the rest on `mode: restore-only`. Otherwise every shard that misses tries to save the same key at the end of the job: all but one fail with `Unable to reserve cache with key ..., another job may be creating this cache`, and each still pays the tar + zstd + upload cost first. @@ -59,7 +59,7 @@ creating this cache`, and each still pays the tar + zstd + upload cost first. # Every parallel consumer (restores only, no post step): - uses: cloudposse/atmos/actions/cache@v1 with: - restore-only: 'true' + mode: restore-only ``` ### Outputs diff --git a/actions/cache/action.yml b/actions/cache/action.yml index 5f7f3801802..6dc3a2127bc 100644 --- a/actions/cache/action.yml +++ b/actions/cache/action.yml @@ -15,16 +15,18 @@ branding: color: 'blue' inputs: - restore-only: + mode: description: >- - When 'true', only restore the cache (actions/cache/restore) and never - save it in the post step. Use this on jobs that fan out into many - parallel consumers of one key (for example test shards) and leave a - single writer job to save. Saves otherwise race for the same key - ("Unable to reserve cache ... another job may be creating this cache") - and each one still pays the tar+zstd upload cost. + `restore-and-save` (default): restore the cache and save it in the post + step, via actions/cache. `restore-only`: restore but never save, via + actions/cache/restore (no post step). Use `restore-only` on jobs that + fan out into many parallel consumers of one key (for example test + shards) and leave a single writer job on `restore-and-save`. Saves + otherwise race for the same key ("Unable to reserve cache ... another + job may be creating this cache") and each one still pays the tar+zstd + upload cost. required: false - default: 'false' + default: 'restore-and-save' outputs: cache-hit: @@ -56,7 +58,7 @@ runs: exit 1 fi - id: cache - if: inputs.restore-only != 'true' + if: inputs.mode != 'restore-only' uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: key: ${{ steps.meta.outputs.key }} @@ -65,7 +67,7 @@ runs: # actions/cache/restore ships in the same repository and tag as # actions/cache, so the same SHA pins both. - id: cache-restore - if: inputs.restore-only == 'true' + if: inputs.mode == 'restore-only' uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: key: ${{ steps.meta.outputs.key }} diff --git a/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md b/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md index 88ca4d96e9d..ce88a7bddfb 100644 --- a/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md +++ b/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md @@ -8,7 +8,7 @@ Part of Phase 2 of the CI-stability plan (see `docs/fixes/2026-09-03-harden-runner-windows-dns-restore-race.md` for Phase 1). The `test` job's ten shards per OS restore the Atmos toolchain cache without ever saving it: the `actions/cache` composite action gained -a `restore-only` input that switches it to `actions/cache/restore`, and `terraform-registry-cache` stays the +a `mode` input (`restore-only`) that switches it to `actions/cache/restore`, and `terraform-registry-cache` stays the single writer of the key per OS. No user-visible behavior changes; the new action input is opt-in and defaults to the previous behavior. @@ -28,10 +28,10 @@ toolchain` costs 43 s on average and up to 3 min per shard for nothing. ## Changes - `.github/workflows/test.yml`: the `test` job's `Cache Atmos toolchain` step (all three OSes) now passes - `restore-only: 'true'`, with a comment carrying the cache data above and naming + `mode: restore-only`, with a comment carrying the cache data above and naming `terraform-registry-cache` as the single writer. The `terraform-registry-cache` job's cache step is unchanged. -- `actions/cache/action.yml`: new optional input `restore-only` (string, default `'false'`). When `'true'` +- `actions/cache/action.yml`: new optional input `mode` (`restore-and-save`, the default, or `restore-only`). When `restore-only` the action runs `actions/cache/restore` (same repository and tag as `actions/cache`, so the same pinned SHA `27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5`) with identical `key`, `path`, and `restore-keys`; otherwise `actions/cache` runs as before. Exactly one of the two steps runs, so the @@ -45,7 +45,7 @@ warmup's writer role. ## Validation - `python3 -c 'import yaml; yaml.safe_load(open(f))'` on `.github/workflows/test.yml` and - `actions/cache/action.yml`: both parse; the action parses with `inputs: [restore-only]` and steps `meta`, + `actions/cache/action.yml`: both parse; the action parses with `inputs: [mode]` and steps `meta`, `validate`, `cache`, `cache-restore`. - `actionlint .github/workflows/test.yml`: clean (exit 0). actionlint does not lint composite actions, so `actions/cache/action.yml` is covered by the YAML parse and the Go regression test below only. From 1497964805707baaa0d1e219c80bba0bd1da7362 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 4 Sep 2026 15:52:27 -0500 Subject: [PATCH 14/19] fix(actions/cache): make the post-step save work when nested in a composite action Post steps of an action nested inside another composite action do not see steps.*.outputs (actions/runner#2800): called from a repo-local toolchain action, actions/cache's save ran with an empty path ("Input required and not supplied: path") and stored nothing. The key, paths and restore-keys now also go through the job env, which post steps do see. Co-Authored-By: Claude Fable 5.1 --- actions/cache/README.md | 8 ++++++++ actions/cache/action.yml | 33 +++++++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/actions/cache/README.md b/actions/cache/README.md index 849e7e70ad7..68fa933f460 100644 --- a/actions/cache/README.md +++ b/actions/cache/README.md @@ -80,6 +80,14 @@ creating this cache`, and each still pays the tar + zstd + upload cost first. If you need Atmos to *own* restore/save (rather than `actions/cache`), use the [`github-runtime`](../github-runtime/README.md) action instead. +## Using it from another composite action + +The action works when called from inside another composite action (for example a repo-local +"set up the toolchain" action). The key, paths and restore-keys are exported to the job +environment (`ATMOS_CACHE_KEY`, `ATMOS_CACHE_PATH`, `ATMOS_CACHE_RESTORE_KEYS`) because the save +runs in `actions/cache`'s post step, which in a nested action cannot see this action's step +outputs (actions/runner#2800); without that the post step ran with an empty path and saved nothing. + ## Versioning This action ships inside the Atmos repository, so the ref is an Atmos release: diff --git a/actions/cache/action.yml b/actions/cache/action.yml index 6dc3a2127bc..72311c47ba9 100644 --- a/actions/cache/action.yml +++ b/actions/cache/action.yml @@ -57,19 +57,40 @@ runs: echo "::error::Atmos cache metadata did not include any cache paths. Check ci.cache configuration and the previous metadata step output." exit 1 fi + # The save happens in actions/cache's post step. When this action runs + # nested inside another composite action, that post step no longer sees + # `steps.meta.outputs` (actions/runner#2800) and would run with an empty + # `path`, saving nothing ("Input required and not supplied: path"). The + # job env is visible to post steps, so the values go through it. + - id: export + shell: bash + env: + ATMOS_CACHE_KEY: ${{ steps.meta.outputs.key }} + ATMOS_CACHE_PATH: ${{ steps.meta.outputs.path }} + ATMOS_CACHE_RESTORE_KEYS: ${{ steps.meta.outputs.restore-keys }} + run: | + { + echo "ATMOS_CACHE_KEY=${ATMOS_CACHE_KEY}" + echo "ATMOS_CACHE_PATH<> "$GITHUB_ENV" - id: cache if: inputs.mode != 'restore-only' uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: - key: ${{ steps.meta.outputs.key }} - path: ${{ steps.meta.outputs.path }} - restore-keys: ${{ steps.meta.outputs.restore-keys }} + key: ${{ env.ATMOS_CACHE_KEY }} + path: ${{ env.ATMOS_CACHE_PATH }} + restore-keys: ${{ env.ATMOS_CACHE_RESTORE_KEYS }} # actions/cache/restore ships in the same repository and tag as # actions/cache, so the same SHA pins both. - id: cache-restore if: inputs.mode == 'restore-only' uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: - key: ${{ steps.meta.outputs.key }} - path: ${{ steps.meta.outputs.path }} - restore-keys: ${{ steps.meta.outputs.restore-keys }} + key: ${{ env.ATMOS_CACHE_KEY }} + path: ${{ env.ATMOS_CACHE_PATH }} + restore-keys: ${{ env.ATMOS_CACHE_RESTORE_KEYS }} From 3dba100cd2d9fcc0dc4e01f67648909e3705fa59 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 4 Sep 2026 15:54:08 -0500 Subject: [PATCH 15/19] docs(fixes): nested post-step save and the mode input Co-Authored-By: Claude Fable 5.1 --- ...6-09-03-restore-only-toolchain-cache-on-shards.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md b/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md index ce88a7bddfb..2ef80f428b0 100644 --- a/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md +++ b/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md @@ -64,3 +64,15 @@ warmup's writer role. existing `build-artifacts-` artifact so the shards stop installing them at all. - Phase 2c (measure the cache-free Windows shard) and the `setup-go` single-writer change are also still open; same tracking note. + +## Addendum (2026-09-04): the save silently did nothing when the action was nested + +Calling `./actions/cache` from inside another composite action (`.github/actions/ci-toolchain`, PR +#3041) showed the post-step save running with `Input required and not supplied: path` and storing +nothing: a post step of an action nested inside a composite cannot see the composite's +`steps.*.outputs` (actions/runner#2800), so `path: ${{ steps.meta.outputs.path }}` evaluated +empty at save time. Direct use from a job never hit this. The action now also exports +`ATMOS_CACHE_KEY`, `ATMOS_CACHE_PATH` and `ATMOS_CACHE_RESTORE_KEYS` to the job env and the cache +steps read those, since the job env is visible to post steps. Also: the `restore-only` boolean +became a `mode` input (`restore-and-save` | `restore-only`) so a caller can pass its own mode +through in one step. From 9349603006defd251c2880ef724466d922b050f7 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 4 Sep 2026 16:07:10 -0500 Subject: [PATCH 16/19] docs: fix CodeRabbit-flagged doc issues on the cache/Defender fix notes - Correct cache entry total to 22, matching the listed 14 setup-go-* + 8 atmos-toolchain-* breakdown. - Fix the python3 YAML-validation snippet: f was never bound, so copying it raised NameError. Wrap it in a shell loop instead. - Add a text fence language to the unfenced diagnostic output block (markdownlint MD040). Co-Authored-By: Claude Sonnet 5 --- .../2026-09-03-restore-only-toolchain-cache-on-shards.md | 7 +++---- .../2026-09-04-windows-defender-exclusions-are-a-noop.md | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md b/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md index 2ef80f428b0..e34be8ce5fd 100644 --- a/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md +++ b/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md @@ -17,7 +17,7 @@ defaults to the previous behavior. Measured on `test.yml` runs from Aug 24 to Sept 3 (80 successful Windows shards, step-level timelines, raw job logs, and `gh api repos/cloudposse/atmos/actions/cache/usage`). -**Toolchain cache.** The repository's Actions cache holds 18.9 GB across 17 entries against a 10 GB LRU +**Toolchain cache.** The repository's Actions cache holds 18.9 GB across 22 entries against a 10 GB LRU quota. Every entry is scoped to a `refs/pull/N/merge` ref and is minutes old (14 `setup-go-*` entries at 1.6 to 1.9 GB, 8 `atmos-toolchain-*` entries at 350 to 470 MB). Each run writes about 5 GB, so nothing saved from `main` survives and the static key `atmos-toolchain---v2` never hits: every shard @@ -44,9 +44,8 @@ warmup's writer role. ## Validation -- `python3 -c 'import yaml; yaml.safe_load(open(f))'` on `.github/workflows/test.yml` and - `actions/cache/action.yml`: both parse; the action parses with `inputs: [mode]` and steps `meta`, - `validate`, `cache`, `cache-restore`. +- `for f in .github/workflows/test.yml actions/cache/action.yml; do python3 -c 'import sys, yaml; yaml.safe_load(open(sys.argv[1]))' "$f"; done`: + both parse; the action parses with `inputs: [mode]` and steps `meta`, `validate`, `cache`, `cache-restore`. - `actionlint .github/workflows/test.yml`: clean (exit 0). actionlint does not lint composite actions, so `actions/cache/action.yml` is covered by the YAML parse and the Go regression test below only. - `go test github.com/cloudposse/atmos/cmd -run TestAtmosCacheActionValidatesMetadataBeforeActionsCache`: diff --git a/docs/fixes/2026-09-04-windows-defender-exclusions-are-a-noop.md b/docs/fixes/2026-09-04-windows-defender-exclusions-are-a-noop.md index 7bd11effb97..c319f29ecc2 100644 --- a/docs/fixes/2026-09-04-windows-defender-exclusions-are-a-noop.md +++ b/docs/fixes/2026-09-04-windows-defender-exclusions-are-a-noop.md @@ -26,7 +26,7 @@ Confirmed live, not just from the build script, by adding a temporary diagnostic `setup-go-cache-warmup.yml` and running it on an actual `windows-latest` runner (queried via `Get-MpComputerStatus`/`Get-MpPreference` immediately before and after the exclusion step): -``` +```text --- before our exclusion step --- RealTimeProtectionEnabled : False DisableRealtimeMonitoring : True From ad0e273ee5b80bc7328fa574d62445626db5493a Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 4 Sep 2026 16:24:07 -0500 Subject: [PATCH 17/19] ci(mock): allow the toolchain download hosts, like the test job atmos toolchain install from .tool-versions installs every pinned tool, so the mock job needs the same download hosts as the test job when the toolchain cache misses (get.helm.sh was blocked and helm failed). With the cache in place the install is a no-op. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 97ea50308b0..cf38927b190 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1792,6 +1792,10 @@ jobs: ocsp.usertrust.com:80 ocsp.digicert.com:80 *.pool.ntp.org:123 + get.helm.sh:443 + proxy.golang.org:443 + sum.golang.org:443 + google.golang.org:443 - name: Check out code into the Go module directory if: ${{ ! ( matrix.flavor.target == 'windows' && github.event.pull_request.draft ) }} From 6e2cf2812a64e7fd0cf7310504049e173bc794da Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 4 Sep 2026 16:35:17 -0500 Subject: [PATCH 18/19] test(testhelpers): keep one record per fake-runtime invocation with multi-line args The fake container runtime wrote each invocation as one tab-joined line; a forwarded -e KEY=VALUE for a multi-line environment variable (ATMOS_CACHE_PATH, exported by the cache action for its nested post step) split the record and the container-override tests lost the interpreter fields on Windows shard 1. Newlines and tabs inside an argument are now escaped in the record. Co-Authored-By: Claude Fable 5.1 --- ...026-09-03-restore-only-toolchain-cache-on-shards.md | 8 ++++++++ tests/testhelpers/fake_container_runtime.go | 10 +++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md b/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md index e34be8ce5fd..386953da06c 100644 --- a/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md +++ b/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md @@ -75,3 +75,11 @@ empty at save time. Direct use from a job never hit this. The action now also ex steps read those, since the job env is visible to post steps. Also: the `restore-only` boolean became a `mode` input (`restore-and-save` | `restore-only`) so a caller can pass its own mode through in one step. + + +Side effect of the env export: `ATMOS_CACHE_PATH` is multi-line (the include glob plus the auth-dir +exclusion), so every later step of the job carries a multi-line environment variable. That is +legitimate in Actions, but the acceptance tests' fake container runtime recorded each invocation +as one tab-joined line and a forwarded `-e ATMOS_CACHE_PATH=...` split the record at the newline, +failing `TestCustomCommandStepContainerOverrideRunsInsideContainer` on Windows shard 1. The +recorder now escapes newlines and tabs inside an argument. diff --git a/tests/testhelpers/fake_container_runtime.go b/tests/testhelpers/fake_container_runtime.go index a5e177ccdad..b532e28771b 100644 --- a/tests/testhelpers/fake_container_runtime.go +++ b/tests/testhelpers/fake_container_runtime.go @@ -122,7 +122,15 @@ func recordArgs(args []string) { return } defer f.Close() - _, _ = fmt.Fprintln(f, strings.Join(args, "\t")) + // One line per invocation, tab-separated. An argument may itself contain a + // newline or a tab (a forwarded -e KEY=VALUE for a multi-line environment + // variable, which GitHub Actions jobs legitimately have), so those are + // escaped inside the field rather than allowed to break the record. + escaped := make([]string, len(args)) + for i, arg := range args { + escaped[i] = strings.NewReplacer("\n", "\\n", "\t", "\\t").Replace(arg) + } + _, _ = fmt.Fprintln(f, strings.Join(escaped, "\t")) } func requiresForwardedEnv(command string) bool { From 8b3e8a25f88ed60b3c7aca28b52b39297f85da57 Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Fri, 4 Sep 2026 17:24:48 -0500 Subject: [PATCH 19/19] docs(fixes): reconcile toolchain-cache size figures with the stated total The per-entry ranges implied a floor above the reported 18.9 GB total (14 entries at a 1.6 GB floor alone is 22.4 GB). State them as upper bounds instead, since the original per-entry measurements are no longer reproducible (transient, already-evicted cache state). --- .../2026-09-03-restore-only-toolchain-cache-on-shards.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md b/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md index 386953da06c..b6ed62c7db8 100644 --- a/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md +++ b/docs/fixes/2026-09-03-restore-only-toolchain-cache-on-shards.md @@ -18,8 +18,8 @@ Measured on `test.yml` runs from Aug 24 to Sept 3 (80 successful Windows shards, job logs, and `gh api repos/cloudposse/atmos/actions/cache/usage`). **Toolchain cache.** The repository's Actions cache holds 18.9 GB across 22 entries against a 10 GB LRU -quota. Every entry is scoped to a `refs/pull/N/merge` ref and is minutes old (14 `setup-go-*` entries at -1.6 to 1.9 GB, 8 `atmos-toolchain-*` entries at 350 to 470 MB). Each run writes about 5 GB, so nothing +quota. Every entry is scoped to a `refs/pull/N/merge` ref and is minutes old (14 `setup-go-*` entries up +to 1.9 GB each, 8 `atmos-toolchain-*` entries up to 470 MB each). Each run writes about 5 GB, so nothing saved from `main` survives and the static key `atmos-toolchain---v2` never hits: every shard logs `Cache not found for input keys`, then all 10 shards race to save the same key and log `Unable to reserve cache with key ..., another job may be creating this cache`. `Post Cache Atmos