diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fd5e3f..d2e37f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,18 @@ on: push: branches: [main] pull_request: + schedule: + - cron: '43 19 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # A newer push to a pull request replaces its running checks; every main + # push, scheduled and manual run completes on its own. + group: ci-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || format('run-{0}', github.run_id) }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: # Incremental data is useless in CI, and debuginfo roughly doubles the @@ -12,6 +24,35 @@ env: CARGO_PROFILE_DEV_DEBUG: "0" jobs: + plan: + name: CI / Plan + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + plan: ${{ steps.plan.outputs.plan }} + rust-clippy: ${{ steps.plan.outputs.rust-clippy }} + rust-clippy-matrix: ${{ steps.plan.outputs.rust-clippy-matrix }} + rust-unit: ${{ steps.plan.outputs.rust-unit }} + rust-unit-matrix: ${{ steps.plan.outputs.rust-unit-matrix }} + rust-linux-isolation: ${{ steps.plan.outputs.rust-linux-isolation }} + rust-integration: ${{ steps.plan.outputs.rust-integration }} + rust-macos: ${{ steps.plan.outputs.rust-macos }} + steps: + - uses: actions/checkout@v4 + with: + submodules: false + fetch-depth: 0 + - name: Verify CI policy + run: python3 -B -m unittest discover -s scripts/tests -p 'test_ci_*.py' + - id: plan + name: Plan affected checks + run: python3 -B scripts/ci_plan.py + - uses: actions/upload-artifact@v4 + with: + name: ci-plan-${{ github.run_attempt }} + path: target/ci-plan/plan.json + if-no-files-found: error + policy-scan: runs-on: ubuntu-latest steps: @@ -51,23 +92,16 @@ jobs: rust-clippy: name: Rust / Clippy (${{ matrix.name }}) + needs: plan + if: ${{ needs.plan.outputs.rust-clippy == 'true' }} runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - # rust-cache prunes the shared target dir once per listed workspace and - # keeps only that workspace's dependencies, so every leg names a single - # anchor workspace whose dependencies cover what the leg compiles. - include: - - name: root - stage: clippy-root dependencies - cache-workspace: ". -> target/upstream-validation" - - name: adapters - stage: clippy-adapters - cache-workspace: "crates/patch -> ../../target/upstream-validation" - - name: codex-adapters - stage: clippy-codex - cache-workspace: "crates/codex-runtime -> ../../target/upstream-validation" + # Legs, stages and cache anchors come from scripts/ci-policy.json. + # rust-cache prunes the shared target dir once per listed workspace and + # keeps only that workspace's dependencies, so every leg names a single + # anchor workspace whose dependencies cover what the leg compiles. + matrix: ${{ fromJSON(needs.plan.outputs.rust-clippy-matrix) }} steps: - uses: actions/checkout@v4 with: @@ -93,29 +127,12 @@ jobs: rust-unit: name: Rust / Unit (${{ matrix.name }}) + needs: plan + if: ${{ needs.plan.outputs.rust-unit == 'true' }} runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - include: - - name: patch - stage: unit-patch - cache-workspace: "crates/patch -> ../../target/upstream-validation" - - name: codex-runtime - stage: unit-codex-runtime - cache-workspace: "crates/codex-runtime -> ../../target/upstream-validation" - - name: pty - stage: unit-pty - cache-workspace: "crates/pty -> ../../target/upstream-validation" - - name: file-system - stage: unit-file-system - cache-workspace: "crates/file-system -> ../../target/upstream-validation" - - name: linux-sandbox-protocol - stage: unit-linux-sandbox-protocol - cache-workspace: ". -> target/upstream-validation" - - name: linux-sandbox - stage: unit-linux-sandbox - cache-workspace: "crates/linux-sandbox -> ../../target/upstream-validation" + matrix: ${{ fromJSON(needs.plan.outputs.rust-unit-matrix) }} steps: - uses: actions/checkout@v4 with: @@ -139,6 +156,8 @@ jobs: rust-linux-isolation: name: Rust / Linux isolation + needs: plan + if: ${{ needs.plan.outputs.rust-linux-isolation == 'true' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -182,6 +201,8 @@ jobs: rust-integration: name: Rust / Integration + needs: plan + if: ${{ needs.plan.outputs.rust-integration == 'true' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -220,6 +241,8 @@ jobs: rust-macos: name: Rust / macOS contracts + needs: plan + if: ${{ needs.plan.outputs.rust-macos == 'true' }} runs-on: macos-latest steps: - uses: actions/checkout@v4 @@ -245,23 +268,31 @@ jobs: rust: name: rust + # Never skippable: it proves the planned legs passed and the rest were skipped. if: ${{ always() }} needs: + - plan + - policy-scan - rust-format - rust-clippy - rust-unit - rust-linux-isolation - rust-integration - rust-macos - - policy-scan runs-on: ubuntu-latest + timeout-minutes: 10 steps: - - name: Require all Rust checks - run: | - test "${{ needs.rust-format.result }}" = "success" - test "${{ needs.rust-clippy.result }}" = "success" - test "${{ needs.rust-unit.result }}" = "success" - test "${{ needs.rust-linux-isolation.result }}" = "success" - test "${{ needs.rust-integration.result }}" = "success" - test "${{ needs.rust-macos.result }}" = "success" - test "${{ needs.policy-scan.result }}" = "success" + - uses: actions/checkout@v4 + with: + submodules: false + - uses: actions/download-artifact@v4 + # A missing report fails the check below, not this download. + continue-on-error: true + with: + pattern: upstream-* + path: target/ci-reports + - name: Require the planned checks + env: + CI_RESULTS: ${{ toJSON(needs) }} + CI_PLAN: ${{ needs.plan.outputs.plan }} + run: python3 -B scripts/check_ci_results.py --reports target/ci-reports diff --git a/docs/ko/upstream-update.md b/docs/ko/upstream-update.md index 82a09e3..a79083f 100644 --- a/docs/ko/upstream-update.md +++ b/docs/ko/upstream-update.md @@ -49,6 +49,16 @@ macOS에서는 Linux 격리를 `not_run`, 전체 결과를 `incomplete`로 표 Linux CI 근거가 별도로 필요하며 macOS CI는 PTY와 파일 시스템 계약도 검사합니다. 한 플랫폼 결과만으로 다른 플랫폼 검증을 대체하지 않습니다. +CI는 변경마다 모든 job을 실행하지 않습니다. 계획 job이 `scripts/ci-policy.json`에 +따라 필요한 leg를 고릅니다. crate를 바꾸면 그 crate를 컴파일하는 모든 leg를, +문서만 바꾸면 아무 leg도 실행하지 않고, 빌드 입력·CI 파일·알 수 없는 경로를 +바꾸면 전체를 실행합니다. policy·Python·pin·format 단계는 항상 실행합니다. +필수 `rust` job은 계획된 모든 leg가 검사한 커밋의 보고서와 함께 성공하고 +나머지 leg는 모두 건너뛰었을 때만 통과합니다. 매일 예약 실행과 수동 실행은 +전체 검증입니다. 계획은 `python3 scripts/ci_plan.py --base `으로 +미리 볼 수 있습니다. CI는 증분 데이터와 debuginfo 없이 빌드하며, 각 job은 +`main`만 저장하는 의존성 캐시를 복원합니다. + 의존성 검사는 `--locked`와 대상 플랫폼 필터를 사용한 Cargo metadata에서 제품 root의 일반·빌드 의존성을 탐색하고 개발용 관계는 제외합니다. 에이전트·제품 crate 및 Runner에서 샌드박스 라이브러리로 향하는 경로는 실패합니다. diff --git a/docs/translations.json b/docs/translations.json index 29d3415..e69ab8a 100644 --- a/docs/translations.json +++ b/docs/translations.json @@ -653,8 +653,8 @@ "제출-전-검증", "후보-검토" ], - "source_sha256": "dcac8e5362953f5ade48326c69d48f4a448509853ec2eb6034fb6400fe0418b3", - "translation_sha256": "692bccd11d607df49a0802b0a37df204af071a23ec826d6e72f8bdd2973fa6ff" + "source_sha256": "b48c10f573a85e5f2b2076a7962de9875eb6a09c603167e97277720cbf27a534", + "translation_sha256": "642490ebece0cc7625b3250a428bb8ebc9880085316810aae8d547ac4cc7ea45" }, { "id": "documentation", diff --git a/docs/upstream-update.md b/docs/upstream-update.md index 14ea331..f68903d 100644 --- a/docs/upstream-update.md +++ b/docs/upstream-update.md @@ -46,6 +46,17 @@ On macOS, Linux isolation is explicitly `not_run` and the overall result is `incomplete`; Linux CI evidence is still required. macOS CI additionally checks PTY and filesystem contracts. Neither result alone replaces the other platform. +CI does not run every job for every change. A planning job selects the legs a +change needs from `scripts/ci-policy.json`: a crate change runs every leg that +compiles that crate, documentation runs none, and build inputs, CI files or +unknown paths run everything. The policy, Python, pin and format stages always +run. The required `rust` job passes only when every planned leg succeeded with +a report for the tested commit and every other leg was skipped. Daily scheduled +and manual runs are full. Preview a plan with +`python3 scripts/ci_plan.py --base `. CI builds without incremental +data or debuginfo, and each job restores a dependency cache that only `main` +saves. + The dependency stage uses locked, target-filtered Cargo metadata and follows normal/build edges from product roots, excluding development edges. It rejects agent/product crates and Runner-to-sandbox-library edges with a dependency path. diff --git a/scripts/check_ci_results.py b/scripts/check_ci_results.py new file mode 100644 index 0000000..ed98bbd --- /dev/null +++ b/scripts/check_ci_results.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Require the planned CI legs to pass with matching reports and every other leg to be skipped.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re + +ROOT = Path(__file__).resolve().parents[1] +POLICY = ROOT / 'scripts' / 'ci-policy.json' +SCHEMA = 'codespace-ci-plan/v1' +REQUIRED_JOBS = {'plan', 'policy-scan', 'rust-format', 'rust-clippy', 'rust-unit', + 'rust-linux-isolation', 'rust-integration', 'rust-macos'} +FULL_EVENTS = {'schedule', 'workflow_dispatch'} + + +def artifacts(reports, leg, attempt): + """Report artifacts of one leg from this and earlier run attempts, oldest first.""" + job, name = leg.split('/') + pattern = re.compile(re.escape(f'upstream-{job}-{name}-') + '([0-9]+)') + found = [] + for path in reports.iterdir() if reports is not None and reports.is_dir() else []: + match = pattern.fullmatch(path.name) + if match and int(match.group(1)) <= attempt: + found.append((int(match.group(1)), path)) + return sorted(found) + + +def report_problems(leg, found, stages, source): + if not found: + return [f'{leg}: no uploaded report'] + files = list(found[-1][1].rglob('report.json')) + if len(files) != 1: + return [f'{leg}: expected one report.json, found {len(files)}'] + report = json.loads(files[0].read_text()) + steps = report.get('stages') or [] + if (report.get('status') != 'passed' or report.get('scope') != stages + or [step.get('name') for step in steps] != stages + or any(step.get('status') != 'passed' for step in steps) + or (report.get('inputs') or {}).get('head') != source): + return [f'{leg}: report does not show {" ".join(stages)} passing at {source[:12]}'] + return [] + + +def problems(results, plan, policy, digest, env, reports): + """Every way this run differs from its plan; an empty list means the gate passes.""" + if not isinstance(results, dict) or set(results) != REQUIRED_JOBS: + return ['the gate does not see exactly the required jobs'] + source, legs, selected = env.get('GITHUB_SHA', ''), policy['legs'], plan.get('legs') + if (plan.get('schema') != SCHEMA or not re.fullmatch('[0-9a-f]{40}', source) + or plan.get('source_sha') != source or plan.get('policy_sha256') != digest + or plan.get('event') != env.get('GITHUB_EVENT_NAME') + or plan.get('profile') not in {'full', 'affected'} + or not isinstance(selected, list) or len(set(selected)) != len(selected) or not set(selected) <= set(legs) + or plan.get('jobs') != sorted({leg.split('/')[0] for leg in selected}) + or (plan['profile'] == 'full' and selected != list(legs)) + or (plan['event'] in FULL_EVENTS and plan['profile'] != 'full')): + return ['the plan is not bound to this source, event and policy'] + always = {leg.split('/')[0] for leg in policy['always']} + found = [] + for name in sorted(REQUIRED_JOBS): + result = (results[name] or {}).get('result') + expected = 'success' if name == 'plan' or name in always or name in plan['jobs'] else 'skipped' + if result != expected: + found.append(f'{name}: {result}, expected {expected}') + attempt = int(env.get('GITHUB_RUN_ATTEMPT') or 1) + planned = dict(policy['always'], **{leg: legs[leg]['stages'] for leg in selected}) + for leg in [*policy['always'], *legs]: + runs = artifacts(reports, leg, attempt) + if leg in planned: + found += report_problems(leg, runs, planned[leg], source) + elif runs: + found.append(f'{leg}: uploaded a report although it was not planned') + return found + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--reports', type=Path, required=True, help='directory of downloaded upstream-* artifacts') + args = parser.parse_args(argv) + env = os.environ + try: + raw = POLICY.read_bytes() + found = problems(json.loads(env['CI_RESULTS']), json.loads(env['CI_PLAN']), json.loads(raw), + hashlib.sha256(raw).hexdigest(), env, args.reports) + except (OSError, ValueError, KeyError, TypeError, AttributeError) as error: + found = [f'cannot evaluate the CI results: {error}'] + text = '\n'.join(['Required CI: ' + ('failed' if found else 'passed')] + ['- ' + line for line in found]) + print(text) + if env.get('GITHUB_STEP_SUMMARY'): + with open(env['GITHUB_STEP_SUMMARY'], 'a') as handle: + handle.write(text + '\n') + raise SystemExit(1 if found else 0) + + +if __name__ == '__main__': + main() diff --git a/scripts/ci-policy.json b/scripts/ci-policy.json new file mode 100644 index 0000000..4b37558 --- /dev/null +++ b/scripts/ci-policy.json @@ -0,0 +1,80 @@ +{ + "schema": "codespace-ci-policy/v1", + "full": [ + "Cargo.toml", "Cargo.lock", "**/Cargo.toml", "**/Cargo.lock", + "rust-toolchain", "rust-toolchain.toml", "**/rust-toolchain", "**/rust-toolchain.toml", + ".cargo/**", "**/.cargo/**", + "third_party/**", ".gitmodules", ".gitignore", + ".github/**", "scripts/**", "docs/upstream-lock.md" + ], + "no_rust": [ + "docs/**", "docs-site/**", "README.md", "README.ko.md", "LICENSE", "NOTICE", ".env.example", ".codex/**" + ], + "components": { + "domain": ["crates/domain/**"], + "policy": ["crates/policy/**"], + "runner": ["crates/runner/**", "deploy/**"], + "store": ["crates/store/**"], + "server": ["crates/server/**", "tests/**"], + "linux-sandbox-protocol": ["crates/linux-sandbox-protocol/**"], + "patch": ["crates/patch/**"], + "codex-runtime": ["crates/codex-runtime/**"], + "pty": ["crates/pty/**"], + "file-system": ["crates/file-system/**"], + "linux-sandbox": ["crates/linux-sandbox/**"] + }, + "always": { + "policy-scan/single": ["policy", "python"], + "rust-format/single": ["pin", "format"] + }, + "legs": { + "rust-clippy/root": { + "stages": ["clippy-root", "dependencies"], "cache": ".", + "compiles": ["domain", "file-system", "linux-sandbox-protocol", "policy", "pty", "runner", "server", "store"] + }, + "rust-clippy/adapters": { + "stages": ["clippy-adapters"], "cache": "crates/patch", + "compiles": ["domain", "file-system", "patch", "policy", "pty"] + }, + "rust-clippy/codex-adapters": { + "stages": ["clippy-codex"], "cache": "crates/codex-runtime", + "compiles": ["codex-runtime", "domain", "file-system", "linux-sandbox", "linux-sandbox-protocol", "policy", "pty", "runner"] + }, + "rust-unit/patch": { + "stages": ["unit-patch"], "cache": "crates/patch", + "compiles": ["domain", "patch", "policy"] + }, + "rust-unit/codex-runtime": { + "stages": ["unit-codex-runtime"], "cache": "crates/codex-runtime", + "compiles": ["codex-runtime", "domain", "file-system", "linux-sandbox-protocol", "policy", "pty", "runner"] + }, + "rust-unit/pty": { + "stages": ["unit-pty"], "cache": "crates/pty", + "compiles": ["pty"] + }, + "rust-unit/file-system": { + "stages": ["unit-file-system"], "cache": "crates/file-system", + "compiles": ["file-system"] + }, + "rust-unit/linux-sandbox-protocol": { + "stages": ["unit-linux-sandbox-protocol"], "cache": ".", + "compiles": ["linux-sandbox-protocol"] + }, + "rust-unit/linux-sandbox": { + "stages": ["unit-linux-sandbox"], "cache": "crates/linux-sandbox", + "compiles": ["linux-sandbox", "linux-sandbox-protocol"] + }, + "rust-linux-isolation/single": { + "stages": ["linux-isolation"], "cache": "crates/linux-sandbox", + "compiles": ["linux-sandbox", "linux-sandbox-protocol"] + }, + "rust-integration/single": { + "stages": ["integration"], "cache": ".", + "compiles": ["codex-runtime", "domain", "file-system", "linux-sandbox", "linux-sandbox-protocol", "patch", "policy", "pty", "runner", "server", "store"] + }, + "rust-macos/single": { + "stages": ["macos-core", "dependencies"], "cache": "crates/file-system", + "compiles": ["file-system", "pty"] + } + } +} diff --git a/scripts/ci_plan.py b/scripts/ci_plan.py new file mode 100644 index 0000000..5131fc0 --- /dev/null +++ b/scripts/ci_plan.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Select the CI legs a change needs and bind the plan to the checked-out source. + +Changed paths are matched against scripts/ci-policy.json. Build inputs, CI +files and unknown paths select every leg; documentation selects none; a crate +selects every leg that compiles it. Scheduled and manual runs are full. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +POLICY = 'scripts/ci-policy.json' +# A change to any planning input runs everything, so a pull request cannot +# narrow the checks of its own run. +PLANNING = ('.github/workflows/ci.yml', POLICY, 'scripts/ci_plan.py', 'scripts/check_ci_results.py') +SCHEMA = 'codespace-ci-plan/v1' +TARGET = 'target/upstream-validation' +SHA = re.compile('[0-9a-f]{40}') +GLOB = {'**/': '(?:.*/)?', '**': '.*', '*': '[^/]*', '?': '[^/]'} + + +class PlanError(Exception): + pass + + +def git(root, *args): + return subprocess.run(['git', *args], cwd=root, check=True, capture_output=True).stdout + + +def read_policy(root=ROOT): + raw = (root / POLICY).read_bytes() + return json.loads(raw), hashlib.sha256(raw).hexdigest() + + +def matches(patterns, path): + """`*` stays inside one path segment; `**` spans segments.""" + for pattern in patterns: + parts = re.split(r'(\*\*/|\*\*|\*|\?)', pattern) + if re.fullmatch(''.join(GLOB.get(part, re.escape(part)) for part in parts), path): + return True + return False + + +def classify(policy, paths): + """Return the changed components and the reasons that force a full run.""" + components, reasons, unknown = set(), set(), [] + for path in paths: + full = [pattern for pattern in policy['full'] if matches([pattern], path)] + if full: + reasons.add('full-path:' + full[0]) + elif not matches(policy['no_rust'], path): + owners = {name for name, patterns in policy['components'].items() if matches(patterns, path)} + components |= owners + if not owners: + unknown.append(path) + reasons |= {'unclassified:' + path for path in unknown[:10]} + return components, reasons + + +def build_plan(policy, digest, event, source, base, paths, reasons): + components, more = classify(policy, paths) + reasons = set(reasons) | more + if base is not None and not paths: + reasons.add('empty-diff') + legs = [leg for leg, spec in policy['legs'].items() if reasons or components & set(spec['compiles'])] + return {'schema': SCHEMA, 'event': event, 'source_sha': source, 'base_sha': base, + 'policy_sha256': digest, 'profile': 'full' if reasons else 'affected', + 'reasons': sorted(reasons), 'components': sorted(components), + 'legs': legs, 'jobs': sorted({leg.split('/')[0] for leg in legs})} + + +def planning_changed(root, base, head): + for path in PLANNING: + try: + if git(root, 'show', f'{base}:{path}') != git(root, 'show', f'{head}:{path}'): + return True + except subprocess.CalledProcessError: + return True + return False + + +def changed_paths(root, base, head): + out = git(root, 'diff', '--name-only', '--no-renames', '-z', base, head) + return sorted(path for path in out.decode().split('\0') if path) + + +def prepare(root, env, event): + name = env['GITHUB_EVENT_NAME'] + source = git(root, 'rev-parse', 'HEAD').decode().strip() + if not SHA.fullmatch(source) or source != env['GITHUB_SHA']: + raise PlanError('the checkout is not GITHUB_SHA') + policy, digest = read_policy(root) + if name in ('schedule', 'workflow_dispatch'): + return build_plan(policy, digest, name, source, None, [], {'event:' + name}) + if name == 'pull_request': + parents = git(root, 'rev-list', '--parents', '-n', '1', source).decode().split() + if len(parents) != 3 or parents[2] != event['pull_request']['head']['sha']: + raise PlanError('the checkout is not the merge of the pull request head') + base = parents[1] + elif name == 'push': + base = event.get('before') or '' + ancestor = SHA.fullmatch(base) and subprocess.run( + ['git', 'merge-base', '--is-ancestor', base, source], cwd=root, capture_output=True).returncode == 0 + if not ancestor or event.get('forced') or base == '0' * 40: + return build_plan(policy, digest, name, source, None, [], {'push-base-untrusted'}) + else: + raise PlanError('unsupported event: ' + name) + reasons = {'planning-changed'} if planning_changed(root, base, source) else set() + return build_plan(policy, digest, name, source, base, changed_paths(root, base, source), reasons) + + +def anchor(workspace): + return f'{workspace} -> {os.path.relpath(TARGET, workspace)}' + + +def outputs(policy, plan): + """GITHUB_OUTPUT lines: the plan, one flag per job and a matrix per matrix job.""" + lines = ['plan=' + json.dumps(plan, separators=(',', ':'), sort_keys=True)] + jobs = {} + for leg in policy['legs']: + job, name = leg.split('/') + jobs.setdefault(job, []).append(name) + for job, names in jobs.items(): + lines.append(f'{job}=' + str(job in plan['jobs']).lower()) + if names != ['single']: + include = [{'name': name, 'stage': ' '.join(policy['legs'][f'{job}/{name}']['stages']), + 'cache-workspace': anchor(policy['legs'][f'{job}/{name}']['cache'])} + for name in names if f'{job}/{name}' in plan['legs']] + lines.append(f'{job}-matrix=' + json.dumps({'include': include}, separators=(',', ':'))) + return lines + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument('--base', help='print the plan for the committed diff BASE..HEAD instead') + parser.add_argument('--head', default='HEAD') + args = parser.parse_args(argv) + try: + policy, digest = read_policy() + if args.base: + base, head = (git(ROOT, 'rev-parse', '--verify', rev + '^{commit}').decode().strip() + for rev in (args.base, args.head)) + reasons = {'planning-changed'} if planning_changed(ROOT, base, head) else set() + plan = build_plan(policy, digest, 'local', head, base, changed_paths(ROOT, base, head), reasons) + print(json.dumps(plan, indent=2)) + return + plan = prepare(ROOT, os.environ, json.loads(Path(os.environ['GITHUB_EVENT_PATH']).read_text())) + out = ROOT / 'target' / 'ci-plan' + out.mkdir(parents=True, exist_ok=True) + (out / 'plan.json').write_text(json.dumps(plan, indent=2) + '\n') + with open(os.environ['GITHUB_OUTPUT'], 'a') as handle: + handle.write('\n'.join(outputs(policy, plan)) + '\n') + if os.environ.get('GITHUB_STEP_SUMMARY'): + with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as handle: + handle.write('### CI plan\n\n```json\n' + json.dumps(plan, indent=2) + '\n```\n') + print(f"CI plan: {plan['profile']}, {len(plan['legs'])} of {len(policy['legs'])} legs") + except (OSError, ValueError, KeyError, TypeError, subprocess.CalledProcessError, PlanError) as error: + raise SystemExit(f'CI planning failed ({error}); no reduced coverage is authorized') + + +if __name__ == '__main__': + main() diff --git a/scripts/tests/test_ci_plan.py b/scripts/tests/test_ci_plan.py new file mode 100644 index 0000000..c18a32a --- /dev/null +++ b/scripts/tests/test_ci_plan.py @@ -0,0 +1,278 @@ +import importlib.util +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import ci_plan +from upstream_dependencies import ADAPTERS, ROOT +spec = importlib.util.spec_from_file_location('validation', ROOT / 'scripts' / 'validate-upstream.py') +validation = importlib.util.module_from_spec(spec) +spec.loader.exec_module(validation) + +POLICY, DIGEST = ci_plan.read_policy() +ALL = list(POLICY['legs']) +PTY = ['rust-clippy/root', 'rust-clippy/adapters', 'rust-clippy/codex-adapters', 'rust-unit/codex-runtime', + 'rust-unit/pty', 'rust-integration/single', 'rust-macos/single'] + + +def plan(*paths): + return ci_plan.build_plan(POLICY, DIGEST, 'local', 'f' * 40, 'e' * 40, list(paths), set()) + + +def ordered(*legs): + return [leg for leg in ALL if leg in legs] + + +class SelectionTests(unittest.TestCase): + def test_documentation_selects_no_leg(self): + docs = plan('docs/guide.md', 'docs/ko/guide.md', 'docs-site/src/index.md', 'README.md', 'README.ko.md', + 'LICENSE', 'NOTICE', '.env.example', '.codex/config.toml') + self.assertEqual((docs['profile'], docs['legs'], docs['jobs']), ('affected', [], [])) + + def test_pty_selects_every_leg_that_compiles_it(self): + self.assertEqual(plan('crates/pty/src/lib.rs')['legs'], PTY) + + def test_components_follow_the_crate_graph(self): + rows = { + 'crates/domain/src/lib.rs': ordered('rust-clippy/root', 'rust-clippy/adapters', 'rust-clippy/codex-adapters', + 'rust-unit/patch', 'rust-unit/codex-runtime', 'rust-integration/single'), + 'crates/runner/src/lib.rs': ordered('rust-clippy/root', 'rust-clippy/codex-adapters', + 'rust-unit/codex-runtime', 'rust-integration/single'), + 'crates/server/src/main.rs': ['rust-clippy/root', 'rust-integration/single'], + 'crates/store/src/lib.rs': ['rust-clippy/root', 'rust-integration/single'], + 'crates/linux-sandbox-protocol/src/lib.rs': ordered( + 'rust-clippy/root', 'rust-clippy/codex-adapters', 'rust-unit/codex-runtime', 'rust-unit/linux-sandbox-protocol', + 'rust-unit/linux-sandbox', 'rust-linux-isolation/single', 'rust-integration/single'), + 'crates/patch/src/lib.rs': ['rust-clippy/adapters', 'rust-unit/patch', 'rust-integration/single'], + 'crates/codex-runtime/src/lib.rs': ['rust-clippy/codex-adapters', 'rust-unit/codex-runtime', + 'rust-integration/single'], + 'crates/file-system/src/lib.rs': [leg.replace('unit/pty', 'unit/file-system') for leg in PTY], + 'crates/linux-sandbox/src/main.rs': ['rust-clippy/codex-adapters', 'rust-unit/linux-sandbox', + 'rust-linux-isolation/single', 'rust-integration/single'], + } + rows['deploy/Dockerfile'] = rows['crates/runner/src/lib.rs'] + rows['tests/e2e/flow.rs'] = rows['crates/server/src/main.rs'] + for path, legs in rows.items(): + with self.subTest(path=path): + self.assertEqual(plan(path)['legs'], legs) + + def test_build_inputs_ci_files_and_unknown_paths_run_everything(self): + for path in ('Cargo.toml', 'Cargo.lock', 'crates/patch/Cargo.toml', 'crates/pty/Cargo.lock', + 'rust-toolchain.toml', 'crates/pty/.cargo/config.toml', 'third_party/codex', '.gitmodules', + '.gitignore', '.github/workflows/docs.yml', 'scripts/check_docs.py', 'docs/upstream-lock.md', + 'Makefile'): + with self.subTest(path=path): + full = plan('docs/guide.md', path) + self.assertEqual((full['profile'], full['legs']), ('full', ALL)) + self.assertIn('unclassified:Makefile', plan('Makefile')['reasons']) + + def test_empty_diff_runs_everything(self): + self.assertEqual(plan()['reasons'], ['empty-diff']) + + def test_globs_respect_segments(self): + self.assertTrue(ci_plan.matches(['**/Cargo.lock'], 'Cargo.lock')) + self.assertTrue(ci_plan.matches(['docs/**'], 'docs/ko/a.md')) + self.assertFalse(ci_plan.matches(['README.md'], 'crates/pty/README.md')) + self.assertFalse(ci_plan.matches(['crates/*/src'], 'crates/a/b/src')) + + +class Repo: + """A throwaway repository holding the real planning inputs.""" + + def __init__(self, root): + self.root = Path(root) + self.git('init', '-q', '-b', 'main') + for path in ci_plan.PLANNING: + self.write(path, (ROOT / path).read_text()) + self.write('docs/guide.md', 'guide\n') + self.base = self.commit('base') + + def git(self, *args): + env = dict(os.environ, GIT_AUTHOR_NAME='CI', GIT_AUTHOR_EMAIL='ci@example.invalid', + GIT_COMMITTER_NAME='CI', GIT_COMMITTER_EMAIL='ci@example.invalid') + command = ['git', '-c', 'commit.gpgsign=false', '-c', 'core.hooksPath=/dev/null', *args] + return subprocess.run(command, cwd=self.root, env=env, check=True, capture_output=True).stdout.decode().strip() + + def write(self, path, text): + (self.root / path).parent.mkdir(parents=True, exist_ok=True) + (self.root / path).write_text(text) + + def commit(self, message): + self.git('add', '-A') + self.git('commit', '-q', '--allow-empty', '-m', message) + return self.git('rev-parse', 'HEAD') + + def pull_request(self, path, text='changed\n'): + self.git('checkout', '-q', '-b', 'topic', self.base) + self.write(path, text) + head = self.commit('head') + self.git('checkout', '-q', 'main') + self.git('merge', '-q', '--no-ff', '-m', 'merge', 'topic') + return head, self.git('rev-parse', 'HEAD') + + def prepare(self, event, name, sha=None): + env = {'GITHUB_EVENT_NAME': name, 'GITHUB_SHA': sha or self.git('rev-parse', 'HEAD')} + return ci_plan.prepare(self.root, env, event) + + +class EventTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.repo = Repo(self.tmp.name) + + def tearDown(self): + self.tmp.cleanup() + + def test_pull_request_plans_its_merge_against_the_base(self): + head, merge = self.repo.pull_request('crates/pty/src/lib.rs') + result = self.repo.prepare({'pull_request': {'head': {'sha': head}}}, 'pull_request') + self.assertEqual((result['source_sha'], result['base_sha']), (merge, self.repo.base)) + self.assertEqual((result['profile'], result['legs']), ('affected', PTY)) + self.assertEqual(result['policy_sha256'], DIGEST) + + def test_changed_planning_input_runs_everything(self): + head, _ = self.repo.pull_request(ci_plan.POLICY, (ROOT / ci_plan.POLICY).read_text() + '\n') + result = self.repo.prepare({'pull_request': {'head': {'sha': head}}}, 'pull_request') + self.assertIn('planning-changed', result['reasons']) + self.assertEqual(result['legs'], ALL) + + def test_pull_request_checkout_must_match_the_event(self): + head, merge = self.repo.pull_request('docs/guide.md') + with self.assertRaises(ci_plan.PlanError): + self.repo.prepare({'pull_request': {'head': {'sha': self.repo.base}}}, 'pull_request') + with self.assertRaises(ci_plan.PlanError): + self.repo.prepare({'pull_request': {'head': {'sha': head}}}, 'pull_request', sha=head) + + def test_push_plans_before_to_after(self): + self.repo.write('docs/guide.md', 'more\n') + after = self.repo.commit('docs') + result = self.repo.prepare({'before': self.repo.base}, 'push') + self.assertEqual((result['source_sha'], result['profile'], result['legs']), (after, 'affected', [])) + + def test_push_without_a_trusted_base_runs_everything(self): + self.repo.write('docs/guide.md', 'more\n') + self.repo.commit('docs') + self.repo.git('checkout', '-q', '--orphan', 'other') + unrelated = self.repo.commit('unrelated') + self.repo.git('checkout', '-q', 'main') + for event in ({'before': '0' * 40}, {'before': '1' * 40}, {'before': unrelated}, {}, + {'before': self.repo.base, 'forced': True}): + with self.subTest(event=event): + result = self.repo.prepare(event, 'push') + self.assertEqual((result['reasons'], result['legs']), (['push-base-untrusted'], ALL)) + + def test_scheduled_and_manual_runs_are_full(self): + for name in ('schedule', 'workflow_dispatch'): + with self.subTest(event=name): + result = self.repo.prepare({}, name) + self.assertEqual((result['profile'], result['base_sha'], result['legs']), ('full', None, ALL)) + + def test_other_events_are_refused(self): + with self.assertRaises(ci_plan.PlanError): + self.repo.prepare({}, 'issue_comment') + + +class OutputTests(unittest.TestCase): + def outputs(self, *paths): + lines = ci_plan.outputs(POLICY, plan(*paths)) + return {key: value for key, value in (line.split('=', 1) for line in lines)} + + def test_flags_and_matrices_carry_only_selected_legs(self): + out = self.outputs('crates/pty/src/lib.rs') + self.assertEqual(json.loads(out['plan']), plan('crates/pty/src/lib.rs')) + self.assertEqual({job: out[job] for job in ('rust-clippy', 'rust-unit', 'rust-linux-isolation', + 'rust-integration', 'rust-macos')}, + {'rust-clippy': 'true', 'rust-unit': 'true', 'rust-linux-isolation': 'false', + 'rust-integration': 'true', 'rust-macos': 'true'}) + unit = json.loads(out['rust-unit-matrix'])['include'] + self.assertEqual(unit, [ + {'name': 'codex-runtime', 'stage': 'unit-codex-runtime', + 'cache-workspace': 'crates/codex-runtime -> ../../target/upstream-validation'}, + {'name': 'pty', 'stage': 'unit-pty', 'cache-workspace': 'crates/pty -> ../../target/upstream-validation'}]) + clippy = json.loads(out['rust-clippy-matrix'])['include'] + self.assertEqual(clippy[0], {'name': 'root', 'stage': 'clippy-root dependencies', + 'cache-workspace': '. -> target/upstream-validation'}) + + def test_documentation_selects_no_job(self): + out = self.outputs('docs/guide.md') + self.assertEqual({out[job] for job in ('rust-clippy', 'rust-unit', 'rust-linux-isolation', 'rust-integration', + 'rust-macos')}, {'false'}) + self.assertEqual(json.loads(out['rust-unit-matrix']), {'include': []}) + + +def path_deps(crate): + return set(re.findall(r'path = "\.\./([a-z-]+)"', (ROOT / 'crates' / crate / 'Cargo.toml').read_text())) + + +def closure(crates): + seen, todo = set(), list(crates) + while todo: + crate = todo.pop() + if crate not in seen: + seen.add(crate) + todo += path_deps(crate) + return seen + + +def compiled(stage): + """Crates a validate-upstream stage compiles, read from its cargo commands.""" + if stage == 'dependencies': + return set() + members = re.search(r'members = \[([^\]]*)\]', (ROOT / 'Cargo.toml').read_text()).group(1) + root = {name.split('/')[-1] for name in re.findall(r'"([^"]+)"', members)} + commands = validation.stages()[stage] + if stage == 'integration': + commands = [validation.cargo('build', area) for area in validation.HELPERS] + [validation.cargo('test')] + crates = set() + for command in commands: + manifest = [arg for arg in command if arg.startswith('crates/') and arg.endswith('/Cargo.toml')] + package = command[command.index('-p') + 1] if '-p' in command else None + if manifest: + crates.add(manifest[0].split('/')[1]) + elif package: + crates.add(package.removeprefix('codespace-')) + else: + crates |= root + return closure(crates) + + +class PolicyTests(unittest.TestCase): + def test_stages_cover_the_validation_entry_point(self): + stages = [stage for spec in POLICY['legs'].values() for stage in spec['stages']] + stages += [stage for always in POLICY['always'].values() for stage in always] + self.assertLessEqual(set(stages), set(validation.stages())) + self.assertEqual(set(stages), set(validation.all_stages()) | {'macos-core'}) + + def test_compiles_matches_the_manifest_graph(self): + for leg, spec in POLICY['legs'].items(): + with self.subTest(leg=leg): + expected = set().union(*(compiled(stage) for stage in spec['stages'])) + self.assertEqual(set(spec['compiles']), expected) + self.assertEqual(spec['compiles'], sorted(spec['compiles'])) + + def test_cache_anchor_is_a_workspace_root(self): + roots = {'.'} | {f'crates/{area}' for area in ADAPTERS} + for leg, spec in POLICY['legs'].items(): + with self.subTest(leg=leg): + self.assertIn(spec['cache'], roots) + self.assertTrue((ROOT / spec['cache'] / 'Cargo.lock').is_file()) + + def test_every_tracked_path_is_classified(self): + paths = subprocess.run(['git', 'ls-files', '-z'], cwd=ROOT, check=True, capture_output=True).stdout + _, reasons = ci_plan.classify(POLICY, [path for path in paths.decode().split('\0') if path]) + self.assertEqual([reason for reason in reasons if reason.startswith('unclassified:')], []) + + def test_components_are_the_crates(self): + crates = sorted(path.name for path in (ROOT / 'crates').iterdir() if (path / 'Cargo.toml').is_file()) + self.assertEqual(sorted(POLICY['components']), crates) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/tests/test_ci_results.py b/scripts/tests/test_ci_results.py new file mode 100644 index 0000000..3027ea6 --- /dev/null +++ b/scripts/tests/test_ci_results.py @@ -0,0 +1,122 @@ +import json +from pathlib import Path +import sys +import tempfile +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import check_ci_results as check +import ci_plan + +POLICY, DIGEST = ci_plan.read_policy() +SOURCE = 'a' * 40 +ENV = {'GITHUB_SHA': SOURCE, 'GITHUB_EVENT_NAME': 'pull_request', 'GITHUB_RUN_ATTEMPT': '1'} + + +def affected(*paths): + return ci_plan.build_plan(POLICY, DIGEST, 'pull_request', SOURCE, 'b' * 40, list(paths), set()) + + +def results(plan): + always = {leg.split('/')[0] for leg in POLICY['always']} + return {job: {'result': 'success' if job == 'plan' or job in always or job in plan['jobs'] else 'skipped'} + for job in check.REQUIRED_JOBS} + + +def upload(reports, leg, stages, attempt=1, status='passed', head=SOURCE): + job, name = leg.split('/') + path = reports / f'upstream-{job}-{name}-{attempt}' / 'stage' / 'report.json' + path.parent.mkdir(parents=True) + path.write_text(json.dumps({'status': status, 'scope': stages, 'inputs': {'head': head}, + 'stages': [{'name': stage, 'status': 'passed'} for stage in stages]})) + + +class GateTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.reports = Path(self.tmp.name) + self.plan = affected('crates/pty/src/lib.rs') + planned = dict(POLICY['always'], **{leg: POLICY['legs'][leg]['stages'] for leg in self.plan['legs']}) + for leg, stages in planned.items(): + upload(self.reports, leg, stages) + + def tearDown(self): + self.tmp.cleanup() + + def problems(self, plan=None, jobs=None, env=None, digest=DIGEST): + plan = plan or self.plan + return check.problems(jobs or results(plan), plan, POLICY, digest, env or ENV, self.reports) + + def test_matching_run_passes(self): + self.assertEqual(self.problems(), []) + + def test_full_run_passes(self): + full = ci_plan.build_plan(POLICY, DIGEST, 'schedule', SOURCE, None, [], {'event:schedule'}) + for leg in full['legs']: + if leg not in self.plan['legs']: + upload(self.reports, leg, POLICY['legs'][leg]['stages']) + self.assertEqual(self.problems(full, env=dict(ENV, GITHUB_EVENT_NAME='schedule')), []) + + def test_job_results_must_follow_the_plan(self): + for job, result in (('rust-unit', 'failure'), ('policy-scan', 'skipped'), ('plan', 'failure'), + ('rust-linux-isolation', 'success'), ('rust-macos', 'cancelled')): + with self.subTest(job=job, result=result): + jobs = results(self.plan) + jobs[job] = {'result': result} + self.assertTrue(self.problems(jobs=jobs)) + + def test_gate_must_see_exactly_the_required_jobs(self): + jobs = results(self.plan) + del jobs['rust-macos'] + self.assertTrue(self.problems(jobs=jobs)) + self.assertTrue(self.problems(jobs=dict(results(self.plan), extra={'result': 'success'}))) + + def test_each_planned_leg_needs_its_passing_report(self): + for leg, change in (('rust-unit/pty', 'missing'), ('rust-format/single', 'missing'), + ('rust-unit/pty', 'incomplete'), ('rust-macos/single', 'scope'), + ('rust-integration/single', 'head')): + with self.subTest(leg=leg, change=change), tempfile.TemporaryDirectory() as tmp: + reports = Path(tmp) + planned = dict(POLICY['always'], **{name: POLICY['legs'][name]['stages'] for name in self.plan['legs']}) + for name, stages in planned.items(): + if name != leg: + upload(reports, name, stages) + elif change == 'incomplete': + upload(reports, name, stages, status='incomplete') + elif change == 'scope': + upload(reports, name, stages[:1]) + elif change == 'head': + upload(reports, name, stages, head='c' * 40) + found = check.problems(results(self.plan), self.plan, POLICY, DIGEST, ENV, reports) + self.assertTrue(any(line.startswith(leg) for line in found), found) + + def test_unplanned_report_fails(self): + upload(self.reports, 'rust-linux-isolation/single', ['linux-isolation']) + self.assertEqual(self.problems(), ['rust-linux-isolation/single: uploaded a report although it was not planned']) + + def test_latest_attempt_up_to_the_current_one_counts(self): + upload(self.reports, 'rust-unit/pty', ['unit-pty'], attempt=2, status='failed') + self.assertTrue(self.problems(env=dict(ENV, GITHUB_RUN_ATTEMPT='2'))) + self.assertEqual(self.problems(), []) + upload(self.reports, 'rust-unit/pty', ['unit-pty'], attempt=3) + self.assertEqual(self.problems(env=dict(ENV, GITHUB_RUN_ATTEMPT='3')), []) + + def test_plan_must_be_bound_to_the_run(self): + schedule = ci_plan.build_plan(POLICY, DIGEST, 'schedule', SOURCE, None, [], set()) + cases = { + 'source': (self.plan, dict(ENV, GITHUB_SHA='d' * 40), DIGEST), + 'event': (self.plan, dict(ENV, GITHUB_EVENT_NAME='push'), DIGEST), + 'policy': (self.plan, ENV, '0' * 64), + 'schedule narrowed': (dict(schedule, legs=self.plan['legs'], jobs=self.plan['jobs']), + dict(ENV, GITHUB_EVENT_NAME='schedule'), DIGEST), + 'unknown leg': (dict(self.plan, legs=self.plan['legs'] + ['rust-unit/other']), ENV, DIGEST), + 'jobs': (dict(self.plan, jobs=['rust-unit']), ENV, DIGEST), + } + for label, (plan, env, digest) in cases.items(): + with self.subTest(case=label): + self.assertEqual(self.problems(plan, env=env, digest=digest), + ['the plan is not bound to this source, event and policy']) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/tests/test_ci_workflow.py b/scripts/tests/test_ci_workflow.py index 02faa8a..9b4b594 100644 --- a/scripts/tests/test_ci_workflow.py +++ b/scripts/tests/test_ci_workflow.py @@ -1,20 +1,18 @@ -import importlib.util -import os from pathlib import Path import re import sys import unittest sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from upstream_dependencies import ADAPTERS, ROOT -spec = importlib.util.spec_from_file_location('validation', ROOT / 'scripts' / 'validate-upstream.py') -validation = importlib.util.module_from_spec(spec) -spec.loader.exec_module(validation) +import check_ci_results +import ci_plan +from upstream_dependencies import ROOT WORKFLOW = (ROOT / '.github' / 'workflows' / 'ci.yml').read_text() +POLICY, _ = ci_plan.read_policy() RUST_CACHE = 'uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2' -COMPILE_JOBS = {'rust-clippy', 'rust-unit', 'rust-linux-isolation', 'rust-integration', 'rust-macos'} -SHARED_TARGET = 'target/upstream-validation' +COMPILE_JOBS = {leg.split('/')[0] for leg in POLICY['legs']} +MATRIX_JOBS = {leg.split('/')[0] for leg in POLICY['legs'] if not leg.endswith('/single')} def jobs(text): @@ -23,16 +21,59 @@ def jobs(text): return dict(zip(parts[1::2], parts[2::2])) -def legs(job): - return re.findall(r'- name: (\S+)\n +stage: (.+)\n +cache-workspace: "([^"]+)"', job) +def run_line(stages): + return 'python3 scripts/validate-upstream.py ' + ' '.join(stages) + ' --output' -class RustCacheTests(unittest.TestCase): +class WorkflowTests(unittest.TestCase): def setUp(self): self.jobs = jobs(WORKFLOW) - def test_job_split(self): - self.assertEqual(set(self.jobs), COMPILE_JOBS | {'policy-scan', 'rust-format', 'rust'}) + def test_jobs_are_the_gate_requirements(self): + self.assertEqual(set(self.jobs), check_ci_results.REQUIRED_JOBS | {'rust'}) + always = {leg.split('/')[0] for leg in POLICY['always']} + self.assertEqual(check_ci_results.REQUIRED_JOBS, {'plan'} | always | COMPILE_JOBS) + + def test_gate_is_unskippable_and_checks_every_job(self): + gate = self.jobs['rust'] + self.assertIn(' if: ${{ always() }}\n', gate) + needs = re.search(r' needs:\n((?: - [a-z-]+\n)+)', gate).group(1) + self.assertEqual(set(re.findall(r'- ([a-z-]+)', needs)), check_ci_results.REQUIRED_JOBS) + self.assertIn('CI_RESULTS: ${{ toJSON(needs) }}', gate) + self.assertIn('CI_PLAN: ${{ needs.plan.outputs.plan }}', gate) + self.assertIn('scripts/check_ci_results.py --reports target/ci-reports', gate) + self.assertIn('pattern: upstream-*', gate) + + def test_plan_exports_every_flag_and_matrix(self): + plan = self.jobs['plan'] + self.assertIn('fetch-depth: 0', plan) + self.assertIn('run: python3 -B scripts/ci_plan.py', plan) + outputs = ['plan', *sorted(COMPILE_JOBS), *(job + '-matrix' for job in sorted(MATRIX_JOBS))] + for output in outputs: + self.assertIn(f' {output}: ${{{{ steps.plan.outputs.{output} }}}}\n', plan) + + def test_compile_jobs_follow_the_plan(self): + for job in COMPILE_JOBS: + text = self.jobs[job] + with self.subTest(job=job): + self.assertIn(' needs: plan\n', text) + self.assertIn(f" if: ${{{{ needs.plan.outputs.{job} == 'true' }}}}\n", text) + if job in MATRIX_JOBS: + self.assertIn(f'matrix: ${{{{ fromJSON(needs.plan.outputs.{job}-matrix) }}}}', text) + self.assertIn('validate-upstream.py ${{ matrix.stage }} --output', text) + self.assertIn('key: ${{ matrix.name }}', text) + self.assertIn('workspaces: ${{ matrix.cache-workspace }}', text) + else: + spec = POLICY['legs'][job + '/single'] + self.assertIn(run_line(spec['stages']), text) + self.assertIn(f'workspaces: "{ci_plan.anchor(spec["cache"])}"', text) + + def test_always_on_jobs_run_their_stages(self): + for leg, stages in POLICY['always'].items(): + text = self.jobs[leg.split('/')[0]] + with self.subTest(leg=leg): + self.assertNotIn('needs:', text) + self.assertIn(run_line(stages), text) def test_one_pinned_main_only_cache_per_compile_job(self): for name, job in self.jobs.items(): @@ -49,42 +90,15 @@ def test_no_shared_cargo_cache_or_checkout_walk(self): self.assertNotIn('actions/cache@', WORKFLOW) self.assertIsNone(re.search(r"hashFiles\([^)]*\*\*", WORKFLOW)) - def test_build_settings(self): + def test_triggers_permissions_and_concurrency(self): header = WORKFLOW.split('\njobs:\n', 1)[0] + self.assertIn(" schedule:\n - cron: '43 19 * * *'\n workflow_dispatch:\n", header) + self.assertIn('permissions:\n contents: read\n', header) + self.assertIn("format('pr-{0}', github.event.pull_request.number) || format('run-{0}', github.run_id)", header) + self.assertIn("cancel-in-progress: ${{ github.event_name == 'pull_request' }}", header) self.assertIn('\n CARGO_INCREMENTAL: "0"\n', header) self.assertIn('\n CARGO_PROFILE_DEV_DEBUG: "0"\n', header) - def test_matrix_legs_are_keyed_and_anchored(self): - for name in ('rust-clippy', 'rust-unit'): - job = self.jobs[name] - with self.subTest(job=name): - self.assertIn('key: ${{ matrix.name }}', job) - self.assertIn('workspaces: ${{ matrix.cache-workspace }}', job) - matrix = job.split('\n steps:\n', 1)[0] - self.assertEqual(len(legs(job)), matrix.count('- name: ')) - - def test_anchor_is_one_workspace_on_the_shared_target(self): - anchors = re.findall(r'(?:workspaces|cache-workspace): "([^"]+)"', WORKFLOW) - self.assertEqual(len(anchors), 3 + 6 + 3) - self.assertIn("ROOT / 'target' / 'upstream-validation'", (ROOT / 'scripts' / 'validate-upstream.py').read_text()) - roots = {'.'} | {f'crates/{area}' for area in ADAPTERS} - for anchor in anchors: - with self.subTest(anchor=anchor): - workspace, target = (part.strip() for part in anchor.split('->')) - self.assertIn(workspace, roots) - self.assertTrue((ROOT / workspace / 'Cargo.lock').is_file()) - self.assertEqual(os.path.normpath(os.path.join(workspace, target)), SHARED_TARGET) - - def test_legs_run_known_stages(self): - known = set(validation.stages()) - units = legs(self.jobs['rust-unit']) - self.assertEqual({stage for _, stage, _ in units}, - {f'unit-{area}' for area in ADAPTERS} | {'unit-linux-sandbox-protocol'}) - for name, stage, _ in units: - self.assertEqual(stage, f'unit-{name}') - for _, stage, _ in legs(self.jobs['rust-clippy']): - self.assertLessEqual(set(stage.split()), known) - if __name__ == '__main__': unittest.main() diff --git a/scripts/validate-upstream.py b/scripts/validate-upstream.py index 37f2bb0..1a64299 100644 --- a/scripts/validate-upstream.py +++ b/scripts/validate-upstream.py @@ -91,7 +91,7 @@ def run(selected, output): before = fingerprint() report['inputs'] = before report['rust'] = capture('rustc', '-vV') - report['build_environment'] = {k: os.environ.get(k) for k in ('DEVELOPER_DIR', 'SDKROOT', 'RUSTFLAGS', 'CARGO_ENCODED_RUSTFLAGS', 'RUSTUP_TOOLCHAIN')} + report['build_environment'] = {k: os.environ.get(k) for k in ('DEVELOPER_DIR', 'SDKROOT', 'RUSTFLAGS', 'CARGO_ENCODED_RUSTFLAGS', 'RUSTUP_TOOLCHAIN', 'CARGO_INCREMENTAL', 'CARGO_PROFILE_DEV_DEBUG')} if before.get('codex_sha') is None and any(s not in ('policy', 'python') for s in selected): raise RuntimeError('Codex submodule is not initialized') target = next(line.split(': ', 1)[1] for line in report['rust'].splitlines() if line.startswith('host: '))