diff --git a/.github/actions/orchestrator-free-sidecar/action.yml b/.github/actions/orchestrator-free-sidecar/action.yml index c6a6cc3919..edddfe1bc3 100644 --- a/.github/actions/orchestrator-free-sidecar/action.yml +++ b/.github/actions/orchestrator-free-sidecar/action.yml @@ -23,16 +23,9 @@ runs: ref: ${{ github.action_ref }} path: ${{ runner.temp }}/cwl-control-plane persist-credentials: false - - name: Set up lock-compatible sidecar Python - id: sidecar_python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - update-environment: false - name: Provision contextual-orchestrator orchestrator/free shell: bash --noprofile --norc -e -o pipefail {0} env: - SIDECAR_PYTHON: ${{ steps.sidecar_python.outputs.python-path }} CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR: ${{ inputs.require_zdr }} ORCHESTRATOR_CATALOG_LIMIT: ${{ inputs.catalog_limit }} ORCHESTRATOR_CATALOG_ACCOUNT_CAP: ${{ inputs.catalog_account_cap }} diff --git a/.github/workflows/agent-mention-noema-dispatch.yml b/.github/workflows/agent-mention-noema-dispatch.yml index c9514a47a1..ad8abc7b25 100644 --- a/.github/workflows/agent-mention-noema-dispatch.yml +++ b/.github/workflows/agent-mention-noema-dispatch.yml @@ -28,7 +28,7 @@ permissions: jobs: validate-and-forward: if: github.repository == 'ContextualWisdomLab/.github' - runs-on: ${{ github.repository == 'ContextualWisdomLab/.github' && endsWith(github.workflow_ref, '@refs/heads/main') && fromJSON('{"group":"CWL MCP remediation","labels":["self-hosted","linux","x64"]}') || fromJSON('["ubuntu-24.04"]') }} + runs-on: ubuntu-24.04 timeout-minutes: 5 permissions: actions: read diff --git a/.github/workflows/agent-mention-opencode-dispatch.yml b/.github/workflows/agent-mention-opencode-dispatch.yml index 3b21667832..05461c9551 100644 --- a/.github/workflows/agent-mention-opencode-dispatch.yml +++ b/.github/workflows/agent-mention-opencode-dispatch.yml @@ -28,7 +28,7 @@ permissions: jobs: validate-and-forward: if: github.repository == 'ContextualWisdomLab/.github' - runs-on: ${{ github.repository == 'ContextualWisdomLab/.github' && endsWith(github.workflow_ref, '@refs/heads/main') && fromJSON('{"group":"CWL MCP remediation","labels":["self-hosted","linux","x64"]}') || fromJSON('["ubuntu-24.04"]') }} + runs-on: ubuntu-24.04 timeout-minutes: 5 permissions: actions: read diff --git a/.github/workflows/agent-mention-router-quality-ci.yml b/.github/workflows/agent-mention-router-quality-ci.yml index 9c36a89119..4e943d74f2 100644 --- a/.github/workflows/agent-mention-router-quality-ci.yml +++ b/.github/workflows/agent-mention-router-quality-ci.yml @@ -14,6 +14,7 @@ on: - "tests/test_agent_mention_*.py" - "tests/test_pr_review_fix_scheduler_coverage.py" - "requirements-opencode-review-ci-hashes.txt" + - "requirements-noema-document-ci-hashes.txt" push: branches: [main] paths: @@ -27,6 +28,7 @@ on: - "tests/test_agent_mention_*.py" - "tests/test_pr_review_fix_scheduler_coverage.py" - "requirements-opencode-review-ci-hashes.txt" + - "requirements-noema-document-ci-hashes.txt" concurrency: group: agent-mention-router-quality-${{ github.repository }}-${{ github.event.pull_request.number || github.ref }} @@ -85,11 +87,14 @@ jobs: with: python-version: "3.14" cache: pip - cache-dependency-path: requirements-opencode-review-ci-hashes.txt + cache-dependency-path: | + requirements-opencode-review-ci-hashes.txt + requirements-noema-document-ci-hashes.txt - name: Install exact hash-locked tooling run: >- python -m pip install --disable-pip-version-check --require-hashes -r requirements-opencode-review-ci-hashes.txt + -r requirements-noema-document-ci-hashes.txt - name: Run complete repository suite and bounded branch coverage shell: bash --noprofile --norc -e -o pipefail {0} run: | diff --git a/.github/workflows/agent-mention-router.yml b/.github/workflows/agent-mention-router.yml index 8b1bb88481..63ec8e3231 100644 --- a/.github/workflows/agent-mention-router.yml +++ b/.github/workflows/agent-mention-router.yml @@ -29,9 +29,7 @@ jobs: concurrency: group: review-agent-mention-router-local-${{ github.repository }}-${{ github.event.issue.number || github.run_id }} cancel-in-progress: true - runs-on: - group: CWL central control - labels: [self-hosted, linux, x64] + runs-on: ubuntu-24.04 timeout-minutes: 5 permissions: actions: read @@ -76,9 +74,7 @@ jobs: concurrency: group: review-agent-mention-router-sweep-${{ github.repository }} cancel-in-progress: false - runs-on: - group: CWL central control - labels: [self-hosted, linux, x64] + runs-on: ubuntu-24.04 timeout-minutes: 15 permissions: actions: read diff --git a/.github/workflows/agent-review-runtime-quality-ci.yml b/.github/workflows/agent-review-runtime-quality-ci.yml index 2b4589a46e..a601e25522 100644 --- a/.github/workflows/agent-review-runtime-quality-ci.yml +++ b/.github/workflows/agent-review-runtime-quality-ci.yml @@ -433,33 +433,7 @@ jobs: --cov=scripts.ci.zdr_policy \ --cov=scripts.ci.contextual_orchestrator_review_policy \ --cov-branch \ - --cov-fail-under=100 \ - tests/test_pr_review_conflict_scope.py \ - tests/test_zdr_policy.py \ - tests/test_contextual_orchestrator_review_policy.py \ - tests/test_contextual_orchestrator_review_sidecar_contract.py \ - tests/test_hourly_review_repair_callers.py \ - tests/test_github_hourly_conflict_repair.py \ - tests/test_hourly_scheduler_runtime_budget.py \ - tests/test_pr_review_conflict_scope_control_files.py \ - tests/test_hourly_autofix_context_quality_gate.py \ - tests/test_pr_review_conflict_scope_git_executable.py \ - tests/test_pr_review_conflict_scope_ignored_paths.py \ - tests/test_pr_review_conflict_scope_symlink_targets.py \ - tests/test_pr_review_fix_hourly_contract.py \ - tests/test_pr_review_fix_scheduler.py \ - tests/test_pr_review_fix_scheduler_source_pin.py \ - tests/test_pr_review_autofix_context_head_binding.py \ - tests/test_pr_review_autofix_nvidia_nim_contract.py \ - tests/test_pr_review_autofix_writer_security_contract.py \ - tests/test_pr_review_autofix_context_failed_checks.py \ - tests/test_pr_review_autofix_context_import_fallback.py \ - tests/test_contextual_orchestrator_free_credential_admission.py \ - tests/test_contextual_orchestrator_bytez_catalog_integration.py \ - tests/test_contextual_orchestrator_review_live_discovery_contract.py \ - tests/test_contextual_orchestrator_review_runtime_preflight.py \ - tests/test_repository_branch_coverage_review_schedulers.py \ - tests/test_repository_branch_coverage_reporting_edges.py + --cov-fail-under=100 python -m interrogate --fail-under 100 \ scripts/ci/pr_review_conflict_scope.py \ scripts/ci/pr_review_autofix_context.py \ diff --git a/.github/workflows/codeql-pr.yml b/.github/workflows/codeql-pr.yml index 8880d63663..cc13d2d87e 100644 --- a/.github/workflows/codeql-pr.yml +++ b/.github/workflows/codeql-pr.yml @@ -6,14 +6,11 @@ # runner, then one coordinator POSTs repository_dispatch to # codeql-scan-dispatch.yml (native, unrestricted, in # ContextualWisdomLab/.github) with the remaining language matrix. The -# handler publishes a base/run/source-bound codeql-dispatch receipt and reruns -# only that exact failed job. On rerun the shard reads the terminal status once. Design: +# handler publishes codeql-dispatch/ and reruns only that exact +# failed job. On rerun the shard reads the terminal status once. Design: # docs/adr/0025-codeql-required-workflow-dispatch-architecture.md. The # merge-preview scan (analyze-merge) is required nowhere (PR #1766) and was # dropped, not migrated. -# Only the trusted main workflow uses the control group. PR-authored workflow -# revisions retain hosted execution; organization group restrictions also enforce -# the exact main path. Heavy scans stay on the dedicated CodeQL runner. name: CodeQL PR on: @@ -59,8 +56,7 @@ jobs: detect-languages: name: Detect CodeQL languages if: github.event.action != 'closed' - runs-on: ${{ fromJSON(github.workflow_ref == 'ContextualWisdomLab/.github/.github/workflows/codeql-pr.yml@refs/heads/main' && '{"group":"CWL central control","labels":["self-hosted","linux","x64"]}' || '"ubuntu-24.04"') }} - timeout-minutes: 5 + runs-on: ubuntu-24.04 permissions: contents: read pull-requests: read @@ -152,11 +148,8 @@ jobs: # dependency exactly; the only case where it's genuinely skipped is a # closed PR, where this job being implicitly skipped too is fine because # closed PRs need no required check. - runs-on: ${{ fromJSON(github.workflow_ref == 'ContextualWisdomLab/.github/.github/workflows/codeql-pr.yml@refs/heads/main' && '{"group":"CWL central control","labels":["self-hosted","linux","x64"]}' || '"ubuntu-24.04"') }} - # Verdict reads have exceeded five minutes; retain the ten-minute control budget. - timeout-minutes: 10 + runs-on: ubuntu-24.04 permissions: - actions: read contents: read id-token: write pull-requests: read @@ -167,7 +160,7 @@ jobs: steps: - name: Read current-head CodeQL dispatch verdict # Shards never dispatch. They re-check the live head, consume an - # authenticated base/run/source-bound CodeQL verdict when one exists, + # authenticated codeql-dispatch/ verdict when one exists, # and otherwise fail pending so the runner is released. One # coordinator job POSTs the remaining language matrix after every # shard has a job id. @@ -186,34 +179,21 @@ jobs: live_pr="$(gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}")" live_head="$(printf '%s' "$live_pr" | jq -r '.head.sha // empty')" live_base="$(printf '%s' "$live_pr" | jq -r '.base.sha // empty')" - live_merge="$(printf '%s' "$live_pr" | jq -r '.merge_commit_sha // empty')" live_state="$(printf '%s' "$live_pr" | jq -r 'if (.state | type) == "string" then .state else empty end')" - if ! [[ "$PR_HEAD_SHA" =~ ^[0-9a-fA-F]{40}$ && "$live_head" =~ ^[0-9a-fA-F]{40}$ ]] || [[ "$live_state" != "open" && "$live_state" != "closed" ]]; then + if [ -z "$live_head" ] || [ -z "$live_state" ]; then echo "::error::Could not validate live pull request state before CodeQL dispatch." exit 1 fi if [ "$live_state" = "closed" ]; then echo "PR is closed on the live exact head; a current-head CodeQL scan is not requested." - echo "verdict=obsolete" >>"$GITHUB_OUTPUT" exit 0 fi if [ "${live_head,,}" != "${PR_HEAD_SHA,,}" ]; then - # A lagging API read or diverged history must not retire the current scan. - comparison="$(gh api "repos/${TARGET_REPOSITORY}/compare/${PR_HEAD_SHA}...${live_head}")" - if ! printf '%s' "$comparison" | jq -e ' - .status == "ahead" and .behind_by == 0 and - ((.ahead_by | type) == "number") and .ahead_by >= 1 - ' >/dev/null; then - echo "::error::Live head does not prove this CodeQL shard was superseded." - exit 1 - fi echo "Pull request head moved on the live open PR; a fresh dispatch will fire for the current head." - echo "verdict=obsolete" >>"$GITHUB_OUTPUT" exit 0 fi - if ! [[ "$live_base" =~ ^[0-9a-fA-F]{40}$ ]] || - ! [[ "$live_merge" =~ ^[0-9a-fA-F]{40}$ ]]; then - echo "::error::Could not validate live pull request base/source SHA before CodeQL verdict read." + if ! [[ "$live_base" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "::error::Could not validate live pull request base SHA before CodeQL verdict read." exit 1 fi if ! [[ "$REQUIRED_RUN_ID" =~ ^[1-9][0-9]*$ ]]; then @@ -222,17 +202,13 @@ jobs: fi statuses="$(gh api "repos/${TARGET_REPOSITORY}/commits/${PR_HEAD_SHA}/statuses")" - expected_context="codeql-dispatch/${LANGUAGE}/${live_base}" - expected_description="cwl1;h=${PR_HEAD_SHA};w=codeql-scan-dispatch;r=${REQUIRED_RUN_ID};s=${live_merge}" - verdict_state="$(printf '%s' "$statuses" | jq -r --arg ctx "$expected_context" --arg description "$expected_description" ' + verdict_state="$(printf '%s' "$statuses" | jq -r --arg ctx "codeql-dispatch/${LANGUAGE}" ' [ .[] | select(.context == $ctx) - | select(.description == $description) | select( (.creator.login // "" | ascii_downcase) as $creator | $creator == "opencode-agent" or $creator == "opencode-agent[bot]" - or $creator == "cwl-noema-review" or $creator == "cwl-noema-review[bot]" ) ] | first // {} | .state // empty @@ -245,15 +221,9 @@ jobs: ;; esac - expected_title="CodeQL Scan Dispatch ${TARGET_REPOSITORY}#${PR_NUMBER}@${PR_HEAD_SHA}/${live_base}/${REQUIRED_RUN_ID}/${live_merge}" + expected_title="CodeQL Scan Dispatch ${TARGET_REPOSITORY}#${PR_NUMBER}@${PR_HEAD_SHA}/${live_base}/${REQUIRED_RUN_ID}" expected_job="CodeQL dispatch scan (${LANGUAGE})" - # A dispatch bound to this required run cannot predate its creation. - required_created_at="$(gh api "repos/${TARGET_REPOSITORY}/actions/runs/${REQUIRED_RUN_ID}" --jq .created_at)" - if ! [[ "$required_created_at" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]; then - echo "::error::Could not validate required run creation time before CodeQL verdict lookup." - exit 1 - fi - runs_json="$(gh api --method GET --paginate -f per_page=100 -f event=repository_dispatch -f created=">=${required_created_at}" "repos/ContextualWisdomLab/.github/actions/workflows/codeql-scan-dispatch.yml/runs" | jq -s .)" + runs_json="$(gh api --paginate --slurp "repos/ContextualWisdomLab/.github/actions/workflows/codeql-scan-dispatch.yml/runs")" run_id="$(printf '%s' "$runs_json" | jq -r --arg title "$expected_title" --arg path ".github/workflows/codeql-scan-dispatch.yml" ' [ .[] | .workflow_runs[] @@ -266,7 +236,7 @@ jobs: | .id // empty ')" if [[ "$run_id" =~ ^[1-9][0-9]*$ ]]; then - jobs_json="$(gh api --paginate "repos/ContextualWisdomLab/.github/actions/runs/${run_id}/jobs" | jq -s .)" + jobs_json="$(gh api --paginate --slurp "repos/ContextualWisdomLab/.github/actions/runs/${run_id}/jobs")" dispatch_job="$(printf '%s' "$jobs_json" | jq -c --arg name "$expected_job" ' [.[] | .jobs[] | select(.name == $name)] | if length == 1 then .[0] else empty end @@ -275,20 +245,11 @@ jobs: gate_conclusion="$(printf '%s' "$dispatch_job" | jq -r ' (.steps[]? | select(.name == "Enforce CodeQL Medium+ SARIF gate") | .conclusion) // empty ')" - ghas_identity_conclusion="$(printf '%s' "$dispatch_job" | jq -r ' - (.steps[]? | select(.name == "Verify GHAS base/head CodeQL configuration identity") | .conclusion) // empty - ')" - sarif_upload_conclusion="$(printf '%s' "$dispatch_job" | jq -r ' - (.steps[]? | select(.name == "Preserve CodeQL SARIF evidence") | .conclusion) // empty - ')" case "$gate_conclusion" in success) - if [ "$ghas_identity_conclusion" = "success" ] && - [ "$sarif_upload_conclusion" = "success" ]; then - echo "verdict=success" >>"$GITHUB_OUTPUT" - echo "Found completed CodeQL dispatch proof for ${LANGUAGE}: gate, GHAS identity, and SARIF evidence succeeded." - exit 0 - fi + echo "verdict=success" >>"$GITHUB_OUTPUT" + echo "Found completed CodeQL dispatch scan gate for ${LANGUAGE}: success." + exit 0 ;; failure|cancelled|skipped) echo "verdict=failure" >>"$GITHUB_OUTPUT" @@ -308,7 +269,7 @@ jobs: fi if [ "$RUN_ATTEMPT" != "1" ]; then - echo "::error::Exact CodeQL job was rerun without an authenticated terminal verdict; GHAS identity and preserved SARIF are required for authenticated terminal proof." + echo "::error::Exact CodeQL job was rerun without an authenticated terminal verdict." exit 1 fi echo "verdict=pending" >>"$GITHUB_OUTPUT" @@ -326,9 +287,6 @@ jobs: exit 1 fi case "$VERDICT_STATE" in - obsolete) - echo "Closed or superseded PR shard; no scan verdict is asserted." - ;; success) echo "Current-head CodeQL dispatch verdict for ${LANGUAGE}: success." ;; @@ -355,8 +313,7 @@ jobs: && github.event.pull_request.state != 'closed' && needs.detect-languages.result == 'success' && needs.detect-languages.outputs.code == 'true' - runs-on: ${{ fromJSON(github.workflow_ref == 'ContextualWisdomLab/.github/.github/workflows/codeql-pr.yml@refs/heads/main' && '{"group":"CWL central control","labels":["self-hosted","linux","x64"]}' || '"ubuntu-24.04"') }} - timeout-minutes: 5 + runs-on: ubuntu-24.04 permissions: contents: read id-token: write @@ -382,7 +339,6 @@ jobs: live_pr="$(gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}")" live_head="$(printf '%s' "$live_pr" | jq -r '.head.sha // empty')" live_base="$(printf '%s' "$live_pr" | jq -r '.base.sha // empty')" - live_merge="$(printf '%s' "$live_pr" | jq -r '.merge_commit_sha // empty')" live_base_ref="$(printf '%s' "$live_pr" | jq -r '.base.ref // empty')" live_head_ref="$(printf '%s' "$live_pr" | jq -r '.head.ref // empty')" live_state="$(printf '%s' "$live_pr" | jq -r 'if (.state | type) == "string" then .state else empty end')" @@ -402,10 +358,8 @@ jobs: echo "::error::CodeQL dispatch requires a canonical current run id." exit 1 fi - if ! [[ "$live_base" =~ ^[0-9a-fA-F]{40}$ ]] || - ! [[ "$live_merge" =~ ^[0-9a-fA-F]{40}$ ]] || - [ -z "$live_base_ref" ] || [ -z "$live_head_ref" ]; then - echo "::error::Could not validate live pull request base/source identity before CodeQL dispatch." + if ! [[ "$live_base" =~ ^[0-9a-fA-F]{40}$ ]] || [ -z "$live_base_ref" ] || [ -z "$live_head_ref" ]; then + echo "::error::Could not validate live pull request base identity before CodeQL dispatch." exit 1 fi @@ -442,17 +396,13 @@ jobs: pending_matrix='[]' while IFS= read -r entry; do language="$(printf '%s' "$entry" | jq -r '.language // empty')" - expected_context="codeql-dispatch/${language}/${live_base}" - expected_description="cwl1;h=${PR_HEAD_SHA};w=codeql-scan-dispatch;r=${REQUIRED_RUN_ID};s=${live_merge}" - verdict_state="$(printf '%s' "$statuses" | jq -r --arg ctx "$expected_context" --arg description "$expected_description" ' + verdict_state="$(printf '%s' "$statuses" | jq -r --arg ctx "codeql-dispatch/${language}" ' [ .[] | select(.context == $ctx) - | select(.description == $description) | select( (.creator.login // "" | ascii_downcase) as $creator | $creator == "opencode-agent" or $creator == "opencode-agent[bot]" - or $creator == "cwl-noema-review" or $creator == "cwl-noema-review[bot]" ) ] | first // {} | .state // empty @@ -510,6 +460,5 @@ jobs: --argjson matrix "$pending_matrix" \ --arg required_run_id "$REQUIRED_RUN_ID" \ --argjson required_jobs "$required_jobs" \ - --arg producer_source_sha "$live_merge" \ - '{event_type:"codeql-scan-v2",client_payload:{target_repository:$target_repository,pr_number:$pr_number,pr_base_ref:$pr_base_ref,pr_base_sha:$pr_base_sha,pr_head:{schema:"1",ref:$pr_head_ref,sha:$pr_head_sha},producer_source_sha:$producer_source_sha,matrix:$matrix,required_run_id:$required_run_id,required_jobs:$required_jobs}}' | + '{event_type:"codeql-scan",client_payload:{target_repository:$target_repository,pr_number:$pr_number,pr_base_ref:$pr_base_ref,pr_base_sha:$pr_base_sha,pr_head_ref:$pr_head_ref,pr_head_sha:$pr_head_sha,matrix:$matrix,required_run_id:$required_run_id,required_jobs:$required_jobs}}' | GH_TOKEN="$app_token" gh api -X POST repos/ContextualWisdomLab/.github/dispatches --input - diff --git a/.github/workflows/codeql-scan-dispatch.yml b/.github/workflows/codeql-scan-dispatch.yml index 04656e5b34..45cfcc75fc 100644 --- a/.github/workflows/codeql-scan-dispatch.yml +++ b/.github/workflows/codeql-scan-dispatch.yml @@ -44,9 +44,7 @@ permissions: jobs: validate-dispatch: name: validate-dispatch - runs-on: - group: CWL central CodeQL - labels: [self-hosted, linux, x64] + runs-on: ubuntu-24.04 timeout-minutes: 8 permissions: contents: read @@ -432,9 +430,7 @@ jobs: scan: name: CodeQL dispatch scan (${{ matrix.language }}) needs: validate-dispatch - runs-on: - group: CWL central CodeQL - labels: [self-hosted, linux, x64] + runs-on: ubuntu-24.04 timeout-minutes: 30 permissions: actions: read @@ -559,14 +555,18 @@ jobs: python3 -c "import ast; ast.parse(open('$RUNNER_TEMP/codeql_ghas_configuration_identity.py').read())" - name: Materialize pull request head for CodeQL scan - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - repository: ${{ needs.validate-dispatch.outputs.target_repository }} - ref: ${{ needs.validate-dispatch.outputs.head_sha }} - token: ${{ steps.target_app_token.outputs.token || secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || github.token }} - persist-credentials: false - clean: true - fetch-depth: 1 + env: + GH_TOKEN: ${{ steps.target_app_token.outputs.token || secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || github.token }} + TARGET_REPOSITORY: ${{ needs.validate-dispatch.outputs.target_repository }} + HEAD_SHA: ${{ needs.validate-dispatch.outputs.head_sha }} + run: | + set -euo pipefail + gh auth setup-git + git init -q . + git remote add origin "$GITHUB_SERVER_URL/$TARGET_REPOSITORY.git" + git fetch --no-tags --depth=1 origin "$HEAD_SHA" + git checkout --detach --quiet "$HEAD_SHA" + git cat-file -e "$HEAD_SHA^{commit}" - name: Initialize CodeQL uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 @@ -587,83 +587,11 @@ jobs: id: gate run: python3 "$RUNNER_TEMP/codeql_sarif_gate.py" codeql-results-dispatch - - name: Detect optional Noema analysis-read credential - id: noema_analysis_config - if: always() && steps.live_metadata.outcome == 'success' - env: - TARGET_REPOSITORY: ${{ needs.validate-dispatch.outputs.target_repository }} - NOEMA_APP_CLIENT_ID: ${{ vars.NOEMA_GITHUB_APP_CLIENT_ID || '' }} - NOEMA_APP_PRIVATE_KEY: ${{ secrets.NOEMA_GITHUB_APP_PRIVATE_KEY || '' }} - run: | - set -euo pipefail - if [ -n "$NOEMA_APP_CLIENT_ID" ] && [ -n "$NOEMA_APP_PRIVATE_KEY" ]; then - printf 'repository=%s\n' "${TARGET_REPOSITORY#*/}" >>"$GITHUB_OUTPUT" - echo "available=true" >>"$GITHUB_OUTPUT" - fi - - - name: Mint target-scoped Noema analysis-read token - id: noema_analysis_token - if: steps.gate.outcome == 'success' && steps.noema_analysis_config.outputs.available == 'true' - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - client-id: ${{ vars.NOEMA_GITHUB_APP_CLIENT_ID }} - private-key: ${{ secrets.NOEMA_GITHUB_APP_PRIVATE_KEY }} - owner: ContextualWisdomLab - repositories: ${{ steps.noema_analysis_config.outputs.repository }} - permission-security-events: read - - - name: Select target CodeQL analysis-read credential - id: ghas_analysis_token - if: steps.gate.outcome == 'success' - env: - TARGET_REPOSITORY: ${{ needs.validate-dispatch.outputs.target_repository }} - TARGET_APP_TOKEN: ${{ steps.target_app_token.outputs.token || '' }} - NOEMA_ANALYSIS_TOKEN: ${{ steps.noema_analysis_token.outputs.token || '' }} - PR_REVIEW_MERGE_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || '' }} - OPENCODE_APPROVE_TOKEN: ${{ secrets.OPENCODE_APPROVE_TOKEN || '' }} - WORKFLOW_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - - probe_analysis_read() { - token_label="$1" - token="$2" - if [ -z "$token" ]; then - return 1 - fi - if GH_TOKEN="$token" gh api \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "repos/${TARGET_REPOSITORY}/code-scanning/analyses?per_page=1&tool_name=CodeQL" \ - >/dev/null 2>&1; then - echo "::add-mask::$token" - { - printf 'token=%s\n' "$token" - printf 'source=%s\n' "$token_label" - } >>"$GITHUB_OUTPUT" - echo "Selected ${token_label} after proving target CodeQL analysis-read access." - return 0 - fi - echo "::notice::${token_label} cannot read target CodeQL analyses; trying the next configured credential." - return 1 - } - - if probe_analysis_read "target-app-token" "$TARGET_APP_TOKEN" || - probe_analysis_read "pr-review-merge-token" "$PR_REVIEW_MERGE_TOKEN" || - probe_analysis_read "opencode-approve-token" "$OPENCODE_APPROVE_TOKEN" || - probe_analysis_read "github-token" "$WORKFLOW_TOKEN" || - probe_analysis_read "noema-analysis-token" "$NOEMA_ANALYSIS_TOKEN"; then - exit 0 - fi - - echo "::error::no configured credential can read target CodeQL analyses; GHAS configuration identity cannot be proven." - exit 1 - - name: Verify GHAS base/head CodeQL configuration identity id: ghas_configuration_identity if: steps.gate.outcome == 'success' env: - GH_TOKEN: ${{ steps.ghas_analysis_token.outputs.token }} + GH_TOKEN: ${{ steps.target_app_token.outputs.token || secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || github.token }} TARGET_REPOSITORY: ${{ needs.validate-dispatch.outputs.target_repository }} PR_NUMBER: ${{ needs.validate-dispatch.outputs.pr_number }} BASE_REF: ${{ needs.validate-dispatch.outputs.base_ref }} @@ -701,23 +629,10 @@ jobs: if-no-files-found: error retention-days: 7 - - name: Mint target-scoped Noema CodeQL status token - id: noema_status_token - if: always() && steps.noema_analysis_config.outputs.available == 'true' - continue-on-error: true - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - client-id: ${{ vars.NOEMA_GITHUB_APP_CLIENT_ID }} - private-key: ${{ secrets.NOEMA_GITHUB_APP_PRIVATE_KEY }} - owner: ContextualWisdomLab - repositories: ${{ steps.noema_analysis_config.outputs.repository }} - permission-statuses: write - - name: Publish CodeQL dispatch status id: publish_status if: always() && steps.live_metadata.outcome == 'success' env: - NOEMA_STATUS_TOKEN: ${{ steps.noema_status_token.outputs.token || '' }} TARGET_APP_STATUS_TOKEN: ${{ steps.target_app_token.outputs.token || '' }} GITHUB_STATUS_READ_TOKEN: ${{ github.token }} PR_REVIEW_MERGE_STATUS_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || '' }} @@ -796,11 +711,6 @@ jobs: actual_creator="$(jq -r '.creator.login // "" | ascii_downcase' "$status_response" 2>/dev/null || true)" creator_trusted=false case "$token_label" in - noema-status-token) - case "$actual_creator" in - cwl-noema-review|cwl-noema-review\[bot\]) creator_trusted=true ;; - esac - ;; target-app-token|pr-review-merge-token|opencode-approve-token) case "$actual_creator" in opencode-agent|opencode-agent\[bot\]) creator_trusted=true ;; @@ -833,9 +743,6 @@ jobs: return 1 } - if post_status "noema-status-token" "${NOEMA_STATUS_TOKEN:-}"; then - exit 0 - fi if post_status "target-app-token" "$TARGET_APP_STATUS_TOKEN"; then exit 0 fi @@ -865,9 +772,7 @@ jobs: && needs.validate-dispatch.result == 'success' && needs.scan.result != 'cancelled' && needs.scan.result != 'skipped' - runs-on: - group: CWL central CodeQL - labels: [self-hosted, linux, x64] + runs-on: ubuntu-24.04 timeout-minutes: 8 permissions: actions: write @@ -941,34 +846,8 @@ jobs: echo "token=$app_token" } >>"$GITHUB_OUTPUT" - - name: Resolve Noema settlement token configuration - id: noema_settlement_config - env: - NOEMA_APP_CLIENT_ID: ${{ vars.NOEMA_GITHUB_APP_CLIENT_ID || '' }} - NOEMA_APP_PRIVATE_KEY: ${{ secrets.NOEMA_GITHUB_APP_PRIVATE_KEY || '' }} - TARGET_REPOSITORY: ${{ needs.validate-dispatch.outputs.target_repository }} - run: | - set -euo pipefail - if [ -n "$NOEMA_APP_CLIENT_ID" ] && [ -n "$NOEMA_APP_PRIVATE_KEY" ]; then - printf 'repository=%s\n' "${TARGET_REPOSITORY#*/}" >>"$GITHUB_OUTPUT" - echo "available=true" >>"$GITHUB_OUTPUT" - fi - - - name: Mint target-scoped Noema CodeQL settlement token - id: noema_settlement_token - if: steps.noema_settlement_config.outputs.available == 'true' - continue-on-error: true - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - client-id: ${{ vars.NOEMA_GITHUB_APP_CLIENT_ID }} - private-key: ${{ secrets.NOEMA_GITHUB_APP_PRIVATE_KEY }} - owner: ContextualWisdomLab - repositories: ${{ steps.noema_settlement_config.outputs.repository }} - permission-actions: write - - name: Settle exact CodeQL required run env: - NOEMA_WAKE_TOKEN: ${{ steps.noema_settlement_token.outputs.token || '' }} TARGET_APP_WAKE_TOKEN: ${{ steps.target_app_token.outputs.token || '' }} PR_REVIEW_MERGE_WAKE_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || '' }} OPENCODE_APPROVE_WAKE_TOKEN: ${{ secrets.OPENCODE_APPROVE_TOKEN || '' }} @@ -1007,8 +886,7 @@ jobs: } github_api() { - run_api "noema-settlement-token" "${NOEMA_WAKE_TOKEN:-}" "$@" || - run_api "target-app-token" "$TARGET_APP_WAKE_TOKEN" "$@" || + run_api "target-app-token" "$TARGET_APP_WAKE_TOKEN" "$@" || run_api "pr-review-merge-token" "$PR_REVIEW_MERGE_WAKE_TOKEN" "$@" || run_api "opencode-approve-token" "$OPENCODE_APPROVE_WAKE_TOKEN" "$@" || run_api "github-token" "$GITHUB_WAKE_TOKEN" "$@" @@ -1057,7 +935,7 @@ jobs: exit 1 fi - if ! required_job_pages="$(github_api --paginate "repos/${TARGET_REPOSITORY}/actions/runs/${REQUIRED_RUN_ID}/jobs?per_page=100" | jq -s .)"; then + if ! required_job_pages="$(github_api --paginate --slurp "repos/${TARGET_REPOSITORY}/actions/runs/${REQUIRED_RUN_ID}/jobs?per_page=100")"; then echo "::error::CodeQL settlement could not read the required jobs." exit 1 fi @@ -1094,8 +972,8 @@ jobs: exit 1 fi - if ! handler_job_pages="$(GH_TOKEN="$HANDLER_READ_TOKEN" gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=100" | jq -s .)" || - ! handler_artifact_pages="$(GH_TOKEN="$HANDLER_READ_TOKEN" gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" | jq -s .)"; then + if ! handler_job_pages="$(GH_TOKEN="$HANDLER_READ_TOKEN" gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs?per_page=100")" || + ! handler_artifact_pages="$(GH_TOKEN="$HANDLER_READ_TOKEN" gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100")"; then echo "::error::CodeQL settlement could not read exact handler evidence." exit 1 fi @@ -1122,26 +1000,6 @@ jobs: echo "::error::CodeQL settlement rejected incomplete handler gate or SARIF evidence for ${language}." exit 1 fi - clean_gate_count="$(printf '%s' "$handler_jobs" | jq --arg name "$expected_job_name" --argjson attempt "$GITHUB_RUN_ATTEMPT" ' - [.[] | select( - .name == $name - and .status == "completed" - and .run_attempt == $attempt - and ([.steps[]? | select(.name == "Enforce CodeQL Medium+ SARIF gate" and .conclusion == "success")] | length) == 1 - )] | length - ')" - ghas_identity_count="$(printf '%s' "$handler_jobs" | jq --arg name "$expected_job_name" --argjson attempt "$GITHUB_RUN_ATTEMPT" ' - [.[] | select( - .name == $name - and .status == "completed" - and .run_attempt == $attempt - and ([.steps[]? | select(.name == "Verify GHAS base/head CodeQL configuration identity" and .conclusion == "success")] | length) == 1 - )] | length - ')" - if [ "$clean_gate_count" -eq 1 ] && [ "$ghas_identity_count" -ne 1 ]; then - echo "::error::CodeQL settlement rejected missing GHAS configuration identity proof for ${language}." - exit 1 - fi done < <(printf '%s' "$REQUIRED_JOBS" | jq -c '.[]') case "$RERUN_MODE" in @@ -1167,8 +1025,7 @@ jobs: return 1 } - if post_wake "noema-settlement-token" "${NOEMA_WAKE_TOKEN:-}" || - post_wake "target-app-token" "$TARGET_APP_WAKE_TOKEN" || + if post_wake "target-app-token" "$TARGET_APP_WAKE_TOKEN" || post_wake "pr-review-merge-token" "$PR_REVIEW_MERGE_WAKE_TOKEN" || post_wake "opencode-approve-token" "$OPENCODE_APPROVE_WAKE_TOKEN" || post_wake "github-token" "$GITHUB_WAKE_TOKEN"; then diff --git a/.github/workflows/exact-artifact-sbom-attestation.yml b/.github/workflows/exact-artifact-sbom-attestation.yml index 198392aa0c..b038c5478e 100644 --- a/.github/workflows/exact-artifact-sbom-attestation.yml +++ b/.github/workflows/exact-artifact-sbom-attestation.yml @@ -78,39 +78,19 @@ jobs: - name: Materialize immutable trusted verifier uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - # Independently reviewed helper snapshot, not caller/called workflow SHA. + # job.workflow_repository/workflow_sha are not real Actions context + # properties (actionlint-flagged); this always resolved to an empty + # repository/ref, silently defaulting checkout away from the pinned + # trusted verifier source. ContextualWisdomLab/.github is this + # workflow's own repository; github.workflow_sha is the real, + # documented property for its pinned commit. repository: ContextualWisdomLab/.github - # Reviewed helper revision; intentionally distinct from workflow revision. - ref: 00c6551183cca101cfc97c43656a17cc2491c1b4 + ref: ${{ github.workflow_sha }} path: trusted-intake persist-credentials: false - sparse-checkout: | - scripts/ci/ - requirements-strix-ci-hashes.txt + sparse-checkout: scripts/ci/verify_exact_artifact_sbom_handoff.py sparse-checkout-cone-mode: false - - name: Verify fixed helper checkout identity - env: - HELPER_ROOT: trusted-intake - CALLER_WORKFLOW_SHA: ${{ github.workflow_sha }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - expected=00c6551183cca101cfc97c43656a17cc2491c1b4 - test "$(git -C "$HELPER_ROOT" rev-parse HEAD)" = "$expected" - origin="$(git -C "$HELPER_ROOT" remote get-url origin)" - case "$origin" in - https://github.com/ContextualWisdomLab/.github|https://github.com/ContextualWisdomLab/.github.git) ;; - *) echo "Foreign helper repository" >&2; exit 1 ;; - esac - test "$(git -C "$HELPER_ROOT" rev-parse HEAD:scripts/ci)" = bf26d3eefdb71fe79b855d941ffb46eb432b2f76 - test "$(git -C "$HELPER_ROOT" rev-parse HEAD:requirements-strix-ci-hashes.txt)" = 9e705850b5ce53c7fe836bc3df3a18771151e3f6 - git -C "$HELPER_ROOT" diff --exit-code HEAD -- scripts/ci requirements-strix-ci-hashes.txt - test -f "$HELPER_ROOT/scripts/ci/release_dependency_gate.py" - test -f "$HELPER_ROOT/scripts/ci/verify_exact_artifact_sbom_handoff.py" - test -f "$HELPER_ROOT/requirements-strix-ci-hashes.txt" - printf 'helper_repository=ContextualWisdomLab/.github helper_sha=%s caller_workflow_sha=%s\n' "$expected" "$CALLER_WORKFLOW_SHA" - - - name: Verify immutable same-run artifact metadata env: GH_TOKEN: ${{ github.token }} @@ -197,39 +177,19 @@ jobs: - name: Materialize immutable trusted verifier uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - # Independently reviewed helper snapshot, not caller/called workflow SHA. + # job.workflow_repository/workflow_sha are not real Actions context + # properties (actionlint-flagged); this always resolved to an empty + # repository/ref, silently defaulting checkout away from the pinned + # trusted verifier source. ContextualWisdomLab/.github is this + # workflow's own repository; github.workflow_sha is the real, + # documented property for its pinned commit. repository: ContextualWisdomLab/.github - # Reviewed helper revision; intentionally distinct from workflow revision. - ref: 00c6551183cca101cfc97c43656a17cc2491c1b4 + ref: ${{ github.workflow_sha }} path: trusted-signer persist-credentials: false - sparse-checkout: | - scripts/ci/ - requirements-strix-ci-hashes.txt + sparse-checkout: scripts/ci/verify_exact_artifact_sbom_handoff.py sparse-checkout-cone-mode: false - - name: Verify fixed helper checkout identity - env: - HELPER_ROOT: trusted-signer - CALLER_WORKFLOW_SHA: ${{ github.workflow_sha }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - expected=00c6551183cca101cfc97c43656a17cc2491c1b4 - test "$(git -C "$HELPER_ROOT" rev-parse HEAD)" = "$expected" - origin="$(git -C "$HELPER_ROOT" remote get-url origin)" - case "$origin" in - https://github.com/ContextualWisdomLab/.github|https://github.com/ContextualWisdomLab/.github.git) ;; - *) echo "Foreign helper repository" >&2; exit 1 ;; - esac - test "$(git -C "$HELPER_ROOT" rev-parse HEAD:scripts/ci)" = bf26d3eefdb71fe79b855d941ffb46eb432b2f76 - test "$(git -C "$HELPER_ROOT" rev-parse HEAD:requirements-strix-ci-hashes.txt)" = 9e705850b5ce53c7fe836bc3df3a18771151e3f6 - git -C "$HELPER_ROOT" diff --exit-code HEAD -- scripts/ci requirements-strix-ci-hashes.txt - test -f "$HELPER_ROOT/scripts/ci/release_dependency_gate.py" - test -f "$HELPER_ROOT/scripts/ci/verify_exact_artifact_sbom_handoff.py" - test -f "$HELPER_ROOT/requirements-strix-ci-hashes.txt" - printf 'helper_repository=ContextualWisdomLab/.github helper_sha=%s caller_workflow_sha=%s\n' "$expected" "$CALLER_WORKFLOW_SHA" - - - name: Verify immutable same-run artifact metadata env: GH_TOKEN: ${{ github.token }} @@ -315,7 +275,9 @@ jobs: - name: Verify online and prepare offline bundles env: GH_TOKEN: ${{ github.token }} - # Signer repository is fixed independently of the caller identity. + # job.workflow_repository is not a real Actions context property + # (actionlint-flagged); ContextualWisdomLab/.github is this workflow's + # own repository, matching the pinned checkout above. SIGNER_REPOSITORY: ContextualWisdomLab/.github PREDICATE_TYPE: ${{ inputs.predicate_type }} SOURCE_REPOSITORY: ${{ inputs.source_repository }} @@ -438,4 +400,4 @@ jobs: name: exact-artifact-sbom-offline-verification path: offline-attestation-evidence if-no-files-found: error - retention-days: 90 + retention-days: 90 \ No newline at end of file diff --git a/.github/workflows/hourly-review-repair.yml b/.github/workflows/hourly-review-repair.yml index d34a8477d8..0b45c7fd37 100644 --- a/.github/workflows/hourly-review-repair.yml +++ b/.github/workflows/hourly-review-repair.yml @@ -136,9 +136,7 @@ permissions: jobs: resolve-target: name: Resolve target(s) for ${{ github.event.schedule }} - runs-on: - group: CWL central control - labels: [self-hosted, linux, x64] + runs-on: ubuntu-24.04 outputs: targets: ${{ steps.lookup.outputs.targets }} steps: diff --git a/.github/workflows/noema-review.yml b/.github/workflows/noema-review.yml index 08ea600538..9be705a50c 100644 --- a/.github/workflows/noema-review.yml +++ b/.github/workflows/noema-review.yml @@ -41,60 +41,21 @@ jobs: && github.event.action != 'converted_to_draft' && github.event.pull_request.head.repo.full_name == github.repository ) - runs-on: ${{ (github.repository == 'ContextualWisdomLab/.github' || github.repository == 'ContextualWisdomLab/contextual-orchestrator' || github.repository == 'ContextualWisdomLab/fast-mlsirm' || github.repository == 'ContextualWisdomLab/cwl-telemetry' || github.repository == 'ContextualWisdomLab/naruon' || github.repository == 'ContextualWisdomLab/late-life-anxiety-reanalysis') && github.workflow_ref == 'ContextualWisdomLab/.github/.github/workflows/noema-review.yml@refs/heads/main' && fromJSON('{"group":"CWL central control","labels":["self-hosted","linux","x64","cwlab-control"]}') || fromJSON('["ubuntu-24.04"]') }} + runs-on: ubuntu-24.04 timeout-minutes: 5 outputs: admitted: ${{ steps.live_head.outputs.admitted }} - base_sha: ${{ steps.live_head.outputs.base_sha }} permissions: contents: read pull-requests: read env: + GH_TOKEN: ${{ secrets.NOEMA_REVIEW_TOKEN || github.token }} TARGET_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name || github.event.client_payload.target_repository || github.repository }} PR_NUMBER: ${{ github.event.pull_request.number || github.event.client_payload.pr_number || '' }} EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.client_payload.pr_head_sha || '' }} steps: - - name: Select native Noema credential for metadata reads - if: env.PR_NUMBER != '' - id: noema_metadata_credential - env: - METADATA_TOKEN: ${{ secrets.NOEMA_REVIEW_TOKEN || secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || '' }} - NOEMA_GITHUB_APP_CLIENT_ID: ${{ vars.NOEMA_GITHUB_APP_CLIENT_ID || '' }} - NOEMA_GITHUB_APP_PRIVATE_KEY: ${{ secrets.NOEMA_GITHUB_APP_PRIVATE_KEY || '' }} - run: | - set -euo pipefail - if ! [[ "$TARGET_REPOSITORY" =~ ^ContextualWisdomLab/[A-Za-z0-9_.-]+$ ]] || - ! [[ "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]] || - ! [[ "$EXPECTED_HEAD_SHA" =~ ^[0-9a-f]{40}$ ]]; then - echo "::error::Noema metadata credential rejected malformed target PR/head metadata." - exit 1 - fi - echo "repository=${TARGET_REPOSITORY#*/}" >>"$GITHUB_OUTPUT" - if [ -n "${METADATA_TOKEN:-}" ]; then - echo "source=pat" >>"$GITHUB_OUTPUT" - elif [ -n "${NOEMA_GITHUB_APP_CLIENT_ID:-}" ] && [ -n "${NOEMA_GITHUB_APP_PRIVATE_KEY:-}" ]; then - echo "source=github-app" >>"$GITHUB_OUTPUT" - else - echo "source=workflow" >>"$GITHUB_OUTPUT" - fi - - - name: Mint read-only native Noema GitHub App token - if: env.PR_NUMBER != '' && steps.noema_metadata_credential.outputs.source == 'github-app' - id: noema_metadata_app_token - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - client-id: ${{ vars.NOEMA_GITHUB_APP_CLIENT_ID }} - private-key: ${{ secrets.NOEMA_GITHUB_APP_PRIVATE_KEY }} - owner: ContextualWisdomLab - repositories: ${{ steps.noema_metadata_credential.outputs.repository }} - permission-contents: read - permission-metadata: read - permission-pull-requests: read - - name: Admit only the exact live Noema head id: live_head - env: - GH_TOKEN: ${{ secrets.NOEMA_REVIEW_TOKEN || secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || steps.noema_metadata_app_token.outputs.token || github.token }} run: | set -euo pipefail echo "admitted=false" >>"$GITHUB_OUTPUT" @@ -111,12 +72,6 @@ jobs: echo "::notice::Noema admission retired a stale trigger before review queue entry." exit 0 fi - live_base="$(jq -r '.base.sha // empty' <<<"$live_pr")" - if ! [[ "$live_base" =~ ^[0-9a-f]{40}$ ]]; then - echo "::error::Noema admission could not bind the live base commit." - exit 1 - fi - echo "base_sha=$live_base" >>"$GITHUB_OUTPUT" echo "admitted=true" >>"$GITHUB_OUTPUT" echo "Exact live Noema head admitted for ${TARGET_REPOSITORY}#${PR_NUMBER}." @@ -137,7 +92,7 @@ jobs: # PR/REPO lookup naturally falls through to "scan everything" for that # path, matching strix.yml's identical repository_dispatch behavior. if: github.event_name != 'pull_request_target' || (github.event.action != 'closed' && github.event.action != 'converted_to_draft') - runs-on: ${{ (github.repository == 'ContextualWisdomLab/.github' || github.repository == 'ContextualWisdomLab/contextual-orchestrator' || github.repository == 'ContextualWisdomLab/fast-mlsirm' || github.repository == 'ContextualWisdomLab/cwl-telemetry' || github.repository == 'ContextualWisdomLab/naruon' || github.repository == 'ContextualWisdomLab/late-life-anxiety-reanalysis') && github.workflow_ref == 'ContextualWisdomLab/.github/.github/workflows/noema-review.yml@refs/heads/main' && fromJSON('{"group":"CWL central control","labels":["self-hosted","linux","x64","cwlab-control"]}') || fromJSON('["ubuntu-24.04"]') }} + runs-on: ubuntu-24.04 timeout-minutes: 5 permissions: contents: read @@ -193,7 +148,7 @@ jobs: if: >- github.event_name == 'pull_request_target' && (github.event.action == 'closed' || github.event.action == 'converted_to_draft') - runs-on: ${{ (github.repository == 'ContextualWisdomLab/.github' || github.repository == 'ContextualWisdomLab/contextual-orchestrator' || github.repository == 'ContextualWisdomLab/fast-mlsirm' || github.repository == 'ContextualWisdomLab/cwl-telemetry' || github.repository == 'ContextualWisdomLab/naruon' || github.repository == 'ContextualWisdomLab/late-life-anxiety-reanalysis') && github.workflow_ref == 'ContextualWisdomLab/.github/.github/workflows/noema-review.yml@refs/heads/main' && fromJSON('{"group":"CWL central control","labels":["self-hosted","linux","x64","cwlab-control"]}') || fromJSON('["ubuntu-24.04"]') }} + runs-on: ubuntu-24.04 # Bound this job well short of GitHub's 360-minute platform default. Its # only step is a single-repository, status-filtered gh api --paginate # list-and-cancel sweep (up to 3 passes x 5 statuses), no branch update @@ -371,7 +326,7 @@ jobs: noema-review: name: noema-review needs: [admit-current-head, changed-scope] - runs-on: ${{ (github.repository == 'ContextualWisdomLab/.github' || github.repository == 'ContextualWisdomLab/contextual-orchestrator' || github.repository == 'ContextualWisdomLab/fast-mlsirm' || github.repository == 'ContextualWisdomLab/cwl-telemetry' || github.repository == 'ContextualWisdomLab/naruon' || github.repository == 'ContextualWisdomLab/late-life-anxiety-reanalysis') && github.workflow_ref == 'ContextualWisdomLab/.github/.github/workflows/noema-review.yml@refs/heads/main' && fromJSON('{"group":"CWL MCP remediation","labels":["self-hosted","linux","x64"]}') || fromJSON('["ubuntu-24.04"]') }} + runs-on: ubuntu-24.04 # No job-level timeout-minutes here, deliberately. This job's "Prepare # Noema model verdict" step calls two_phase.py's call_llm synchronously # via the contextual-orchestrator gateway and blocks on the model's own @@ -405,13 +360,6 @@ jobs: contents: read id-token: write pull-requests: read - outputs: - transport_capacity_unavailable: ${{ steps.noema_prepare.outputs.transport_capacity_unavailable || steps.noema_sidecar_failure.outputs.transport_capacity_unavailable }} - transport_retry_eligible: ${{ steps.noema_prepare.outputs.transport_retry_eligible || steps.noema_sidecar_failure.outputs.transport_retry_eligible }} - transport_retry_delay_seconds: ${{ steps.noema_prepare.outputs.transport_retry_delay_seconds || steps.noema_sidecar_failure.outputs.transport_retry_delay_seconds }} - transport_retry_next_attempt: ${{ steps.noema_prepare.outputs.transport_retry_next_attempt || steps.noema_sidecar_failure.outputs.transport_retry_next_attempt }} - provider_attempt_count: ${{ steps.noema_prepare.outputs.provider_attempt_count || steps.noema_sidecar_failure.outputs.provider_attempt_count }} - transport_http_status: ${{ steps.noema_prepare.outputs.transport_http_status || steps.noema_sidecar_failure.outputs.transport_http_status }} env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true TARGET_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name || github.event.client_payload.target_repository || github.repository }} @@ -500,47 +448,10 @@ jobs: tar -xzf "$trusted_archive" -C "$GITHUB_WORKSPACE" --strip-components=1 test -f scripts/ci/noema_review_gate.py - - name: Select native Noema credential for metadata reads - if: env.PR_NUMBER != '' - id: noema_metadata_credential - env: - METADATA_TOKEN: ${{ secrets.NOEMA_REVIEW_TOKEN || secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || '' }} - NOEMA_GITHUB_APP_CLIENT_ID: ${{ vars.NOEMA_GITHUB_APP_CLIENT_ID || '' }} - NOEMA_GITHUB_APP_PRIVATE_KEY: ${{ secrets.NOEMA_GITHUB_APP_PRIVATE_KEY || '' }} - run: | - set -euo pipefail - if ! [[ "$TARGET_REPOSITORY" =~ ^ContextualWisdomLab/[A-Za-z0-9_.-]+$ ]] || - ! [[ "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]] || - ! [[ "$EXPECTED_HEAD_SHA" =~ ^[0-9a-f]{40}$ ]]; then - echo "::error::Noema metadata credential rejected malformed target PR/head metadata." - exit 1 - fi - echo "repository=${TARGET_REPOSITORY#*/}" >>"$GITHUB_OUTPUT" - if [ -n "${METADATA_TOKEN:-}" ]; then - echo "source=pat" >>"$GITHUB_OUTPUT" - elif [ -n "${NOEMA_GITHUB_APP_CLIENT_ID:-}" ] && [ -n "${NOEMA_GITHUB_APP_PRIVATE_KEY:-}" ]; then - echo "source=github-app" >>"$GITHUB_OUTPUT" - else - echo "source=workflow" >>"$GITHUB_OUTPUT" - fi - - - name: Mint read-only native Noema GitHub App token - if: env.PR_NUMBER != '' && steps.noema_metadata_credential.outputs.source == 'github-app' - id: noema_metadata_app_token - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - client-id: ${{ vars.NOEMA_GITHUB_APP_CLIENT_ID }} - private-key: ${{ secrets.NOEMA_GITHUB_APP_PRIVATE_KEY }} - owner: ContextualWisdomLab - repositories: ${{ steps.noema_metadata_credential.outputs.repository }} - permission-contents: read - permission-metadata: read - permission-pull-requests: read - - name: Reject a stale trigger before credential or model setup if: env.PR_NUMBER != '' env: - GH_TOKEN: ${{ secrets.NOEMA_REVIEW_TOKEN || secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || steps.noema_metadata_app_token.outputs.token || github.token }} + GH_TOKEN: ${{ github.token }} run: | set -euo pipefail if [[ ! "$EXPECTED_HEAD_SHA" =~ ^[0-9a-f]{40}$ ]]; then @@ -803,61 +714,9 @@ jobs: ;; esac - - name: Check live pull request draft state before sidecar provisioning - # two_phase.py's verdict preparation already reads the live PR and - # skips a draft ("PR is draft; Noema verdict preparation skipped."), - # but only after the 10-13 minute contextual-orchestrator sidecar - # provisioning below has held a runner (e.g. newsdom-api job - # 108077744310, .github job 106665379126). This moves that same - # runtime decision ahead of provisioning; it deliberately reads the - # live PR instead of the event payload's draft flag, because the - # organization ruleset runs this workflow in other repositories only - # on opened/synchronize/reopened, so the event snapshot is not the - # authority. Fails OPEN: an unreadable or malformed live PR yields - # live_draft=false and keeps today's full review path, where - # two_phase.py still performs its own draft check. A draft conclusion - # skips the model steps, leaves noema_prepare's outputs unset (the - # same "publication skipped" state a draft produced before), and the - # job still succeeds. - if: env.PR_NUMBER != '' - id: live_draft - env: - GH_TOKEN: ${{ secrets.NOEMA_REVIEW_TOKEN || steps.noema_github_app_token.outputs.token || steps.noema_oidc_token.outputs.token }} - run: | - set -uo pipefail - live_draft=false - if pull_request_json="$(gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}" 2>/tmp/noema-live-draft-error)"; then - if jq -e -s 'length == 1 and (.[0] | type == "object" and .draft == true)' <<<"$pull_request_json" >/dev/null 2>&1; then - live_draft=true - fi - else - echo "::warning::Noema could not read the live pull request draft state; continuing with the model review." - sed 's/^/ /' /tmp/noema-live-draft-error >&2 || true - fi - echo "live_draft=${live_draft}" >>"$GITHUB_OUTPUT" - if [ "$live_draft" = "true" ]; then - echo "::notice::PR is draft; Noema model review skipped before sidecar provisioning." - fi - - - name: Provision pinned Node.js for Noema document review - if: env.PR_NUMBER != '' && steps.live_draft.outputs.live_draft != 'true' - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 - with: - node-version: "22.23.3" - - - name: Set up lock-compatible sidecar Python - if: env.PR_NUMBER != '' && steps.live_draft.outputs.live_draft != 'true' - id: sidecar_python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - update-environment: false - - name: Provision contextual-orchestrator review sidecar - id: noema_sidecar - if: env.PR_NUMBER != '' && steps.live_draft.outputs.live_draft != 'true' + if: env.PR_NUMBER != '' env: - SIDECAR_PYTHON: ${{ steps.sidecar_python.outputs.python-path }} BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }} NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} @@ -866,22 +725,10 @@ jobs: CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR: ${{ steps.target_visibility.outputs.require_zdr }} run: | set -euo pipefail - test ! -L "$GITHUB_WORKSPACE/strix_runs" - rm -f "$GITHUB_WORKSPACE/strix_runs/contextual-orchestrator-preflight.json" bash "$GITHUB_WORKSPACE/scripts/ci/contextual_orchestrator_review_sidecar.sh" - - name: Classify sidecar provider-capacity failure - id: noema_sidecar_failure - if: failure() && steps.noema_sidecar.outcome == 'failure' - env: - NOEMA_TRANSPORT_RETRY_ATTEMPT: ${{ toJSON(github.event.client_payload.transport_retry_attempt) }} - run: | - python3 "$GITHUB_WORKSPACE/scripts/ci/noema_preflight_capacity.py" \ - --expected-head "$EXPECTED_HEAD_SHA" \ - --preflight-report "$GITHUB_WORKSPACE/strix_runs/contextual-orchestrator-preflight.json" - - name: Provision local reviewed HWP document reader - if: env.PR_NUMBER != '' && steps.live_draft.outputs.live_draft != 'true' + if: env.PR_NUMBER != '' env: NPM_CONFIG_IGNORE_SCRIPTS: "true" run: | @@ -909,14 +756,14 @@ jobs: echo "NOEMA_HWP_MCP_SOURCE=$reader_root/node_modules/hwp-mcp" >>"$GITHUB_ENV" - name: Prepare Noema model verdict - if: env.PR_NUMBER != '' && steps.live_draft.outputs.live_draft != 'true' + if: env.PR_NUMBER != '' id: noema_prepare env: GH_TOKEN: ${{ secrets.NOEMA_REVIEW_TOKEN || steps.noema_github_app_token.outputs.token || steps.noema_oidc_token.outputs.token }} NOEMA_REVIEW_TOKEN_SOURCE: ${{ steps.noema_credential.outputs.source == 'pat' && 'noema-review-pat' || steps.noema_credential.outputs.source == 'github-app' && 'noema-review-github-app' || 'noema-review-app-oidc' }} NOEMA_REVIEW_ACTOR: ${{ steps.noema_github_app_token.outputs['app-slug'] && format('{0}[bot]', steps.noema_github_app_token.outputs['app-slug']) || '' }} NOEMA_REVIEW_INSTALLATION_ID: ${{ steps.noema_github_app_token.outputs['installation-id'] }} - NOEMA_TRANSPORT_RETRY_ATTEMPT: ${{ toJSON(github.event.client_payload.transport_retry_attempt) }} + NOEMA_TRANSPORT_RETRY_ATTEMPT: ${{ github.event.client_payload.transport_retry_attempt || 0 }} run: | set -euo pipefail if [ -z "${PR_NUMBER:-}" ]; then @@ -947,8 +794,55 @@ jobs: echo "::notice::Noema model phase produced no publishable envelope; publication is skipped." fi - - name: Upload contextual-orchestrator sidecar evidence - if: always() && env.PR_NUMBER != '' + - name: Schedule bounded Noema transport re-dispatch + if: >- + failure() + && env.PR_NUMBER != '' + && steps.noema_prepare.outputs.transport_capacity_unavailable == 'true' + && steps.noema_prepare.outputs.transport_retry_eligible == 'true' + env: + GH_TOKEN: ${{ secrets.NOEMA_REVIEW_TOKEN || steps.noema_github_app_token.outputs.token || github.token }} + DELAY_SECONDS: ${{ steps.noema_prepare.outputs.transport_retry_delay_seconds }} + NEXT_ATTEMPT: ${{ steps.noema_prepare.outputs.transport_retry_next_attempt }} + PROVIDER_ATTEMPT_COUNT: ${{ steps.noema_prepare.outputs.provider_attempt_count || '' }} + TRANSPORT_HTTP_STATUS: ${{ steps.noema_prepare.outputs.transport_http_status || '' }} + run: | + set -euo pipefail + if ! [[ "${DELAY_SECONDS}" =~ ^[1-9][0-9]*$ ]] || [ "${DELAY_SECONDS}" -gt 300 ]; then + echo "::error::Noema transport re-dispatch refused a non-bounded delay." + exit 1 + fi + if ! [[ "${NEXT_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::Noema transport re-dispatch refused a malformed attempt counter." + exit 1 + fi + echo "::notice::Noema provider capacity unavailable (http_status=${TRANSPORT_HTTP_STATUS:-unknown}, provider_attempt_count=${PROVIDER_ATTEMPT_COUNT:-unknown}); waiting ${DELAY_SECONDS}s before same-head continuation re-dispatch ${NEXT_ATTEMPT}." + sleep "${DELAY_SECONDS}" + live_pr="$(gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}")" + live_head="$(jq -r '.head.sha // empty' <<<"$live_pr")" + live_state="$(jq -r '.state // empty' <<<"$live_pr")" + if [ "${live_head,,}" != "${EXPECTED_HEAD_SHA,,}" ] || [ "$live_state" != "open" ]; then + echo "::notice::Noema transport re-dispatch retired because the live head moved or closed." + exit 0 + fi + jq -n \ + --arg target_repository "$TARGET_REPOSITORY" \ + --argjson pr_number "$PR_NUMBER" \ + --arg pr_head_sha "$EXPECTED_HEAD_SHA" \ + --argjson transport_retry_attempt "$NEXT_ATTEMPT" \ + '{ + event_type: "noema-review", + client_payload: { + target_repository: $target_repository, + pr_number: $pr_number, + pr_head_sha: $pr_head_sha, + transport_retry_attempt: $transport_retry_attempt + } + }' | gh api -X POST "repos/${TARGET_REPOSITORY}/dispatches" --input - + echo "::notice::Scheduled Noema transport continuation re-dispatch for ${TARGET_REPOSITORY}#${PR_NUMBER} at ${EXPECTED_HEAD_SHA} (attempt ${NEXT_ATTEMPT})." + + - name: Upload contextual-orchestrator sidecar evidence on failure + if: failure() && env.PR_NUMBER != '' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: noema-sidecar-evidence @@ -995,78 +889,3 @@ jobs: exit 1 fi python3 "$GITHUB_WORKSPACE/.github/actions/noema-review/two_phase.py" --repo "$TARGET_REPOSITORY" --pr-number "$PR_NUMBER" --expected-head "$EXPECTED_HEAD_SHA" --publish-verdict-file "$verdict_file" - - continue-noema-transport: - needs: [admit-current-head, noema-review] - if: >- - always() - && needs.admit-current-head.outputs.admitted == 'true' - && needs.noema-review.result == 'failure' - && needs.noema-review.outputs.transport_capacity_unavailable == 'true' - && needs.noema-review.outputs.transport_retry_eligible == 'true' - runs-on: ${{ (github.repository == 'ContextualWisdomLab/.github' || github.repository == 'ContextualWisdomLab/contextual-orchestrator' || github.repository == 'ContextualWisdomLab/fast-mlsirm' || github.repository == 'ContextualWisdomLab/cwl-telemetry' || github.repository == 'ContextualWisdomLab/naruon' || github.repository == 'ContextualWisdomLab/late-life-anxiety-reanalysis') && github.workflow_ref == 'ContextualWisdomLab/.github/.github/workflows/noema-review.yml@refs/heads/main' && fromJSON('{"group":"CWL central control","labels":["self-hosted","linux","x64","cwlab-control"]}') || fromJSON('["ubuntu-24.04"]') }} - timeout-minutes: 10 - permissions: - contents: write - pull-requests: read - env: - # Consumer required workflows need the existing central dispatch credential. - # The central handler can use its repository-scoped token as fallback. - GH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || github.token }} - TARGET_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name || github.event.client_payload.target_repository || github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number || github.event.client_payload.pr_number || '' }} - EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.client_payload.pr_head_sha || '' }} - EXPECTED_BASE_SHA: ${{ needs.admit-current-head.outputs.base_sha }} - DELAY_SECONDS: ${{ needs.noema-review.outputs.transport_retry_delay_seconds }} - NEXT_ATTEMPT: ${{ needs.noema-review.outputs.transport_retry_next_attempt }} - PROVIDER_ATTEMPT_COUNT: ${{ needs.noema-review.outputs.provider_attempt_count }} - TRANSPORT_HTTP_STATUS: ${{ needs.noema-review.outputs.transport_http_status }} - steps: - - name: Schedule bounded Noema transport re-dispatch - run: | - set -euo pipefail - if { [ "$GITHUB_REPOSITORY" != "ContextualWisdomLab/.github" ] && - [ "$TARGET_REPOSITORY" != "$GITHUB_REPOSITORY" ]; } || - ! [[ "$TARGET_REPOSITORY" =~ ^ContextualWisdomLab/[A-Za-z0-9_.-]+$ ]] || - ! [[ "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]] || - ! [[ "$EXPECTED_HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] || - ! [[ "$EXPECTED_BASE_SHA" =~ ^[0-9a-f]{40}$ ]]; then - echo "::error::Noema transport re-dispatch rejected an unrelated origin or malformed PR identity." - exit 1 - fi - if ! [[ "$DELAY_SECONDS" =~ ^[1-9][0-9]*$ ]] || [ "$DELAY_SECONDS" -gt 300 ] || - ! [[ "$NEXT_ATTEMPT" =~ ^[12]$ ]]; then - echo "::error::Noema transport re-dispatch refused an unbounded delay or attempt." - exit 1 - fi - echo "::notice::Noema provider capacity unavailable (http_status=${TRANSPORT_HTTP_STATUS:-unknown}, provider_attempt_count=${PROVIDER_ATTEMPT_COUNT:-unknown}); waiting ${DELAY_SECONDS}s before same-head continuation re-dispatch ${NEXT_ATTEMPT}." - sleep "$DELAY_SECONDS" - live_pr="$(gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}")" - live_head="$(jq -r '.head.sha // empty' <<<"$live_pr")" - live_head_repo="$(jq -r '.head.repo.full_name // empty' <<<"$live_pr")" - live_base="$(jq -r '.base.sha // empty' <<<"$live_pr")" - live_base_repo="$(jq -r '.base.repo.full_name // empty' <<<"$live_pr")" - live_state="$(jq -r '.state // empty' <<<"$live_pr")" - if [ "$live_head" != "$EXPECTED_HEAD_SHA" ] || - [ "$live_head_repo" != "$TARGET_REPOSITORY" ] || - [ "$live_base" != "$EXPECTED_BASE_SHA" ] || - [ "$live_base_repo" != "$TARGET_REPOSITORY" ] || - [ "$live_state" != "open" ]; then - echo "::notice::Noema transport re-dispatch retired because the live PR head or base moved or closed." - exit 0 - fi - jq -n \ - --arg target_repository "$TARGET_REPOSITORY" \ - --argjson pr_number "$PR_NUMBER" \ - --arg pr_head_sha "$EXPECTED_HEAD_SHA" \ - --argjson transport_retry_attempt "$NEXT_ATTEMPT" \ - '{ - event_type: "noema-review", - client_payload: { - target_repository: $target_repository, - pr_number: $pr_number, - pr_head_sha: $pr_head_sha, - transport_retry_attempt: $transport_retry_attempt - } - }' | gh api -X POST "repos/ContextualWisdomLab/.github/dispatches" --input - - echo "::notice::Scheduled Noema transport continuation re-dispatch for ${TARGET_REPOSITORY}#${PR_NUMBER} at ${EXPECTED_HEAD_SHA} (attempt ${NEXT_ATTEMPT})." diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index 58f11efc90..cbc8d21439 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -53,9 +53,7 @@ jobs: # inside a 13h57m run, ~97.5% of which was queue wait between exactly # these job boundaries). if: github.event_name == 'repository_dispatch' - runs-on: - group: CWL central OpenCode - labels: [self-hosted, linux, x64] + runs-on: ubuntu-24.04 timeout-minutes: 20 permissions: contents: read @@ -396,9 +394,7 @@ jobs: if: >- needs.validate-pr-metadata.result == 'success' && github.event_name == 'repository_dispatch' - runs-on: - group: CWL central OpenCode - labels: [self-hosted, linux, x64] + runs-on: ubuntu-24.04 timeout-minutes: 300 permissions: # The PR tree arrives through a same-run artifact. No repository-content, @@ -454,15 +450,6 @@ jobs: TRUSTED_SOURCE_REF: ${{ steps.trusted_source.outputs.ref }} run: | set -euo pipefail - # Persistent runners retain old files and Git configuration between jobs. - if [ -z "${GITHUB_WORKSPACE:-}" ] || [ -z "${RUNNER_WORKSPACE:-}" ] || - [ -L "$GITHUB_WORKSPACE" ] || [ "$RUNNER_WORKSPACE" = / ] || - [ "$(realpath "$GITHUB_WORKSPACE")" != "$(pwd -P)" ] || - [ "$(dirname "$(realpath "$GITHUB_WORKSPACE")")" != "$(realpath "$RUNNER_WORKSPACE")" ]; then - echo "::error::Coverage workspace is outside the current runner job directory." - exit 1 - fi - find "$GITHUB_WORKSPACE" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + git init "$GITHUB_WORKSPACE" git -C "$GITHUB_WORKSPACE" remote add trusted-source https://github.com/ContextualWisdomLab/.github.git git -C "$GITHUB_WORKSPACE" fetch --depth=1 --no-tags trusted-source "$TRUSTED_SOURCE_REF" @@ -479,11 +466,11 @@ jobs: - name: Prepare pull request merge tree for coverage measurement env: COVERAGE_SOURCE_ARCHIVE: ${{ runner.temp }}/opencode-coverage-artifact/opencode-coverage-source.tar - COVERAGE_SOURCE_WORKDIR: ${{ runner.temp }}/opencode-coverage-${{ github.run_id }}-${{ github.run_attempt }} + COVERAGE_SOURCE_WORKDIR: ${{ runner.temp }}/pr-head run: | set -euo pipefail - # Each attempt owns a fresh tree; never delete another job's checkout. - mkdir "$COVERAGE_SOURCE_WORKDIR" + rm -rf "$COVERAGE_SOURCE_WORKDIR" + mkdir -p "$COVERAGE_SOURCE_WORKDIR" # The archive contains pull-request-controlled paths. Validate every # member before extraction so a symlink, hardlink, device, FIFO, or # traversal path cannot redirect a later trusted host-side parser. @@ -533,7 +520,7 @@ jobs: env: PR_BASE_SHA: ${{ needs.validate-pr-metadata.outputs.base_sha }} PR_HEAD_SHA: ${{ needs.validate-pr-metadata.outputs.head_sha }} - COVERAGE_SOURCE_WORKDIR: ${{ runner.temp }}/opencode-coverage-${{ github.run_id }}-${{ github.run_attempt }} + COVERAGE_SOURCE_WORKDIR: ${{ runner.temp }}/pr-head # Dependency resolution may consume wheels/packages, but PR-defined # install/build hooks are never executed implicitly. UV_NO_BUILD: "1" @@ -568,7 +555,7 @@ jobs: - name: Enforce changed-file syntax gate env: PR_BASE_SHA: ${{ needs.validate-pr-metadata.outputs.base_sha }} - COVERAGE_SOURCE_WORKDIR: ${{ runner.temp }}/opencode-coverage-${{ github.run_id }}-${{ github.run_attempt }} + COVERAGE_SOURCE_WORKDIR: ${{ runner.temp }}/pr-head run: | set -euo pipefail # Deterministic per-file syntax check on the PR's changed files. The @@ -600,7 +587,7 @@ jobs: env: PR_BASE_SHA: ${{ needs.validate-pr-metadata.outputs.base_sha }} PR_HEAD_SHA: ${{ needs.validate-pr-metadata.outputs.head_sha }} - COVERAGE_SOURCE_WORKDIR: ${{ runner.temp }}/opencode-coverage-${{ github.run_id }}-${{ github.run_attempt }} + COVERAGE_SOURCE_WORKDIR: ${{ runner.temp }}/pr-head # Apply wheel-only resolution in the same step that consumes # pull-request dependency metadata. A value on an earlier step does # not cross the GitHub Actions step boundary. @@ -645,17 +632,12 @@ jobs: coverage_tool_image="opencode-coverage-tools:${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" coverage_build_dir="${RUNNER_TEMP}/opencode-coverage-tool-build" trusted_ci_requirements="${GITHUB_WORKSPACE}/requirements-opencode-review-ci-hashes.txt" - trusted_noema_document_requirements="${GITHUB_WORKSPACE}/requirements-noema-document-ci-hashes.txt" trusted_base_python_installer="${GITHUB_WORKSPACE}/scripts/ci/install_base_python_locks.py" trusted_vcs_import_root_resolver="${GITHUB_WORKSPACE}/scripts/ci/resolve_opencode_base_vcs_import_root.sh" if [ ! -f "$trusted_ci_requirements" ] || [ -L "$trusted_ci_requirements" ]; then echo "::error::Trusted coverage requirements must be a regular non-symlink file." exit 1 fi - if [ ! -f "$trusted_noema_document_requirements" ] || [ -L "$trusted_noema_document_requirements" ]; then - echo "::error::Trusted Noema document requirements must be a regular non-symlink file." - exit 1 - fi if [ ! -f "$trusted_base_python_installer" ] || [ -L "$trusted_base_python_installer" ]; then echo "::error::Trusted base Python lock installer must be a regular non-symlink file." exit 1 @@ -669,26 +651,10 @@ jobs: chmod 0700 "$coverage_build_dir" install -m 0644 "$trusted_ci_requirements" \ "$coverage_build_dir/requirements-opencode-review-ci-hashes.txt" - install -m 0644 "$trusted_noema_document_requirements" \ - "$coverage_build_dir/requirements-noema-document-ci-hashes.txt" install -m 0755 "$trusted_base_python_installer" \ "$coverage_build_dir/install-base-python-locks.py" install -m 0755 "$trusted_vcs_import_root_resolver" \ "$coverage_build_dir/resolve-opencode-base-vcs-import-root.sh" - trusted_cargo_fixture="${GITHUB_WORKSPACE}/tests/fixtures/coverage-cargo" - if [ -L "$trusted_cargo_fixture" ] || [ -L "$trusted_cargo_fixture/src" ]; then - echo "::error::Trusted Cargo coverage fixture directories must not be symlinks." - exit 1 - fi - for fixture_file in Cargo.toml Cargo.lock src/lib.rs; do - if [ ! -f "$trusted_cargo_fixture/$fixture_file" ] || - [ -L "$trusted_cargo_fixture/$fixture_file" ]; then - echo "::error::Trusted Cargo coverage fixture is missing or not a regular file." - exit 1 - fi - install -D -m 0644 "$trusted_cargo_fixture/$fixture_file" \ - "$coverage_build_dir/coverage-cargo-fixtures/$fixture_file" - done python_change_files="${RUNNER_TEMP}/opencode-python-change-files" if ! git -C "$COVERAGE_SOURCE_WORKDIR" diff \ --name-only --diff-filter=ACMRTUXBD -z "$PR_BASE_SHA" HEAD \ @@ -747,30 +713,10 @@ jobs: # the generic Python coverage path failed at collection with `ImportError: cannot # import name '_core'`, both surfacing as an indistinguishable "Coverage gate: failure" # even when the pull request itself introduced no regression. - rust_lock_args=() - rust_change_files="${RUNNER_TEMP}/opencode-rust-change-files" - if ! git -C "$COVERAGE_SOURCE_WORKDIR" diff --no-renames --name-only -z \ - "$PR_BASE_SHA" "$PR_HEAD_SHA" >"$rust_change_files"; then - echo "::error::Could not classify exact-head Cargo changes." >&2 - exit 1 - fi - rust_lock_changed=0 - rust_manifest_changed=0 - while IFS= read -r -d '' changed_path; do - case "${changed_path##*/}" in - Cargo.lock) rust_lock_changed=1 ;; - Cargo.toml) rust_manifest_changed=1 ;; - esac - done <"$rust_change_files" - if [ "$rust_lock_changed" -eq 1 ] && [ "$rust_manifest_changed" -eq 0 ]; then - # The trusted materializer still rejects non-base pins and changed graphs. - rust_lock_args=(--head-sha "$PR_HEAD_SHA") - fi python3 -I "$GITHUB_WORKSPACE/scripts/ci/materialize_base_rust_dependencies.py" \ --repo-root "$COVERAGE_SOURCE_WORKDIR" \ --base-sha "$PR_BASE_SHA" \ --output-dir "$coverage_build_dir/base-rust-dependencies" \ - "${rust_lock_args[@]}" \ --vendor-dir-for-config /opt/base-rust-dependencies/vendor cat >"$coverage_build_dir/Dockerfile" <<'DOCKERFILE' FROM docker.io/library/python:3.14-slim@sha256:b877e50bd90de10af8d82c57a022fc2e0dc731c5320d762a27986facfc3355c1 @@ -797,11 +743,6 @@ jobs: vulkan-tools \ xz-utils \ && rm -rf /var/lib/apt/lists/* - COPY coverage-cargo-fixtures /tmp/coverage-cargo-fixtures - RUN CARGO_HOME=/opt/coverage-cargo-home cargo fetch --locked \ - --manifest-path /tmp/coverage-cargo-fixtures/Cargo.toml \ - && rm -rf /tmp/coverage-cargo-fixtures \ - && chmod -R a+rX /opt/coverage-cargo-home ENV LLVM_COV=/usr/bin/llvm-cov-19 ENV LLVM_PROFDATA=/usr/bin/llvm-profdata-19 ENV COREPACK_HOME=/opt/corepack @@ -992,7 +933,6 @@ jobs: chown -R root:root /work/.git chmod -R go-w /work/.git fi - rm -rf -- /work/.opencode-sandbox-home mkdir -p "$RUNNER_TEMP" /work/.opencode-sandbox-home /work/.opencode-sandbox-cache chown "$OPENCODE_SANDBOX_UID:$OPENCODE_SANDBOX_GID" /work/.opencode-sandbox-home /work/.opencode-sandbox-cache # `run_and_capture`/`run_and_capture_advisory` below pin CARGO_HOME to @@ -1000,15 +940,12 @@ jobs: # the read-only image -- so the baked offline vendor config from # /opt/base-rust-dependencies (see materialize_base_rust_dependencies.py) has to be # copied there explicitly rather than set as an image ENV default. - mkdir -p /work/.opencode-sandbox-home/.cargo - cp -a /opt/coverage-cargo-home/. /work/.opencode-sandbox-home/.cargo/ if [ -s /opt/base-rust-dependencies/cargo-config.toml ]; then - install -m 0644 /opt/base-rust-dependencies/cargo-config.toml \ + mkdir -p /work/.opencode-sandbox-home/.cargo + install -m 0444 /opt/base-rust-dependencies/cargo-config.toml \ /work/.opencode-sandbox-home/.cargo/config.toml + chown -R "$OPENCODE_SANDBOX_UID:$OPENCODE_SANDBOX_GID" /work/.opencode-sandbox-home/.cargo fi - printf '\n[net]\noffline = true\n' >>/work/.opencode-sandbox-home/.cargo/config.toml - chmod 0444 /work/.opencode-sandbox-home/.cargo/config.toml - chown -R "$OPENCODE_SANDBOX_UID:$OPENCODE_SANDBOX_GID" /work/.opencode-sandbox-home/.cargo chmod 0700 "$RUNNER_TEMP" : >"$GITHUB_OUTPUT" chmod 0600 "$GITHUB_OUTPUT" @@ -2403,9 +2340,7 @@ jobs: needs.validate-pr-metadata.outputs.target_repository }}-${{ needs.validate-pr-metadata.outputs.pr_number || github.run_id }} cancel-in-progress: true - runs-on: - group: CWL central OpenCode - labels: [self-hosted, linux, x64] + runs-on: ubuntu-24.04 # Coverage and current-head evidence are prepared before the model pool. # A single legitimate review may need a full hour. The enclosing job must # contain the 12-minute evidence step, 205-minute provider-pool step, the @@ -2522,16 +2457,8 @@ jobs: printf 'Validated exact-head OpenCode review source for %s#%s (%s).\n' \ "$GH_REPOSITORY" "$PR_NUMBER" "$head_repository" - - name: Set up lock-compatible sidecar Python - id: sidecar_python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - update-environment: false - - name: Provision contextual-orchestrator review sidecar env: - SIDECAR_PYTHON: ${{ steps.sidecar_python.outputs.python-path }} BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }} NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} @@ -4063,7 +3990,7 @@ jobs: "npm": "@ai-sdk/openai-compatible", "name": "Contextual Orchestrator", "options": { - "baseURL": "{env:CONTEXTUAL_ORCHESTRATOR_BASE_URL}/v1", + "baseURL": "{env:CONTEXTUAL_ORCHESTRATOR_BASE_URL}", "apiKey": "{env:CONTEXTUAL_ORCHESTRATOR_TOKEN}" }, "models": { @@ -5372,9 +5299,13 @@ jobs: publish_fallback_diff_review() { local body_file event body_file="$(mktemp)" - # Infrastructure failure is not a source-backed product verdict. - # COMMENT preserves diagnostics while coverage and receipt gates fail closed. - event="COMMENT" + # A COMMENT here can never satisfy opencode_review_receipt_gate.py's + # FORMAL_STATES, so the required workflow's "Fail closed without a + # current-head OpenCode verdict" job never sees a receipt, the + # rerun step gated on that receipt is skipped, and the required + # check fails closed forever instead of settling on an honest + # verdict. + event="REQUEST_CHANGES" python3 scripts/ci/opencode_review_surfaces.py build-fallback-review \ --changed-files-file "${OPENCODE_CHANGED_FILES_FILE}" \ --source-root "${OPENCODE_SOURCE_WORKDIR}" \ @@ -5385,7 +5316,10 @@ jobs: >"$body_file" printf '\n%s\n\n%s\n' "## Review outcome" "Coverage is a gate, not the review. This body reviews the changed product files." >>"$body_file" create_pull_review "$event" "$(cat "$body_file")" - # Retain the coverage blocker independently of the diagnostic review. + # create_pull_review REQUEST_CHANGES rewrites the status comment to + # Gate result: REQUEST_CHANGES. Restore the coverage gate so a miss + # never looks finished; next action stays "fix coverage evidence, + # then rerun". request_changes_for_coverage_evidence_failure rm -f "$body_file" } diff --git a/.github/workflows/opencode-review.yml b/.github/workflows/opencode-review.yml index a3fd70a014..ec94e6d24e 100644 --- a/.github/workflows/opencode-review.yml +++ b/.github/workflows/opencode-review.yml @@ -43,7 +43,7 @@ jobs: # queue wait between two single-digit-second jobs. See # docs/doctoring/actions-capacity-root-cause-20260917.md for the # underlying measurement methodology. - runs-on: ${{ github.workflow_ref == 'ContextualWisdomLab/.github/.github/workflows/opencode-review.yml@refs/heads/main' && fromJSON('{"group":"CWL central control","labels":["self-hosted","linux","x64"]}') || fromJSON('["ubuntu-24.04"]') }} + runs-on: ubuntu-24.04 permissions: contents: read pull-requests: read @@ -304,7 +304,7 @@ jobs: # gap using the identical classifier. Fails OPEN: an unreadable, empty, # or truncated file list reviews everything. if: github.event_name != 'pull_request_target' || (github.event.action != 'closed' && github.event.action != 'converted_to_draft') - runs-on: ${{ github.workflow_ref == 'ContextualWisdomLab/.github/.github/workflows/opencode-review.yml@refs/heads/main' && fromJSON('{"group":"CWL central control","labels":["self-hosted","linux","x64"]}') || fromJSON('["ubuntu-24.04"]') }} + runs-on: ubuntu-24.04 timeout-minutes: 5 permissions: contents: read @@ -358,7 +358,7 @@ jobs: name: coverage-source-tree needs: [required-workflow-bootstrap] if: needs.required-workflow-bootstrap.outputs.admitted == 'true' - runs-on: ${{ github.workflow_ref == 'ContextualWisdomLab/.github/.github/workflows/opencode-review.yml@refs/heads/main' && fromJSON('{"group":"CWL central control","labels":["self-hosted","linux","x64"]}') || fromJSON('["ubuntu-24.04"]') }} + runs-on: ubuntu-24.04 steps: - run: >- echo "PR-head source and coverage execution are delegated to the @@ -378,7 +378,7 @@ jobs: # through coverage-source-tree, so an unadmitted head still skips it. needs: [required-workflow-bootstrap] if: needs.required-workflow-bootstrap.outputs.admitted == 'true' - runs-on: ${{ github.workflow_ref == 'ContextualWisdomLab/.github/.github/workflows/opencode-review.yml@refs/heads/main' && fromJSON('{"group":"CWL central control","labels":["self-hosted","linux","x64"]}') || fromJSON('["ubuntu-24.04"]') }} + runs-on: ubuntu-24.04 steps: - run: >- echo "This required-workflow job preserves the stable branch-protection @@ -397,7 +397,7 @@ jobs: # directly by this job's own `if:` below, not inherited through that edge. needs: [required-workflow-bootstrap, changed-scope] if: needs.required-workflow-bootstrap.outputs.admitted == 'true' - runs-on: ${{ github.workflow_ref == 'ContextualWisdomLab/.github/.github/workflows/opencode-review.yml@refs/heads/main' && fromJSON('{"group":"CWL central control","labels":["self-hosted","linux","x64"]}') || fromJSON('["ubuntu-24.04"]') }} + runs-on: ubuntu-24.04 permissions: contents: read pull-requests: read @@ -607,7 +607,7 @@ jobs: # head no longer matches the live one. The target job also revalidates the # live PR before dispatch and verdict admission. if: github.event_name == 'pull_request_target' && github.event.action == 'synchronize' - runs-on: ${{ github.workflow_ref == 'ContextualWisdomLab/.github/.github/workflows/opencode-review.yml@refs/heads/main' && fromJSON('{"group":"CWL central control","labels":["self-hosted","linux","x64"]}') || fromJSON('["ubuntu-24.04"]') }} + runs-on: ubuntu-24.04 permissions: actions: write contents: read diff --git a/.github/workflows/pr-review-autofix.yml b/.github/workflows/pr-review-autofix.yml index a0cf3642d4..1b7849a0c5 100644 --- a/.github/workflows/pr-review-autofix.yml +++ b/.github/workflows/pr-review-autofix.yml @@ -263,16 +263,8 @@ jobs: python3 "$GITHUB_WORKSPACE/trusted-autofix-source/scripts/ci/pr_review_autofix_context.py" \ "${context_args[@]}" - - name: Set up lock-compatible sidecar Python - id: sidecar_python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - update-environment: false - - name: Provision contextual-orchestrator review sidecar env: - SIDECAR_PYTHON: ${{ steps.sidecar_python.outputs.python-path }} BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }} NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} @@ -372,7 +364,7 @@ jobs: "npm": "@ai-sdk/openai-compatible", "name": "Contextual Orchestrator", "options": { - "baseURL": "{env:CONTEXTUAL_ORCHESTRATOR_BASE_URL}/v1", + "baseURL": "{env:CONTEXTUAL_ORCHESTRATOR_BASE_URL}", "apiKey": "{env:CONTEXTUAL_ORCHESTRATOR_TOKEN}" }, "models": { diff --git a/.github/workflows/pr-review-merge-scheduler.yml b/.github/workflows/pr-review-merge-scheduler.yml index aa020f7c6f..d98a72e605 100644 --- a/.github/workflows/pr-review-merge-scheduler.yml +++ b/.github/workflows/pr-review-merge-scheduler.yml @@ -110,11 +110,7 @@ jobs: github.event_name != 'repository_dispatch' || github.event.client_payload.org_sweep != true ) - # The group admits only trusted central main workflows, including reusable - # callers. Keep admission/dispatch off pools occupied by model execution. - runs-on: - group: CWL central control - labels: [self-hosted, linux, x64] + runs-on: ubuntu-24.04 # Bound scan-pr-queue to a wall-clock ceiling well short of GitHub's # 360-minute platform default. This is a single-repository queue scan # (paginated GraphQL reads plus at most one review dispatch and one diff --git a/.github/workflows/release-dependency-license-strix-gate.yml b/.github/workflows/release-dependency-license-strix-gate.yml deleted file mode 100644 index 68d48d15ef..0000000000 --- a/.github/workflows/release-dependency-license-strix-gate.yml +++ /dev/null @@ -1,1098 +0,0 @@ -name: Release Dependency License and Strix Gate - -# Central pre-publish dependency gate (issue #2342). A release workflow calls -# this BEFORE it publishes anything. There is no neutral outcome: the gate -# either succeeds or the release is refused. The organization's scheduled SBOM -# roll-up (scripts/ci/sbom_inventory_aggregator.py) is informational governance -# reporting and is deliberately not reused here. -# -# On success the job returns the complete distribution verdict alongside the -# existing inputs of .github/workflows/exact-artifact-sbom-attestation.yml: -# -# gate: -# uses: ContextualWisdomLab/.github/.github/workflows/release-dependency-license-strix-gate.yml@ -# secrets: inherit -# attest: -# needs: gate -# uses: ContextualWisdomLab/.github/.github/workflows/exact-artifact-sbom-attestation.yml@ -# with: -# source_repository: ${{ needs.gate.outputs.source_repository }} -# ... -# -# The caller must verify the complete verdict separately before admitting the -# full distribution set; the existing attestation still seals one wheel/sdist pair. - -on: - workflow_call: - inputs: - source_repository: - description: Release repository in owner/name form. - required: true - type: string - source_sha: - description: Exact release head commit SHA. - required: true - type: string - ecosystems: - description: Comma-separated ecosystems to enumerate (python and/or cargo). - required: true - type: string - python_lock_path: - description: Hash-pinned Python lock installed into the build environment. - required: false - type: string - default: "" - cargo_manifest_path: - description: Release Cargo.toml whose Cargo.lock and build graph are gated. - required: false - type: string - default: "" - cargo_dev_manifest_path: - description: Development Cargo member whose workspace lock must also be gated. - required: false - type: string - default: "" - distribution_set_artifact_id: - description: Immutable same-run reproducibility record artifact ID containing the complete distribution set manifest. - required: true - type: string - distribution_set_artifact_digest: - description: Expected sha256-prefixed reproducibility record artifact digest from the producer upload. - required: true - type: string - wheel_filename: - description: Exact wheel filename inside the build artifact. - required: true - type: string - sdist_filename: - description: Exact source distribution filename inside the build artifact. - required: true - type: string - evidence_artifact_name: - description: Unique per-call sealed evidence name; also namespaces diagnostic reports. The default retains legacy report names. - required: false - type: string - default: release-dependency-sealed-evidence - # The five provider credentials are declared optional so the licence stage, - # which needs none of them, can run on a review-only negative fixture that - # never reaches Strix. Optional is not lenient: the Strix stage refuses to - # start unless all five are present (STRIX_CREDENTIALS_ABSENT), so an allowed - # input that reaches Strix without credentials fails closed rather than being - # skipped, neutralized, or passed. `required: false` exists so a caller that - # passes no secrets fails on the *licence decision* rather than on - # `workflow_call` schema validation — a schema error is not evidence of a - # licence denial or of Strix being blocked. It is not an invitation to supply - # dummy secrets, and it does not widen any caller's secret exposure. - secrets: - BYTEZ_API_KEY: - required: false - NVIDIA_NIM_API_KEY: - required: false - NVIDIA_NIM_API_KEY_SUB: - required: false - OPENROUTER_API_KEY: - required: false - OPENAI_API_KEY: - required: false - outputs: - full_set_verdict_artifact_id: - description: Immutable same-run complete distribution and dependency verdict artifact ID. - value: ${{ jobs.gate.outputs.full_set_verdict_artifact_id }} - full_set_verdict_artifact_digest: - description: SHA-256 digest of the complete verdict artifact archive. - value: ${{ jobs.gate.outputs.full_set_verdict_artifact_digest }} - source_repository: - description: Gated release repository. - value: ${{ jobs.gate.outputs.source_repository }} - source_sha: - description: Gated release head SHA. - value: ${{ jobs.gate.outputs.source_sha }} - evidence_artifact_id: - description: Immutable same-run sealed evidence artifact ID. - value: ${{ jobs.gate.outputs.evidence_artifact_id }} - evidence_artifact_name: - description: Sealed evidence artifact name. - value: ${{ jobs.gate.outputs.evidence_artifact_name }} - evidence_artifact_digest: - description: Sealed evidence artifact digest in sha256: form. - value: ${{ jobs.gate.outputs.evidence_artifact_digest }} - wheel_filename: - description: Exact gated wheel filename. - value: ${{ jobs.gate.outputs.wheel_filename }} - wheel_sha256: - description: SHA-256 of the exact gated wheel. - value: ${{ jobs.gate.outputs.wheel_sha256 }} - wheel_sbom_filename: - description: CycloneDX SBOM filename for the gated wheel. - value: ${{ jobs.gate.outputs.wheel_sbom_filename }} - wheel_sbom_sha256: - description: SHA-256 of the gated wheel's CycloneDX SBOM. - value: ${{ jobs.gate.outputs.wheel_sbom_sha256 }} - sdist_filename: - description: Exact gated source distribution filename. - value: ${{ jobs.gate.outputs.sdist_filename }} - sdist_sha256: - description: SHA-256 of the exact gated source distribution. - value: ${{ jobs.gate.outputs.sdist_sha256 }} - sdist_sbom_filename: - description: CycloneDX SBOM filename for the gated source distribution. - value: ${{ jobs.gate.outputs.sdist_sbom_filename }} - sdist_sbom_sha256: - description: SHA-256 of the gated source distribution's CycloneDX SBOM. - value: ${{ jobs.gate.outputs.sdist_sbom_sha256 }} - source_identity_sha256: - description: SHA-256 of the sealed source-identity.json. - value: ${{ jobs.gate.outputs.source_identity_sha256 }} - checksum_sha256: - description: SHA-256 of the sealed checksums.sha256. - value: ${{ jobs.gate.outputs.checksum_sha256 }} - predicate_type: - description: Canonical CycloneDX in-toto predicate type. - value: ${{ jobs.gate.outputs.predicate_type }} - cyclonedx_schema: - description: Canonical CycloneDX 1.7 schema URL. - value: ${{ jobs.gate.outputs.cyclonedx_schema }} - -permissions: - contents: read - -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true - -jobs: - prepare: - name: Verify exact distributions and approve the licence closure - runs-on: ubuntu-24.04 - timeout-minutes: 180 - permissions: - contents: read - actions: read - outputs: - matrix_json: ${{ steps.fanout.outputs.matrix_json }} - matrix_overflow_json: ${{ steps.fanout.outputs.matrix_overflow_json }} - has_overflow: ${{ steps.fanout.outputs.has_overflow }} - steps: - - name: Harden runner - uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 - with: - egress-policy: audit - - - name: Materialize immutable trusted gate - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - repository: ContextualWisdomLab/.github - # Reviewed helper revision; intentionally distinct from workflow revision. - ref: e45f1b144aef900d734ff4c900f9e0010fd5a32d - path: trusted-gate - persist-credentials: false - # The whole scripts/ci tree, not an enumerated file list: the trusted - # Strix gate, the orchestrator sidecar and the token loader each source - # siblings by their own directory (strix_model_utils.sh, - # sanitize_contextual_orchestrator_sidecar_stream.py, - # install_strix_timeout_compat.py, strix_timeout_compat.py, …), and an - # enumeration silently breaks the moment one of them gains another. - sparse-checkout: | - scripts/ci/ - requirements-strix-ci-hashes.txt - sparse-checkout-cone-mode: false - - - name: Verify fixed helper checkout identity - env: - HELPER_ROOT: trusted-gate - CALLER_WORKFLOW_SHA: ${{ github.workflow_sha }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - expected=e45f1b144aef900d734ff4c900f9e0010fd5a32d - test "$(git -C "$HELPER_ROOT" rev-parse HEAD)" = "$expected" - origin="$(git -C "$HELPER_ROOT" remote get-url origin)" - case "$origin" in - https://github.com/ContextualWisdomLab/.github|https://github.com/ContextualWisdomLab/.github.git) ;; - *) echo "Foreign helper repository" >&2; exit 1 ;; - esac - test "$(git -C "$HELPER_ROOT" rev-parse HEAD:scripts/ci)" = 7f902df89a925f89c4fae69a842508406cd0207c - test "$(git -C "$HELPER_ROOT" rev-parse HEAD:requirements-strix-ci-hashes.txt)" = eb83beda177c9d2e4ca9b7e2888a1ccb55a123ac - git -C "$HELPER_ROOT" diff --exit-code HEAD -- scripts/ci requirements-strix-ci-hashes.txt - test -f "$HELPER_ROOT/scripts/ci/release_dependency_gate.py" - test -f "$HELPER_ROOT/scripts/ci/verify_release_distribution_set.py" - test -f "$HELPER_ROOT/scripts/ci/verify_release_scope_evidence_set.py" - test -f "$HELPER_ROOT/scripts/ci/prescreen_release_runtime_archives.py" - test -f "$HELPER_ROOT/scripts/ci/collect_release_strix_bindings.py" - test -f "$HELPER_ROOT/scripts/ci/verify_exact_artifact_sbom_handoff.py" - test -f "$HELPER_ROOT/requirements-strix-ci-hashes.txt" - printf 'helper_repository=ContextualWisdomLab/.github helper_sha=%s caller_workflow_sha=%s\n' "$expected" "$CALLER_WORKFLOW_SHA" - - - name: Validate the exact release identity before anything else runs - env: - SOURCE_REPOSITORY: ${{ inputs.source_repository }} - SOURCE_SHA: ${{ inputs.source_sha }} - EVIDENCE_ARTIFACT_NAME: ${{ inputs.evidence_artifact_name }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - # Keep sealed evidence outside both diagnostic artifact namespaces. - case "$EVIDENCE_ARTIFACT_NAME" in - release-dependency-sealed-evidence|license-evidence-?*) ;; - *) echo "evidence artifact name must use the license-evidence- namespace" >&2; exit 1 ;; - esac - # `workflow_call` can only type these inputs as `string`, so a branch - # name or a short SHA would otherwise be accepted here and only caught - # by the gate's own 40-hex check after Strix had already run. The shape - # is therefore checked by the trusted gate before the release head is - # even fetched, and long before any credential is materialized. - python3 -I trusted-gate/scripts/ci/release_dependency_gate.py validate-inputs \ - --source-repository "$SOURCE_REPOSITORY" \ - --source-sha "$SOURCE_SHA" - - - name: Check out the exact release head - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - repository: ${{ inputs.source_repository }} - ref: ${{ inputs.source_sha }} - path: release-source - persist-credentials: false - - - name: Set up the release build interpreter - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.13" - - - name: List the current run and attempt artifacts - env: - GH_TOKEN: ${{ github.token }} - SOURCE_REPOSITORY: ${{ inputs.source_repository }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - test "$SOURCE_REPOSITORY" = "$GITHUB_REPOSITORY" - gh api --paginate "repos/${SOURCE_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \ - --jq '.artifacts[]' > "${RUNNER_TEMP}/release-artifacts.jsonl" - gh api "repos/${SOURCE_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}" \ - > "${RUNNER_TEMP}/release-attempt.json" - - - name: Verify every immutable distribution before dependency capture - env: - GH_TOKEN: ${{ github.token }} - SOURCE_REPOSITORY: ${{ inputs.source_repository }} - SOURCE_SHA: ${{ inputs.source_sha }} - CONTROL_SHA: ${{ github.sha }} - RECORD_ID: ${{ inputs.distribution_set_artifact_id }} - RECORD_DIGEST: ${{ inputs.distribution_set_artifact_digest }} - WHEEL_FILENAME: ${{ inputs.wheel_filename }} - SDIST_FILENAME: ${{ inputs.sdist_filename }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - python3 -I trusted-gate/scripts/ci/verify_release_distribution_set.py \ - --repository "$SOURCE_REPOSITORY" \ - --source-sha "$SOURCE_SHA" \ - --control-sha "$CONTROL_SHA" \ - --run-id "$GITHUB_RUN_ID" \ - --run-attempt "$GITHUB_RUN_ATTEMPT" \ - --record-artifact-id "$RECORD_ID" \ - --record-artifact-digest "$RECORD_DIGEST" \ - --wheel-filename "$WHEEL_FILENAME" \ - --sdist-filename "$SDIST_FILENAME" \ - --metadata "${RUNNER_TEMP}/release-artifacts.jsonl" \ - --attempt "${RUNNER_TEMP}/release-attempt.json" \ - --output release-distributions > "${RUNNER_TEMP}/verified-distributions.json" - - - name: Inventory exact release wheel native links - env: - SOURCE_SHA: ${{ inputs.source_sha }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - python3 -I trusted-gate/scripts/ci/scan_release_native_links.py \ - --verified-distributions "${RUNNER_TEMP}/verified-distributions.json" \ - --distribution-root release-distributions \ - --source-sha "$SOURCE_SHA" \ - --output "${RUNNER_TEMP}/release-native-links.json" - - - name: Verify every immutable scope archive before Strix - env: - GH_TOKEN: ${{ github.token }} - SOURCE_REPOSITORY: ${{ inputs.source_repository }} - SOURCE_SHA: ${{ inputs.source_sha }} - CONTROL_SHA: ${{ github.sha }} - RECORD_ID: ${{ inputs.distribution_set_artifact_id }} - RECORD_DIGEST: ${{ inputs.distribution_set_artifact_digest }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - python3 -I trusted-gate/scripts/ci/verify_release_scope_evidence_set.py \ - --repository "$SOURCE_REPOSITORY" \ - --source-sha "$SOURCE_SHA" \ - --control-sha "$CONTROL_SHA" \ - --run-id "$GITHUB_RUN_ID" \ - --run-attempt "$GITHUB_RUN_ATTEMPT" \ - --record-artifact-id "$RECORD_ID" \ - --record-artifact-digest "$RECORD_DIGEST" \ - --verified-distributions "${RUNNER_TEMP}/verified-distributions.json" \ - --metadata "${RUNNER_TEMP}/release-artifacts.jsonl" \ - --attempt "${RUNNER_TEMP}/release-attempt.json" \ - --output release-scope-evidence > "${RUNNER_TEMP}/verified-scope-evidence.json" - - - name: Recheck exact maturin release assets and native links - shell: bash --noprofile --norc -e -o pipefail {0} - run: python3 -I trusted-gate/scripts/ci/verify_release_maturin_tool_assets.py - - - name: Refuse denied runtime wheel licences before Strix - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - python3 -I trusted-gate/scripts/ci/prescreen_release_runtime_archives.py \ - --verified-scope "${RUNNER_TEMP}/verified-scope-evidence.json" \ - --scope-root release-scope-evidence \ - --output "${RUNNER_TEMP}/runtime-archive-license-report.json" - - - name: Collect the release closure without installing or executing it - env: - ECOSYSTEMS: ${{ inputs.ecosystems }} - PYTHON_LOCK_PATH: ${{ inputs.python_lock_path }} - CARGO_MANIFEST_PATH: ${{ inputs.cargo_manifest_path }} - CARGO_DEV_MANIFEST_PATH: ${{ inputs.cargo_dev_manifest_path }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - python_lock="" - cargo_manifest="" - cargo_dev_manifest="" - if [ -n "$PYTHON_LOCK_PATH" ]; then - python_lock="${PWD}/release-source/${PYTHON_LOCK_PATH}" - fi - if [ -n "$CARGO_MANIFEST_PATH" ]; then - cargo_manifest="${PWD}/release-source/${CARGO_MANIFEST_PATH}" - fi - if [ -n "$CARGO_DEV_MANIFEST_PATH" ]; then - cargo_dev_manifest="${PWD}/release-source/${CARGO_DEV_MANIFEST_PATH}" - fi - bash trusted-gate/scripts/ci/release_dependency_capture_raw.sh \ - --raw-root "${RUNNER_TEMP}/raw" \ - --capture-root "${RUNNER_TEMP}/capture" \ - --ecosystems "$ECOSYSTEMS" \ - --python-lock "$python_lock" \ - --download-root "${RUNNER_TEMP}/collected" \ - --cargo-manifest "$cargo_manifest" \ - --cargo-dev-manifest "$cargo_dev_manifest" - - - name: Assemble per-dependency evidence and isolated synthetic fixtures - env: - SOURCE_REPOSITORY: ${{ inputs.source_repository }} - SOURCE_SHA: ${{ inputs.source_sha }} - ECOSYSTEMS: ${{ inputs.ecosystems }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - jq -n \ - --arg repository "$SOURCE_REPOSITORY" \ - --arg sha "$SOURCE_SHA" \ - --arg ecosystems "$ECOSYSTEMS" \ - '{source_repository: $repository, source_sha: $sha, - ecosystems: ($ecosystems | split(","))}' \ - > "${RUNNER_TEMP}/capture/release.json" - python3 -I trusted-gate/scripts/ci/release_dependency_gate.py capture \ - --raw "${RUNNER_TEMP}/raw" \ - --capture "${RUNNER_TEMP}/capture" - python3 -I trusted-gate/scripts/ci/release_dependency_gate.py capture-license-selections \ - --source release-source --source-sha "$SOURCE_SHA" \ - --capture "${RUNNER_TEMP}/capture" - - - name: Refuse a denied or unverifiable licence before any credential exists - id: license-stage - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - # The licence determination runs here, ahead of every credentialed and - # model step, using the *same* evaluate_dependency_license path the final - # gate uses — so a GPL/LGPL/AGPL dependency, an UNKNOWN licence, or an - # `OR` expression with no recorded permissive selection refuses the - # release before a provider secret is ever read. `capture` alone does not - # reject a licence; it only assembles evidence and fixtures. This stage - # also performs the full-set scope comparison, so an ecosystem whose - # membership cannot be established fails here too. - python3 -I trusted-gate/scripts/ci/release_dependency_gate.py prescreen \ - --source release-source \ - --capture "${RUNNER_TEMP}/capture" \ - --report "${RUNNER_TEMP}/license-report.json" - - - name: Install the prescreened closure into a lock-only environment - if: ${{ inputs.python_lock_path != '' }} - env: - PYTHON_LOCK_PATH: ${{ inputs.python_lock_path }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - # Installing runs dependency code, so it happens only after the licence - # stage above has passed, and only from the bytes that stage judged: - # --no-index --find-links over the collected distributions, with - # --require-hashes so pip proves each file against the lock. Nothing is - # re-resolved or re-downloaded, so the installed bytes are the inspected - # bytes even when the lock records several hashes for a project. - # --without-pip keeps the environment's contents exactly what the lock - # installed, so LOCK_ENV_MISMATCH means a real disagreement. - python3 -m venv --without-pip "${RUNNER_TEMP}/gate-venv" - bash trusted-gate/scripts/ci/release_dependency_capture_raw.sh \ - --install-gated \ - --python-lock "${PWD}/release-source/${PYTHON_LOCK_PATH}" \ - --python-interpreter "${RUNNER_TEMP}/gate-venv/bin/python" \ - --capture-root "${RUNNER_TEMP}/capture" \ - --download-root "${RUNNER_TEMP}/collected" \ - --license-report "${RUNNER_TEMP}/license-report.json" - - - name: Publish the exact licence-approved fixture matrix - id: fanout - env: - CONTROL_SHA: ${{ github.sha }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - python3 -I trusted-gate/scripts/ci/release_dependency_gate.py fanout-plan \ - --capture "${RUNNER_TEMP}/capture" \ - --license-report "${RUNNER_TEMP}/license-report.json" \ - --runtime-archive-license-report "${RUNNER_TEMP}/runtime-archive-license-report.json" \ - --control-sha "$CONTROL_SHA" \ - --run-id "$GITHUB_RUN_ID" \ - --run-attempt "$GITHUB_RUN_ATTEMPT" \ - --output "${RUNNER_TEMP}/strix-fanout-plan.json" - - - name: Export the pre-credential licence report - if: ${{ !cancelled() && steps.license-stage.conclusion != 'skipped' }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.0 - with: - name: ${{ inputs.evidence_artifact_name == 'release-dependency-sealed-evidence' && 'release-dependency-license-report' || format('release-dependency-license-report--{0}', inputs.evidence_artifact_name) }} - path: ${{ runner.temp }}/license-report.json - if-no-files-found: error - - strix: - name: Strix ${{ matrix.key }} - needs: prepare - runs-on: ubuntu-24.04 - timeout-minutes: 360 - permissions: - contents: read - strategy: - fail-fast: false - max-parallel: ${{ needs.prepare.outputs.has_overflow == 'true' && 4 || 8 }} - matrix: ${{ fromJSON(needs.prepare.outputs.matrix_json) }} - steps: &strix_steps - - name: Harden runner - uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 - with: - egress-policy: audit - - - name: Materialize immutable trusted gate - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - repository: ContextualWisdomLab/.github - # Reviewed helper revision; intentionally distinct from workflow revision. - ref: e45f1b144aef900d734ff4c900f9e0010fd5a32d - path: trusted-gate - persist-credentials: false - # The whole scripts/ci tree, not an enumerated file list: the trusted - # Strix gate, the orchestrator sidecar and the token loader each source - # siblings by their own directory (strix_model_utils.sh, - # sanitize_contextual_orchestrator_sidecar_stream.py, - # install_strix_timeout_compat.py, strix_timeout_compat.py, …), and an - # enumeration silently breaks the moment one of them gains another. - sparse-checkout: | - scripts/ci/ - requirements-strix-ci-hashes.txt - sparse-checkout-cone-mode: false - - - name: Verify fixed helper checkout identity - env: - HELPER_ROOT: trusted-gate - CALLER_WORKFLOW_SHA: ${{ github.workflow_sha }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - expected=e45f1b144aef900d734ff4c900f9e0010fd5a32d - test "$(git -C "$HELPER_ROOT" rev-parse HEAD)" = "$expected" - origin="$(git -C "$HELPER_ROOT" remote get-url origin)" - case "$origin" in - https://github.com/ContextualWisdomLab/.github|https://github.com/ContextualWisdomLab/.github.git) ;; - *) echo "Foreign helper repository" >&2; exit 1 ;; - esac - test "$(git -C "$HELPER_ROOT" rev-parse HEAD:scripts/ci)" = 7f902df89a925f89c4fae69a842508406cd0207c - test "$(git -C "$HELPER_ROOT" rev-parse HEAD:requirements-strix-ci-hashes.txt)" = eb83beda177c9d2e4ca9b7e2888a1ccb55a123ac - git -C "$HELPER_ROOT" diff --exit-code HEAD -- scripts/ci requirements-strix-ci-hashes.txt - test -f "$HELPER_ROOT/scripts/ci/release_dependency_gate.py" - test -f "$HELPER_ROOT/scripts/ci/verify_release_distribution_set.py" - test -f "$HELPER_ROOT/scripts/ci/verify_release_scope_evidence_set.py" - test -f "$HELPER_ROOT/scripts/ci/prescreen_release_runtime_archives.py" - test -f "$HELPER_ROOT/scripts/ci/collect_release_strix_bindings.py" - test -f "$HELPER_ROOT/scripts/ci/verify_exact_artifact_sbom_handoff.py" - test -f "$HELPER_ROOT/requirements-strix-ci-hashes.txt" - printf 'helper_repository=ContextualWisdomLab/.github helper_sha=%s caller_workflow_sha=%s\n' "$expected" "$CALLER_WORKFLOW_SHA" - - - name: Require every Strix provider credential before the Strix stage starts - env: - BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }} - NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} - NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} - OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - # The secrets are optional on the contract so the licence stage above can - # run without them. An allowed input that gets this far must still be - # scanned, so absence is a refusal with STRIX_CREDENTIALS_ABSENT. This is - # deliberately not an `if:` condition: a condition would *skip* the Strix - # stage and let the release proceed unscanned. The reason code names only - # the absent variables and never echoes or measures a present value. - python3 -I trusted-gate/scripts/ci/release_dependency_gate.py require-strix-credentials - - - name: Provision the zero-cost review gateway for Strix - env: - BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }} - NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} - NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} - OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - bash trusted-gate/scripts/ci/contextual_orchestrator_review_sidecar.sh - - # Scope boundary, recorded rather than left implicit. The steps below install - # the *gate's own* toolchain — this repository's hash-pinned - # requirements-strix-ci-hashes.txt, materialized from github.workflow_sha, and - # the orchestrator sidecar's own pinned lock. They are a different trust domain - # from the caller's release closure: they are pinned and reviewed in this - # repository, and the licence stage that judges the closure cannot judge the - # scanner it has to run first without a cycle. They are therefore NOT covered by - # the prescreen above, and that is a stated limit, not an exemption: bringing - # the gate's own dependencies under a licence verdict is an owner decision, - # tracked separately, and nothing here may be read as evidence that it happened. - - name: Install the pinned Strix toolchain - working-directory: trusted-gate - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - # Mirrors .github/workflows/strix.yml's install invariants: a private - # umask so the credential-bearing console script is not group-writable, - # --no-deps because strix-agent declares cryptography<49 against this - # repository's cryptography==50.0.0 security pin (see - # requirements-strix-ci-overrides.txt, #952), and an absolute, - # non-symlinked executable inside the interpreter's own scripts root. - umask 022 - python3 -m pip install --disable-pip-version-check --no-cache-dir \ - --require-hashes --no-deps -r requirements-strix-ci-hashes.txt - strix_executable="" - if command -v strix >/dev/null 2>&1; then - strix_executable="$(command -v strix)" - fi - if [ -z "$strix_executable" ] || [[ "$strix_executable" != /* ]] \ - || [ ! -f "$strix_executable" ] || [ -L "$strix_executable" ] \ - || [ ! -x "$strix_executable" ]; then - echo "::error::Pinned Strix installation did not produce a trusted absolute executable path." - exit 1 - fi - case "$strix_executable" in - "$GITHUB_WORKSPACE"/*|"$RUNNER_TEMP"/*) - echo "::error::Refusing a Strix executable from a workspace or runner-temp path." - exit 1 - ;; - esac - strix_scripts_root="$(python3 -c 'import sysconfig; print(sysconfig.get_path("scripts"))')" - if [ -z "$strix_scripts_root" ] || [[ "$strix_scripts_root" != /* ]] \ - || [ ! -d "$strix_scripts_root" ] || [ -L "$strix_scripts_root" ]; then - echo "::error::Pinned Strix installation did not produce a trusted absolute scripts root." - exit 1 - fi - case "$strix_executable" in - "$strix_scripts_root"/*) ;; - *) - echo "::error::Pinned Strix executable is outside the trusted scripts root." - exit 1 - ;; - esac - chmod go-w -- "$strix_scripts_root" "$strix_executable" - { - printf 'STRIX_EXECUTABLE_PATH=%s\n' "$strix_executable" - printf 'STRIX_EXECUTABLE_ROOT=%s\n' "$strix_scripts_root" - printf 'STRIX_EXECUTABLE_SHA256=%s\n' \ - "$(sha256sum "$strix_executable" | cut -d' ' -f1)" - } >> "$GITHUB_ENV" - - - name: Bind the zero-cost model, key, and API base for Strix - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - source trusted-gate/scripts/ci/load_contextual_orchestrator_token.sh - sanitized="$(printf '%s' "${CONTEXTUAL_ORCHESTRATOR_TOKEN:-}" | tr -d '\r\n')" - trimmed="$(printf '%s' "$sanitized" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" - if [ -z "$trimmed" ]; then - echo '::error::CONTEXTUAL_ORCHESTRATOR_TOKEN is required for Strix scans.' - exit 1 - fi - echo "::add-mask::${trimmed}" - sidecar_base="${CONTEXTUAL_ORCHESTRATOR_BASE_URL:-}" - if [ "$sidecar_base" != "http://127.0.0.1:18080" ]; then - echo '::error::Strix sidecar base URL is not the pinned local gateway origin.' - exit 1 - fi - umask 077 - printf '%s' "$trimmed" > "${RUNNER_TEMP}/llm_api_key.txt" - printf '%s/v1' "${sidecar_base%/}" > "${RUNNER_TEMP}/llm_api_base.txt" - printf '%s' 'orchestrator/free' > "${RUNNER_TEMP}/strix_llm.txt" - { - printf 'LLM_API_KEY_FILE=%s\n' "${RUNNER_TEMP}/llm_api_key.txt" - printf 'LLM_API_BASE_FILE=%s\n' "${RUNNER_TEMP}/llm_api_base.txt" - printf 'STRIX_LLM_FILE=%s\n' "${RUNNER_TEMP}/strix_llm.txt" - } >> "$GITHUB_ENV" - - - name: Run Strix against this isolated synthetic fixture - env: - STRIX_LLM_DEFAULT_PROVIDER: contextual_orchestrator - STRIX_REASONING_EFFORT: none - STRIX_FALLBACK_MODELS: "" - STRIX_FAIL_ON_PROVIDER_SIGNAL: "1" - STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM - STRIX_DISABLE_PR_SCOPING: "1" - STRIX_TARGET_PATH: fixture - STRIX_SOURCE_DIRS: "." - IS_PR_EVIDENCE_RUN: "false" - NPM_CONFIG_IGNORE_SCRIPTS: "true" - PNPM_CONFIG_IGNORE_SCRIPTS: "true" - YARN_ENABLE_SCRIPTS: "false" - BUN_CONFIG_IGNORE_SCRIPTS: "true" - SOURCE_SHA: ${{ inputs.source_sha }} - CONTROL_SHA: ${{ github.sha }} - FIXTURE_JSON: ${{ toJSON(matrix.fixture) }} - FIXTURE_KEY: ${{ matrix.key }} - FIXTURE_DIGEST: ${{ matrix.fixture_sha256 }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - export LLM_TIMEOUT=0 - export STRIX_MEMORY_COMPRESSOR_TIMEOUT=0 - export STRIX_PROCESS_TIMEOUT_SECONDS=0 - export STRIX_TOTAL_TIMEOUT_SECONDS=0 - trusted_gate_root="${PWD}/trusted-gate" - workspace="${RUNNER_TEMP}/strix-workspace" - mkdir -p "$workspace/scripts/ci" "$workspace/fixture" - printf '%s\n' "$FIXTURE_JSON" > "$workspace/fixture/fixture.json" - python3 -I - "$workspace/fixture/fixture.json" "$FIXTURE_KEY" "$FIXTURE_DIGEST" <<'PYCODE' - import json, sys - sys.path.insert(0, 'trusted-gate/scripts/ci') - import release_dependency_gate as gate - fixture = json.load(open(sys.argv[1], encoding='utf-8')) - dependency = fixture['dependency'] - key = fixture.get('id') or f"{dependency['ecosystem']}/{dependency['name']}@{dependency['version']}" - if key != sys.argv[2] or gate.fixture_digest(fixture) != sys.argv[3]: - raise SystemExit('matrix fixture differs from the licence-approved plan') - PYCODE - cp "$trusted_gate_root/scripts/ci/strix_evidence_binding.py" \ - "$workspace/scripts/ci/strix_evidence_binding.py" - (cd "$workspace" && STRIX_REPO_ROOT="$workspace" \ - bash "$trusted_gate_root/scripts/ci/strix_quick_gate.sh") - vulnerabilities="" - if [ -d "$workspace/strix_runs" ]; then - vulnerabilities="$(find "$workspace/strix_runs" -type f -name 'vulnerabilities.json' -print -quit)" - fi - test -n "$vulnerabilities" - findings_json="$(jq -c ' - if type == "array" then . - elif type == "object" and (.vulnerabilities? | type) == "array" then .vulnerabilities - else null end' "$vulnerabilities")" - test "$findings_json" != null - mkdir -p "${RUNNER_TEMP}/binding" - jq -n --argjson fixture "$FIXTURE_JSON" --argjson findings "$findings_json" \ - --arg key "$FIXTURE_KEY" \ - --arg sha "$SOURCE_SHA" --arg control "$CONTROL_SHA" \ - --arg digest "$FIXTURE_DIGEST" --argjson run_id "$GITHUB_RUN_ID" \ - --argjson run_attempt "$GITHUB_RUN_ATTEMPT" ' - {schema: "cwl.release-dependency-strix-binding/1", - dependency: $fixture.dependency, - fixture: {id: $key, - sha256: $digest, scenarios: ($fixture.scenarios | keys)}, - source_sha: $sha, control_sha: $control, - run_id: $run_id, run_attempt: $run_attempt, - findings: $findings, - verdict: (if ($findings | length) == 0 then "no_exploitable_findings" - else "findings_present" end)}' > "${RUNNER_TEMP}/binding/${{ matrix.slug }}.json" - - - name: Upload this run-attempt binding - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.0 - with: - name: ${{ matrix.artifact_name }} - path: ${{ runner.temp }}/binding/${{ matrix.slug }}.json - if-no-files-found: error - - - strix_overflow: - name: Strix overflow ${{ matrix.key }} - needs: prepare - if: ${{ needs.prepare.outputs.has_overflow == 'true' }} - runs-on: ubuntu-24.04 - timeout-minutes: 360 - permissions: - contents: read - strategy: - fail-fast: false - max-parallel: 4 - matrix: ${{ fromJSON(needs.prepare.outputs.matrix_overflow_json) }} - steps: *strix_steps - - gate: - needs: [prepare, strix, strix_overflow] - if: >- - ${{ !cancelled() && needs.prepare.result == 'success' && needs.strix.result == 'success' && - ((needs.prepare.outputs.has_overflow == 'true' && needs.strix_overflow.result == 'success') || - (needs.prepare.outputs.has_overflow == 'false' && needs.strix_overflow.result == 'skipped')) }} - name: Collect every Strix binding and seal the complete verdict - runs-on: ubuntu-24.04 - timeout-minutes: 180 - permissions: - contents: read - actions: read - outputs: - full_set_verdict_artifact_id: ${{ steps.full-set-verdict.outputs.artifact-id }} - full_set_verdict_artifact_digest: sha256:${{ steps.full-set-verdict.outputs.artifact-digest }} - source_repository: ${{ steps.seal.outputs.source_repository }} - source_sha: ${{ steps.seal.outputs.source_sha }} - evidence_artifact_id: ${{ steps.sealed-evidence.outputs.artifact-id }} - evidence_artifact_name: ${{ steps.seal.outputs.evidence_artifact_name }} - evidence_artifact_digest: sha256:${{ steps.sealed-evidence.outputs.artifact-digest }} - wheel_filename: ${{ steps.seal.outputs.wheel_filename }} - wheel_sha256: ${{ steps.seal.outputs.wheel_sha256 }} - wheel_sbom_filename: ${{ steps.seal.outputs.wheel_sbom_filename }} - wheel_sbom_sha256: ${{ steps.seal.outputs.wheel_sbom_sha256 }} - sdist_filename: ${{ steps.seal.outputs.sdist_filename }} - sdist_sha256: ${{ steps.seal.outputs.sdist_sha256 }} - sdist_sbom_filename: ${{ steps.seal.outputs.sdist_sbom_filename }} - sdist_sbom_sha256: ${{ steps.seal.outputs.sdist_sbom_sha256 }} - source_identity_sha256: ${{ steps.seal.outputs.source_identity_sha256 }} - checksum_sha256: ${{ steps.seal.outputs.checksum_sha256 }} - predicate_type: ${{ steps.seal.outputs.predicate_type }} - cyclonedx_schema: ${{ steps.seal.outputs.cyclonedx_schema }} - steps: - - name: Harden runner - uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 - with: - egress-policy: audit - - - name: Materialize immutable trusted gate - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - repository: ContextualWisdomLab/.github - # Reviewed helper revision; intentionally distinct from workflow revision. - ref: e45f1b144aef900d734ff4c900f9e0010fd5a32d - path: trusted-gate - persist-credentials: false - # The whole scripts/ci tree, not an enumerated file list: the trusted - # Strix gate, the orchestrator sidecar and the token loader each source - # siblings by their own directory (strix_model_utils.sh, - # sanitize_contextual_orchestrator_sidecar_stream.py, - # install_strix_timeout_compat.py, strix_timeout_compat.py, …), and an - # enumeration silently breaks the moment one of them gains another. - sparse-checkout: | - scripts/ci/ - requirements-strix-ci-hashes.txt - sparse-checkout-cone-mode: false - - - name: Verify fixed helper checkout identity - env: - HELPER_ROOT: trusted-gate - CALLER_WORKFLOW_SHA: ${{ github.workflow_sha }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - expected=e45f1b144aef900d734ff4c900f9e0010fd5a32d - test "$(git -C "$HELPER_ROOT" rev-parse HEAD)" = "$expected" - origin="$(git -C "$HELPER_ROOT" remote get-url origin)" - case "$origin" in - https://github.com/ContextualWisdomLab/.github|https://github.com/ContextualWisdomLab/.github.git) ;; - *) echo "Foreign helper repository" >&2; exit 1 ;; - esac - test "$(git -C "$HELPER_ROOT" rev-parse HEAD:scripts/ci)" = 7f902df89a925f89c4fae69a842508406cd0207c - test "$(git -C "$HELPER_ROOT" rev-parse HEAD:requirements-strix-ci-hashes.txt)" = eb83beda177c9d2e4ca9b7e2888a1ccb55a123ac - git -C "$HELPER_ROOT" diff --exit-code HEAD -- scripts/ci requirements-strix-ci-hashes.txt - test -f "$HELPER_ROOT/scripts/ci/release_dependency_gate.py" - test -f "$HELPER_ROOT/scripts/ci/verify_release_distribution_set.py" - test -f "$HELPER_ROOT/scripts/ci/verify_release_scope_evidence_set.py" - test -f "$HELPER_ROOT/scripts/ci/prescreen_release_runtime_archives.py" - test -f "$HELPER_ROOT/scripts/ci/collect_release_strix_bindings.py" - test -f "$HELPER_ROOT/scripts/ci/verify_exact_artifact_sbom_handoff.py" - test -f "$HELPER_ROOT/requirements-strix-ci-hashes.txt" - printf 'helper_repository=ContextualWisdomLab/.github helper_sha=%s caller_workflow_sha=%s\n' "$expected" "$CALLER_WORKFLOW_SHA" - - - name: Validate the exact release identity before anything else runs - env: - SOURCE_REPOSITORY: ${{ inputs.source_repository }} - SOURCE_SHA: ${{ inputs.source_sha }} - EVIDENCE_ARTIFACT_NAME: ${{ inputs.evidence_artifact_name }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - # Keep sealed evidence outside both diagnostic artifact namespaces. - case "$EVIDENCE_ARTIFACT_NAME" in - release-dependency-sealed-evidence|license-evidence-?*) ;; - *) echo "evidence artifact name must use the license-evidence- namespace" >&2; exit 1 ;; - esac - # `workflow_call` can only type these inputs as `string`, so a branch - # name or a short SHA would otherwise be accepted here and only caught - # by the gate's own 40-hex check after Strix had already run. The shape - # is therefore checked by the trusted gate before the release head is - # even fetched, and long before any credential is materialized. - python3 -I trusted-gate/scripts/ci/release_dependency_gate.py validate-inputs \ - --source-repository "$SOURCE_REPOSITORY" \ - --source-sha "$SOURCE_SHA" - - - name: Check out the exact release head - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - repository: ${{ inputs.source_repository }} - ref: ${{ inputs.source_sha }} - path: release-source - persist-credentials: false - - - name: Set up the release build interpreter - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.13" - - - name: List the current run and attempt artifacts - env: - GH_TOKEN: ${{ github.token }} - SOURCE_REPOSITORY: ${{ inputs.source_repository }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - test "$SOURCE_REPOSITORY" = "$GITHUB_REPOSITORY" - gh api --paginate "repos/${SOURCE_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \ - --jq '.artifacts[]' > "${RUNNER_TEMP}/release-artifacts.jsonl" - gh api "repos/${SOURCE_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}" \ - > "${RUNNER_TEMP}/release-attempt.json" - - - name: Verify every immutable distribution before dependency capture - env: - GH_TOKEN: ${{ github.token }} - SOURCE_REPOSITORY: ${{ inputs.source_repository }} - SOURCE_SHA: ${{ inputs.source_sha }} - CONTROL_SHA: ${{ github.sha }} - RECORD_ID: ${{ inputs.distribution_set_artifact_id }} - RECORD_DIGEST: ${{ inputs.distribution_set_artifact_digest }} - WHEEL_FILENAME: ${{ inputs.wheel_filename }} - SDIST_FILENAME: ${{ inputs.sdist_filename }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - python3 -I trusted-gate/scripts/ci/verify_release_distribution_set.py \ - --repository "$SOURCE_REPOSITORY" \ - --source-sha "$SOURCE_SHA" \ - --control-sha "$CONTROL_SHA" \ - --run-id "$GITHUB_RUN_ID" \ - --run-attempt "$GITHUB_RUN_ATTEMPT" \ - --record-artifact-id "$RECORD_ID" \ - --record-artifact-digest "$RECORD_DIGEST" \ - --wheel-filename "$WHEEL_FILENAME" \ - --sdist-filename "$SDIST_FILENAME" \ - --metadata "${RUNNER_TEMP}/release-artifacts.jsonl" \ - --attempt "${RUNNER_TEMP}/release-attempt.json" \ - --output release-distributions > "${RUNNER_TEMP}/verified-distributions.json" - - - name: Inventory exact release wheel native links - env: - SOURCE_SHA: ${{ inputs.source_sha }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - python3 -I trusted-gate/scripts/ci/scan_release_native_links.py \ - --verified-distributions "${RUNNER_TEMP}/verified-distributions.json" \ - --distribution-root release-distributions \ - --source-sha "$SOURCE_SHA" \ - --output "${RUNNER_TEMP}/release-native-links.json" - - - name: Verify every immutable scope evidence archive before dependency capture - env: - GH_TOKEN: ${{ github.token }} - SOURCE_REPOSITORY: ${{ inputs.source_repository }} - SOURCE_SHA: ${{ inputs.source_sha }} - CONTROL_SHA: ${{ github.sha }} - RECORD_ID: ${{ inputs.distribution_set_artifact_id }} - RECORD_DIGEST: ${{ inputs.distribution_set_artifact_digest }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - python3 -I trusted-gate/scripts/ci/verify_release_scope_evidence_set.py \ - --repository "$SOURCE_REPOSITORY" \ - --source-sha "$SOURCE_SHA" \ - --control-sha "$CONTROL_SHA" \ - --run-id "$GITHUB_RUN_ID" \ - --run-attempt "$GITHUB_RUN_ATTEMPT" \ - --record-artifact-id "$RECORD_ID" \ - --record-artifact-digest "$RECORD_DIGEST" \ - --verified-distributions "${RUNNER_TEMP}/verified-distributions.json" \ - --metadata "${RUNNER_TEMP}/release-artifacts.jsonl" \ - --attempt "${RUNNER_TEMP}/release-attempt.json" \ - --output release-scope-evidence > "${RUNNER_TEMP}/verified-scope-evidence.json" - - - name: Recheck exact maturin release assets and native links - shell: bash --noprofile --norc -e -o pipefail {0} - run: python3 -I trusted-gate/scripts/ci/verify_release_maturin_tool_assets.py - - - name: Refuse denied or unknown licences in transported runtime wheels - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - python3 -I trusted-gate/scripts/ci/prescreen_release_runtime_archives.py \ - --verified-scope "${RUNNER_TEMP}/verified-scope-evidence.json" \ - --scope-root release-scope-evidence \ - --output "${RUNNER_TEMP}/runtime-archive-license-report.json" - - - name: Collect the release closure without installing or executing it - env: - ECOSYSTEMS: ${{ inputs.ecosystems }} - PYTHON_LOCK_PATH: ${{ inputs.python_lock_path }} - CARGO_MANIFEST_PATH: ${{ inputs.cargo_manifest_path }} - CARGO_DEV_MANIFEST_PATH: ${{ inputs.cargo_dev_manifest_path }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - python_lock="" - cargo_manifest="" - cargo_dev_manifest="" - if [ -n "$PYTHON_LOCK_PATH" ]; then - python_lock="${PWD}/release-source/${PYTHON_LOCK_PATH}" - fi - if [ -n "$CARGO_MANIFEST_PATH" ]; then - cargo_manifest="${PWD}/release-source/${CARGO_MANIFEST_PATH}" - fi - if [ -n "$CARGO_DEV_MANIFEST_PATH" ]; then - cargo_dev_manifest="${PWD}/release-source/${CARGO_DEV_MANIFEST_PATH}" - fi - bash trusted-gate/scripts/ci/release_dependency_capture_raw.sh \ - --raw-root "${RUNNER_TEMP}/raw" \ - --capture-root "${RUNNER_TEMP}/capture" \ - --ecosystems "$ECOSYSTEMS" \ - --python-lock "$python_lock" \ - --download-root "${RUNNER_TEMP}/collected" \ - --cargo-manifest "$cargo_manifest" \ - --cargo-dev-manifest "$cargo_dev_manifest" - - - name: Assemble per-dependency evidence and isolated synthetic fixtures - env: - SOURCE_REPOSITORY: ${{ inputs.source_repository }} - SOURCE_SHA: ${{ inputs.source_sha }} - ECOSYSTEMS: ${{ inputs.ecosystems }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - jq -n \ - --arg repository "$SOURCE_REPOSITORY" \ - --arg sha "$SOURCE_SHA" \ - --arg ecosystems "$ECOSYSTEMS" \ - '{source_repository: $repository, source_sha: $sha, - ecosystems: ($ecosystems | split(","))}' \ - > "${RUNNER_TEMP}/capture/release.json" - python3 -I trusted-gate/scripts/ci/release_dependency_gate.py capture \ - --raw "${RUNNER_TEMP}/raw" \ - --capture "${RUNNER_TEMP}/capture" - python3 -I trusted-gate/scripts/ci/release_dependency_gate.py capture-license-selections \ - --source release-source --source-sha "$SOURCE_SHA" \ - --capture "${RUNNER_TEMP}/capture" - - - name: Refuse a denied or unverifiable licence before any credential exists - id: license-stage - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - # The licence determination runs here, ahead of every credentialed and - # model step, using the *same* evaluate_dependency_license path the final - # gate uses — so a GPL/LGPL/AGPL dependency, an UNKNOWN licence, or an - # `OR` expression with no recorded permissive selection refuses the - # release before a provider secret is ever read. `capture` alone does not - # reject a licence; it only assembles evidence and fixtures. This stage - # also performs the full-set scope comparison, so an ecosystem whose - # membership cannot be established fails here too. - python3 -I trusted-gate/scripts/ci/release_dependency_gate.py prescreen \ - --source release-source \ - --capture "${RUNNER_TEMP}/capture" \ - --report "${RUNNER_TEMP}/license-report.json" - - - name: Install the prescreened closure into a lock-only environment - if: ${{ inputs.python_lock_path != '' }} - env: - PYTHON_LOCK_PATH: ${{ inputs.python_lock_path }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - # Installing runs dependency code, so it happens only after the licence - # stage above has passed, and only from the bytes that stage judged: - # --no-index --find-links over the collected distributions, with - # --require-hashes so pip proves each file against the lock. Nothing is - # re-resolved or re-downloaded, so the installed bytes are the inspected - # bytes even when the lock records several hashes for a project. - # --without-pip keeps the environment's contents exactly what the lock - # installed, so LOCK_ENV_MISMATCH means a real disagreement. - python3 -m venv --without-pip "${RUNNER_TEMP}/gate-venv" - bash trusted-gate/scripts/ci/release_dependency_capture_raw.sh \ - --install-gated \ - --python-lock "${PWD}/release-source/${PYTHON_LOCK_PATH}" \ - --python-interpreter "${RUNNER_TEMP}/gate-venv/bin/python" \ - --capture-root "${RUNNER_TEMP}/capture" \ - --download-root "${RUNNER_TEMP}/collected" \ - --license-report "${RUNNER_TEMP}/license-report.json" - - - name: Recompute the exact licence-approved fixture matrix - env: - CONTROL_SHA: ${{ github.sha }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - python3 -I trusted-gate/scripts/ci/release_dependency_gate.py fanout-plan \ - --capture "${RUNNER_TEMP}/capture" \ - --license-report "${RUNNER_TEMP}/license-report.json" \ - --runtime-archive-license-report "${RUNNER_TEMP}/runtime-archive-license-report.json" \ - --control-sha "$CONTROL_SHA" \ - --run-id "$GITHUB_RUN_ID" \ - --run-attempt "$GITHUB_RUN_ATTEMPT" \ - --output "${RUNNER_TEMP}/strix-fanout-plan.json" - - - name: Refuse unless every current-attempt binding and full gate passes - id: full-stage - env: - GH_TOKEN: ${{ github.token }} - SOURCE_REPOSITORY: ${{ inputs.source_repository }} - SOURCE_SHA: ${{ inputs.source_sha }} - CONTROL_SHA: ${{ github.sha }} - RECORD_ID: ${{ inputs.distribution_set_artifact_id }} - RECORD_DIGEST: ${{ inputs.distribution_set_artifact_digest }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - gh api --paginate "repos/${SOURCE_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \ - --jq '.artifacts[]' > "${RUNNER_TEMP}/release-artifacts.jsonl" - python3 -I trusted-gate/scripts/ci/collect_release_strix_bindings.py \ - --source release-source \ - --capture "${RUNNER_TEMP}/capture" \ - --license-report "${RUNNER_TEMP}/license-report.json" \ - --plan "${RUNNER_TEMP}/strix-fanout-plan.json" \ - --metadata "${RUNNER_TEMP}/release-artifacts.jsonl" \ - --attempt "${RUNNER_TEMP}/release-attempt.json" \ - --repository "$SOURCE_REPOSITORY" --source-sha "$SOURCE_SHA" \ - --control-sha "$CONTROL_SHA" --run-id "$GITHUB_RUN_ID" \ - --run-attempt "$GITHUB_RUN_ATTEMPT" \ - --verified-distributions "${RUNNER_TEMP}/verified-distributions.json" \ - --runtime-archive-license-report "${RUNNER_TEMP}/runtime-archive-license-report.json" \ - --native-report "${RUNNER_TEMP}/release-native-links.json" \ - --verified-scope "${RUNNER_TEMP}/verified-scope-evidence.json" \ - --record-artifact-id "$RECORD_ID" --record-artifact-digest "$RECORD_DIGEST" \ - --report "${RUNNER_TEMP}/gate-report.json" \ - --verdict "${RUNNER_TEMP}/full-set-verdict.json" - - - name: Export the complete distribution and dependency verdict - id: full-set-verdict - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.0 - with: - name: ${{ inputs.evidence_artifact_name == 'release-dependency-sealed-evidence' && 'release-dependency-sealed-evidence--full-set-verdict' || format('release-dependency-full-set-verdict--{0}', inputs.evidence_artifact_name) }} - path: | - ${{ runner.temp }}/full-set-verdict.json - ${{ runner.temp }}/gate-report.json - ${{ runner.temp }}/runtime-archive-license-report.json - ${{ runner.temp }}/release-native-links.json - if-no-files-found: error - - - name: Seal exactly the gated bytes for attestation - id: seal - env: - EVIDENCE_ARTIFACT_NAME: ${{ inputs.evidence_artifact_name }} - WHEEL_FILENAME: ${{ inputs.wheel_filename }} - SDIST_FILENAME: ${{ inputs.sdist_filename }} - shell: bash --noprofile --norc -e -o pipefail {0} - run: | - python3 -I trusted-gate/scripts/ci/release_dependency_gate.py seal \ - --report "${RUNNER_TEMP}/gate-report.json" \ - --wheel "release-distributions/${WHEEL_FILENAME}" \ - --sdist "release-distributions/${SDIST_FILENAME}" \ - --evidence-root "${RUNNER_TEMP}/sealed-evidence" \ - --evidence-artifact-name "$EVIDENCE_ARTIFACT_NAME" - - - name: Export the sealed evidence as one immutable same-run artifact - id: sealed-evidence - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.0 - with: - name: ${{ inputs.evidence_artifact_name }} - path: ${{ runner.temp }}/sealed-evidence - if-no-files-found: error - - - name: Export the per-dependency gate report - if: ${{ !cancelled() && steps.full-stage.conclusion != 'skipped' }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.0 - with: - name: ${{ inputs.evidence_artifact_name == 'release-dependency-sealed-evidence' && 'release-dependency-gate-report' || format('release-dependency-gate-report--{0}', inputs.evidence_artifact_name) }} - path: ${{ runner.temp }}/gate-report.json - if-no-files-found: error diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 88fc1c42c4..f15b29f564 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -130,7 +130,7 @@ jobs: # docs/doctoring/required-workflow-path-filter-boundary.md. # Fails OPEN: an unreadable, empty, or truncated file list scans everything. if: github.event_name != 'pull_request_target' || (github.event.action != 'closed' && github.event.action != 'converted_to_draft') - runs-on: ${{ (github.repository == 'ContextualWisdomLab/.github' || github.repository == 'ContextualWisdomLab/fast-mlsirm') && github.workflow_ref == 'ContextualWisdomLab/.github/.github/workflows/strix.yml@refs/heads/main' && fromJSON('{"group":"CWL central control","labels":["self-hosted","linux","x64","cwlab-control"]}') || fromJSON('["ubuntu-24.04"]') }} + runs-on: ubuntu-24.04 timeout-minutes: 5 permissions: contents: read @@ -187,93 +187,21 @@ jobs: if: >- github.event_name != 'pull_request_target' || (github.event.action != 'closed' && github.event.action != 'converted_to_draft') - runs-on: ${{ (github.repository == 'ContextualWisdomLab/.github' || github.repository == 'ContextualWisdomLab/fast-mlsirm') && github.workflow_ref == 'ContextualWisdomLab/.github/.github/workflows/strix.yml@refs/heads/main' && fromJSON('{"group":"CWL central control","labels":["self-hosted","linux","x64","cwlab-control"]}') || fromJSON('["ubuntu-24.04"]') }} + runs-on: ubuntu-24.04 timeout-minutes: 5 permissions: contents: read pull-requests: read - id-token: write outputs: admitted: ${{ steps.admission.outputs.admitted }} target_repository: ${{ steps.admission.outputs.target_repository }} pr_number: ${{ steps.admission.outputs.pr_number }} steps: - - name: Exchange OpenCode app token for Strix target repository metadata reads - id: metadata_read_app_token - if: >- - github.event_name == 'repository_dispatch' - && github.event.client_payload.target_repository != '' - && github.event.client_payload.target_repository != github.repository - && startsWith(github.event.client_payload.target_repository, format('{0}/', github.repository_owner)) - env: - OIDC_AUDIENCE: opencode-github-action - OPENCODE_API_BASE_URL: https://api.opencode.ai - run: | - set -euo pipefail - - mark_unavailable() { - echo "available=false" >>"$GITHUB_OUTPUT" - } - - if [ -z "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:-}" ] || - [ -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]; then - echo "OpenCode app token exchange unavailable: OIDC request environment is missing." - mark_unavailable - exit 0 - fi - - request_url="${ACTIONS_ID_TOKEN_REQUEST_URL}" - separator="&" - case "$request_url" in - *\?*) ;; - *) separator="?" ;; - esac - - if ! oidc_response="$( - curl -fsS \ - -H "Authorization: Bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \ - "${request_url}${separator}audience=${OIDC_AUDIENCE}" - )"; then - echo "OpenCode app token exchange unavailable: OIDC token request did not complete." - mark_unavailable - exit 0 - fi - - if ! oidc_token="$(jq -ser 'select(length == 1 and (.[0] | type == "object")) | .[0].value | select(type == "string" and length > 0 and (test("[[:space:]]") | not))' <<<"$oidc_response")"; then - echo "OpenCode app token exchange unavailable: OIDC token response was empty." - mark_unavailable - exit 0 - fi - - if ! token_response="$( - curl -fsS \ - -X POST \ - -H "Authorization: Bearer ${oidc_token}" \ - "${OPENCODE_API_BASE_URL}/exchange_github_app_token" - )"; then - echo "OpenCode app token exchange unavailable: app token request did not complete." - mark_unavailable - exit 0 - fi - - if ! app_token="$(jq -ser 'select(length == 1 and (.[0] | type == "object")) | .[0].token | select(type == "string" and length > 0 and (test("[[:space:]]") | not))' <<<"$token_response")"; then - echo "OpenCode app token exchange unavailable: app token response was empty." - mark_unavailable - exit 0 - fi - - echo "::add-mask::$app_token" - { - echo "available=true" - echo "token=$app_token" - } >>"$GITHUB_OUTPUT" - - name: Verify event metadata against the live pull request id: admission env: - GH_TOKEN: ${{ steps.metadata_read_app_token.outputs.token || secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || github.token }} + GH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || github.token }} EVENT_NAME: ${{ github.event_name }} - EXPECTED_REPOSITORY_OWNER: ${{ github.repository_owner }} TARGET_REPOSITORY: ${{ github.event.client_payload.target_repository || github.event.pull_request.base.repo.full_name || github.repository }} TARGET_PR_NUMBER: ${{ github.event.client_payload.pr_number || github.event.pull_request.number }} EXPECTED_BASE_REF: ${{ github.event.client_payload.pr_base_ref || github.event.pull_request.base.ref }} @@ -299,11 +227,6 @@ jobs: echo "::error::Strix event metadata is incomplete or malformed." exit 1 fi - if [ "$EVENT_NAME" = "repository_dispatch" ] && - [ "${TARGET_REPOSITORY%%/*}" != "$EXPECTED_REPOSITORY_OWNER" ]; then - echo "::error::Strix dispatch target is outside the workflow repository owner." - exit 1 - fi pull_request_json="$(gh api "repos/${TARGET_REPOSITORY}/pulls/${TARGET_PR_NUMBER}")" live_tuple="$(jq -r '[.state // "", .base.repo.full_name // "", .base.ref // "", .base.sha // "", .head.repo.full_name // "", .head.sha // ""] | @tsv' <<<"$pull_request_json")" expected_tuple="$(printf 'open\t%s\t%s\t%s\t%s\t%s' "$TARGET_REPOSITORY" "$EXPECTED_BASE_REF" "$EXPECTED_BASE_SHA" "$EXPECTED_HEAD_REPOSITORY" "$EXPECTED_HEAD_SHA")" @@ -337,7 +260,7 @@ jobs: github.event.pull_request.base.repo.full_name || github.repository }}-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: true - runs-on: ${{ (github.repository == 'ContextualWisdomLab/.github' || github.repository == 'ContextualWisdomLab/fast-mlsirm') && github.workflow_ref == 'ContextualWisdomLab/.github/.github/workflows/strix.yml@refs/heads/main' && fromJSON('{"group":"CWL central control","labels":["self-hosted","linux","x64","cwlab-control"]}') || fromJSON('["ubuntu-24.04"]') }} + runs-on: ubuntu-24.04 # Bound this gh-api-only cleanup job so a stuck call (rate limit, hung # `gh api --paginate`) cannot silently occupy a runner for GitHub's # 360-minute platform default -- exactly the window when a busy PR is @@ -400,7 +323,7 @@ jobs: --arg action "$PR_ACTION" --arg repo "$TARGET_REPOSITORY" --arg current "$CURRENT_RUN_ID" ' .workflow_runs[] | select((.id | tostring) != $current) - | select(.path == ".github/workflows/strix.yml") + | select(.name == "Strix Security Scan") | select(.event == "pull_request_target") | ((.display_title // "") | startswith("Strix Security Scan " + $repo + "#" + $pr + "@")) as $title_matches | ((.pull_requests // []) | any((.number | tostring) == $pr)) as $metadata_matches @@ -444,13 +367,6 @@ jobs: done strix: - outputs: - transport_capacity_unavailable: ${{ steps.strix_scan.outputs.transport_capacity_unavailable || steps.strix_sidecar_failure.outputs.transport_capacity_unavailable }} - transport_retry_eligible: ${{ steps.strix_scan.outputs.transport_retry_eligible || steps.strix_sidecar_failure.outputs.transport_retry_eligible }} - transport_retry_delay_seconds: ${{ steps.strix_scan.outputs.transport_retry_delay_seconds || steps.strix_sidecar_failure.outputs.transport_retry_delay_seconds }} - transport_retry_next_attempt: ${{ steps.strix_scan.outputs.transport_retry_next_attempt || steps.strix_sidecar_failure.outputs.transport_retry_next_attempt }} - provider_attempt_count: ${{ steps.strix_sidecar_failure.outputs.provider_attempt_count }} - transport_http_status: ${{ steps.strix_sidecar_failure.outputs.transport_http_status }} needs: [changed-scope, admit-current-head] if: needs.changed-scope.outputs.code == 'true' && needs.admit-current-head.outputs.admitted == 'true' # Large, actively-growing repositories (e.g. contextual-orchestrator) can @@ -854,19 +770,9 @@ jobs: echo 'provider_mode=contextual_orchestrator' } >> "$GITHUB_OUTPUT" - - name: Set up lock-compatible sidecar Python - if: steps.gate.outputs.enabled == 'true' - id: sidecar_python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - update-environment: false - - name: Provision contextual-orchestrator Strix sidecar - id: strix_sidecar if: steps.gate.outputs.enabled == 'true' env: - SIDECAR_PYTHON: ${{ steps.sidecar_python.outputs.python-path }} BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }} NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} @@ -876,26 +782,8 @@ jobs: CONTEXTUAL_ORCHESTRATOR_POOL: free run: | set -euo pipefail - report_parent="$GITHUB_WORKSPACE/strix_runs" - if [ -L "$report_parent" ]; then - echo '::error::Strix preflight report directory must not be a symbolic link.' - exit 1 - fi - mkdir -p "$report_parent" - rm -f "$report_parent/contextual-orchestrator-preflight.json" bash "$TRUSTED_STRIX_SOURCE/scripts/ci/contextual_orchestrator_review_sidecar.sh" - - name: Classify all-429 Strix sidecar failure - id: strix_sidecar_failure - if: failure() && steps.strix_sidecar.outcome == 'failure' - env: - NOEMA_TRANSPORT_RETRY_ATTEMPT: ${{ toJSON(github.event.client_payload.transport_retry_attempt) }} - EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.client_payload.pr_head_sha || '' }} - run: | - python3 "$TRUSTED_STRIX_SOURCE/scripts/ci/noema_preflight_capacity.py" \ - --preflight-report "$GITHUB_WORKSPACE/strix_runs/contextual-orchestrator-preflight.json" \ - --expected-head "$EXPECTED_HEAD_SHA" - - name: Set up Python if: steps.gate.outputs.enabled == 'true' uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -1041,7 +929,6 @@ jobs: echo "STRIX_LLM_FILE=$strix_llm_file" >> "$GITHUB_ENV" - name: Run Strix (quick) - id: strix_scan if: steps.gate.outputs.enabled == 'true' # Security invariant for pull_request_target: execute only from the # trusted base checkout. The gate copies PR-head blobs into an isolated @@ -1079,7 +966,6 @@ jobs: PR_BASE_SHA: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.event.client_payload.pr_base_sha }} PR_HEAD_SHA: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.event.client_payload.pr_head_sha }} IS_PR_EVIDENCE_RUN: ${{ (github.event_name == 'pull_request_target' || github.event.client_payload.pr_number != '') && 'true' || 'false' }} - NOEMA_TRANSPORT_RETRY_ATTEMPT: ${{ toJSON(github.event.client_payload.transport_retry_attempt) }} run: | export LLM_TIMEOUT=0 export STRIX_MEMORY_COMPRESSOR_TIMEOUT=0 @@ -1090,15 +976,6 @@ jobs: # Defined before the gate loop so the bounded retry decision below # can classify outcomes without duplicating the patterns later. backend_unavailable_signal='STRIX_PROVIDER_UNAVAILABLE|RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*500[^[:cntrl:]]*internal_error|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|Error during penetration test: loginAsGuest failed after [0-9]+ attempts: curl exit 7: curl: \(7\) Failed to connect to 127\.0\.0\.1 port 48080' - # Only explicit connection/rate-limit failures qualify for a new - # attempt; scanner defects and sandbox bootstrap failures do not. - runtime_transport_signal='LLM CONNECTION FAILED|Could not establish connection to the language model|RateLimitError|Too many requests\. For more on scraping GitHub' - # Scanner tooling breakage (Caido GraphQL query/cursor errors) is - # not a provider outcome. It can occur while providers are healthy - # and it can coexist with genuine provider rate limits, so it gets - # its own typed notice instead of being folded into the provider - # verdict. See docs/doctoring/review-failure-taxonomy.md. - tooling_error_signal='Invalid HTTPQL query|Failed to parse cursor|TransportQueryError|caido_sdk_client\.errors' model_behavior_error_signal='(^|[^A-Za-z0-9_])(agents|pydantic_ai|strix)(\.[A-Za-z_][A-Za-z0-9_]*)*\.ModelBehaviorError([^A-Za-z0-9_]|$)' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the @@ -1152,22 +1029,10 @@ jobs: # Classify provider/backend exhaustion only when no vulnerability # finding was emitted. Classification improves diagnosis; it never # converts an incomplete scan into passing security evidence. - # Report scanner tooling breakage on its own, whatever the provider - # verdict turns out to be. This never changes the exit code: an - # incomplete scan stays non-passing either way. - if grep -Eq "$tooling_error_signal" "$strix_neutralization_scope_log"; then - echo "::error title=STRIX_TOOLING_ERROR::Strix scanner tooling failed (Caido GraphQL query or cursor error). This is a scanner defect, not a provider outage; a provider notice may also follow. See the strix-reports artifact and run log." - fi - if ( grep -Eiq "$backend_unavailable_signal" "$strix_neutralization_scope_log" \ || grep -Eq "$model_behavior_error_signal" "$strix_neutralization_scope_log" ) \ && ! grep -Eiq "$reported_vulnerability_signal" "$strix_neutralization_scope_log"; then echo "::error title=STRIX_PROVIDER_UNAVAILABLE::Strix could not complete authoritative vulnerability analysis because its provider/backend was unavailable (rate limit, token cap, connection, warm-up, or model-behavior failure). See the strix-reports artifact and run log." - if grep -Eiq "$runtime_transport_signal" "$strix_neutralization_scope_log" \ - && ! grep -Eq "$tooling_error_signal" "$strix_neutralization_scope_log" \ - && ! grep -Fq 'STRIX_SANDBOX_UNAVAILABLE' "$strix_neutralization_scope_log"; then - PYTHONPATH="$TRUSTED_STRIX_SOURCE" python3 -m scripts.ci.strix_runtime_capacity --expected-head "$PR_HEAD_SHA" - fi exit "$strix_rc" fi @@ -1294,7 +1159,7 @@ jobs: name: publish-manual-pr-evidence-status needs: strix if: ${{ always() && !cancelled() && github.event_name == 'repository_dispatch' && github.event.client_payload.pr_head_sha != '' }} - runs-on: ${{ (github.repository == 'ContextualWisdomLab/.github' || github.repository == 'ContextualWisdomLab/fast-mlsirm') && github.workflow_ref == 'ContextualWisdomLab/.github/.github/workflows/strix.yml@refs/heads/main' && fromJSON('{"group":"CWL central control","labels":["self-hosted","linux","x64","cwlab-control"]}') || fromJSON('["ubuntu-24.04"]') }} + runs-on: ubuntu-24.04 # Single-shot OIDC exchange plus a handful of curl/gh api calls, no loop # or pagination -- same shape as the agent-mention-*-dispatch.yml # validate-and-forward jobs, which bound at timeout-minutes: 5. Without @@ -1507,88 +1372,3 @@ jobs: echo "::error::Could not publish manual Strix status from follow-up job after all configured credentials failed after a non-successful scan; the target PR head is missing required Strix status evidence. See the preceding notices for token-specific reasons." exit 1 - - continue-strix-transport: - needs: [admit-current-head, strix] - if: >- - always() - && needs.admit-current-head.outputs.admitted == 'true' - && !cancelled() - && needs.strix.result == 'failure' - && needs.strix.outputs.transport_capacity_unavailable == 'true' - && needs.strix.outputs.transport_retry_eligible == 'true' - runs-on: ${{ (github.repository == 'ContextualWisdomLab/.github' || github.repository == 'ContextualWisdomLab/fast-mlsirm') && github.workflow_ref == 'ContextualWisdomLab/.github/.github/workflows/strix.yml@refs/heads/main' && fromJSON('{"group":"CWL central control","labels":["self-hosted","linux","x64","cwlab-control"]}') || fromJSON('["ubuntu-24.04"]') }} - timeout-minutes: 10 - permissions: - contents: write - pull-requests: read - env: - # Consumer required workflows need the existing central dispatch credential. - # The central handler can use its repository-scoped token as fallback. - GH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || github.token }} - TARGET_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name || github.event.client_payload.target_repository || github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number || github.event.client_payload.pr_number || '' }} - EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.client_payload.pr_head_sha || '' }} - EXPECTED_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.client_payload.pr_base_sha || '' }} - EXPECTED_BASE_REF: ${{ github.event.pull_request.base.ref || github.event.client_payload.pr_base_ref || '' }} - DELAY_SECONDS: ${{ needs.strix.outputs.transport_retry_delay_seconds }} - NEXT_ATTEMPT: ${{ needs.strix.outputs.transport_retry_next_attempt }} - PROVIDER_ATTEMPT_COUNT: ${{ needs.strix.outputs.provider_attempt_count }} - TRANSPORT_HTTP_STATUS: ${{ needs.strix.outputs.transport_http_status }} - steps: - - name: Schedule bounded Strix transport re-dispatch - run: | - set -euo pipefail - if { [ "$GITHUB_REPOSITORY" != "ContextualWisdomLab/.github" ] && - [ "$TARGET_REPOSITORY" != "$GITHUB_REPOSITORY" ]; } || - ! [[ "$TARGET_REPOSITORY" =~ ^ContextualWisdomLab/[A-Za-z0-9_.-]+$ ]] || - ! [[ "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]] || - ! [[ "$EXPECTED_HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] || - ! [[ "$EXPECTED_BASE_SHA" =~ ^[0-9a-f]{40}$ ]]; then - echo "::error::Strix transport re-dispatch rejected an unrelated origin or malformed PR identity." - exit 1 - fi - if ! [[ "$DELAY_SECONDS" =~ ^[1-9][0-9]*$ ]] || [ "$DELAY_SECONDS" -gt 300 ] || - ! [[ "$NEXT_ATTEMPT" =~ ^[12]$ ]]; then - echo "::error::Strix transport re-dispatch refused an unbounded delay or attempt." - exit 1 - fi - echo "::notice::Strix provider capacity unavailable (http_status=${TRANSPORT_HTTP_STATUS:-unknown}, provider_attempt_count=${PROVIDER_ATTEMPT_COUNT:-unknown}); waiting ${DELAY_SECONDS}s before same-head continuation re-dispatch ${NEXT_ATTEMPT}." - sleep "$DELAY_SECONDS" - live_pr="$(gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}")" - live_head="$(jq -r '.head.sha // empty' <<<"$live_pr")" - live_head_repo="$(jq -r '.head.repo.full_name // empty' <<<"$live_pr")" - live_base="$(jq -r '.base.sha // empty' <<<"$live_pr")" - live_base_repo="$(jq -r '.base.repo.full_name // empty' <<<"$live_pr")" - live_base_ref="$(jq -r '.base.ref // empty' <<<"$live_pr")" - live_ready="$(jq -r '.draft == false' <<<"$live_pr")" - live_state="$(jq -r '.state // empty' <<<"$live_pr")" - if [ "$live_head" != "$EXPECTED_HEAD_SHA" ] || - [ "$live_head_repo" != "$TARGET_REPOSITORY" ] || - [ "$live_base" != "$EXPECTED_BASE_SHA" ] || - [ "$live_base_repo" != "$TARGET_REPOSITORY" ] || - [ "$live_base_ref" != "$EXPECTED_BASE_REF" ] || - [ "$live_ready" != "true" ] || - [ "$live_state" != "open" ]; then - echo "::notice::Strix transport re-dispatch retired because the live PR head or base moved or closed." - exit 0 - fi - jq -n \ - --arg target_repository "$TARGET_REPOSITORY" \ - --argjson pr_number "$PR_NUMBER" \ - --arg pr_head_sha "$EXPECTED_HEAD_SHA" \ - --arg pr_base_ref "$EXPECTED_BASE_REF" \ - --arg pr_base_sha "$EXPECTED_BASE_SHA" \ - --argjson transport_retry_attempt "$NEXT_ATTEMPT" \ - '{ - event_type: "strix-scan", - client_payload: { - target_repository: $target_repository, - pr_number: $pr_number, - pr_head_sha: $pr_head_sha, - pr_base_ref: $pr_base_ref, - pr_base_sha: $pr_base_sha, - transport_retry_attempt: $transport_retry_attempt - } - }' | gh api -X POST "repos/ContextualWisdomLab/.github/dispatches" --input - - echo "::notice::Scheduled Strix transport continuation re-dispatch for ${TARGET_REPOSITORY}#${PR_NUMBER} at ${EXPECTED_HEAD_SHA} (attempt ${NEXT_ATTEMPT})." diff --git a/.jules/bolt.md b/.jules/bolt.md index 4f20b36047..088c43d3f8 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -54,3 +54,7 @@ ## 2026-09-01 - 대용량 문자열 서브스트링 스캐닝 루프 최적화 **Learning:** 긴 텍스트에서 여러 기준 문자열(`candidate`)을 탐색하여 다음 구역의 시작점을 찾을 때, 텍스트 전체에 대해 반복적으로 `text.find(candidate)`를 호출하면 O(N)의 비효율적인 중복 스캐닝 오버헤드가 발생합니다. 특히 가장 가까운 시작점을 찾기 위해 모든 후보를 스캔할 때 이 문제가 심화됩니다. **Action:** 기준점(`start`)을 잡은 후, `idx = text.find(candidate, start, end)`를 사용하여 검색 범위를 동적으로 축소(`end = min(end, idx)`)하십시오. 이렇게 하면 불필요한 스캐닝 오버헤드를 막고 검색 범위를 안전하게 줄여 매우 큰 성능 향상을 얻을 수 있습니다. + +## 2026-09-17 - ThreadPoolExecutor shutdown latency +**Learning:** Using `executor.shutdown(wait=False)` to speed up generator cleanup (e.g. `list_recent_pull_requests`) inside a Python script only hides the blocking until process exit. CPython will still forcefully join all non-daemon worker threads when terminating, so the overall job wall-clock time is not reduced, and the GitHub API requests continue running unconstrained in the background until completion or network timeout. +**Action:** Do not use `wait=False` micro-optimizations to abandon running I/O workers. Instead, rely on cooperative cancellation (e.g., passing a `threading.Event` to interrupt network waits) so workers cleanly abort, allowing both the generator and the interpreter to exit quickly. diff --git a/CHANGELOG.d/20260923-release-dependency-license-strix-gate.md b/CHANGELOG.d/20260923-release-dependency-license-strix-gate.md deleted file mode 100644 index 784333b9c3..0000000000 --- a/CHANGELOG.d/20260923-release-dependency-license-strix-gate.md +++ /dev/null @@ -1,186 +0,0 @@ -### Central pre-publish dependency gate: parsed license denial, resolved-graph reconciliation, per-dependency Strix bindings - -- `origin/main` had **no** fail-closed pre-publish dependency gate. The only license signal was - `scripts/ci/sbom_inventory_aggregator.py`, a *scheduled, informational* org SBOM roll-up that - flags GPL/AGPL/NOASSERTION for governance: it is not per-dependency, not fail-closed, and not - bound to a release head. That gap blocked fast-mlsirm's 0.11.5 PyPI release and - contextual-orchestrator's VCS-pin removal (#2342). -- New reusable `workflow_call` workflow `.github/workflows/release-dependency-license-strix-gate.yml` - runs **before** a release workflow publishes. It has no `continue-on-error`, no `if: always()`, - no neutral outcome, and no bypass; `permissions` is `contents: read` at both workflow and job - scope, and every action is pinned to the same commits `exact-artifact-sbom-attestation.yml` uses. - The decision code is materialized from `ContextualWisdomLab/.github` at `github.workflow_sha` - into `trusted-gate/`, so a caller's tree can never supply it. -- New `scripts/ci/spdx_license_policy.py` is a recursive-descent SPDX 2.3 expression parser - (`AND`/`OR`/`WITH`/parentheses/legacy `+`). Policy is applied to the parsed tree, never by - substring matching: GPL, LGPL, and AGPL are denied in every version and in both the `-only` and - `-or-later` spellings, an exception never rescues a denied base (`GPL-2.0-only WITH - Classpath-exception-2.0` stays denied), and `missing`, `NOASSERTION`, `NONE`, `UNKNOWN`, - `custom`, `LicenseRef-*`, and any unparseable expression fail closed. A dual-licensed dependency - passes only when a non-denied operand is explicitly selected with a written rationale, which is - copied into the artifact provenance as a CycloneDX component property. Bundled `LICENSE`, - `COPYING`, and `NOTICE` text *is* substring-scanned — correct for prose — so metadata claiming - MIT while shipping GPL text fails as a disagreement. -- New `scripts/ci/release_dependency_gate.py` enumerates both ecosystems and refuses any - asymmetry: the hash-pinned Python lock against `pip inspect` of the build environment - (`LOCK_ENV_MISMATCH`), and `Cargo.lock` against the full resolved build graph including - build-dependencies and every `cfg()`-gated target (`CARGO_LOCK_GRAPH_MISMATCH`, - `CARGO_CHECKSUM_MISSING`). It records name, version, source hash, license, license source, and - distribution inclusion per dependency; verifies the captured source hash against the pin - (`SOURCE_HASH_MISMATCH`); evaluates static and dynamic linking targets of shipped native - libraries against an explicit, auditable platform-runtime soname allowlist (glibc, the GCC - runtime-library-exception libraries, `libpython`) so a real compiled wheel can pass at all; and - runs deterministic archive-escape and install-hook detectors (`ARCHIVE_PATH_ESCAPE`, - `INSTALL_HOOK`). -- Strix evidence is accepted **only** as a machine-readable binding, one isolated synthetic - fixture per dependency, simulating file parsing, install hooks, archive traversal, native library - loading, credential/network attempts, and known-vulnerability surface. A textual "0 findings" or - "No exploitable vulnerabilities detected" is rejected (`STRIX_TEXTUAL_PASS_REJECTED`), and a - missing or malformed binding is a failure, never neutral (`STRIX_BINDING_MISSING`, - `STRIX_BINDING_MALFORMED`, `STRIX_BINDING_UNBOUND`). The trusted binder is resolved next to the - gate script's **own** directory, adopting `strix_quick_gate.sh`'s trusted-path semantics in new - code without touching that file (PR #2291 owns its one-line repair). -- On success the gate seals exactly the six members - `scripts/ci/verify_exact_artifact_sbom_handoff.py` expects — wheel, sdist, their CycloneDX 1.7 - SBOMs, `source-identity.json`, `checksums.sha256` — and emits all 17 inputs of - `exact-artifact-sbom-attestation.yml` as workflow outputs, so provenance covers exactly the bytes - that were gated. `tests/test_release_dependency_gate_capture_and_seal.py` proves the sealed - directory is accepted verbatim by that verifier. -- Strix itself is invoked through the organization's existing trusted entry point - `scripts/ci/strix_quick_gate.sh`, once per isolated fixture workspace via `STRIX_REPO_ROOT`, - with `strix.yml`'s bootstrap invariants mirrored verbatim (private install umask, - `--require-hashes --no-deps` against the unmodified `requirements-strix-ci-hashes.txt`, absolute - non-symlinked executable inside the interpreter's scripts root, `chmod go-w`, digest pinned into - `GITHUB_ENV`, sidecar-provided `LLM_API_KEY_FILE`/`LLM_API_BASE_FILE`/`STRIX_LLM_FILE`, and - `orchestrator/free` as the only accepted model). The trusted binder is copied into each fixture - workspace so the gate's binder lookup resolves both on current `main` and after #2291, without - editing that file. `strix_runs/**/vulnerabilities.json` is normalized to an array only when it - already is one (or carries a `vulnerabilities` array); any other shape writes no binding, so the - gate refuses with `STRIX_BINDING_MISSING` rather than inventing a result. -- `scripts/ci/release_dependency_capture_raw.sh` runs the runner-only tools (`pip inspect`, - `pip download`, `cargo metadata --locked`, `cargo fetch`, archive listing, `readelf -d`) and - writes their output verbatim; every decision lives in the unit-tested Python that reads it. It - inspects a `python3 -m venv --without-pip` environment holding exactly the lock, so the - no-exemption lock/environment rule is not defeated by setup-python's preinstalled `pip`, and it - fetches by exact pin with hash checking deliberately disabled so `SOURCE_HASH_MISMATCH` is - observable rather than pre-empted by pip. The gate adds no Python dependency and does not touch - any `anyio` pin or `requirements-strix-ci*` (#2278 owns that lane). Refs #2342. -- The gate now runs in **two stages**, so the licence determination precedes every credential and - model step. `release_dependency_gate.py prescreen` (`stage: license`) enumerates the full - dependency scope and applies the *same* `evaluate_dependency_license` decision the final gate - uses, reading no Strix binding and requiring no provider credential: a GPL/LGPL/AGPL dependency, - an `UNKNOWN`/missing licence, or an `OR` expression with no recorded permissive selection refuses - the release before a secret is read. `capture` alone never rejected a licence — it only assembles - evidence and fixtures — so making the secrets optional would not by itself have produced a - pre-Strix rejection. The five provider secrets are therefore declared `required: false`, which is - not leniency: `require-strix-credentials` refuses the Strix stage with `STRIX_CREDENTIALS_ABSENT` - when any is absent, as a failing command rather than an `if:` condition, because a condition would - *skip* the scan and let the release proceed unscanned. The reason code names only the absent - variables and never echoes or measures a present value. Only a `full`-stage report may be sealed, - so a passing prescreen can never stand in for the Strix stage. -- Dependency **scope is compared as a whole set**, per ecosystem, with `expected_count`, - `enumerated_count`, `collected_count`, and `matched_count` recorded in the report and equality - required. CO#1226 accepted coverage because one component of one ecosystem existed; an ecosystem - this gate cannot enumerate is now `SCOPE_UNVERIFIABLE` rather than silently skipped, a collected - set that is a subset of the producer's declared set is `SCOPE_SET_MISMATCH`, and so is capture - material for something no declared ecosystem expects. Scope is direct, transitive, build, dev, - optional and platform: `resolve_cargo_graph` walks every `resolve.nodes` edge regardless of - `dep_kind` or target `cfg`, so a UEFI-only crate such as `r-efi` is an expected member and gets no - target-based exemption. -- Licence metadata is read from **each fetched distribution's own** `METADATA`/`PKG-INFO`, by the - trusted gate's `distribution-metadata`, which also re-checks that the archive declares the pinned - project and version. It cannot come from `pip inspect` of the lock-only environment any more, - because no such environment exists yet when the licence is judged; the enumeration is built from - the same fetched set, in the `pip inspect` shape the lock/environment reconciliation already reads, - so identity and licence stay consistent by construction and an entry that is not present exactly - once is an error rather than a default. The previous metadata step ran `python3 -m pip show` - without the `--python` target its neighbours carried, so it inspected the *runner's* global - interpreter where the release dependencies are not installed at all. -- The exact release identity is shape-checked **first**. `workflow_call` can only type - `source_sha` as `string`, and the gate's own 40-hex check was reached only after Strix had run, so - `validate-inputs` now refuses a branch name or a short SHA before the release head is fetched. -- Failure evidence survives the failure that produced it: each report upload is bound to the step - that writes it, running whether that step passed or failed but not when it never ran and not on - cancellation. This is deliberately narrower than a blanket `always()`, and with - `if-no-files-found: error` a report that should have been written but was not stays a failure - instead of being masked. Neither upload can rescue the run. Refs #2342. -- **Install and capture now resolve from the same validated sources.** `pip install -r ` reads - the real lock and honors `--index-url`, `--extra-index-url` and `--find-links` in it, while the - capture step's `pip download` used a reconstructed plain requirements file built with - `grep -oE '^[A-Za-z0-9._-]+==[^ ;]+'`, which dropped every `-`-prefixed directive. Collection could - therefore resolve from a different source than install, and any release lock using a private or - extra index failed capture outright. The fix never forwards what the lock says: `lock-source-options` - parses each directive, validates it, and only then emits an explicit option list, reusing the - trusted-origin and bounded-path policy `materialize_base_python_requirements.py` already applies - (HTTPS, default port, host allowlist, no userinfo; normalized relative path with no `.`/`..` and - none of `\\ : ? #`). An unlisted origin is `LOCK_SOURCE_ORIGIN_DENIED`, a URL carrying userinfo is - `LOCK_SOURCE_CREDENTIAL_IN_URL` and withholds the whole URL from both the message and the report, a - path leaving the release tree is `LOCK_SOURCE_PATH_ESCAPE`, and a nested `-r`/`-c` include, an - environment marker, or any other directive form is `LOCK_SOURCE_UNSUPPORTED`. Nothing is dropped - silently, because silent dropping was the defect. The supported dialect is deliberately narrow and - this organization's own `requirements-*-hashes.txt` files use none of these forms. The options are - read into a bash array with the validator's exit status checked explicitly — *not* through - `mapfile < <(…)`, where `set -e` discards a refusal and it would read as "no options" and resolve - from the default index anyway. Source resolution decides only where pip looks: the hash pin still - decides what is acceptable, so `SOURCE_HASH_MISMATCH` remains observable and an offline - `--find-links` root cannot substitute different bytes. Refs #2342. -- **Nothing is installed before it has been adjudicated.** The gate's premise is that a denied, - unknown or untrusted dependency is refused before any of it runs, but the workflow installed the - whole release closure in a step that preceded *both* the lock-source validation and the licence - prescreen. A GPL/LGPL/AGPL or `UNKNOWN` dependency therefore reached the environment first, and a - lock pointing at an untrusted index had its directives honoured by that install while only the - later capture validated them — so the first network action of the run was the unvalidated one. The - order is now: validate the lock's sources (no network), collect the closure with - `pip download --no-deps --only-binary=:all:` (wheels only, because `pip download` executes an - sdist's build backend for metadata even with `--no-deps`), judge the licence, and only then - install. The install is `--require-hashes --only-binary=:all: --no-index --find-links ` - over the very bytes that were inspected, so nothing is re-resolved or re-downloaded and the - installed bytes are the judged bytes even where the lock records several hashes for one project — - which a second hash-less download could not have established. `install-authorized` refuses the - install unless a prescreen report records a passed `license` stage, so a missing, malformed or - failing report fails closed instead of defaulting to permitted. - One consequence is stated plainly rather than papered over: `LOCK_ENV_MISMATCH` is now evaluated - against the *collected* closure, because no installed environment exists when the gate reads its - capture. Agreement between that closure and the environment is enforced at install time instead, - by pip itself: `--require-hashes` with `--no-index --find-links ` can only install a - file from the collected root that matches a hash the lock records, so a disagreement fails the - install rather than being reported by a later inspect. - `tests/test_release_dependency_install_ordering.py` pins the wiring rather than the parser: with - `RELEASE_GATE_PIP` pointed at a recorder, a refused lock directive performs **no** pip call at all, - an unauthorized licence stage performs **no** `install`, an authorized release performs exactly one - offline hash-checked `install` from the collected root, and the workflow's step order is asserted - because the defect lived there. Refs #2342. -- **Three release-blocking defects found by independent review of `03ba1777`, each with its own - regression.** (1) *A permissive declaration was accepted as licence evidence.* The decision - allowed the declared SPDX expression and then only looked for a **denied** title in the bundled - text, so `scan_license_text` returning `None` was read as "the text is fine" — it only means no - GPL/LGPL/AGPL title was found. Reproduced: MIT metadata with `license_texts = {}`, with - `LICENSE = UNKNOWN`, and with `LICENSE = Commercial redistribution is prohibited.` each passed - the licence stage with an empty failure list. `recognize_license_text` is the positive half — - it returns the SPDX identifiers a body actually supports — so absent text is now - `LICENSE_TEXT_MISSING`, an unrecognizable body is `LICENSE_TEXT_UNVERIFIED`, and a recognized - body naming none of the declared identifiers is `LICENSE_TEXT_DISAGREEMENT`. Two of this - repository's own fixtures were declaring one licence while bundling another and are corrected. - (2) *The approval was not bound to what was installed.* `install_is_authorized` checked only - `stage` and `result`, and the install re-read the original lock, so a two-field report authorized - it and a lock recording several hashes for one project let `--require-hashes` accept an artifact - whose licence and contents were never judged. The verdict now records `python_lock_sha256`, and - `bind-install` refuses unless that lock still digests to what the verdict read, every judged - artifact is present in the collected root **by digest**, and the root holds no other - distribution; it then writes a requirements file pinning each project to the one judged digest, - which is what the install reads. A swapped artifact, an extra unjudged wheel, an edited lock and - a failing report each install nothing. (3) *The install could never run.* `python3 -m venv` - symlinks `bin/python` on POSIX, and the interpreter guard refused symlinks outright, so a normal - virtual environment exited 2 before pip was reached. The guard now resolves the link and requires - the resolved target to be a regular executable file, which a real venv satisfies while a dangling - link and a directory still fail. Refs #2342. -- **The gate's own toolchain is out of the prescreen's scope, and that limit is now written down - instead of being implicit.** The same review noted that the pinned Strix toolchain - (`requirements-strix-ci-hashes.txt`, materialized from `github.workflow_sha`) and the orchestrator - sidecar's own lock are installed without passing through the licence stage. They are a different - trust domain from the caller's release closure — pinned and reviewed in this repository — and the - stage that judges the closure cannot judge the scanner it must run first without a cycle. The - workflow says so at the install step: not an automatic exception for CI/build/dev dependencies, - but a stated limit whose removal is an owner decision tracked separately. Nothing in this gate's - output may be read as evidence that the gate's own dependencies were licence-judged. Refs #2342. - diff --git a/CHANGELOG.d/20260926-noema-draft-before-sidecar.md b/CHANGELOG.d/20260926-noema-draft-before-sidecar.md deleted file mode 100644 index 3baf00837f..0000000000 --- a/CHANGELOG.d/20260926-noema-draft-before-sidecar.md +++ /dev/null @@ -1,18 +0,0 @@ -### Noema checks live draft state before provisioning the orchestrator sidecar - -- `noema-review.yml`'s `noema-review` job provisioned the contextual-orchestrator review - sidecar (10-13 minutes) before `two_phase.py --prepare-verdict-file` read the live PR and - printed `PR is draft; Noema verdict preparation skipped.`, so every draft run held a runner - for ~13 minutes and produced nothing (newsdom-api job 108077744310 on 2026-09-25, `.github` - job 106665379126 on 2026-09-22) while the organization's Actions concurrency is saturated. - A new `live_draft` step, placed after `Validate current pull request head` / `Resolve Noema - target repository visibility`, reads the live PR with the same reviewer token and REST lookup - and gates sidecar provisioning, the HWP document reader, and `Prepare Noema model verdict` on - `steps.live_draft.outputs.live_draft != 'true'`. The decision stays runtime-only (no trigger - filter, no `github.event.pull_request.draft`), fails open to today's full path on a lookup - error, leaves `noema_prepare` outputs unset so publication stays skipped exactly as before, - and the job still concludes success for drafts. Ruleset-launched runs in other repositories - keep identical outcomes: a draft never produced a Noema verdict at runtime; only the check - moved earlier. `tests/test_noema_draft_admission_before_sidecar.py` pins ordering, gating, - and the fail-open step behavior; `docs/doctoring/noema-draft-before-sidecar.md` records the - rationale. diff --git a/CHANGELOG.d/20260927-codeql-obsolete-pr-verdict.md b/CHANGELOG.d/20260927-codeql-obsolete-pr-verdict.md deleted file mode 100644 index 0846ecd8a2..0000000000 --- a/CHANGELOG.d/20260927-codeql-obsolete-pr-verdict.md +++ /dev/null @@ -1,8 +0,0 @@ -### Correct CodeQL compatibility results after a PR closes or changes - -Closed PRs and superseded changes no longer produce a missing-verdict failure -when a queued compatibility check starts later. Current changes still require -a verified scan result; absent or failed evidence continues to block them. - -The scanner’s AnyIO dependency is pinned to the patched 4.14.2 release, with -verified release hashes and a source/lock parity guard. diff --git a/CHANGELOG.d/20260927-codeql-terminal-proof.md b/CHANGELOG.d/20260927-codeql-terminal-proof.md deleted file mode 100644 index ea4154fcec..0000000000 --- a/CHANGELOG.d/20260927-codeql-terminal-proof.md +++ /dev/null @@ -1,9 +0,0 @@ -## Fixed - -- Require a successful GHAS base/head configuration-identity proof and preserved - SARIF before a clean central CodeQL gate may settle or satisfy an exact required - run. A failed post-gate identity check can no longer be promoted to GREEN by a - wake-only fallback. -- Bind CodeQL terminal receipts to the live base, required run, head, and merge - source through the v2 dispatch protocol, preventing a trusted but stale commit - status from satisfying a retargeted or later required run. diff --git a/CHANGELOG.md b/CHANGELOG.md index d90fa0c899..b8c5d19aa2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,72 +1,3 @@ -### Intel macOS native archives are bound to x86_64 bytes - -- The release prescreener now requires every native member in an Intel macOS - continuation wheel to contain x86_64 code. Architecture inspection happens - before package/hash deduplication, so a wheel already reviewed for the - universal2 release leg cannot bypass the Intel-specific check. Universal2 - binaries that contain x86_64 remain valid; aarch64-only binaries fail closed. - Exact-tree evidence is 4,061 passed, 8 skipped, and 40 subtests passed, with - all 17,383 production statements and 7,098 branches covered. - -### Intel macOS runtime archives enter the exact release dependency gate - -- Require three same-run Intel macOS install receipts for the universal2 wheels. - The central verifier checks each artifact ZIP digest, source and distribution - identity, x86_64 interpreter, and dependency archive bytes before licence - prescreen. Distinct x86_64 dependency wheels join the Strix fixture matrix; - the final verdict seals the three artifact IDs and digests. The thirteen - publishable distribution identities remain unchanged. Local focused tests - are 77 passed, the full suite is 4,063 passed and 4 skipped, and the three - changed production modules have 100% statement and branch coverage. Release - admission remains HOLD pending the fast-mlsirm consumer and hosted checks. - -### Exact native-link review is bound before release verdict sealing - -- Release wheel and build-interpreter native links now fail closed unless each - target is a reviewed operating-system, interpreter, self-install-name, or - named external runtime. The immutable report advances to - `cwl.release-native-links/2` and records the review basis beside every needed - library. Concurrent coverage work was preserved by an ordinary two-parent - merge, including its exact Maturin release-asset verifier. That integration - first reproduced a 99% coverage failure with 22 missing statements and 10 - partial branches; behavior contracts now cover bounded downloads, archive - shapes, executable identity, native-link review, CLI dispatch, and prescreen - rejection paths. Current-tree evidence is 4,049 passed, 8 skipped, and 40 - subtests passed; all 17,302 production statements and 7,058 branches are - covered. Ruff E9/F/I, compileall, and diff checks also pass. Hosted exact-head - Checks and independent review remain required before admission. - -### Native release prescreen coverage remains fail-closed - -- Added behavior-level contracts for directory entries, cached analyzer reuse, oversized and unreadable native members, build-snapshot files omitted from package receipts, runtime wheels with unknown dynamic links, and malformed static-link evidence. This repairs the coverage regression introduced when runtime wheels and build-interpreter snapshots began using the pinned native-link analyzer. The exact-tree suite is 4,037 passed, 8 skipped, and 40 subtests passed; all 17,186 production statements and 7,000 branches are covered. Release admission remains Draft/HOLD pending fresh exact-head hosted Checks and qualifying independent review. - -### Canonical Rust materializer integration closes the repository coverage gate - -- Ordinary-merged the complete `ContextualWisdomLab/.github#2360` owner branch into the release-control stack, preserving its foundation ancestry, multi-root `cargo vendor --sync --locked` implementation, target-path confinement, real-Cargo integration cases, and toolchain-independent mock/error/CLI contracts. The focused materializer suite is 26 passed and 3 real-Cargo skips with `materialize_base_rust_dependencies.py` at 155/155 statements and 60/60 branches. The merged exact tree is 4,030 passed, 8 skipped, and 40 subtests passed; all 17,144 production statements and 6,982 branches are covered. Draft remains required until fresh exact-head hosted Checks and qualifying independent review complete. - -### Noema document-reader trust boundaries reach 100% executable coverage - -- Added behavior-level coverage for unsupported and oversized inputs, bounded DOCX ZIP/XML structure, empty documents, visible Word controls, ragged and escaped tables, missing or unstartable local HWP readers, oversized/non-UTF-8/empty adapter output, UTF-8-safe prompt truncation, and both CLI outcomes. Production reader behavior is unchanged. The focused suite is 11 passed and 2 optional real-fixture skips with `noema_review_document.py` at 144/144 statements and 52/52 branches. The warnings-as-errors full suite is 3,988 passed, 28 skipped, and 40 subtests passed; only the independently owned Rust dependency materializer on `ContextualWisdomLab/.github#2360` remains below 100%, so the repository gate remains RED and this PR remains Draft. - -### Queue-health ownership matches the documented boundary and reaches 100% coverage - -- Removed the dead duplicate `collect_snapshot()` and CLI `main()` from `actions_queue_health_core.py`; the executable `actions_queue_health.py` remains the single owner of collection, retry, exact-head reconciliation, and process exit behavior, while the core retains bounded parsing and report primitives. New boundary cases cover both pre-evidence identity retries, malformed active and terminal run IDs, obsolete target cancellations, and remediation-action deduplication. The focused queue-health suite is 80 passed with both queue-health modules at 100% statement and branch coverage. The full exact tree is 3,982 passed, 28 skipped, and 40 subtests passed; uncovered statements fell from 249 to 163 and partial branches from 26 to 19, leaving only the Noema document reader and Rust dependency materializer owners. - -### Release dependency gate trust boundaries reach executable 100% coverage - -- `release_dependency_gate.py` now has behavior-level coverage for bounded archive reads, unsafe or absent declared licence files, symlink/special members, archive-member limits, raw-capture and destination symlinks, Cargo workspace identity, Strix fanout identity/fixture/runtime-report validation, and install-time licence rebinding. The no-caller `parse_member_listing` helper and its isolated test were removed; immutable archive bytes remain the sole member authority. Focused evidence is 442 passed with 1,126/1,126 statements and 472/472 branches; the warnings-as-errors repository suite is 3,976 passed and 28 skipped. Repository-wide coverage rises from 98% to 99%, so the overall 100% release gate remains RED and the PR stays Draft. - -### Pingora declared binary artifacts reject readable runtime directives - -- A file under a base-owned declared research/data prefix no longer gains binary admission merely by adding an invalid UTF-8 byte to readable Nginx runtime content. For suffixes without recognized format magic, the bounded replacement-decoded bytes must also contain no prohibited runtime pattern; `.github#2386` covers `.sh`, `.dat`, and `.txt` names through the production evaluation boundary. -### Queue-health permission contract rejects aggregate token grants - -- The queue-health workflow contract now pins both workflow-level and collector-job permissions to exactly `contents: read` plus `actions: read`, rejecting scalar `read-all`/`write-all`, quoting/spacing variants, inline maps, and unexpected write scopes. - -### OpenCode coverage image materializes every Dockerfile lock input - -- Required OpenCode run `35370902053` for `.github#2266@12621f75e` failed before executing PR code because its trusted Dockerfile copied `requirements-noema-document-ci-hashes.txt` while the isolated build context contained only the OpenCode lockfile. The coverage owner now validates both lockfiles as regular non-symlink files and copies both into the trusted build context before the networked image build. `tests/test_opencode_agent_contract.py` pins the complete input boundary. Hosted exact-head acceptance remains Proposed until the new run reaches the image-build and coverage steps. - ### Noema transport capacity schedules a bounded continuation re-dispatch - After gateway failover, HTTP 429/5xx no longer end only as a permanent required-check failure with `caller attempts=1`. ADR-0031 classifies that class as `provider_capacity_unavailable`, keeps the single gateway request per job, surfaces `provider_attempt_count` from the orchestrator error envelope, and authorizes at most two same-head `repository_dispatch` retries after a capped `Retry-After` or deterministic 60–180 s jitter. Review is never skipped. Refs #2165. @@ -165,6 +96,12 @@ - Raised `hourly-review-repair.yml`'s discovery ceiling from 50 to 200 while rotating deterministic 50-PR deep-inspection windows by hourly run number. The scheduler hydrates only the selected window and stops immediately after its single dispatch, preserving access to newer PRs without quadrupling expensive review/check/comment work. See `docs/doctoring/hourly-review-repair-single-file-consolidation.md`'s 2026-09-03 follow-up. ## [Unreleased] +- **Repair the Agent Mention full-suite dependency closure.** The quality + workflow now triggers on, caches, and installs both the OpenCode and Noema + hash locks, so repository-wide collection can import `defusedxml`. The + stale `ThreadPoolExecutor.shutdown(wait=False)` proposal is removed because + CPython still joins those workers at process exit while the early return + would let GitHub API work continue after generator cleanup. - **Bind GitHub REST redirect evidence to both production opener chains.** `.github#2279` now feeds a synthetic same-authority 302 through the CodeQL identity and Strix evidence clients' real module-level openers, proving the redirect target is never contacted and the bearer header is never forwarded. Removing `_RejectRedirects` from either opener makes the contract fail on the forbidden second request. Four stale Strix HTTP/transport/JSON fixtures now patch that same production seam; direct handler unit cases and standalone CodeQL materialization remain unchanged. - **Define an evidence-backed repository README quality standard.** Added `docs/repository-readme-quality-standard.md` as the shared review contract for product-first structure, code-current onboarding, authority boundaries, durable quality signals, and repository/source/dependency license due diligence. Product repositories continue to own their own README prose; the standard is linked from the root documentation map and does not centralize or generate product claims. - Include merge-scheduler entrypoint, core, and regression-test changes in diff --git a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md index 7543f736e8..6629675f14 100644 --- a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md +++ b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md @@ -24,7 +24,7 @@ all five, and auto-optimize routing by cost. 1. **Vendoring, pinned**: `scripts/ci/contextual_orchestrator_review_sidecar.sh` clones `ContextualWisdomLab/contextual-orchestrator` at an exact SHA - (`01bf92a3ec67a0e1f9b68978eb16b60301e985fd` today) into `RUNNER_TEMP`. The + (`767e67fbc6b881a452761f32abb69b9971b9b03b` today) into `RUNNER_TEMP`. The source's `requirements.lock` is installed with `--require-hashes` and `--no-deps`, so dependency resolution cannot silently move the reviewed runtime. @@ -294,24 +294,3 @@ all five, and auto-optimize routing by cost. per-agent attempt; it changes only *which* agent gets tried next, never any per-attempt timeout, consistent with the 2026-08-31 amendment above. No other contextual-orchestrator behavior changes with this pin advance. -- **2026-09-25 amendment: adopt bounded 429 recovery in the review runtime.** - Advance the vendored pin from `767e67fbc6b881a452761f32abb69b9971b9b03b` - to `0d0637d032560417a9a08a8477c4aaf3a5942e0a`, the protected-main - revision containing the merged rate-limit admission repair (#1179). The - old runtime advanced to another provider after one 429 but returned a 429 - when all eligible free routes were cooling. The new runtime honors a - provider cooldown within its bounded request budget and returns a typed - 429 when no eligible route can recover in time. It keeps - `orchestrator/free` inside the admitted free pool and retains the default - null model timeout. Both revisions have byte-identical `requirements.lock`. - This pin change still needs protected delivery and a successful exact-head - Noema or OpenCode review; preflight success alone is not that evidence. - -- **2026-09-27 amendment: retain cooldown recovery with a patched dependency lock.** - The deployed pin is `01bf92a3ec67a0e1f9b68978eb16b60301e985fd`, a merged CO main revision containing - #1179 recovery and AnyIO 4.14.2. Auditing the earlier proposed `0d0637d0` - pin with pip-audit 2.10.1 found CVE-2026-63374, CVE-2026-64847, and - CVE-2026-63349 in AnyIO 4.14.1. The replacement hash lock has no known - vulnerabilities in the same audit. The earlier byte-identical-lock claim - describes the superseded proposal, not this amended target. No review - completion or runtime provider success is inferred from the lock audit. diff --git a/docs/adr/0029-sidecar-preflight-lazy-fill.md b/docs/adr/0029-sidecar-preflight-lazy-fill.md index 15001441cb..166d49f9a8 100644 --- a/docs/adr/0029-sidecar-preflight-lazy-fill.md +++ b/docs/adr/0029-sidecar-preflight-lazy-fill.md @@ -81,77 +81,3 @@ That competes directly with the org's 60-job ceiling work, and `#1949`'s measure The report adds `postponed_probed_count`; `skipped_count` now means "postponed and never reached", and `candidate_count − probed_count − skipped_count` keeps its meaning. A refused probe additionally records `retry_after_s` when the response carried a whole-seconds `Retry-After` header (the HTTP-date form and out-of-range values record nothing). Nothing waits on that value; it exists so the next census can answer the question this amendment could not. **Discriminator.** `postponed_probed_count > 0` marks any boot that reached a second pass, which includes the `12 / 12 / 3` class as well as the burst class. To isolate the all-429 class, read the first `probed_count − postponed_probed_count` rows of `routes` (they are in probe order) and require every one to carry `http_status` 429. The next census asks (a) whether such boots end with `ready_count ≥ 1`, (b) what fraction of 429 rows carry `retry_after_s` and how long the refusals claim to last, (c) whether the healthy-minute figures (`ready 5–6`) are unchanged, and (d) the provisioning step's duration on those boots, so the benefit in (a) and the cost above are read from one table. If (a) is consistently 0 **and** (b) shows providers publishing a usable delay, the follow-up is to spend the second pass after that delay rather than immediately — a decision this ADR deliberately leaves to that data. `#1948`'s shared rate ledger remains the lever above all of it. - -## 2026-09-27 amendment — concurrent readiness (Proposed) - -### Context - -The 90-second probe-duration examples above predate ADR-0003's 2026-09-13 -runtime pin update. They are historical measurements, not current wall-time -bounds. At pin `767e67fbc6b881a452761f32abb69b9971b9b03b`, model inference -has no configured deadline. In fast-mlsirm run `36237188327`, retained artifact -`10919666896` shows discovery completed and several serial probes finished, -then `nvidia_nim` `meta/llama-3.2-90b-vision-instruct` began at -2026-09-26T19:06:49.173Z without a later outcome before hosted cancellation -at 2026-09-27T01:00:56.653Z. Health readiness and scanning were never reached. - -### Decision - -In the shared review startup, facing a pending provider probe that prevents -later candidates from being validated, we use concurrent validation with the -existing total probe budget, in order to reach the same eight-ready target -without classifying slow inference as failure, accepting more simultaneous -provider traffic within the unchanged request-count budget. - -The shared launcher uses the existing per-route validator and payload. It -processes available completions before scheduling more candidates, postpones -future candidates after observed consecutive account 429s, and spends at most -16 base probes per stage and four escalations per run (shared across primary -and fallback). The same probe budget bounds outstanding calls; no new numeric -limit is introduced. Already outstanding calls cannot be retroactively -postponed when another call reports 429. Only completed validated routes and -explicitly retryable responses enter the serving pool. Pending rows are -recorded separately, never rejected or admitted. The snapshot seals further -escalations; pending base calls may complete but cannot spend another retry. -No model call is cancelled when the readiness target is reached. Their threads -remain within the sidecar lifecycle and end when that process is explicitly -terminated or its host ends. The priced fallback still begins only after the -primary stage terminates with no ready route. - -### Consequences - -A pending probe no longer serializes all later candidates. The ready target, -free/ZDR selection, validation, and global request budgets remain intact. -A pool without enough responding routes can still wait indefinitely; this -change makes no inference deadline or hosted-capacity guarantee. Simultaneous -traffic can expose provider capacity limits sooner. Readiness membership follows -completion order, while serving priority and evidence rows retain catalog -scheduling order. The -snapshot may contain pending calls that subsequently finish; it is startup -admission evidence, not a final verdict on every candidate. - -### Alternatives considered - -- A fixed inference timeout conflicts with ADR-0003 and was rejected. -- Lowering the eight-ready target weakens the intended validated pool and was - rejected. -- Admitting unprobed candidates removes provider validation and was rejected. -- Eight outstanding calls recreate the same obstruction when eight pending - probes precede eight healthy ones; the regression oracle demonstrated this, - so the existing total probe budget also bounds outstanding calls. -- Runner cancellation discards valid current-head work and does not repair - startup scheduling. - -### Verification - -Event-controlled tests keep one or eight probes pending while eight later -routes become ready. The scheduler must return before the test releases those -calls. Separate checks cover all-429 failure, global escalation budget, -primary/fallback ordering, deferred-route admission, and unexpected worker -faults. Hosted acceptance is required; this amendment is not a claim that the -repair has been deployed. - -Implementation uses only the standard library's [Thread and Lock contracts](https://docs.python.org/3/library/threading.html) -and [synchronized Queue](https://docs.python.org/3/library/queue.html). Pending -probe threads deliberately share the sidecar process lifecycle; interpreter -shutdown is not a resumable-provider guarantee. diff --git a/docs/adr/0031-noema-transport-capacity-redispatch.md b/docs/adr/0031-noema-transport-capacity-redispatch.md index 793fb4e8c0..c0ecc51b81 100644 --- a/docs/adr/0031-noema-transport-capacity-redispatch.md +++ b/docs/adr/0031-noema-transport-capacity-redispatch.md @@ -35,8 +35,7 @@ model-failure verdict or restoring fixed model-path attempt ceilings. bound (`MAX_TRANSPORT_REDISPATCH_ATTEMPTS = 2`), the workflow schedules exactly one same-head `repository_dispatch` (`noema-review`) with an incremented attempt counter after a short jitter delay. The new job is a fresh admission/continuation; the failed - job remains failed evidence for that attempt. A malformed supplied counter exhausts - the budget rather than starting it over. + job remains failed evidence for that attempt. 4. **Jitter is post-failure scheduling, not a model timeout.** Prefer a whole-seconds `Retry-After` from the gateway error when present and in `[1, 300]`. Otherwise use a deterministic delay in `[60, 180]` seconds derived from the exact head SHA and attempt @@ -47,11 +46,6 @@ model-failure verdict or restoring fixed model-path attempt ceilings. include `provider_attempt_count=` alongside the existing last-attempt fields so capacity incidents are distinguishable from code-review verdicts without dumping raw provider bodies. -6. **Isolate dispatch authority.** The failed review job exports only typed retry - evidence and retains read-only repository contents access. A dependent job alone - receives repository-scoped Contents write through `GITHUB_TOKEN`; it has no - checkout or model inputs, and rechecks the live repository, PR head, and base - before dispatch. The reviewer App token remains limited to Contents read. ## Consequences @@ -75,22 +69,3 @@ model-failure verdict or restoring fixed model-path attempt ceilings. - **Rely only on the merge scheduler's next tick.** Deferred as a complementary path; it does not give the Noema workflow its own bounded, evidence-typed recovery when the scheduler is not looking at that head. - -## Proposed Strix startup extension — 2026-09-27 - -- **Status:** Proposed; deployment and independent review remain unverified. -- **Context:** Strix all-429 preflight fails before its model gate can retry; - late-life-anxiety-reanalysis #257/#269 have exact-job evidence of this path. -- **Decision:** Reuse the bounded classifier in a separate post-failure dispatch - job, with live repository/head/base/ref/Ready validation and the same two-attempt - ceiling. Retain the failed scan and status; never infer approval from recovery. -- **Consequences:** Automatic recovery can enter a healthier provider window and - uses up to two additional scan admissions. Persistent capacity failure still - requires operator action. Consumer execution needs the existing central - dispatch credential; its absence remains visible and fail-closed. -- **Alternatives:** Model-gate retries cannot run before successful startup; - in-job startup loops hold a scan runner; unbounded dispatch amplifies capacity - pressure; neutral/success status would weaken the required security gate. - -See `../doctoring/strix-preflight-capacity-continuation-20260927.md` for evidence -and the local-versus-hosted verification boundary. diff --git a/docs/adr/adr-0032-owned-codeql-status-and-settlement-authority.md b/docs/adr/adr-0032-owned-codeql-status-and-settlement-authority.md deleted file mode 100644 index 50c6392edc..0000000000 --- a/docs/adr/adr-0032-owned-codeql-status-and-settlement-authority.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -title: "ADR-0032: Owned CodeQL status and settlement authority" -status: Proposed -date: "2026-09-27" -authors: "Codex" -tags: [architecture, ci, security] -supersedes: "" -superseded_by: "" ---- - -# ADR-0032: Owned CodeQL status and settlement authority - -## Status - -Proposed. Requires #2405 complete terminal-proof foundation, owned-app installation permission acceptance, and an unchanged-head live canary before protected deployment is accepted. - -## Context - -DiskSage #473 dispatch 36305375849 encountered cross-repository HTTP403 during status publication and required-run settlement. Public app and organization installation metadata confirm opencode-agent is owned by anomalyco and has Actions/read and statuses/read. A consumer cannot change the external owner's app permissions. The organization-owned cwl-noema-review (app4291520) is already installed on all repositories with security_events/read; its private-key organization secret is available to central workflows. The existing target-scoped analysis-read token remains the GHAS reader. - -## Decision - -Use the existing owned Noema app for separate target-repository tokens: statuses/write solely for authenticated CodeQL receipt publication, and Actions/write solely for exact required-run settlement. Keep security_events/read in its existing separate read token. The installation must authorize those two write permissions; credentials cannot mint permissions the installation lacks. Optional mint failures retain existing fallback credentials and never create validation success. - -The owned status writer must publish as cwl-noema-review or cwl-noema-review[bot]; another returned creator is rejected. No arbitrary actor is added. Complete base/head/run/source/workflow receipt and terminal SARIF/GHAS proof from #2405 remain prerequisites; do not deploy the new receiver trust before that foundation. Preserve exact-run identity, supersession, rerun budget, SARIF preservation and Medium+ gates. - -## Consequences - -- POS-001: Removes dependence on an external app owner's unavailable write grants. -- POS-002: Reuses an installed app and keeps analysis, publication and lifecycle tokens separate and target scoped. -- NEG-001: Expands the owned installation's capabilities and therefore the impact of its private-key compromise. Restrict key access and retain the trusted default-branch workflow boundary; never export keys into reviewed source or logs. -- NEG-002: Needs owner-authenticated app settings and installation acceptance plus live verification. Unit contracts do not prove deployment or permission availability. - -## Alternatives Considered - -- ALT-001: Change the external OpenCode app. Rejected because anomalyco owns that app and its current grants cannot satisfy writes. -- ALT-002: Transfer a user's CLI token into CI. Rejected: broad personal credentials are unnecessary and not copied. -- ALT-003: Bypass identity/receipt checks or synthesize success. Rejected because that removes the security proof. -- ALT-004: Reuse the analysis-read token for mutations. Rejected because its read-only contract must remain unchanged. - -## Implementation Notes - -- IMP-001: Pin the existing create-github-app-token action and request exactly one target repository and one write permission per writer token. -- IMP-002: Grant Actions/write and Commit statuses/write to the owned app and accept the installation update; do not add Code Scanning writes. -- IMP-003: Accept deployment only after real current-head scan, GHAS identity, receipt creator, one exact run-wide wake and terminal required verdict are verified. References: ContextualWisdomLab/.github#2276, #1929 and #2405. - -## References - -GitHub. (n.d.). *Create GitHub App token*. https://github.com/actions/create-github-app-token -GitHub. (n.d.). *Choosing permissions for a GitHub App*. https://docs.github.com/en/apps/creating-github-apps/setting-up-a-github-app/choosing-permissions-for-a-github-app diff --git a/docs/adr/adr-0032-release-gate-exact-set-fanout.md b/docs/adr/adr-0032-release-gate-exact-set-fanout.md deleted file mode 100644 index 331b669203..0000000000 --- a/docs/adr/adr-0032-release-gate-exact-set-fanout.md +++ /dev/null @@ -1,149 +0,0 @@ ---- -title: "ADR-0032: Bind release dependency verdicts to the complete artifact set" -status: "Proposed" -date: "2026-09-26" -authors: "CWL release gate maintainers" -tags: ["architecture", "decision", "release", "supply-chain"] -supersedes: "" -superseded_by: "" ---- - -# ADR-0032: Bind release dependency verdicts to the complete artifact set - -## Status - -**Proposed**. No release HOLD may be removed on the strength of this record. The -implementation and exact-head hosted evidence are still required by #2342. - -## Context - -The reusable gate in #2347 scans all resolved dependencies sequentially in one -360-minute job and seals one wheel and one sdist. ContextualWisdomLab/fast-mlsirm#2135 builds twelve -wheels and one sdist. Its admission job currently exits with an unconditional HOLD -because the existing handoff does not authenticate a same-run full licence and -Strix verdict or the complete release artifact set. A successful two-file seal -cannot establish a verdict for the other eleven wheels. - -GitHub Actions reusable-workflow outputs from a matrix contain the value from -the last successful completing call that set a value. That output cannot -represent a complete verdict set. A matrix job's aggregate result can prove -that every invocation succeeded, but still does not identify which artifacts -each invocation examined. An uploaded JSON field claiming `PASS` is likewise -not a trusted job conclusion. - -## Decision - -- **DEC-001**: The protected release workflow uses its existing - `reproducibility-record` job to produce one immutable, same-run manifest of - all thirteen publishable fast-mlsirm distributions. Each row carries target, - filename, file SHA-256, upload artifact ID, name, and archive digest. The - trusted job enforces the expected twelve-wheel-plus-one-sdist set before it - passes the manifest's ID and digest to the central gate. -- **DEC-002**: The central reusable gate takes that manifest by immutable - artifact ID and digest, checks its run ID and attempt against the current - invocation, downloads every referenced artifact by ID, and recomputes every - file digest. It derives the dependency set and synthetic fixtures from the - exact release source and collected build evidence before any Strix credential - exists. Missing, duplicate, extra, expired, wrong-run, wrong-attempt, or - wrong-digest evidence fails the gate. -- **DEC-003**: Strix runs in a dynamic matrix with exactly one dependency - fixture per job. Each job uses the pinned trusted helper, the same source SHA, - and an isolated fixture; it uploads a uniquely named immutable binding that - includes dependency identity, fixture digest, source SHA, run ID, and attempt. - The matrix fan-out has a reviewable concurrency bound and refuses a fixture - set above GitHub Actions' 256-job matrix limit. Elapsed model time is not - converted into a passing or failing security verdict. -- **DEC-004**: A downstream collector runs only when the licence stage and - every matrix job succeeded. It compares the exact expected dependency keys - with the binding-artifact keys, checks every binding and digest, and produces - one full-set verdict artifact with its own ID and digest. It does not aggregate - matrix job outputs and it cannot turn a failed or skipped scan into success. -- **DEC-005**: fast-mlsirm admission depends on the pinned central reusable - gate's job result in the same workflow run. It verifies the returned verdict - artifact by ID and digest, source SHA, run ID and attempt, and equality of all - thirteen distribution rows to its locally verified manifest. It also - requires a trusted, target-specific closure inventory for runtime, build, - dev, optional, native, and bundled scopes. An `UNKNOWN` scope or a - declaration identity without resolved dependency evidence refuses - admission. Only then may it write `admitted-manifest.tsv`; the existing tag - and publish jobs remain downstream of admission. -- **DEC-006**: The unconditional admission HOLD remains until hosted RED and - GREEN runs on exact current heads prove this entire path, including a real - Strix binding. Unit fixtures alone do not authorize its removal. - -## Consequences - -### Positive - -- **POS-001**: The release verdict covers the bytes of every distribution the - publish job can consume, including each wheel target. -- **POS-002**: An incomplete matrix, forged `PASS` document, or artifact from - another run cannot satisfy the collector and admission contracts. -- **POS-003**: Each Strix scan has its own job lifetime, while the matrix's - concurrency bound limits organization runner occupancy. - -### Negative - -- **NEG-001**: Fan-out and exact-set collection add jobs, artifacts, and - validation code to a security-sensitive workflow. -- **NEG-002**: The full closure may occupy the Actions queue for many hours; - queued jobs are pending evidence, not a passing verdict. -- **NEG-003**: The existing six-member, seventeen-output wheel/sdist - attestation contract does not itself cover thirteen distributions. The - full-set verdict must be verified separately until a reviewed generalized - attestation contract replaces it. -- **NEG-004**: Source declaration hashes and one Ubuntu dependency capture do - not establish the native and bundled closure of Linux, macOS, and Windows - wheel build environments. Per-target collection and verification add work - before the current scope HOLD can be removed. - -## Alternatives Considered - -### One sequential Strix job - -- **ALT-001**: Keep the current single job and increase its timeout. -- **ALT-002**: Rejected because the job is already at GitHub's 360-minute - ceiling, while one dependency's model path may take more than two hours. - -### One reusable gate invocation per wheel - -- **ALT-003**: Call the existing two-file gate twelve times, pairing each wheel - with the same sdist. -- **ALT-004**: Rejected as the final design because it repeats the entire - dependency scan twelve times. A caller can use the matrix job result and - exact same-run artifact set without relying on its last-wins outputs, so - this remains a possible intermediate wiring step while the release HOLD - stays in force. - -### Trust a seal or report by its filename - -- **ALT-005**: Download a named artifact and accept its declared `PASS` field. -- **ALT-006**: Rejected because a name and a payload do not prove that the - pinned gate succeeded in this run on these thirteen bytes. - -## Implementation Notes - -- **IMP-001**: First add RED cases for missing, duplicate, extra, stale-run, - stale-attempt, wrong-source, altered archive, altered distribution, and - omitted matrix binding. Include a scope record that remains `UNKNOWN` or - substitutes declarations for a resolved platform inventory. Each must - refuse before admission or publication. -- **IMP-002**: Keep source validation, pre-credential licence refusal, and - immutable build-artifact intake from #2347. Preserve diagnostic artifacts on - failures without an `always()` path that could allow downstream release jobs. -- **IMP-003**: The hosted GREEN case must exercise real capture, Strix, - collection, sealing, and fast-mlsirm admission on an exact head. Record run - and job IDs, all thirteen file digests, and the pinned central workflow SHA. -- **IMP-004**: Keep #2135 draft and release HOLD until #2342 acceptance and - both repositories' exact-head required checks are terminal green. Merge, - tag, and PyPI publication are separate later decisions. -- **IMP-005**: Preserve the existing unconditional refusal in - `verify_scope_identities` as well as the workflow's final admission HOLD - until the collector verifies all six scopes for every wheel target and the - sdist. Removing only the workflow HOLD cannot make admission succeed. - -## References - -- **REF-001**: ContextualWisdomLab/.github#2342 and #2347; ContextualWisdomLab/fast-mlsirm#2135. -- **REF-002**: [GitHub reusable workflow matrix output behavior](https://docs.github.com/en/actions/how-tos/reuse-automations/reuse-workflows#using-a-matrix-strategy-with-a-reusable-workflow). -- **REF-003**: [GitHub Actions matrix output rules](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#jobsjob_idoutputs) and [immutable upload artifact IDs and digests](https://github.com/actions/upload-artifact/blob/main/README.md#outputs). diff --git a/docs/doctoring/20260924-release-gate-negative-fixture-verification-plan.md b/docs/doctoring/20260924-release-gate-negative-fixture-verification-plan.md deleted file mode 100644 index ca4508b2b7..0000000000 --- a/docs/doctoring/20260924-release-gate-negative-fixture-verification-plan.md +++ /dev/null @@ -1,196 +0,0 @@ -# Negative-fixture verification plan for the central release dependency gate (#2342, #2347) - -Prepared, **not approved to run**. No hosted run, publish, merge, approval or re-run is authorized -by this document. It closes the two written gaps the coordinator required alongside the reviewable -exact head, and records the source-policy constraints on the proposed fixture. - -Historical central head described by the original plan: `65727fa8411ec92672e03b1c6447b3a47d2616fc` on -`feat/release-dependency-license-strix-gate-2342`, on top of the reviewed -`3c3ca9b1445d4a73a9d47216ff88996f012f1757`. - -Source-directive assessment updated after integration `79be9bd3d2d1aeea62f0c32659d22318521b0a6c`; hosted-run claims below remain unverified by this document. - -Two claims are kept apart throughout, and must stay apart in any report that cites this file: - -- **Reason codes verified locally.** Unit results from `pytest`, which prove a decision outcome and - its reason code and nothing else. -- **Run-level facts.** "The Strix step did not start", "the gate job concluded `failure`", "the - publish job did not start". None of these is established here. A failing unit-test wrapper is - never a real release-gate failure, and a passing test is never a release PASS. - -## Gap 1 — how a negative case enters the real gate - -### The path, by step, subcommand and function - -| # | Workflow step (`release-dependency-license-strix-gate.yml`) | Runs | Decides | -|---|---|---|---| -| 1 | `Validate the exact release identity before anything else runs` | `release_dependency_gate.py validate-inputs` | `validate_release_identity` — 40-hex `source_sha`, `owner/name` repository | -| 2 | `Install the release dependency closure into a lock-only environment` | `pip install --require-hashes --only-binary=:all: -r release-source/` into a `--without-pip` venv | — | -| 3 | `Download the exact distributions the caller intends to publish` | `actions/download-artifact` → `release-distributions/` | — | -| 4 | `Collect raw resolved-dependency evidence from both ecosystems` | `release_dependency_capture_raw.sh` | — (writes tool output verbatim; `capture_python` derives each `metadata.json` from `python/installed.json`) | -| 5 | `Assemble per-dependency evidence and isolated synthetic fixtures` | `release_dependency_gate.py capture` | `capture` → `build_evidence` → `evidence/.json`, `strix/fixtures/.json` | -| 6 | `Refuse a denied or unverifiable licence before any credential exists` | `release_dependency_gate.py prescreen` | **`gate(stage="license")` → `evaluate_dependency_license` → `declared_license_expression` → `spdx_license_policy.evaluate_license_expression`** | -| 7 | `Require every Strix provider credential before the Strix stage starts` | `release_dependency_gate.py require-strix-credentials` | `require_strix_credentials` → `STRIX_CREDENTIALS_ABSENT` | -| 8–11 | gateway, toolchain, credential binding, Strix | `strix_quick_gate.sh` per fixture workspace | — | -| 12 | `Refuse the release unless every dependency passes` | `release_dependency_gate.py gate` | `gate(stage="full")` — the same licence decision **plus** `validate_strix_binding` | -| 13 | `Seal exactly the gated bytes for attestation` | `release_dependency_gate.py seal` | `seal` — refuses a non-`PASS` **and** a non-`full` report | - -The licence decision in step 6 is the same function the final gate calls in step 12. There is one -decision implementation, not a prescreen copy of one. - -`needs` path traversed: the gate is a single `workflow_call` job (`jobs.gate`). A caller composes -`gate` → `attest`, and any mock job models only the edge out of `jobs.gate`. - -### No collection-bypass input exists - -Verified by reading the current source: - -- The workflow's `workflow_call` inputs are the release identity, ecosystems, lock/manifest paths, - artifact and filenames. **None of them skips capture, skips the licence stage, or injects a - verdict.** `test_workflow_is_reusable_and_never_branch_selectable` and - `test_gate_has_no_bypass_of_any_kind` pin the absence of a bypass shape. -- Step 4 always runs; step 5 always runs; step 6 always runs. The only `if:` conditions in the - workflow are the lock-only install guard and the two evidence-retention uploads - (`test_failure_evidence_survives_the_failure_that_produced_it` asserts there are exactly three). -- A hand-written `evidence/.json` cannot manufacture a case. The expected set comes from the - producer's own lock (`_enumerate_python`) and `Cargo.lock` (`_enumerate_cargo`), and - `_scope_rows` refuses collected material that no declared ecosystem expects with - `SCOPE_SET_MISMATCH`. Test: `test_collected_material_outside_every_expected_set_is_a_scope_mismatch`. - -**Consequence, stated plainly: a denial case cannot be fed in as a bare JSON blob.** It must arrive -as something the real capture path genuinely collects — a distribution present in the lock, with a -real `sha256`, whose own metadata carries the case. - -### The fixture-distribution shape: bounded source policy - -The earlier dropped-directive defect has been fixed. The capture script validates -`lock-source-options` and passes the resulting `source_options` to `pip download`. -It does not silently discard source directives. - -The supported sources remain deliberately narrow: - -- Index URLs must use HTTPS, the default port, and no user information, with a host - of `pypi.org` or `files.pythonhosted.org`. -- `--find-links` must name a normalized, bounded relative directory inside the - lock-file directory. The separately downloaded `release-distributions/` - directory in this plan does not meet that constraint. -- Environment markers, `-r`/`--requirement`, and `-c`/`--constraint` are rejected - with `LOCK_SOURCE_UNSUPPORTED`. - -A locally authored fixture therefore requires a hash-bound wheel under a permitted -lock-relative directory. This document supplies no hosted collection result and -establishes no release acceptance. - -### A GPL-declaring fixture package is rejected - -The coordinator has **retracted** the idea of authoring or installing a fixture package whose -metadata declares a copyleft identifier. It is not to be built. The two evidence classes are split -instead: - -- **Per-reason denial codes stay unit-level.** A self-authored, **data-only** SPDX string is a valid - input to the production decision functions, and `LICENSE_DENIED_GPL`, `LICENSE_DENIED_LGPL`, - `LICENSE_DENIED_AGPL`, `LICENSE_UNPARSEABLE`, `LICENSE_UNRECOGNIZED`, `LICENSE_MISSING`, - `LICENSE_SELECTION_REQUIRED` and `LICENSE_SELECTION_INVALID` are proven exactly there, by direct - calls to `evaluate_dependency_license` / `declared_license_expression` in the existing - `tests/test_release_dependency_gate.py`. Those SPDX strings are **not** extended into the real - package-install path. -- **The real capture path is exercised with a self-authored artifact containing no forbidden - source**, verified through a `LICENSE_MISSING` rejection. Nothing copyleft is fetched, declared or - installed at any point. - -The `LICENSE_MISSING` fixture must also satisfy the bounded source policy above. -Its current placement outside the lock directory is unsuitable; collectibility -requires compliant placement and an actual non-deploy collection run. - -### Naming discipline for the eventual run - -What such a run can prove, and the only way it may be described: -**real collection → licence-missing rejection → Strix blocked → `mock_publish` gated by `needs`.** - -It is **not** a "GPL real-collection-refusal E2E" and must never be called one. The per-reason -copyleft denials are unit-level decision evidence and belong in a separate, separately labelled -section of any report. The link between the real capture path and the decision function is for the -coordinator to review from the exact-head source; it is not established by this prose. - -## Gap 2 — the `mock_publish` job's contract and its limits - -Name: **`mock_publish`**. Never `publish`, `release`, or `deploy`, so no reader or later script -mistakes it for the release job. - -What it verifies: **only that the gating edge behaves as the real caller's publish job would.** Its -`needs` and `if` must be character-identical to the real release workflow's publish job, and both -must be quoted side by side in the run's evidence. It models the *edge*, nothing else: it does not -show that a real publish job would not start, because it is not that job and does not share its -environment, permissions or triggers. - -Prohibited in `mock_publish`, and unnecessary for the contract: any `permissions:` beyond -`contents: read`, any token or secret, any `environment:`, any tag creation, any release creation, -any registry credential, any upload to a registry. Its steps are `echo` only. - -**The real publish job's `needs`/`if` cannot be quoted here.** The caller workflow is FMLS-owned and -is not present at this head, so the two conditions must be quoted from the FMLS caller at its exact -SHA when the run is proposed. This plan does not invent them. - -### Judgement rule and the exact fields to read - -A skipped job can still carry a `started_at` in GitHub's payload, so **nothing may be inferred from -an absent or present `started_at` alone.** Judge from the raw payload plus whether steps actually -executed: - -From `GET /repos/{owner}/{repo}/actions/runs/{run_id}` — `id`, `head_sha` (must equal the fixture -commit exactly), `status`, `conclusion`. - -From `GET /repos/{owner}/{repo}/actions/runs/{run_id}/jobs` per job — `name`, `status`, -`conclusion`, and the full `steps[]` array, reading each step's `name`, `status`, `conclusion` and -`number`. - -Decision rules: - -- **Gate refused**: the `gate` job has `conclusion == "failure"`, and the step named - `Refuse a denied or unverifiable licence before any credential exists` has - `conclusion == "failure"`. The reason code is read from the - `release-dependency-license-report` artifact's `failures[].code`, not from log prose. -- **Strix never started**: every step from `Provision the zero-cost review gateway for Strix` - through `Run Strix against one isolated synthetic fixture per dependency` has - `conclusion == "skipped"`. A step that ran and failed is a different outcome and must not be - reported as "did not start". -- **Credentials were never required for the licence decision**: the step - `Require every Strix provider credential before the Strix stage starts` also has - `conclusion == "skipped"`, which places it after the licence refusal. -- **`mock_publish` did not execute**: its `conclusion == "skipped"` **and** its `steps[]` is empty or - every entry has `conclusion == "skipped"`. `started_at` is recorded verbatim and explicitly **not** - used as evidence either way. -- **Evidence survived the failure**: the `release-dependency-license-report` artifact exists on the - failed run, which is the behavior the bound-to-producing-step upload condition exists to provide. - -Anything not on this list stays unverified. - -## Remaining end-to-end verification scope - -Splitting the evidence into unit-level denials and one `LICENSE_MISSING` collection run does **not** -shrink the requirement, and nothing here may be marked fully complete. Still unproven, with the kind -of run that would prove each: - -| Unproven | What would prove it | -|---|---| -| A copyleft dependency is refused by the **real collection path** | A run whose collected metadata carries a denied licence. No such run is planned, because authoring or installing a copyleft-declaring package is rejected. This gap stays open by policy. | -| `release_dependency_capture_raw.sh` executes at all | Any hosted run that reaches step 4. No step of that script has ever executed, here or in CI. | -| A locally authored fixture distribution is collectible | A hash-bound wheel in a permitted lock-relative directory, then one non-deploy run. | -| Strix succeeds and produces a real binding | A credentialed run that reaches step 12 with `verdict` and `findings` from an actual scan. | -| `seal` output is accepted by the real attestation workflow on real bytes | A run composing `gate` → `attest` on a real wheel and sdist. Locally only the *shape* is checked, against a synthetic sealed directory. | -| Capture/hash/metadata/artifact binding agree end to end | The same composed run, comparing `wheel_sha256` and `sdist_sha256` against the published artifact digests. | -| A real publish job would not start | Nothing planned proves this. `mock_publish` models the gating edge only. | - -## What remains unverified without a hosted run - -Verified locally by execution: every reason code above, produced by the production functions through -the existing harness in `tests/test_release_dependency_gate.py` and its siblings. - -Not verified, and not claimable until a single approved non-deploy run exists: that the gate job -concludes `failure` on a real runner; that the Strix steps report `skipped`; that `mock_publish` -does not execute; that the capture script's real `pip inspect`/`pip download`/`cargo metadata` -invocations behave as read (no step of `release_dependency_capture_raw.sh` has ever been executed, -here or in CI); that `seal` and `exact-artifact-sbom-attestation.yml` agree on real bytes; and that -the fixture distribution is collectible under the bounded source policy described above. - -Refs #2342, #2347. diff --git a/docs/doctoring/central-dedicated-runner-routing-20260927.md b/docs/doctoring/central-dedicated-runner-routing-20260927.md deleted file mode 100644 index aa84465589..0000000000 --- a/docs/doctoring/central-dedicated-runner-routing-20260927.md +++ /dev/null @@ -1,133 +0,0 @@ -# Central dedicated runner routing - -## Status - -Proposed workflow change; organization runner groups and five S1 runners are -already deployed. This document does not assert protected-main adoption. - -## Context - -The 2026-09-27T09:41:06.945544+00:00 collection recorded 610 unique queued central runs. -Trusted-main dispatch queues included 61 CodeQL and 31 OpenCode runs; three -control workflows had 18 main-branch runs. Older runs may be stale, so these -counts are allocation evidence, not exact-head merge evidence. - -An exact-rational linear relaxation plus exhaustive integer allocation selected -one additional runner for each lane under a four-core host CPU-quota budget, -32 GiB additional guest RAM and 160 GiB sparse-disk budget. Existing two runners -were preserved. Historical successful job duration sums excluded queue waits. -The forecast is sensitive to service times; measured latency improvement remains -unverified. The detailed calculation is retained in the system-management task's -`central-runner-optimization-20260927.md` and JSON input/output receipts. - -## Decision - -Declare dedicated routing in the five workflow files: - -| Workflow | Runner selection | -| --- | --- | -| CodeQL scan dispatch | Group `CWL central CodeQL`, labels `self-hosted`, `linux`, `x64` | -| OpenCode review dispatch | Group `CWL central OpenCode`, labels `self-hosted`, `linux`, `x64` | -| Agent mention router | Group `CWL central control`, labels `self-hosted`, `linux`, `x64` | -| Hourly review recovery | Group `CWL central control`, labels `self-hosted`, `linux`, `x64` | -| PR review merge scheduler | Central caller: self-hosted Linux X64 with `cwlab-control`; other callers: `ubuntu-24.04` | - -Groups 4/5/6 allow only the central repository and their selected workflow paths -at `refs/heads/main`. Keep those restrictions and external contributor approval. -The control runner has the `cwlab-control` label. A reusable workflow inherits -the caller's repository context, so its consumer branch must retain hosted access. - -The OpenCode guest exposes four virtual CPUs and 20 GiB RAM to satisfy the -existing four-CPU/14-GiB Docker sandbox, with host CPUQuota 100%. CodeQL has two -CPUs/eight GiB; control has one CPU/four GiB. Existing guests remain running; -their group excludes future OpenCode dispatch because their three visible CPUs -cannot satisfy that Docker request. A real container resource check passed on -the new OpenCode guest. Do not change model deadlines, providers, permissions, -concurrency, or protected review requirements to compensate for admission delay. - -## Consequences and alternatives - -Explicit selectors keep long reviews separate from short control work. Native -organization group restrictions remain the trust boundary. A missing dedicated -runner now queues the central job instead of silently choosing a hosted runner. - -Generic Ubuntu labels alone served existing queued jobs, but did not express -durable lane selection in source. Expanding central groups to every consumer -repository would weaken their access boundary; caller-aware scheduler routing -avoids that expansion. No new manual dispatch trigger or PR-branch group access -is added. - -## Verification - -Check workflow syntax, runner-selection contracts, the independent OpenCode -workflow blob pin, and existing affected contracts. Native assignment receipts -already show CodeQL dispatch and OpenCode review execution on the new runners -and a successful control queue job; they do not prove this proposed source has -landed or that all required review gates pass. - - -## Noema control admission follow-up - -At 2026-09-27 10:38 UTC the organization API listed five online runners, -while the central repository still listed 455 queued runs. Group 6's control -runner was idle in the subsequent group-membership observation; these are -point-in-time observations, not a measured capacity forecast. - -Noema's admission, changed-scope, closed-run cleanup, and post-failure -re-dispatch jobs now select `self-hosted`, `Linux`, `X64`, `cwlab-control` -only in the central repository. The concurrent #2421 allocation for contextual-orchestrator consumers is preserved; other consumer repositories retain Ubuntu 24.04. -The concurrent #2421 model-review allocation to the MCP remediation pool is preserved; this change allocates short -control work and does not assert compatibility of a model sandbox with the -one-core/four-GiB control guest. - -Deployment requires group 6's existing trusted-main workflow allowlist to -include `ContextualWisdomLab/.github/.github/workflows/noema-review.yml@refs/heads/main`. -Preserve every existing allowlist entry, repository restriction, and external -contributor approval. Do not allow a feature-branch ref. Until that grant is -verified, the source change is not an operational routing repair. - - -## Issue 1565 review admission follow-up - -The SDK and naruon consumer Noema jobs still selected hosted Ubuntu after the -initial runner rollout. Extend the existing repository allowlist to -`ContextualWisdomLab/cwl-telemetry` and `ContextualWisdomLab/naruon`, only when -`github.workflow_ref` is exactly the central Noema workflow at `refs/heads/main`. -Metadata and continuation use the control pool; model review uses MCP remediation. -PR-authored workflow refs retain hosted execution and existing fork admission, -credentials, review publication, concurrency and inference-time policy remain. - -At 2026-09-27 12:12 UTC, group 3 repository membership was verified after two -repository-specific PUT requests. Its selected-workflow restrictions remain; -group 6 already allows repositories subject to its selected-workflow restrictions. -This is runner admission, not approval or evidence of a completed model review. -Existing queued runs retain their original workflow revision and may still wait -until event-driven current-head recovery creates a new run. - -The allocation calculation from the initial rollout is reused; no new host -capacity or independent service-time measurement justifies another solver. -The routing regression fails against the unchanged baseline. Workflow syntax -and affected contracts passed: 238 passed, 2 skipped with `GITHUB_ACTIONS=true`; -`actionlint` and `git diff --check` passed. - - -## fast-mlsirm Strix control admission - -Current fast-mlsirm PR #2220 head `4eaeb799a6647ea29f3f4902d9ca79a1377e795c` -queued Strix admission job `108617323217` with `ubuntu-24.04`, despite the -self-hosted rollout. Route only changed-scope, current-head admission, -superseded-run cleanup and manual status publication through group 6 when -the source is exactly central `strix.yml@refs/heads/main` and the caller is -the central repository or fast-mlsirm. These jobs do not check out PR code. -The model scan keeps its existing hosted image and all evidence, credentials, -fork handling and live-head validation remain intact. - -Reuse the deployed allocation; no new service-time or capacity measurement -justifies a different solver result. Deployment requires adding only central -`strix.yml@refs/heads/main` to group 6's selected workflows, preserving all -existing restrictions and grants. Old queued jobs keep their original source. - -The routing test failed on the unmodified workflow. The affected runner, -changed-scope and dependency-hash tests passed (21 tests); actionlint and -diff whitespace checks passed. This is local source proof, not completed -consumer gate evidence. diff --git a/docs/doctoring/co-noema-control-allocation-20260927.md b/docs/doctoring/co-noema-control-allocation-20260927.md deleted file mode 100644 index d9a5922f6f..0000000000 --- a/docs/doctoring/co-noema-control-allocation-20260927.md +++ /dev/null @@ -1,26 +0,0 @@ -# Contextual Orchestrator Noema control allocation - -## Structure and gap - -After #2420, the four metadata-only Noema jobs used the control group only for -`.github`. Contextual Orchestrator still placed admission and scope detection -in the two-runner model pool. On 2026-09-27, run 36314265013 had both jobs queued -with no runner assignment while both remediation runners were busy. All five -organization runners were online; the idle CodeQL runner was workflow-restricted. - -## Allocation - -Apply the existing control allocation to both already admitted repositories. -Keep the trusted-main guard, hosted fallback, exact-head admission, permissions, -and model job unchanged. The control group permits all repositories but limits -execution to explicit central workflows at main; its allowlist already includes -Noema. These four jobs do not check out PR code. No optimizer or broader runner -access is needed for this fixed eligibility partition. - -## Verification and limits - -The five focused runner, queue, admission and Noema contract files completed -222 tests locally. Actionlint passed. Hosted current-head execution and actual -queue drainage must be checked after deployment; local tests do not establish -runner capacity or independent model approval. Rollback restores only the four -runner expressions from parent revision efe71f4. diff --git a/docs/doctoring/codeql-metadata-admission-bound-20260927.md b/docs/doctoring/codeql-metadata-admission-bound-20260927.md deleted file mode 100644 index 60c9f5f9f2..0000000000 --- a/docs/doctoring/codeql-metadata-admission-bound-20260927.md +++ /dev/null @@ -1,53 +0,0 @@ -# CodeQL metadata job admission bound - -On 2026-09-27, central Strix admission job108624881622 was queued with the -correct self-hosted/cwlab-control labels while all three control runners were -busy. Contextual-orchestrator CodeQL job108620193038 occupied cwlab-s2-01 in -`Read current-head CodeQL dispatch verdict`. This establishes a shared control -lane and live metadata work; it does not prove which individual API call stalled. - -The central codeql-pr jobs detect languages, read verdicts, or coordinate -dispatch, with no job execution bound. Language detection checks out source -for trusted classification; it does not execute PR-authored code. A stalled gh call can -therefore occupy a control slot for the platform default six hours. Add the -existing operational budgets: five minutes for detection/dispatch and ten -minutes for verdict reads, as used by control cleanup. -The separately dispatched scan and model inference keep their own contracts; -no elapsed inference time becomes a model-failure verdict. A timed-out metadata -job remains non-passing and cannot authorize a merge. - -The new assertion fails against unchanged source. The CodeQL and runner -contracts pass in local and GITHUB_ACTIONS=true modes (36 each); actionlint -and whitespace checks pass. Runner access, source-ref guards, permissions, -head revalidation, concurrency and authenticated verdict checks are unchanged. -Existing runs retain their original source and were not cancelled. Native -post-merge execution is needed to prove slot recovery and queue latency. - -## Terminal and idle-runner revalidation - -At 2026-09-27 14:46 UTC, job108620193038 was verified terminal: started -13:16:43, completed 13:23:14, failure. Its verdict-read step succeeded from -13:16:48 to 13:23:06 (378 seconds), then its enforcement step failed. It is -not a currently stuck slot, nor proof of a particular stalled API call. The -initial five-minute proposal would interrupt this observed orderly path; -only the verdict-reader budget is therefore revised to the existing ten-minute -control budget. This is an operational bound, not a calibrated latency optimum. - -Strix job108624881622 remains queued with cwlab-control labels while group6 -reports an online idle cwlab-s1-05. Its run36321072667 has no pending deployment -approval. The selected-workflow allowlist includes trusted-main Strix. This -contradicts treating every wait as simply all control runners being busy; -workflow eligibility, concurrency and organization admission still require -current evidence. These observations do not authorize cancellation or runner -access expansion. Some REST reads succeed while run-list reads return quota -errors, so no complete active-job census or current global-ceiling claim is made. - -## Current-main integration, 2026-09-28 - -Replayed only the three job budgets onto central main `5b0024a9`. -Existing trusted-source routing, current-head validation and scan contracts remain. -The CodeQL workflow, runner-image and required-queue contract suites pass: -112 tests locally and 112 with `GITHUB_ACTIONS=true`. Actionlint (ShellCheck -disabled) and `git diff --check` pass. These checks establish local source -contracts; native queue recovery remains unverified. Earlier runner observations -above are historical and do not describe current occupancy. diff --git a/docs/doctoring/codeql-obsolete-pr-verdict.md b/docs/doctoring/codeql-obsolete-pr-verdict.md deleted file mode 100644 index 82b27beae1..0000000000 --- a/docs/doctoring/codeql-obsolete-pr-verdict.md +++ /dev/null @@ -1,75 +0,0 @@ -# Closed and superseded CodeQL compatibility shards - -## Incident and root cause - -ContextualWisdomLab/fast-mlsirm#2172 merged at 2026-09-26 11:05:49 UTC. -The actions compatibility shard in [run 36237658142](https://github.com/ContextualWisdomLab/fast-mlsirm/actions/runs/36237658142/job/108414341704) -started its live PR read at 19:47 UTC. It correctly observed the closed PR and -returned without requesting a scan. The next step saw a successful read with -an empty verdict and failed with `CodeQL shard has no authenticated current-head -verdict or dispatch receipt.` The same producer/consumer mismatch existed when -the live open PR head differed from the event head. - -The queue delay exposed this bug; delay itself does not explain the failed -verdict contract. The missing output is the causal defect. - -## Repair and boundaries - -The live read now emits `verdict=obsolete` for a closed PR or a live head proven to descend from the event head. -Enforcement accepts that state without asserting a successful scan and without -publishing a security status. A lagging or diverged head, failed/incomplete comparison, malformed SHA, or unknown PR state fails -before retirement. Open PRs at the event head still require the existing -trusted terminal verdict; pending, failed, missing and unauthenticated evidence -remain failures. No permissions, security severity or required gates change. - -This repairs the required workflow compatibility layer. It does not replace -#2382's separate dispatch-handler stale-run repair or change an already-recorded -historical check result. - -## Verification - -Tests execute the actual workflow shell blocks with a stubbed GitHub API. -Closed and superseded targets reproduce the missing-output failure on the -baseline and pass with the repair. Unknown states, malformed SHAs and unproven forward ancestry fail in -both the read and enforcement steps. Existing exact-head verdict tests cover -trusted failure, spoofed success, missing evidence and terminal dispatch receipts. - -## Primary platform basis - -GitHub. (n.d.). *Workflow commands for GitHub Actions: Setting an output parameter*. -https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-output-parameter - -GitHub. (n.d.). *Contexts reference: Steps context*. -https://docs.github.com/en/actions/reference/workflows-and-actions/contexts#steps-context - -## Existing security baseline repaired with the consumer - -The repository's open Dependabot alerts 11–13 identify AnyIO 4.14.0 in -`requirements-strix-ci-hashes.txt`. The Critical and High advisories are -GHSA-82r6-8w77-94w6 and GHSA-3w57-8xmc-8v26; the patched version is 4.14.2. -The source pin, two release hashes and source/lock parity test are reused from -#2385 at `372f5b8bb1ae1bb32ab29e9afbe363d81aed81e3`, without claiming that PR's -other changes or checks have been inherited. Both release digests were verified -against PyPI's version-specific JSON. This removes the known vulnerable lock -entry while preserving the repository-wide security gate. - -GitHub. (2026). *AnyIO: TLSStream IDNA 2003 host name encoding enables potential -TLS certificate spoofing* (GHSA-82r6-8w77-94w6). -https://github.com/advisories/GHSA-82r6-8w77-94w6 - -Python Package Index. (2026). *AnyIO 4.14.2*. -https://pypi.org/project/anyio/4.14.2/ - -## Dedicated control admission - -The five-runner allocation already documented in -[central dedicated routing](central-dedicated-runner-routing-20260927.md) separates -heavy CodeQL scans, long OpenCode reviews, and small control work. Compatibility -language detection, verdict reads, and dispatch coordination use the control -lane when `github.workflow_ref` identifies this exact trusted main workflow. -PR-authored revisions retain hosted execution. The existing control group adds -only this main workflow path to its allowlist; no PR ref or repository access -is broadened. Heavy scans continue using the dedicated CodeQL group. - -GitHub. (n.d.). *Using self-hosted runners in a workflow: Using labels and groups*. -https://docs.github.com/en/actions/how-tos/manage-runners/self-hosted-runners/use-in-a-workflow diff --git a/docs/doctoring/codeql-terminal-proof-2352.md b/docs/doctoring/codeql-terminal-proof-2352.md deleted file mode 100644 index cfd2b356a3..0000000000 --- a/docs/doctoring/codeql-terminal-proof-2352.md +++ /dev/null @@ -1,66 +0,0 @@ -# CodeQL terminal-proof settlement (#2352) - -## Incident - -On `.github#2352@f1a8dc813e6dba4e4905bf3e1b770b6d44344944`, required CodeQL -run `35805450471` initially failed pending and was later rerun. Attempt 2 jobs -`107353895415` (Actions) and `107353895562` (Python) became GREEN by reading -the successful `Enforce CodeQL Medium+ SARIF gate` step from producer run -`35841640640`. - -The producer jobs were nevertheless terminal failures: the later -`Verify GHAS base/head CodeQL configuration identity` step received HTTP 403. -The gate-only fallback therefore hid the exact credential/permission defect -tracked by `#2275` and `#2276`. - -## Root cause and boundary - -The required receiver and settlement contract treated one successful SARIF -gate step as terminal success even when a later mandatory proof failed. This -was originally allowed so a wake-only API failure could not invalidate an -otherwise complete scan, but the contract did not distinguish that harmless -late failure from GHAS identity or SARIF-preservation failure. - -A clean result recovered from a producer job whose overall conclusion is -failure now requires the same three proof units in both paths: - -1. `Enforce CodeQL Medium+ SARIF gate` succeeds; -2. `Verify GHAS base/head CodeQL configuration identity` succeeds; and -3. `Preserve CodeQL SARIF evidence` succeeds. - -A later failure confined to waking the exact required job remains outside the -scan verdict and may still be reconciled. A Medium+ gate failure remains a -terminal security failure and does not require a successful GHAS identity -step. A producer job whose overall conclusion is success remains authenticated -terminal proof because GitHub completed its non-optional steps successfully. -Missing, duplicate, skipped, cancelled, or failed proof on the failed-job clean -fallback stays fail-closed. - -An independent review found a second boundary defect before merge: the -required receiver and coordinator trusted the legacy -`codeql-dispatch/` commit status using only head SHA and publisher. -GitHub retains statuses on a commit, so the same head could reuse a success -from an earlier base, required run, or producer protocol after a PR retarget. -The current producer and consumers now use the v2 receipt exclusively: - -- context: `codeql-dispatch//`; -- description: exact head SHA, required run ID, workflow identity, and live - merge-source SHA; and -- publisher: the existing allowlisted app identity. - -The coordinator dispatches `codeql-scan-v2` with the versioned `pr_head` -envelope and live merge source. A legacy or otherwise stale status is ignored, -so the exact run performs or reuses only its own base/source-bound scan. - -## Verification and ownership - -Executable regressions reproduce the direct receiver and run-wide settlement -false-GREEN surfaces plus stale trusted-status reuse. They are RED on protected -`main` and GREEN with the proof contract. Focused workflow tests pass 91/91; -the complete repository suite passes 3,372 tests with 28 skips and 40 subtests. - -The central `.github` workflow remains the canonical owner. Do not copy the -workflow into a consumer, synthesize a status, accept clean SARIF alone, or -weaken the GHAS identity proof. `#2275`/`#2276` still own the real credential -and target permission repair; this change prevents that missing authority from -being mislabeled as a successful required check. diff --git a/docs/doctoring/codeql-verdict-history-scope.md b/docs/doctoring/codeql-verdict-history-scope.md deleted file mode 100644 index c25f395608..0000000000 --- a/docs/doctoring/codeql-verdict-history-scope.md +++ /dev/null @@ -1,21 +0,0 @@ -# CodeQL verdict history scope - -## Structure and gap - -Required CodeQL shards consume an authenticated status or an exact completed dispatch bound to target repository, PR, head, base and required run ID. The fallback previously paginated the complete central dispatch history. On 2026-09-27 the public workflow API reported 10,311 runs; contextual-orchestrator#1031 Python verdict job 108609837545 was executing the lookup on cwlab-s1-05. This proves the lookup workload, not that it alone caused all queue delay. - -## Repair and invariant - -Read the canonical required run creation timestamp with Actions read permission. Fail closed if it is missing or malformed. Query repository_dispatch runs created at or after that timestamp, retaining pagination and exact identity and terminal gate checks. A producer bound to the required run cannot exist before the required run. No elapsed-time model verdict, synthetic approval, runner-group relaxation or security exemption is introduced. - -The same live API query with created >= 2026-09-27T11:08:00Z returned 3 runs. This is query cardinality evidence, not deployed latency or completed CodeQL proof. - -## Verification - -Real extracted Bash verdict scripts retain successful completed-dispatch recovery, later-page recovery, stale base/run rejection, unknown-state rejection and no-dispatch pending behavior. Added invalid timestamp failure coverage. The focused contract suite passed 29 tests before the explicit Actions read grant. Final combined verification is recorded in the PR. - -The original extended runner-image oracle expected three literal ubuntu-24.04 jobs while protected main routes trusted workflow jobs to the central control group. The exact oracle failed on unmodified base c3e86141c. It now requires all three jobs to compare the exact trusted main workflow ref, select the control group with self-hosted/linux/x64 labels, and retain the explicit ubuntu-24.04 fallback for other refs. The six runner-image tests pass locally; combined final receipt is recorded in the PR. - -## Reference - -GitHub. (n.d.). *REST API endpoints for workflow runs*. Retrieved September 27, 2026, from https://docs.github.com/en/rest/actions/workflow-runs#list-workflow-runs-for-a-workflow . The created filter uses date-time search syntax; per_page supports 100. Filtered searches return up to 1,000 runs; overflow cannot authorize a false success because exact receipt matching remains mandatory. diff --git a/docs/doctoring/fmls-preflight-readiness-20260927.md b/docs/doctoring/fmls-preflight-readiness-20260927.md deleted file mode 100644 index 3b65f3f68b..0000000000 --- a/docs/doctoring/fmls-preflight-readiness-20260927.md +++ /dev/null @@ -1,160 +0,0 @@ -# fast-mlsirm review-gate continuation: readiness repair - -## Original intent and boundaries - -Independently review ContextualWisdomLab/fast-mlsirm#2114 and -ContextualWisdomLab/fast-mlsirm#2120, then report shared CI failures to the -main or owning coordinator. Those PRs were merged on 2026-09-24. The previous -session stopped after the five-item report and correction of Cargo ownership. -This continuation preserves the numerical formulas, source transcript, -other agents' dirty worktrees, and protected review/security gates. It makes -no merge, tag, release, provider-availability, or hosted-acceptance claim. - -## Current scope - -DOI URL tests are now in ContextualWisdomLab/fast-mlsirm#2171 at -`cffb90fb742d0ebcb9c5aa49c8323ce349138eab`; six focused tests pass on that head -and after an isolated conflict-free merge with main `6dd48140`. - -Existing central repairs were independently checked and received scoped -COMMENT reviews: - -| Repair | Exact reviewed head | Local evidence | -| --- | --- | --- | -| #2360, committed Rust roots via `cargo vendor --sync --locked` | `fc9c8d2c8537e9a0582299d5b26eef31d6309710` | 26 passed; three real Cargo integrations excluded | -| #2385, proven target GHAS-read credential selection | `372f5b8bb1ae1bb32ab29e9afbe363d81aed81e3` | 59 focused tests passed | -| #2387, Noema retry dispatch credential separation | `33b9028318ddd0cf6c34d1816b09b94e95346c10` | 85 focused tests passed | - -Those local results do not prove hosted credentials, provider capacity, -whole-PR approval, or deployment. The Noema repair addresses the observed -eligible HTTP 504 followed by integration HTTP 403 in run `36237188317`. -The Strix binder successor remains separately owned by #2291. The original -CodeQL-status permission and live governance requirements must be evaluated -from terminal producer evidence, not inferred from the GHAS-read test. - -## New root cause and primary evidence - -[Strix run 36237188327](https://github.com/ContextualWisdomLab/fast-mlsirm/actions/runs/36237188327), -job `108408520367`, artifact `10919666896` (`strix-reports`): discovery ends -at 19:06:31Z on 2026-09-26. Two OpenRouter probes return 429; gemma-3 probes -on both NVIDIA accounts return 404. Later probes complete (the next sequential -invocation demonstrates completion). `nvidia_nim` llama-3.2-90b begins at -19:06:49.173Z without any later outcome before hosted cancellation at -01:00:56.653Z on 2026-09-27. This is about 5h54m before readiness, gateway -preflight, or scanning. The preflight JSON is empty; sanitized stderr retains -the route invocation. ZIP SHA-256: -`7bfd559ac1abda65c150fc3d5ec99562d8c83fca1a8d9dc7b444f7de6a4304e7`. - -The executed shared base is `e6334e229581a918e2f22de18733b76fa65d7e71`, -vendoring contextual-orchestrator `767e67fbc6b881a452761f32abb69b9971b9b03b`. -That runtime intentionally removed the 90-second inference deadline. -The sequential readiness walk consequently prevents later eligible routes -from being checked while a provider remains pending. Historical ADR-0029 -wall-time bounds no longer describe that pin. - -An event-controlled check against the exact base launcher confirmed that a -pending first call prevents all eight later readiness calls. An initial -concurrent implementation with only eight outstanding calls reproduced the -same obstruction with eight pending candidates, so the final scheduling uses -the existing sixteen-base-probe budget to bound outstanding calls as well. -It processes available completions before scheduling more work, preserves -catalog priority among admitted routes, and shares the four escalation -reservations across all probes and fallback. Pending calls continue without -admission or a synthetic failure verdict. - -## Verification and remaining acceptance - -The original runtime preflight suite and eight new concurrency checks pass: -143 tests with warnings as errors, both locally and with `GITHUB_ACTIONS=true`. -The event tests hold one or eight calls pending and require eight subsequent -ready routes to be admitted before releasing the pending calls. Other checks -cover all-429 failure, escalation bounds, deferred-route admission, fallback -ordering, fault propagation, and production wiring. Ruff and diff whitespace -checks pass. No live provider credential or model API was used in tests. - -After this repair lands through normal protection, a fresh exact-head review -must show completed readiness and a valid reviewer receipt. Pools without -enough responding routes can still wait; hosting loss and durable resumption -remain distinct concerns. The snapshot's pending rows are startup evidence, -not final outcomes of those model calls. More simultaneous calls may expose -provider rate limits sooner, within unchanged total request budgets. - -Organization-wide evidence found 51 assigned running jobs (30 Strix, 19 Noema, -two compatibility), while the #2171 OpenCode coverage and CodeQL producer jobs -were unassigned. This establishes occupancy, not the exact concurrency ceiling. -The Actions budget does not halt usage. Five oldest sampled review runs still -matched open current-head PRs; no stale cancellation was justified. - -Owner report: [fast-mlsirm #2171 comment](https://github.com/ContextualWisdomLab/fast-mlsirm/pull/2171#issuecomment-5853298639). -Repair tracking: #2408, Project #1 In Progress. Hosted current-head acceptance -and qualifying independent review remain required. - -## 2026-09-27 security prerequisite integration - -Noema #2387's hosted pip-audit job `108414598334` is a real shared-lock -failure: `requirements-strix-ci-hashes.txt` still selects AnyIO 4.14.0. -The audit lists CVE-2026-63374, CVE-2026-64847 and CVE-2026-63349, each with -4.14.2 as the patched version. This is separate from the startup scheduling -failure and the retry-dispatch credential defect. - -The canonical dependency repair is #2278 at -`8a5251bf409fe84b3dd0cba1e48992f5b8d9eda5`. Its complete three-dot delta -against protected main is exactly the six-line AnyIO pin/hash change. The -current-head requested-changes review cites failed coverage and contains no -source-backed lock finding; there are no inline review comments. That review -is retained, and no approval or main merge is inferred from the dependency -verification. - -An ordinary two-parent integration carries the canonical owner's exact commit -into this isolated repair branch. The integration changes only that lockfile; -it does not modify the owner's branch or copy unrelated foundation repairs. -The release wheel and sdist were downloaded from PyPI's official distribution -host and their actual SHA-256 bytes matched both committed hashes: - -- wheel: `9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494` -- sdist: `cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f` - -A hash-pinned pip-audit 2.10.1 installed in an isolated project venv audited -all 106 distributions listed in the original and repaired Strix lock, with -`--strict --disable-pip --no-deps --format json`. The original returns exit 1 -with exactly those three AnyIO findings; the repaired lock returns exit 0 -with zero findings. Both JSON results contain 106 dependencies and zero -skipped entries. Target dependencies were not installed or executed. HTTP -cache entries that could not be decoded were ignored by the tool; the audit -completed. This establishes the changed lock's advisory result, not the -security of every repository input or a live Strix run. - -Primary advisory basis: [AnyIO process-pool stderr advisory](https://github.com/agronholm/anyio/security/advisories/GHSA-5p39-cfhj-2xmp) -and [supplementary-group advisory](https://github.com/agronholm/anyio/security/advisories/GHSA-3w57-8xmc-8v26). -The existing gate and its severity/ignore policy remain intact. - -## Current governance audit and correction of the historical diagnosis - -Live ruleset `18156473` still requires seven `.github@main` workflows, -including `codeql-pr.yml`. That CodeQL entry is intentional: the protected -rollout document's 2026-09-04 correction restored a dispatch-safe entrypoint -that does not directly invoke `github/codeql-action`. The earlier transcript's -claim that its presence disagreed with the removal policy is superseded by -that correction. The July inventory warning still described removal/native -setup as the current posture; this continuation repairs that stale wording -without changing a required gate. - -The live organization payload also reveals a separate approval-policy -mismatch. Its approving-review count is one and last-push approval is false; -protected main's audit contract and July 23 rollout evidence require two and -true. Running the existing auditor on the actual payload returns exactly those -two errors. All seven workflow identities, required source ref, exclusions, -stale-review dismissal, review-thread resolution, and branch protection rules -pass that audit. The stacked ruleset `21732164` separately passes its audit. -Code-owner review remains false as the maintainer requires. - -An unapplied candidate changing only those two approval fields passes the -existing auditor. The current payload's SHA-256 is -`d6e6efd8c67027ae4a3625753c0e90198f8f92c67c691a0857231bd81d8ce412`. -The audit-log endpoint returned HTTP 404, so the reason or authority for the -live approval settings cannot be established from that endpoint. The user has -been asked which approval policy is intended before changing organization-wide -merge conditions or the repository contract. No live ruleset has been changed. -This mismatch does not explain an unassigned CI runner; job -`108568126406` and the original OpenCode coverage job `108521250487` are -separately confirmed queued with runner_id zero and no executed steps. diff --git a/docs/doctoring/fmls-release-sidecar-runtime-adoption-20260928.md b/docs/doctoring/fmls-release-sidecar-runtime-adoption-20260928.md deleted file mode 100644 index fa68aac4ed..0000000000 --- a/docs/doctoring/fmls-release-sidecar-runtime-adoption-20260928.md +++ /dev/null @@ -1,7 +0,0 @@ -# Release sidecar runtime adoption - -The immutable release helper at 4b0c6b75 predates the shared-runtime fix from #2468. Updating central main or rerunning an old immutable workflow cannot make that checkout consume the fix. - -All three release helper checkouts and their identity guards now pin protected-main ancestor e45f1b144aef900d734ff4c900f9e0010fd5a32d. The scripts/ci tree is 7f902df89a925f89c4fae69a842508406cd0207c; the requirements blob remains eb83beda177c9d2e4ca9b7e2888a1ccb55a123ac. The entire scripts delta from the prior helper is the six-line shared Python-library binding. Origin, commit, tree, clean-file and sibling checks remain intact. - -Independent guest-02 probes confirmed the selected 3.12.14 executable loaded the 3.12.3 runtime under inherited paths; the exact patched prefix selects 3.12.14 and passes logging/asyncio imports. The shared sidecar contracts passed 31 tests locally and in CI mode before #2468 merged; its whole merged tree matched the tested tree. This adoption does not claim hosted Noema acceptance, source grant clearance, release publication or a twelve-wheel verdict. The fast caller must separately adopt this callee revision. diff --git a/docs/doctoring/fmls-reviewed-release-helper-adoption-20260928.md b/docs/doctoring/fmls-reviewed-release-helper-adoption-20260928.md deleted file mode 100644 index 5eca9ef7df..0000000000 --- a/docs/doctoring/fmls-reviewed-release-helper-adoption-20260928.md +++ /dev/null @@ -1,9 +0,0 @@ -# Reviewed release helper adoption - -The release callee continued checking out helper `5a29e0a5` after central #2457 and #2465 were merged. Advancing the fast caller workflow alone would therefore not execute the reviewed artifact recognition or complete libfuzzer source-obligation checks. - -All three callee jobs now pin protected-main ancestor `4b0c6b754fc30a0d0bf77f9c41650e1451476c26`. Their identity guards require scripts tree `f1b96f0a0af5cc30f8c8f2bb662727d41129f7dd`; requirements blob remains `eb83beda177c9d2e4ca9b7e2888a1ccb55a123ac`. Foreign origin, dirty/missing files, incorrect commit/tree and caller-controlled sources remain rejected. - -Reviewed #2465 source d0abbd63 integrated on a2ba7972: full merge tree `5b92f72696aae71cfe35ce5d765bf280f4f7d504` exactly equals merged 4b0c6b75. Independently fetched 59 immutable source/grant bodies with exact hash/size agreement; full LLVM modern/legacy terms and CREDITS were read. The 957 affected license tests pass locally and under GITHUB_ACTIONS=true. Adoption workflow/identity suites pass 58 tests in each mode; actionlint (ShellCheck disabled) and diff checks pass. - -The scripts-tree delta also includes separate Noema/materializer changes; these are not release-gate entry points. The release sidecar delta enables fatal stack-location diagnostics without frame locals. Strix requirements are unchanged. This is fixed-helper adoption only: fast caller adoption, native execution, original HOLD clearance, and published 12-wheel acceptance remain distinct requirements. diff --git a/docs/doctoring/noema-central-transport-continuation.md b/docs/doctoring/noema-central-transport-continuation.md deleted file mode 100644 index f53277998d..0000000000 --- a/docs/doctoring/noema-central-transport-continuation.md +++ /dev/null @@ -1,34 +0,0 @@ -# Noema central transport continuation - -## Failure - -The 429-capacity continuation sent repository dispatch to the product repository. -Organization-required workflows do not supply a local repository-dispatch handler -there. A central review of another repository also failed its same-repository -origin guard, even though the review itself had admitted that target. - -## Repair - -Send the existing `noema-review` event to the central `.github` handler. Preserve -its target repository, PR, exact head and retry count. Permit only the central -origin or the target repository's own required workflow. Re-fetch an open PR and -require matching head, base, base repository and head repository before sending. -Fork, stale, closed and unrelated-origin continuations retire or fail closed. - -The central handler can dispatch with its repository-scoped GitHub token. -A consumer continuation requires the existing `PR_REVIEW_MERGE_TOKEN` to read the -product PR and create a central repository dispatch; absence or insufficient -permission fails explicitly. This change does not assert that every consumer has -that credential. Native trusted-main runner restrictions, independent review, -publication fencing and the existing post-failure retry bound remain unchanged. -No model inference deadline is added. - -## Evidence - -The shell regression fails on the baseline because the dispatch endpoint is the -consumer repository. It executes the actual workflow step against a fake API, -checks the central endpoint and preserved payload, permits central-origin retry, -rejects unrelated origin and fork or changed-base evidence, and proves a rejected -POST cannot report successful continuation. The unchanged reviewer contracts -are run alongside this regression. Live provider recovery and approval still -require successful current-head hosted execution. diff --git a/docs/doctoring/noema-draft-before-sidecar.md b/docs/doctoring/noema-draft-before-sidecar.md deleted file mode 100644 index a0452db11e..0000000000 --- a/docs/doctoring/noema-draft-before-sidecar.md +++ /dev/null @@ -1,60 +0,0 @@ -# Noema live draft check before sidecar provisioning - -Date: 2026-09-26 -Repository: `ContextualWisdomLab/.github` -Workflow: `.github/workflows/noema-review.yml`, job `noema-review` - -## Root cause - -`Provision contextual-orchestrator review sidecar` (`scripts/ci/contextual_orchestrator_review_sidecar.sh`) -takes 10-13 minutes. Only afterwards did `Prepare Noema model verdict` run -`.github/actions/noema-review/two_phase.py --prepare-verdict-file`, whose `prepare_verdict` reads the -live pull request and returns early with `PR is draft; Noema verdict preparation skipped.` without an -envelope, so publication was skipped. Every draft run therefore held a hosted runner for ~13 minutes to -reach a decision that was available from one API call. Observed examples: newsdom-api job -108077744310 (2026-09-25) and `.github` job 106665379126 (2026-09-22). With organization Actions -concurrency saturated, that runner time delays other required checks. - -## Repair - -- New step `Check live pull request draft state before sidecar provisioning` (`id: live_draft`), - placed after `Validate current pull request head` and `Resolve Noema target repository visibility`, - reads `repos//pulls/` with the same selected reviewer token and REST lookup the validate - step already uses, and writes `live_draft=true|false`. -- `Provision contextual-orchestrator review sidecar`, `Provision local reviewed HWP document reader`, - and `Prepare Noema model verdict` are gated on `steps.live_draft.outputs.live_draft != 'true'`. -- Fail open: a lookup error, malformed body, or any `draft` value other than JSON `true` yields - `live_draft=false`, which is exactly today's path; `two_phase.py` keeps its own draft check. -- Downstream steps are unchanged. They already require `steps.noema_prepare.outputs.prepared == 'true'` - (publication token refresh, publish) or `failure()` (transport re-dispatch, sidecar evidence upload), - so unset prepare outputs mean "publication skipped", the state a draft already produced. The job - concludes success for drafts, as before. - -## Why ruleset repositories are unaffected - -The organization ruleset launches this required workflow in other repositories only for -opened/synchronize/reopened, never `ready_for_review`. The repair therefore adds no trigger-level or -event-payload draft filter; it moves the existing runtime live-PR draft decision earlier. A draft -never produced a Noema verdict at runtime, and a ready PR follows the identical path. - -One narrow timing difference remains: a PR that was draft when the check ran but was marked ready -during what used to be the 10-13 minute provisioning window would previously have been reviewed by -that same run; it now needs the next run (a `ready_for_review` event here, or the next push in a -ruleset repository). - -## Regression coverage - -`tests/test_noema_draft_admission_before_sidecar.py` pins step ordering, the gate on each -model-heavy step, the live REST lookup and token reuse, the unchanged trigger list, the absence of -`github.event.pull_request.draft`, and executes the step with a fake `gh` for draft, ready, lookup -failure, and malformed/non-boolean `draft` bodies. - -## Complete response parsing repair — 2026-09-27 - -Independent exact-head review of `dfa41ab4` found that jq can print `true` before -returning a nonzero status on trailing malformed input. A stdout-only comparison -therefore skipped review for an invalid response. Draft admission now requires a -successful complete slurped parse containing exactly one object with boolean -`draft: true`. Invalid trailing bytes and a second JSON value keep the full review -path. The actual workflow shell regression failed before the fix and passes after -it; normal Draft/Ready behavior and bounded startup continuation remain covered. diff --git a/docs/doctoring/noema-self-hosted-node-bootstrap.md b/docs/doctoring/noema-self-hosted-node-bootstrap.md deleted file mode 100644 index 3a6d6943f4..0000000000 --- a/docs/doctoring/noema-self-hosted-node-bootstrap.md +++ /dev/null @@ -1,18 +0,0 @@ -# Noema document-reader runtime on self-hosted workers - -The exact-head Noema job for .github#2373, run 36256598579 job 108504745563, -terminated before model review on 2026-09-27 with exit 127: the local HWP reader -version probe could not find `node`. This is a runtime prerequisite failure, -not provider capacity exhaustion or a product review verdict. - -The workflow now provisions Node.js 22.23.3 through the exact-pinned setup-node -v4 action before provisioning the gateway sidecar. The reader already accepts -Node 20 or 22 and uses its reviewed local npm lock with lifecycle scripts -disabled. This removes an implicit hosted-image prerequisite without modifying -providers, model deadlines, reader dependencies, review sufficiency, or retry -limits. It also avoids occupying a runner for gateway discovery before learning -that the local document reader cannot start. - -The regression asserts the pinned version, action revision and preparation -order. A local workflow contract pass is not proof of hosted review approval; -a fresh exact-head run still must execute the reader and publish a real verdict. diff --git a/docs/doctoring/opencode-coverage-cargo-fixtures-20260928.md b/docs/doctoring/opencode-coverage-cargo-fixtures-20260928.md deleted file mode 100644 index 1b5d0b65fd..0000000000 --- a/docs/doctoring/opencode-coverage-cargo-fixtures-20260928.md +++ /dev/null @@ -1,55 +0,0 @@ -# OpenCode coverage Cargo fixture intake (2026-09-28) - -## Incident - -The current-head `.github#1026` OpenCode dispatch run `36348910783`, job -`108722448709`, reached the isolated coverage sandbox. Its full suite reported -four failures in `test_materialize_base_rust_dependencies.py` and -`test_maturin_offline_build_contract.py`, each before the tested behavior at -`cargo generate-lockfile` (3,538 passed, 4 failed, 4 skipped). The PR does not -change either test file. Both files construct registry-backed crates (`itoa`, -`ryu`, and PyO3) during the test, while the sandbox runs with `--network=none` -and its base-repository Rust materializer finds no Cargo lock in this PR's -validated base tree. - -With a fresh `CARGO_HOME` and `CARGO_NET_OFFLINE=true`, the two representative -tests failed at the same command. A direct `cargo generate-lockfile` on the -`itoa` fixture reported `no matching package named itoa found` in the offline -crates.io index. This establishes missing registry fixture input, rather than -a product-code assertion failure. The original hosted test helper captures -Cargo stderr, so the hosted log alone does not identify the missing crate. - -## Repair and trust boundary - -A trusted, lockfile-pinned fixture manifest covers the three registry crates -used by these tests. The networked coverage image build fetches that exact -closure with `cargo fetch --locked`. The PR tree never enters that build -context. The later untrusted test container still has `--network=none` and -receives only the trusted image's cached registry artifacts, copied into its -isolated `CARGO_HOME` after removing any PR-supplied sandbox home. Its trusted -Cargo config enables offline registry resolution, so a lockfile generated by a -test cannot attempt an index refresh against the disconnected network. Tests -that intentionally create a separate Cargo home can still resolve local Git -fixtures before their own offline build step. Existing base-repository Rust -vendor configuration and the sandbox's credentials and network restrictions -remain in force. - -The image build fails if the trusted fixture files are absent, symbolic links, -or cannot be fetched against their checksummed lock. It does not turn a failed -test into a pass. - -## Verification boundary - -An initial PyO3 extension build exceeded its 600-second limit on a loaded -macOS host. A later run with the project-local pinned maturin completed that -test in 32 seconds. With a fresh `CARGO_HOME` populated only by the locked -fixture and its trusted offline config, the complete two-file Rust dependency -and PyO3 suite passed all 29 tests with `GITHUB_ACTIONS=true`. The first -attempt at global `CARGO_NET_OFFLINE=true` failed one local Git dependency -fixture; scoping offline resolution to the default trusted Cargo home fixed -that failure. A terminal hosted rerun is still required before claiming the -coverage gate repaired. The targeted workflow contract suite passed 77 tests -with `GITHUB_ACTIONS=true` after this change. -The fixture crate's own `cargo llvm-cov --all-features --fail-under-lines 100` -run passed locally with one test and 100% line coverage, exercising its cached -`itoa` and `ryu` dependencies. diff --git a/docs/doctoring/opencode-infrastructure-review-state.md b/docs/doctoring/opencode-infrastructure-review-state.md deleted file mode 100644 index eb54c2fe6e..0000000000 --- a/docs/doctoring/opencode-infrastructure-review-state.md +++ /dev/null @@ -1,28 +0,0 @@ -# OpenCode infrastructure failures and review state - -Status: Proposed; protected delivery and downstream exact-head review are unverified. - -## Causal evidence - -CO #1223 at 90911687cb1ee49325ddd1f2bdfd29284a526028 was returned to Draft -because older OpenCode reviews remained CHANGES_REQUESTED. Their bodies explicitly -reported no source-backed product finding. Central run 36081670283, coverage job -107989271158, failed its trusted Docker image build before any CO test executed: -requirements-noema-document-ci-hashes.txt was missing from the build context. -That independent source defect is owned by ContextualWisdomLab/.github#2286 and #2385. - -The fallback publisher unconditionally used REQUEST_CHANGES to satisfy the formal -receipt gate. This promoted missing infrastructure evidence into a product verdict. -The correction publishes COMMENT instead and retains COVERAGE_BLOCKED separately. -COMMENTED still fails the formal receipt gate, never grants approval, and never -satisfies merge acceptance. Real source-backed model findings retain REQUEST_CHANGES. -Old reviews are not dismissed. After infrastructure delivery, a fresh model review -and required checks must settle against the exact current target head and base. - -## Verification - -The new event regression fails on unmodified main (1 failed, exit 1). Focused -publisher, coverage identity, receipt, pinned-workflow and toolchain contracts pass: -88 passed, 1 skipped, exit 0. actionlint and git diff --check pass. No hosted Docker -build or protected merge is claimed. The deterministic fallback source body is -unchanged; only its GitHub event is diagnostic rather than a fabricated verdict. diff --git a/docs/doctoring/persistent-runner-workspace-reuse-20260927.md b/docs/doctoring/persistent-runner-workspace-reuse-20260927.md deleted file mode 100644 index 18914b4fd6..0000000000 --- a/docs/doctoring/persistent-runner-workspace-reuse-20260927.md +++ /dev/null @@ -1,44 +0,0 @@ -# Persistent runner workspace reuse - -## Observed failures - -On 2026-09-27, CodeQL scan job 108599677231 on `cwlab-s1-03` -failed with `remote origin already exists` before analysis. The manual Git -initialization reused a repository from an earlier job. Anonymous OpenCode -coverage materialization used the same pattern with `trusted-source` and -also failed a real local repeated-workspace regression. - -The subsequent cross-repository status publication returned HTTP 403. -Live organization installation metadata shows `opencode-agent` has only -`statuses: read`. Adding `statuses: write` to the workflow cannot elevate -that installation permission. Existing successful-scan artifact settlement -remains the supported authenticated fallback; this repair does not invent -a trusted status author or widen application permissions. - -Scheduler job 108601462359 failed with `invalid UTF-8 string` on the first -GraphQL page. Its query is ASCII; the next same-source job 108601744765 -succeeded, and a current read-only request for the same 25-PR page succeeds. -No encoding mutation is justified by this non-reproducing observation. - -## Repair - -Use the repository's pinned native checkout action for CodeQL's validated -repository and exact head, with cleanup and without persisting credentials. -For anonymous OpenCode coverage bootstrap, discard only the current job -workspace contents after verifying it is not a symlink, matches the physical -current directory, and is directly under the runner-provided job directory. -The directory itself remains. Old Git hooks, configuration, and untracked -files cannot survive this anonymous bootstrap; child symlink targets and -sibling directories remain untouched. The Git fetch stays anonymous and -pinned to the validated trusted source ref. - -## Verification - -Before repair, three focused regressions failed: repeated anonymous checkout, -linked workspace refusal, and the native CodeQL checkout contract. -After repair, the focused real-Git regressions and existing CodeQL, OpenCode -shell, coverage toolchain, and paired workflow blob contracts report -**105 passed, 1 skipped**. Actionlint validates the two modified workflows -with ShellCheck and Pyflakes disabled; no claim about those engines is made. -Hosted execution and downstream CodeQL analysis are separate acceptance -steps. No active runner is restarted and no unrelated working copy is cleaned. diff --git a/docs/doctoring/release-build-artifact-identity.md b/docs/doctoring/release-build-artifact-identity.md deleted file mode 100644 index bf00920ee3..0000000000 --- a/docs/doctoring/release-build-artifact-identity.md +++ /dev/null @@ -1,62 +0,0 @@ -# Immutable same-run build artifact intake - -Base: 1916e95a8ee3b0dbd1c84011d88fc580700430e3. Previously the dependency -gate selected the caller's build artifact by name only. The gate now requires -`build_artifact_id` and `build_artifact_digest` as well as the expected name. -The producer must pass its upload result ID and `sha256:`-prefixed digest. -Missing values cannot fall back to name selection. - -The shipped shell reuses the existing exact-artifact-sbom-attestation workflow's -same-repository/same-run metadata comparison and the same pinned download action. -It also checks the returned ID explicitly and requires canonical positive -decimal ID and sha256 digest input. Metadata identity, name, digest, run and -unexpired status must agree before download. An API error blocks consumption. -No second generic verifier, new token, or helper revision is introduced. - -The gate requests only the additional `actions: read` permission required by -the metadata API. The caller must grant it; a called workflow cannot elevate -the caller's token permissions. Existing name-only callers must provide both -new required inputs when adopting this revision. Repository-local inspection -finds no executable caller of this reusable workflow; external caller adoption -is not exhaustively verified. FMLS's trusted matrix aggregation remains unwired. - -## Pinned download action contract, source inspection only - -The actual pinned revision is -`actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c`. -Its action.yml lines42–46 declare `digest-mismatch: error` as the default; -this candidate explicitly selects error. Its src/download-artifact.ts -lines94–136 select immutable IDs from current-run artifacts, lines171–183 pass -the selected artifact's digest as expectedHash, and lines215–231 throw and fail -the action on mismatch. A single requested ID avoids the multi-ID partial-match -warning path. A single selected artifact uses the existing destination root. - -Primary sources opened directly: -https://raw.githubusercontent.com/actions/download-artifact/3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c/action.yml -https://raw.githubusercontent.com/actions/download-artifact/3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c/src/download-artifact.ts - -The pre-download API comparison is not an independent hash of downloaded -bytes. Byte verification relies on this pinned action's implementation; its -download library/bundled execution and hosted transport are not executed in -the local tests. No warning-only revision is treated as equivalent. The API -record and downloaded record must refer to the same immutable ID; this does -not establish a release-source build proof or trusted gate success by itself. - -## Scoped evidence and remaining holds - -Tests execute the actual shell with inert gh output and installed jq. They -cover a valid record, same-name different ID, different run, absent/modified -digest, expiry, absent ID, invalid expected digest, different repository and -API failure. Rejected cases cannot reach the next-step marker. Static checks -bind download to ID and error-on-digest-mismatch; no real download occurs. -The first test run failed10 cases because its declaration extractor split at -child indentation; after anchoring the next input key correctly, the same -cases execute the shipped shell. This is a test harness repair, not acceptance -of the failed run. - -Source/control equality, fixed helper00c655, full licence/Strix policy, -platform closure, same-run trusted success aggregation, resource bounds, and -final R5 HOLD remain unchanged. API rate exhaustion prevents a fresh broad -external ownership/Project census; no inference of absent external callers or -approval follows. CodeGraph indexed38 workflow files with0nodes/edges, so all -workflow call paths are inspected as source rather than graph completeness. diff --git a/docs/doctoring/release-fixed-helper-source.md b/docs/doctoring/release-fixed-helper-source.md deleted file mode 100644 index f93ef9f0c1..0000000000 --- a/docs/doctoring/release-fixed-helper-source.md +++ /dev/null @@ -1,34 +0,0 @@ -# Fixed helper source for release gates - -The three trusted checkouts use literal repository `ContextualWisdomLab/.github` -and reviewed helper revision `00c6551183cca101cfc97c43656a17cc2491c1b4`. -This is an independent helper revision, not an assertion that helper and called -workflow revisions are equal. A future workflow change does not silently update -these helper bytes. Updating the pin and content identities requires review. - -All three checkouts materialize `scripts/ci/` and -`requirements-strix-ci-hashes.txt`, preserving helper siblings. Before executing -helpers they verify checkout HEAD, canonical origin URL, the scripts tree -`bf26d3eefdb71fe79b855d941ffb46eb432b2f76`, lock blob -`9e705850b5ce53c7fe836bc3df3a18771151e3f6`, tracked-file cleanliness and required -entrypoints. Missing, foreign or mismatched source rejects. The step reports -helper repository/SHA separately from caller workflow SHA. The latter is only -provenance context, never a checkout selector or authorization input. - -GitHub's [current context reference](https://docs.github.com/en/actions/reference/workflows-and-actions/contexts#job-context) -documents called-job workflow identity fields, but actionlint 1.7.12 and the -inspected upstream main strict schema do not yet support them. This alternative -uses neither those expressions nor an ignored diagnostic or permissive schema. -It changes the contract from called-self checkout to an explicitly adopted -fixed helper snapshot. The earlier called-self candidate remains separate. - -Local synthetic guards exercise valid identity, another caller SHA, missing -git source, foreign origin, wrong HEAD/tree, dirty tracked files and a missing -entrypoint. They do not perform checkout or network access. Hosted checkout and -attestation behavior remain unexecuted. - -`SOURCE_SHA == GITHUB_SHA`, full licence/Strix authorization, twelve-platform -closure and complete resource intake budgets remain unresolved independently. -No condition is relaxed by this source-selection fix. The pinned snapshot -retains its existing licence/tool-dependency limitations; exact-byte provenance -does not imply policy acceptance. diff --git a/docs/doctoring/release-license-archive-binding-20260924.md b/docs/doctoring/release-license-archive-binding-20260924.md deleted file mode 100644 index 745e843727..0000000000 --- a/docs/doctoring/release-license-archive-binding-20260924.md +++ /dev/null @@ -1,64 +0,0 @@ -# Archive-bound license evidence candidate - -Base: `a78b1c9f788d1a89fd7c8ab39d6347152b7e3065`. - -## Reproduced defect - -`/private/tmp/pr2347-archive-binding-repro.py` exercises the real capture, license -gate and install-lock binder. Its synthetic wheel contains academic-only terms, -while the separately supplied raw `licenses/LICENSE` contains the reviewed pytest -MIT text. The wheel and lock SHA-256 both equal -`659169bde33b6d27bf4cf927c01d69418cc97241bf728627448542f781c2771d`. -The base gate returns PASS and the binder writes that restrictive archive's hash. -This is a synthetic exploit, not a claim about any upstream package. -The raw receipt is `/private/tmp/pr2347-archive-binding-repro.md`. - -## Candidate contract - -- Raw capture retains `source.archive` for both wheel and crate inputs. -- One bounded byte read supplies both the archive digest and in-memory license - extraction. No archive extraction or package execution occurs in this helper. -- Conventional license/notice names at every depth and declared custom - `License-File` / Cargo `license-file` members are inspected. Missing declared - members, duplicate normalized paths, traversal, archive links, invalid text - and malformed archives refuse the capture. -- Separate raw license sidecars no longer supply the license decision. -- The gate reopens the retained archive and compares the source digest, full - license text mapping and raw member SHA-256 mapping against the evidence. -- The install binder verifies those member hashes against the collected wheel - and repeats the license decision on its actual member bytes before writing - the pinned install lock. Forged permissive report fields cannot authorize - an unrecognized restrictive body. - -## Verification - -The existing eight targeted test files plus -`tests/test_release_dependency_archive_binding.py` produce **296 passed, -1 skipped**, raw exit **0**, in 4.67 seconds. The skip is the existing GNU-find -Linux capture integration case; it is not a new skip. - -The 16 added cases cover Python/Cargo sidecar forgery, evidence alteration, -archive replacement, archive absence, duplicate archive members, nested/raw-byte -hash preservation, custom wheel license paths and missing declarations, plus -two real shell install-binder refusals. A fake pip recorder establishes zero -install calls in both new install-negative cases. No real install or download -is used. Existing fixture archives now carry the same source-bound full texts -used by their license tests; one formerly permissive missing-archive capture -expectation changes to an explicit refusal. - -`git diff --check` and shell syntax checking also return 0. This is a selected -regression result, not full-suite, coverage, hosted execution or release approval. - -## Remaining boundaries - -Raw shell capture still performs its existing extraction/native/hook collection -before assembly. Those scanners and separate metadata are not all reconstructed -from the retained snapshot by this patch. This candidate closes the demonstrated -license-sidecar binding defect; it does not certify every captured evidence field -or shell extraction as safe. The final install trusts the protected workflow -workspace to prevent mutation between binding and pip's hash-checked read. - -Only the previously reviewed exact full texts are recognized. Unsupported texts, -MPL/BSL complex provenance, NumPy bundle questions, tools/sidecar pre-install -coverage and complete dependency closure remain held. No license exception or -legal conclusion follows from this candidate. diff --git a/docs/doctoring/release-license-fixture-recovery-20260924.md b/docs/doctoring/release-license-fixture-recovery-20260924.md deleted file mode 100644 index ed010bf912..0000000000 --- a/docs/doctoring/release-license-fixture-recovery-20260924.md +++ /dev/null @@ -1,52 +0,0 @@ -# 기존 18실패의 원문 근거 복원 - -기준 `4fe66efdcee6bb6b68e5a6c386feea7280ecea8d`에서 새 브랜치 `codex/pr2347-source-fixture-recovery-20260924`를 사용한다. 기존 여섯 원문의 JSON 값과 provenance 첫 여섯 행은 Git blob 대조로 불변을 확인한다. - -공통 fixture는 Python MIT→실제 pytest 전체 MIT, Cargo Apache→실제 atheris 전체 Apache로 연결한다. 패키지 자체에 atheris라는 이름을 붙이거나 metadata 누락을 고친다는 주장이 아니라, synthetic gate fixture의 동일 SPDX 본문을 검증 가능한 전체 원문으로 복원한다. Cargo를 MIT로 바꾸지 않는다. 원문 hash는 기존 provenance에 있다. - -## 18개 before/after 기대와 근거 - -아래 이름은 `test_release_dependency_license_text_evidence.py` 기준이다. P=Python 원문, C=Cargo 원문이다. 기존 실패의 별도 C UNVERIFIED는 실제 Apache 전체 원문으로 해소한다. 검사의 핵심 실패 사유를 삭제하지 않는다. - -|번호|사례|before expected|after expected 및 변경 근거| -|---|---|---|---| -|1|no_bundled_text|MISSING 1개|동일. C만 전체 Apache로 복원| -|2|unrecognizable[unknown]|UNVERIFIED 1개|동일. UNKNOWN 입력 유지| -|3|unrecognizable[commercial-prohibited]|UNVERIFIED 1개|동일. 상업 금지 입력 유지| -|4|unrecognizable[empty]|UNVERIFIED 1개|동일. 빈 원문 유지| -|5|unrecognizable[pointer]|UNVERIFIED 1개|동일. 링크-only 입력 유지| -|6|unrecognizable[all-rights-reserved]|UNVERIFIED 1개|동일. 권리 유보 입력 유지| -|7|recognized_text_contradicts_declaration|DISAGREEMENT 1개|동일. MIT 선언에 실제 전체 Apache를 넣음| -|8|denied_title_disagreement|DISAGREEMENT 1개|동일. 정상 LICENSE는 실제 MIT, 별도 GPL COPYING 유지| -|9|matching_declaration|실패 없음|동일. 전체 pytest MIT 사용| -|10|permissive_family[Apache]|실패 없음|동일. 전체 atheris Apache 사용| -|11|permissive_family[BSD3]|실패 없음|동일. 전체 colorama BSD3 사용| -|12|permissive_family[ISC]|실패 없음|동일. 전체 libloading ISC 사용| -|13|permissive_family[MPL]|실패 없음|제목-only 원문은 UNVERIFIED. unsupported_title_only[MPL]로 목적을 명시하며 동일 입력을 유지. 실제 hypothesis 복합 적용 범위는 아래 별도 HOLD| -|14|permissive_family[BSL]|실패 없음|제목-only 원문은 UNVERIFIED. unsupported_title_only[BSL]로 목적을 명시하며 동일 입력을 유지. 기존 cache 인벤토리에 BSL 원문 mapping이 없음| -|15|permissive_family[Unlicense]|실패 없음|동일. memchr의 실제 전체 UNLICENSE 사용| -|16|dual_selection_disagreement|DISAGREEMENT 1개|동일. BSD/GPL 선언·BSD 선택에 실제 전체 MIT를 넣어 불일치 유지| -|17|recognizer_positive_half|MIT 포함|동일. 전체 MIT를 사용하고 UNKNOWN 반환 검사는 유지| -|18|install_binding cargo_only_release|실패 없음, lock digest 없음|동일. C에 전체 Apache를 연결. lock/hash 구현은 변경하지 않음| - -BSD 선택 양성과 sealed SBOM 선택 rationale 테스트도 동일 colorama BSD3 전체 원문으로 복원한다. 앞 후보의 부분 Apache 거부 테스트는 공통 fixture 변경에 영향받지 않도록 그 테스트에서 부분 Apache를 명시한다. 기존 음성을 정상으로 변경하지 않는다. - -## 추가 실제 원문과 보류 - -memchr2.8.3 `.crate` SHA256 `cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98`, member `memchr-2.8.3/UNLICENSE`, 원시 `7e12e5df4bae12cb21581ba157ced20e1986a0508dd10d0e8a4ab9a4cf94e85c`, 정규화 `2069c208cba553e43cd0b730df8a0c10bf1b1101b96f661e2f1307c73b9722e3`다. 전체 본문에서 copy/modify/publish/use/compile/sell/distribute, commercial or non-commercial, public-domain dedication 및 면책을 직접 읽는다. 다른 추가 조건을 발견하지 않는다. 이 파일만 registry에 추가하며 crate의 COPYING pointer·MIT/Unlicense 선택 전체를 자동 수용하지 않는다. - -hypothesis6.156.6 wheel SHA256 `b4e66aaa7385538a5d617174d47c198ee807f06de99e282a67c6cb724c69340d`, member `hypothesis-6.156.6.dist-info/licenses/LICENSE.txt`, raw `ac89037bac63550644dce8cf32c6765e5fab9dc1a1ce94b89f8a805f341a6750`이다. 전체 1–10절과 Exhibits A/B를 읽는다. 앞부분은 명시된 예외 외 MPL 적용, 다른 프로젝트 코드의 원래 license와 수정 dual license를 설명한다. METADATA의 License-Expression=MPL-2.0/License-File=LICENSE.txt이며, archive의 license/copying/notice 이름 member는 이 파일 하나다. 개별 코드의 다른 원래 라이선스 적용 범위는 이 한 파일로 확정되지 않아 후속 mapping 검토가 필요하다. - -1.12절의 GPL/LGPL/AGPL 명칭은 Secondary License 정의다. 그 이름만으로 실제 금지 의존성이나 선택된 copyleft라고 판정하지 않는다. 기존 `scan_license_text`가 이 명칭에서 거부하는 문제는 의미 구분이 없는 별도 한계다. 이번 원문은 `unsupported-hypothesis.json`에 원시 hash와 함께 보존하고 recognizer UNKNOWN을 확인하며, keyword 거부를 실제 GPL 확정 판정으로 승인하지 않는다. 이번 범위에서 scanner 예외를 새로 허용하지 않는다. - -## 실행 - -``` -PYTHONDONTWRITEBYTECODE=1 PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python3 -B -m pytest --noconftest -p no:cacheprovider -o addopts= tests/test_release_dependency_license_text_evidence.py tests/test_release_dependency_install_binding.py tests/test_release_dependency_install_ordering.py tests/test_release_dependency_full_text_contract.py tests/test_release_dependency_reviewed_artifact_texts.py tests/test_spdx_license_policy.py tests/test_release_dependency_gate.py tests/test_release_dependency_gate_capture_and_seal.py -q --tb=short -280 passed, 1 skipped in 2.94s -raw exit 0 -``` - -최초 실행은 sealed SBOM 사례의 REVIEWED_TEXTS import 누락으로 1 failed/279 passed/1 skipped/exit1이다. 실제 해당 함수 import를 고쳐 위 결과를 얻는다. 기존 skip은 macOS의 GNU find capture 경로이며 새 skip을 추가하지 않는다. 검사 삭제 없이 MPL/BSL 두 parameter를 별도 UNKNOWN 음성으로 유지하고, 실원문과 추가 제한·다중 파일·미선언 거부 회귀를 함께 실행한다. 마지막 unused import 제거는 실행 경로와 무관하다. - -`git diff --check` exit0. 기존6개 텍스트/provenance 불변 대조 true. 전체 suite·coverage·hosted·tooldeps·현재 release closure는 이번 소형 성공으로 수용하지 않으며 HOLD다. diff --git a/docs/doctoring/release-license-six-artifact-texts-20260924.md b/docs/doctoring/release-license-six-artifact-texts-20260924.md deleted file mode 100644 index c477bcb52c..0000000000 --- a/docs/doctoring/release-license-six-artifact-texts-20260924.md +++ /dev/null @@ -1,30 +0,0 @@ -# 실제 artifact 원문 여섯 개의 증분 인식 - -이 후보는 `4329ebb84ddac1752e5c4149fd3c94731b262f2e` 뒤에 여섯 전체 원문 hash를 추가한다. 일반적인 라이선스 판별기나 전체 배포 closure 승인으로 확대하지 않는다. production 변경은 기존 `_VERIFIED_LICENSE_TEXT_DIGESTS` 추가뿐이다. - -## 출처와 직접 읽은 범위 - -자료는 기존 로컬 FMLS license evidence archive다. artifact filename/SHA256, 내부 member, 원시 SHA256, 정규화 SHA256, SPDX 대응은 `tests/fixtures/release_license_texts/provenance.json`에 기록한다. 원문은 기억에서 재작성하지 않고 archive member를 UTF-8로 읽는다. 전체 본문을 줄 생략 없이 확인한다. `texts.json`의 각 문자열을 UTF-8로 인코딩한 바이트가 원시 member hash와 일치하는지 테스트한다. `fixture` 필드는 이 JSON 안의 키다. 끝 개행이 없는 원문도 그대로 보존한다. - -|실제 원문|읽은 허용·조건과 경계| -|---|---| -|pytest9.1.1 LICENSE / MIT|전체 grant에 use/copy/modify/merge/publish/distribute/sublicense/sell과 without restriction이 있다. copyright·permission notice 보존 및 보증 면책을 읽는다. 정확한 Holger Krekel 머리말도 hash에 포함한다. 다른 MIT 머리말을 자동 인정하지 않는다.| -|atheris3.1.0 LICENSE / Apache-2.0|1–9절과 적용 부록 전체다. 2절 copyright grant, 3절 patent grant·소송 종료조건, 4절 재배포·수정·고지, 5절 contribution, 6절 trademark, 7–9절 보증·책임을 읽는다. 별도 NC/학술 전용 부속문구는 없다. 이는 파일의 인식이며 실제 atheris의 metadata 선언 누락은 계속 HOLD다.| -|Rust numpy0.29.0 LICENSE / BSD-2-Clause|source·binary 재배포 허용, 두 고지 보존 조건, 전체 면책을 읽는다. PyPI NumPy 복합 원문과 다른 파일이다.| -|colorama0.4.6 LICENSE.txt / BSD-3-Clause|source·binary 재배포 허용, 고지 보존 두 조건, 이름을 허가 없이 endorsement에 사용하지 않는 세 번째 조건과 면책을 읽는다. 추가 상업 이용 금지는 없다.| -|libloading0.8.9 LICENSE / ISC|any purpose with or without fee의 use/copy/modify/distribute 허용, copyright·permission notice 보존, 전체 면책을 읽는다. Simonas Kazlauskas 머리말을 포함한다.| -|foldhash0.2.0 LICENSE / Zlib|any purpose including commercial applications, alter/redistribute 허용과 출처 오인 금지·변형 표시·고지 제거 금지 세 조건을 읽는다. 전체 면책도 포함한다.| - -모두 저장된 실제 소스에 대한 인식 지원이다. 상용 제품의 모든 법적 의무 충족을 확정하는 판단이 아니다. 선택된 OR의 pointer 문서, PyPI NumPy의 GPL/LGPL 복합 원문, upstream 16개, 다른 copyright/원문 변형, 수집 누락은 별도 HOLD다. 추가 파일마다 기존 consumer의 검사가 계속 적용된다. - -## 검사와 실패 보존 - -명령 공통 환경: `PYTHONDONTWRITEBYTECODE=1 PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python3 -B -m pytest --noconftest -p no:cacheprovider -o addopts=`. - -- `tests/test_release_dependency_reviewed_artifact_texts.py tests/test_release_dependency_full_text_contract.py tests/test_spdx_license_policy.py -q`: 99 passed, raw exit0. 여섯 실제 원문의 hash·정상 consumer, 앞/뒤/중간 추가 조건, 별도 NOTICE 제한, atheris 선언 누락 유지가 포함된다. -- 첫 fixture 생성은 끝 개행을 추가해 원시 hash 6건이 실패(6 failed/93 passed)하고, 첫 보정은 개행 없는 2개 파일 끝 문자를 훼손해 4 failed/95 passed다. 원문 문자열을 JSON에 그대로 보존하는 방식으로 보정하고 모든 원시 hash를 다시 확인한다. 정규화 hash만 일치한다는 이유로 이 실패를 무시하지 않는다. -- 기존 비교군 `test_release_dependency_license_text_evidence.py`, `test_release_dependency_install_binding.py`, `test_release_dependency_install_ordering.py`: 18 failed/27 passed/1 skipped, raw exit1. 이전과 같은 미지원 부분 원문 기대값 실패를 보존한다. 기존 fixture 수정은 없다. - -이전18건 중 잘못된 양성 기대는 짧은 MIT/Apache/BSD/ISC/MPL/BSL/Unlicense 제목·일부 grant를 완전 원문으로 취급하는 부분이다. 정상 corpus를 바꾸려면 그 라이선스의 실제 전체 자료와 hash를 같은 SPDX로 연결하는 별도 diff가 필요하다. Cargo Apache 제목-only fixture도 Apache 전체 원문으로 복원해야 하며, MIT로 바꾸는 방식은 허용하지 않는다. 이번 변경은 그 기대값을 편의상 고치지 않는다. - -전체 suite·coverage·hosted·tooldeps 검사와 publish는 미실행·HOLD다. 여섯 원문 추가로 범위 전체를 수용하지 않는다. diff --git a/docs/doctoring/release-license-whole-text-candidate-20260924.md b/docs/doctoring/release-license-whole-text-candidate-20260924.md deleted file mode 100644 index ffa9db7855..0000000000 --- a/docs/doctoring/release-license-whole-text-candidate-20260924.md +++ /dev/null @@ -1,34 +0,0 @@ -# PR2347 전체 원문 확인 중간 후보 - -기준은 `48caafec7160dd0cb9bafc58b28a884dc4c35cbb`이다. 원문 제목/부분 문자열만으로 허용하는 P1을 닫는 로컬 후보이며, 전체 의존성 정책 구현 완료나 병합·배포 수용을 뜻하지 않는다. - -## 근거와 지원 경계 - -직접 읽은 저장소 `LICENSE` 전체를 근거로 삼는다. 원시 SHA256은 `08f1fd81fb120bc468b69dc3e58ea0dc23c216305c766e45e107f56c76559e3f`이다. ASCII 공백·탭·CR·LF만 연속 공백 하나로 정규화한 전체 본문 SHA256은 `f5ac0308cf2b3f96a0f49a8c0c9e4a2a02c483afc72a646af8de1f356983de06`이다. - -검증 지원은 이 MIT 원문 한 개이며 Copyright 문구까지 포함한다. 다른 저작권자 머리말도 아직 UNKNOWN이다. 임의 머리말·추가 조건·접미사·유니코드 제어 문자를 지우지 않는다. SPDX 선언, 제목, 허용 구절만으로 확인된 원문이 되지 않는다. 이 레지스트리는 새로운 의존성을 라이선스 이름만으로 승인하는 수단이 아니다. - -실제 closure에 필요한 BSD, Apache, CC0 및 다른 라이선스 원문·변형 지원은 미완료다. 각 원문과 전체 일치 계약을 독립 검토한 뒤 별도 증분으로 추가해야 한다. 현재 인벤토리 전체 PASS는 불가능하다. 설치 전 도구 의존성 검사는 별도 미해결이다. - -`recognize_license_text`의 production caller는 `release_dependency_gate.evaluate_dependency_license`다. CodeGraph는 해당 트리에 index가 없다고 반환하며, 소스 참조를 직접 대조한다. caller가 파일마다 판정하므로 허용 LICENSE와 별도 제한 NOTICE를 함께 넣어도 거부한다. - -## 검증 - -모든 명령은 이 별도 작업 트리에서 실행한다. 환경은 `PYTHONDONTWRITEBYTECODE=1 PYTEST_DISABLE_PLUGIN_AUTOLOAD=1`, 공통 인자는 `python3 -B -m pytest --noconftest -p no:cacheprovider -o addopts=`다. - -1. `tests/test_release_dependency_full_text_contract.py tests/test_spdx_license_policy.py -q`: **68 passed, raw exit 0**. 최초 실행은 새 테스트에서 상수 소유 모듈을 잘못 적어 1 failed/67 passed/exit1이다. `policy.LICENSE_TEXT_DISAGREEMENT`를 실제 소유자 `gate`로 고친 뒤 위 결과를 얻는다. -2. 기존 소형 비교군 `tests/test_release_dependency_license_text_evidence.py tests/test_release_dependency_install_binding.py tests/test_release_dependency_install_ordering.py -q --tb=no`: **18 failed, 27 passed, 1 skipped, raw exit 1**. 기존 fixture는 수정하지 않는다. -3. `git diff --check`: exit0. - -새 회귀는 실제 gate의 MIT 추가 상업 제한과 CC0/NonCommercial 반례를 거부하고, 완전한 확인 MIT 원문은 동일 dependency caller에서 통과시킨다. gate 전체의 정상 Python 사례에서도 기존 Cargo Apache fixture가 UNKNOWN으로 남아 전체 통과를 주장하지 않는다. GPL 별도 파일, 추가 NOTICE, 본문 변조·앞뒤 조건·NUL·zero-width suffix도 확인한다. - -### 보존하는 기존 실패 분류 - -- 원문 누락/UNKNOWN/별도 GPL 등 기존 원인 자체는 계속 거부한다. 같은 capture의 Cargo Apache 제목-only fixture가 추가 `LICENSE_TEXT_UNVERIFIED`를 내므로 기존 exact-single-failure 기대와 다르다. -- MIT·Apache·BSD·ISC·MPL·BSL·Unlicense 부분 원문을 정상으로 기대한 사례와 recognizer 직접 호출의 부분 MIT 기대는 더 이상 충족하지 않는다. -- 기존 Apache-vs-MIT 제목-only 불일치는 확인된 Apache가 아니므로 UNKNOWN으로 분류한다. 지원하지 않는 본문에서 라이선스 종류를 확정하지 않는다. -- Cargo-only binding 테스트는 그 Apache fixture 원문의 미확인 때문에 실패한다. lock hash 결속 구현의 변경은 아니다. - -실제 gate 분류 재확인: Python 원문 없음은 MISSING + Cargo UNVERIFIED, Python UNKNOWN/부분 MIT/부분 Apache는 각각 Python UNVERIFIED + Cargo UNVERIFIED다. 합성 Cargo를 MIT로 바꿔 실패를 숨기지 않는다. - -전체 suite·coverage·hosted CI·Linux capture·원문 수집 완전성·tooldeps 설치 전 검사는 이번 수용 밖이며 HOLD다. 이 후보는 공개 push 없이 다른 reviewer의 검토에 인계한다. diff --git a/docs/doctoring/required-review-control-runner.md b/docs/doctoring/required-review-control-runner.md deleted file mode 100644 index c7bb9a3987..0000000000 --- a/docs/doctoring/required-review-control-runner.md +++ /dev/null @@ -1,29 +0,0 @@ -# Required review control-runner admission - -## Cause and scope - -On 2026-09-27, CO #1222 at `048d90b3715f792bd6a779d0b013c665fdb01385` still had queued required review entrypoints although five organization self-hosted runners were online. Central #2385 and #2417 are merged. Their scanner and scheduler routing does not change the required OpenCode/Noema entrypoints: these still explicitly request hosted Ubuntu. Existing queued attempts retain their original workflow configuration. - -## Constrained assignment - -Let x_j be 1 when an eligible admission job uses the control pool and 0 when it uses hosted capacity. Minimize sum(1 - x_j) over the six jobs, subject to 0 <= x_j <= 1, trusted central-main workflow identity, no PR-source execution, and separation of model/scanner work from control. For the exact central-main workflow identity, the unique admissible self-hosted pool is `CWL central control`; its one registered worker permits at most one executing job at a time, which GitHub enforces natively. Setting all six x_j to 1 attains the lower bound zero hosted admission jobs. This is a direct linear assignment, not an estimated optimum for completion time: model durations and historical queue positions are not reliable cost coefficients. No solver dependency or learned-policy claim is introduced. - -Non-main and unrecognized workflow identities retain hosted Ubuntu 24.04; the selected-workflow group must not strand PR/branch-ref validation jobs. All six OpenCode entrypoint jobs read metadata, retain required context names, dispatch, or clean superseded runs. They never checkout PR code. Noema control admission is independently owned by #2420. This PR leaves its worker and transport continuation unchanged. - -## Runner policy and rollout - -Group 6 must preserve `visibility=all`, `allows_public_repositories=true`, and `restricted_to_workflows=true`, retaining all existing selected workflows and adding only this exact central-main path: - -- `ContextualWisdomLab/.github/.github/workflows/opencode-review.yml@refs/heads/main` - -Read the live group immediately before PATCH and include the complete policy so omitted fields cannot erase existing restrictions. Read it back after mutation. The permission remains limited to jobs defined by these trusted central workflows, rather than arbitrary consumer workflows. - -Tests pin the six assignments and absence of PR checkout, and retain the docs-only and exact-head dispatch contracts. Run affected contracts both normally and with `GITHUB_ACTIONS=true`, then actionlint. The maintainer explicitly authorized bypass merge for this CI admission repair; missing hosted checks must remain recorded as missing evidence. - -After merge, verify a new targeted review/scheduler attempt uses `cwlab-s1-05`. Old queued runs are not deployment proof. A new workflow event or trusted central dispatch is needed to adopt the new configuration. Rollback restores the runner selectors and removes only the added OpenCode group path after verifying no dependent jobs need them. - -## References - -GitHub. *Choosing the runner for a job*. https://docs.github.com/en/actions/how-tos/write-workflows/choose-where-workflows-run/choose-the-runner-for-a-job - -GitHub. *Managing access to self-hosted runners using groups*. https://docs.github.com/en/actions/how-tos/manage-runners/self-hosted-runners/manage-access diff --git a/docs/doctoring/reusable-scheduler-control-runner.md b/docs/doctoring/reusable-scheduler-control-runner.md deleted file mode 100644 index a5e2aef4f4..0000000000 --- a/docs/doctoring/reusable-scheduler-control-runner.md +++ /dev/null @@ -1,21 +0,0 @@ -# Reusable scheduler control runner - -## Cause and assignment - -On 2026-09-27, fast-mlsirm#2199 still had 21 queued checks despite five online self-hosted runners. Four runners were busy; cwlab-s1-05 was idle. The central control group admitted only the .github repository, while the reusable scheduler explicitly sent consumer repositories to hosted Ubuntu. Adding runners alone did not remove either access/routing constraint. - -The assignment minimizes hosted admission for trusted scheduler work subject to one dedicated control runner and separation from long model, scanner, and PR build execution. With one eligible control pool, the assignment is direct; no optimizer dependency or speculative duration weights are needed. Existing CodeQL/OpenCode pools and live inference are preserved. - -## Change and trust boundary - -The reusable scheduler uses group `CWL central control` and labels `[self-hosted, linux, x64]` for every caller. Runner group `CWL central control` must grant organization repository access while retaining `restricted_to_workflows=true` and exactly the three existing central `@refs/heads/main` workflow paths: agent-mention-router, hourly-review-repair, and pr-review-merge-scheduler. This permits only jobs directly defined in trusted central workflows, not arbitrary caller jobs. The scheduler materializes only the immutable central workflow source; PR source execution is unchanged. Security gates, review verdicts, provider policy, and model duration remain unchanged. - -## Verification - -Run the scheduler runner-image contract, required-workflow queue contracts, and actionlint. After integration, inspect the actual runner name of a targeted dry-run dispatch; config acceptance is not execution proof. No skipped/queued checks are represented as successful tests. User explicitly authorized bypass merge for blocked CI on this task. - -## References - -GitHub. (n.d.). *Managing access to self-hosted runners using groups*. https://docs.github.com/en/actions/how-tos/manage-runners/self-hosted-runners/manage-access - -GitHub. (n.d.). *Reusing workflow configurations*. https://docs.github.com/en/actions/reference/workflows-and-actions/reusing-workflow-configurations diff --git a/docs/doctoring/review-failure-taxonomy.md b/docs/doctoring/review-failure-taxonomy.md deleted file mode 100644 index 45c064d337..0000000000 --- a/docs/doctoring/review-failure-taxonomy.md +++ /dev/null @@ -1,51 +0,0 @@ -# Review failure taxonomy: gateway routing, scanner tooling, dispatch admission - -On 2026-09-21 the central review pipeline looked like a provider outage. It was not. -Three unrelated failures were being read as one. - -## What the hosted evidence showed - -Every run reached the vendored contextual-orchestrator sidecar and every run reported -`provider secrets present: 5 of 5`, including runs that predate any credential change. -The gateway answered its own preflight with `status: ready` and `finish_reason: stop`. -Provider credentials were never the blocker. - -## 1. OpenCode: gateway routing, reported as `Error: not found` - -The sidecar exports `CONTEXTUAL_ORCHESTRATOR_BASE_URL` as a bare `scheme://host:port`. -Noema and Strix append `/v1/chat/completions` themselves. OpenCode's -`@ai-sdk/openai-compatible` provider appends only `/chat/completions`, so it posted to an -unprefixed path. The gateway serves `/v1/chat/completions` and answers anything else with -`route_not_found`, whose message is the bare string `not found` — which OpenCode printed -verbatim as `Error: not found`, half a second after its banner had already resolved the -agent and model. - -Reproduced locally against a stub gateway with the installed OpenCode CLI: an unprefixed -`baseURL` produced `POST /chat/completions`, and `{env:CONTEXTUAL_ORCHESTRATOR_BASE_URL}/v1` -produced `POST /v1/chat/completions`. The `/v1` belongs in the OpenCode provider options, -never in the sidecar export — moving it there would double-prefix Noema and Strix. - -The banner is the tell: once `> · ` has printed, agent and model already -resolved, so a later `not found` is a transport answer, not configuration lookup. - -## 2. Strix: scanner tooling, previously folded into the provider verdict - -A failing scan emitted Caido GraphQL errors (`Invalid HTTPQL query`, `Failed to parse -cursor`, `TransportQueryError`) while the gateway was healthy. Genuine provider rate -limits appeared in the same log, so the single `STRIX_PROVIDER_UNAVAILABLE` notice was not -wrong — it was incomplete, and it hid a scanner defect behind an infrastructure label. -`strix.yml` now emits `STRIX_TOOLING_ERROR` on its own whenever a tooling signature -appears. Both notices can appear together. Neither changes the exit code: an incomplete -scan stays non-passing. - -## 3. Dispatch admission: never a gateway outcome - -`repository_dispatch authorization rejected` and `repository_dispatch metadata does not -match the live pull request` fire before the sidecar is provisioned. Counting them as -review-pipeline outages inflates the apparent provider failure rate. - -## Rule - -Attribute a review failure to the provider only after the gateway request itself failed. -Name the gateway's served path, the scanner's own errors, and admission gates as separate -classes. `tests/test_review_failure_taxonomy_contract.py` pins all three. diff --git a/docs/doctoring/sidecar-python-shared-library-20260928.md b/docs/doctoring/sidecar-python-shared-library-20260928.md deleted file mode 100644 index 70b928440e..0000000000 --- a/docs/doctoring/sidecar-python-shared-library-20260928.md +++ /dev/null @@ -1,48 +0,0 @@ -# Sidecar CPython shared-library binding - -## Status - -Proposed common startup repair; protected hosted acceptance remains unverified. - -## Evidence and root cause - -Naruon #1795 Noema and Strix both exited 139 in the offline gateway fixture on -`cwlab-s1-02`, before live provider calls, at pinned orchestrator -`01bf92a3ec67a0e1f9b68978eb16b60301e985fd`. -Their selected executable was toolcache Python `3.12.14/x64/bin/python`. -The composite action intentionally uses `update-environment: false`; Strix -retained the consumer `3.13.15/x64/lib` in `LD_LIBRARY_PATH`. - -A read-only runtime comparison on that same guest on 2026-09-27 UTC -found that the selected executable, without its matching library path, reported -Python **3.12.3** while reading the toolcache 3.12.14 standard library and -`_asyncio` extension. The complete offline fixture, exact source and binary-only -hash-pinned dependencies installed in a new owned environment, passed under -system Python 3.12.3. Keeping those dependency bytes and switching to the -selected toolcache executable reproduced SIGSEGV at `logging.LogRecord`, -`asyncio.current_task()` in the HTTP request thread. Only `_cffi_backend` was -listed as an external extension in the fatal trace; this is not evidence of a -provider or fast-mlsirm defect. - -Prepending the selected toolcache's `lib` directory made the entire fixture -pass. The actual patched shell selection also passed with the stale consumer -3.13 library path supplied. No shared installation, service, runner registration, -group grant or existing job was modified. - -## Repair and verification - -Resolve the selected executable (including symlinks and PATH lookup), then -prepend its adjacent library directory only when `libpython3.12.so.1.0` exists. -Keep the existing library search path as a suffix and keep this environment -inside the sidecar shell process. Python 3.12 validation, dependency hashes, -all offline assertions, provider discovery and review gates remain enforced. - -The behavioral regression fails on unmodified main and passes after the repair; -it covers symlink resolution, matching-library precedence and absent-library -fallback. Local sidecar contracts: 31 passed. With `GITHUB_ACTIONS=true`, warnings -as errors and pytest plugin autoload disabled: 168 sidecar, runtime-preflight -and composite-action contracts passed in 16.68 seconds. Bash syntax, Ruff and -`git diff --check` pass. Linux real-fixture comparison additionally exercised -imports, server creation, rejection logging, large request, tool descriptions -and shutdown. Hosted exact-head review and full live-provider acceptance are -still required. diff --git a/docs/doctoring/strix-preflight-capacity-continuation-20260927.md b/docs/doctoring/strix-preflight-capacity-continuation-20260927.md deleted file mode 100644 index 8e44924901..0000000000 --- a/docs/doctoring/strix-preflight-capacity-continuation-20260927.md +++ /dev/null @@ -1,36 +0,0 @@ -# Strix all-429 startup continuation — 2026-09-27 - -## Evidence and cause - -Current-head late-life-anxiety-reanalysis #257 (`3936039d8406275e754fc518f70fefa491d3d8bb`) -Strix job `108504580446` and #269 (`78617f3160cfe8fbf7a4dae2ca3f3fdaca44db8e`) -job `108303563901` failed before sidecar health. Sanitized producer evidence reported -`strix-plain-chat-preflight-v2`, ready=0, rejected=3, probed=3, and HTTP 429 -for all selected routes. This was startup capacity loss, not a completed security review. - -At main `23f36cd56fbe245a06e7a9727cb28d9511154645`, Strix's model retry -lives after sidecar startup and cannot recover this failure. PR #2440 repaired -Noema's corresponding boundary; this change reuses its stdlib-only classifier. - -## Proposed boundary - -The failed scan exports only typed capacity evidence. A separate job with minimal -Contents write permission sends at most two automatic `strix-scan` continuations. -It has no checkout, model inputs, or provider secrets. It rechecks the live open, -Ready PR's repository, head SHA, base SHA, and base ref after bounded scheduling -jitter, and retires if any changed. The payload includes all identity fields needed -by the existing Strix dispatch validator. The scan remains failed and its existing -status publication is unchanged. Cancellation cannot start a continuation. - -Missing, malformed, non-429, linked, or oversized preflight reports are ineligible; -malformed retry counters exhaust the shared budget. Stale reports are removed -before startup. Private-target ZDR, free-route policy, gateway failover, and model -inference time limits are unchanged. - -## Verification boundary - -The actual dispatch shell is exercised with local GitHub/sleep stubs. It verifies -one exact payload for a valid Ready PR and zero dispatches for moved head/base ref, -Draft, malformed Draft, closed state, or invalid attempts. Existing classifier and -Strix sidecar contract tests also pass. This is local evidence only; fresh hosted -review and an observed same-head continuation are still required after deployment. diff --git a/docs/noema-sidecar-evidence-1218.md b/docs/noema-sidecar-evidence-1218.md deleted file mode 100644 index 555c5218b4..0000000000 --- a/docs/noema-sidecar-evidence-1218.md +++ /dev/null @@ -1,45 +0,0 @@ -# Noema startup evidence for contextual-orchestrator PR #1218 - -On 2026-09-27, run `36025318452`, attempt 3, job `108389560765` -was inspected at consumer head `2fed942d378cd959250f648a16b35276279c9603`. -The job used gateway pin `767e67fbc6b881a452761f32abb69b9971b9b03b`. -Dependencies installed successfully. At 2026-09-26T12:37:08Z the sidecar -started; no health/preflight-ready confirmation followed. The job was cancelled -at 18:35:14Z, approximately six hours after admission. This does not establish -that a Noema review request or any particular provider attempt occurred. - -Artifact `10877250808` was created on 2026-09-25T17:04:14Z and belongs to an -earlier attempt. It must not be attributed to attempt 3 merely because the -workflow run ID and consumer head are equal. - -The workflow uploaded its two existing sanitized evidence files only under -`failure()`. Using `always()` preserves those same files after successful, -failed and normally cancelled execution, without collecting raw provider logs. -The pinned uploader, file allowlist, five-day retention and absent-file behavior -remain intact. A hard runner timeout may prevent cleanup/upload entirely; this -change cannot recover the missing attempt-3 evidence or prove its internal -startup cause. Gateway inference timeouts must not be invented to hide it. - -Verification: the changed workflow contract fails on the previous source; -the Noema workflow contract suite and actionlint verify the revised step. -Hosted execution and independent review remain required before integration. -# Startup progress follow-up for CO #1083 - -ContextualWisdomLab/contextual-orchestrator#1209 run `36138543702`, job -`108153123179`, logged sidecar start at 19:11:54Z on 2026-09-25, then runner -shutdown at 23:12:37Z without readiness confirmation. The run's artifact API -returned no artifacts. This proves loss of startup evidence, not a particular -provider deadlock or a model failure. - -The shared readiness loop now logs every 60 failed health polls whether its -discovery, catalog, policy, and preflight report files are nonempty. These are -presence observations only: no report content, provider response, or credential -is printed. Poll count is not elapsed time and does not impose an inference -deadline. A successful health check still ends the loop, and sidecar process -exit retains the existing failure handling. - -The executable regression runs the real health loop with absent, partially -completed, and completed report stages; verifies exact output and secret -non-disclosure; and verifies readiness can succeed after the diagnostic. It -does not establish that the unknown startup cause is repaired. A fresh hosted -run after protected integration is still needed to locate that cause. diff --git a/docs/org-required-workflow-rollout.md b/docs/org-required-workflow-rollout.md index 674a5d0b5a..88f6cc4deb 100644 --- a/docs/org-required-workflow-rollout.md +++ b/docs/org-required-workflow-rollout.md @@ -136,19 +136,21 @@ gate only when they do not compete to upload the same SARIF. The central native dispatch handler analyzes the target head without making the target repository's default-setup upload path its source of truth. -### Repository-local CodeQL inventory (2026-07-04) — HISTORICAL - -**This subsection records the original July rollout, not current guidance.** -That plan invoked `github/codeql-action` directly inside a required workflow, -which GitHub does not support. The old entrypoint was removed on 2026-09-03. -The 2026-09-04 correction above restores a different, dispatch-safe -`codeql-pr.yml`: it sends the scan to a native workflow and consumes an -app-authored exact-head status. This restored entrypoint is in the current -seven-workflow ruleset. Native default setup is a repository-local safety net, -not a replacement for that central gate. The table below remains only the -2026-07-04 snapshot of repositories with a local `codeql.yml`; it does not -identify present-day adoption gaps. - +### Repository-local CodeQL inventory (2026-07-04) — HISTORICAL, superseded 2026-09-03 + +**This entire subsection describes a plan that did not work and is not +current guidance.** It assumed `codeql-pr.yml` would become a functioning +central required check once ruleset `18156473` included it; the "Correction +(2026-09-03)" note under "Code scanning required workflow posture" above +explains why that assumption was wrong — `codeql-action` cannot run inside a +required workflow at all, so `codeql-pr.yml` was removed from the ruleset, +not fixed. "Centralizing through `codeql-pr.yml` fixes every inherited +repository in one ruleset change" (below) never happened and never could. +Coverage for repositories without a local CodeQL workflow now comes from +GitHub's native `code-scanning/default-setup` instead (see the 2026-09-03 +"Evidence from this rollout" entry) — do not read the table below as +"repositories still needing the ruleset update to land"; treat it only as a +2026-07-04 point-in-time snapshot of which repositories had a local `codeql.yml`. Org audit of default-branch workflow files as of 2026-07-04. diff --git a/docs/policies/PINGORA_EDGE_POLICY.md b/docs/policies/PINGORA_EDGE_POLICY.md index f7a6e6a5ee..e7fd78c563 100644 --- a/docs/policies/PINGORA_EDGE_POLICY.md +++ b/docs/policies/PINGORA_EDGE_POLICY.md @@ -98,11 +98,9 @@ UTF-8 is still fully content-scanned, never silently admitted. A file whose suffix has a known magic byte (`.hwpx`, `.pdf`, `.png`) is verified by that format's structural evidence; a file with no known magic entry (most research-data formats) is admitted only on the stricter combination of "no -diff patch", "the fetched bytes are not valid UTF-8", and "the -replacement-decoded content contains no prohibited runtime pattern". A text -file cannot be mistaken for a binary artefact merely by sitting under a -declared prefix, and a stray invalid byte cannot hide a readable runtime -directive. +diff patch" and "the fetched bytes are not valid UTF-8" -- a text file can +never be mistaken for a binary artefact merely by sitting under a declared +prefix. **Bounds.** The declaration is capped at 64 entries and 8 path segments of depth per entry (`MAX_DECLARED_ARTIFACT_PREFIXES` / diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 6e5f1c549a..c617e3ad73 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -7,24 +7,11 @@ 이 문서는 제품·기술·운영 Gap을 현재 문서와 현재 GitHub 상태에 묶어 두는 기준선이다. 새 작업은 먼저 이 문서의 Gap ID를 PR 설명과 테스트 증거에 연결하고, PR의 정확한 exact HEAD·Checks·리뷰를 다시 수집한 뒤 구현한다. 표의 상태는 작성 시점의 관측값이므로, 병합 판단에는 재사용하지 않는다. 이 인벤토리는 스냅샷이며 merge authorization이 아니다. -### 2026-09-19 exact-head incident delta - -| Gap ID | 상태 | exact-head evidence | causal owner / next gate | -|---|---|---|---| -| CONTROL-OPENCODE-COVERAGE-LOCK-CONTEXT-01 | **Proposed — PR-bound incident register; GitHub Project #1 roadmap item이 아님; `.github#2385@950ab885…` source convergence, hosted acceptance pending** | Required OpenCode run `35370902053`의 `coverage-evidence` job `105778600365`은 PR source 실행 전에 `COPY requirements-opencode-review-ci-hashes.txt requirements-noema-document-ci-hashes.txt /tmp/`에서 두 번째 파일을 찾지 못해 종료했다. RED `9b9f5edcd`는 Dockerfile의 모든 lock input이 trusted build context에 존재해야 한다는 계약을 고정했다. 이 행은 live Project 상태를 주장하지 않고 exact-head PR evidence만 추적하며, protected integration 뒤 제거 여부를 재평가한다. | Canonical owner는 중앙 `.github/.github/workflows/opencode-review-dispatch.yml`이고 complete successor는 `.github#2385`이다. 두 lockfile을 각각 regular non-symlink로 검증하고 build context로 복사한 뒤 exact-head focused/full suite와 새 hosted `coverage-evidence`를 통과해야 한다. PR 제품 source나 coverage 비율의 결함으로 오인하지 않으며 synthetic status·manual rerun·bypass를 사용하지 않는다. | - ### 2026-09-13 current-head incident delta | Gap ID | 상태 | exact-head evidence | causal owner / next gate | |---|---|---|---| | CONTROL-OPENCODE-VCS-PYROOT-01 | **Source repaired on `main` (#2123 `ebc69a401`); image-path helper extracted + offline-proven under #2157 follow-up; hosted consumer step-#17 link still required to close the issue** | `ContextualWisdomLab/contextual-orchestrator#1149@684cf28f`의 중앙 [OpenCode run 34701472466](https://github.com/ContextualWisdomLab/.github/actions/runs/34701472466) `coverage-evidence` job `103574547257`은 PR 코드를 실행하기 전에 immutable `ContextualWisdomLab/fast-mlsirm@09f762d`의 `python/fast_mlsirm` import root를 찾지 못해 종료했다. 같은 head의 제품 테스트는 `3602 passed, 2 skipped`, native CodeQL·fuzz·SBOM·SAST·Strix는 성공했다. | `.github`의 `opencode-review-dispatch.yml`이 root/`src/`만 허용한 계약 drift를 소유했다. #2123이 `python/` candidates를 추가해 `main`에 병합했고, #2157 follow-up은 동일 로직을 `scripts/ci/resolve_opencode_base_vcs_import_root.sh`로 추출해 `tests/test_opencode_vcs_python_source_root_contract.py` fixture로 증명한다. Issue #2157 종료는 post-`ebc69a401` consumer `coverage-evidence`가 docker step #17을 통과한 job id를 문서에 링크한 뒤에만 한다. | -| CONTROL-PINGORA-DECLARED-BINARY-RUNTIME-01 | **Source repaired on `.github#2386@dea7532e`; protected integration pending** | A base-owned artifact-prefix declaration admitted a no-patch file after any non-UTF-8 byte, even when readable bytes contained `nginx -c /etc/nginx/nginx.conf`. The production-bound regression covers `.sh`, `.dat`, and `.txt`; the focused suite is the exact-head acceptance target. | `.github` owns `scripts/ci/pingora_edge_policy.py`. Replacement-decoded content must contain no `CONTENT_RULES` match before an unrecognized binary suffix is admitted. Current-head hosted security Checks, qualifying independent approval, ordinary protected merge, and downstream `late-life-anxiety-reanalysis#269` revalidation remain required. | - -### 2026-09-27 CodeQL compatibility retirement delta - -| Gap ID | Status | Evidence and remaining gate | -|---|---|---| -| CONTROL-CODEQL-OBSOLETE-VERDICT-01 | Source repair under verification | ContextualWisdomLab/fast-mlsirm#2172 closed before compatibility job 108414341704 began. The live read returned no verdict and enforcement failed. Explicit obsolete output repairs closed/superseded target retirement without weakening exact-head security evidence. See [RCA and regression checks](doctoring/codeql-obsolete-pr-verdict.md); protected merge and hosted current-head gates remain required. | ## 1. 근거와 범위 @@ -3437,214 +3424,3 @@ alone -- it is a documented multi-PR hot-file collision zone. Contract: **Action.** Exact `57477289ebec5631b0c48f0bc419f336dbe19deb` adds a dependency-free synthetic-302 transport to `tests/test_github_api_url_boundary.py`. For both actual production openers, the case drives a canonical bearer request through the real HTTPS open/response chain, requires the typed HTTP-302 failure mapping, and proves transport receives exactly one original request; lookalike HTTPS, HTTP, `file:`, and same-authority redirect targets never receive a second request or bearer. Exact `e0b0b4d4fff5b6ea88236a1e91dcd7dbb3be09b5` repairs the doctoring claim so direct-handler coverage is not mislabeled as production-chain proof. **Evidence / remaining condition.** The standalone fixture mechanism was executed locally against Python stdlib and produced one canonical request followed by terminal HTTP 302 for every hostile target. This is mechanism evidence, not repository acceptance. Final authority requires focused/full exact-tree GREEN, fresh exact-head Security/SAST/Python Security/CodeQL/runtime-quality checks, no unresolved actionable review, ordinary protected-main integration, and downstream consumer validation. No scanner suppression, redirect allowlist widening, provider fallback, workflow gate weakening, or credential-boundary change is included. - -## 2026-09-27 exact release distribution/scope evidence coverage - -**Status:** Proposed on `ContextualWisdomLab/.github#2400`; the current -architecture-binding repair starts from reviewed parent -`51db1d0c00c2eab36d051b5307541558bbc735c2`. The PR body—not a -self-referential SHA in this file—is the authority for the current exact head. -The PR remains Draft. - -**Context Map / owner.** The central `.github` release-control bounded context -owns same-run distribution/scope artifact verification and the immutable -licence/Strix verdict contract. Product release workflows consume only the -pinned central workflow and helper commits; product repositories do not copy -the verifier source or read central transient state. - -**Gap.** The release prescreener was already complete, but the adjacent -distribution and scope evidence verifiers still had unexecuted fail-closed -paths. At predecessor `27cf2f339393aa08b9f8a26c3a9bd0da47de33c1`, -`verify_release_distribution_set.py` covered 148/200 statements with 20 -partial branches (71%). At predecessor -`eb8130c5573b4bfc59bdc725be5e1466f24c25db`, -`verify_release_scope_evidence_set.py` covered 197/242 statements with 33 -partial branches (77%). The repository-wide mandatory 100% coverage gate was -therefore RED even though the positive release path passed. - -**Action.** Two ordinary, non-force commits add test-only boundary evidence for -duplicate/non-finite/oversized controls, canonical time and digest identity, -unsafe and oversized ZIP members, download failure/termination, build snapshot -inventory and byte binding, runtime wheel identity, consumer native layout, -lock drift, scope envelope/row identity, aggregate size, and both CLI entry -paths. Production release code and workflow admission policy are unchanged. - -A same-PR continuation covers the adjacent release gate's real trust -boundaries: bounded and nonregular archive input, declared Python/Cargo licence -paths, archive links and member counts, raw-capture/destination symlinks, Cargo -workspace identity, Strix fanout identity/fixture/runtime-report binding, and -install-time licence rebinding. `parse_member_listing` and its isolated test -were removed after repository-wide caller search proved that immutable archive -bytes—not the unused shell listing—are the member authority. The redundant -post-read length branch was also removed because both stdlib ZIP and tar readers -already clamp reads to the entry size checked immediately beforehand. - -**Exact-tree evidence / remaining condition.** Distribution focused tests are -15 passed with 200/200 statements and 84/84 branches; scope focused tests are -48 passed with 242/242 statements and 120/120 branches. The warnings-as-errors -full suite is 3,953 passed, 28 skipped, and 40 subtests passed. Against the -pre-repair full-repository run, uncovered statements fell 386→289 and partial -branches 112→59, but the total remains 98%; the 100% gate is still RED. Fresh -exact-head CodeQL PR run `36280393614`, SAST run `36280393599`, and Security -Scan run `36280393621` were queued on that repair head. Adding this baseline -record creates a documentation-only successor with its own fresh runs; their -current IDs and conclusions are tracked in the PR body and must not inherit -the predecessor's status. Qualifying independent approval is absent. Do not -merge, tag, publish, or create an admission manifest until the remaining -production surfaces reach 100%, all required checks are terminal GREEN on one -exact head, and an independent current-head approval exists. - -The continuation's focused release-dependency suite is 442 passed with -`release_dependency_gate.py` at 1,126/1,126 statements and 472/472 branches. -The warnings-as-errors full suite is 3,976 passed and 28 skipped; uncovered -repository statements fell 289→249 and partial branches 59→26, raising the -rounded total to 99% but not satisfying the fail-under-100 gate. The remaining -misses belong to queue health, Noema document review, and the separately owned -Rust materializer work on `ContextualWisdomLab/.github#2360`; no duplicate Rust -repair is introduced here. Current exact-head hosted runs and conclusions remain -PR-body authority after the next ordinary-forward update. - -The queue-health continuation removes a responsibility contradiction rather -than preserving it with tests: `actions_queue_health_core.py` still contained -a second collector and CLI even though the Context Map assigns collection, -identity reconciliation, and process exit to `actions_queue_health.py`. The -duplicate was unreachable after the executable imported the core and replaced -those names. A source-shape RED contract now prevents either entrypoint from -returning to the core; the executable owns its `time.sleep` retry dependency -directly. Boundary cases cover both pre-evidence identity retry outcomes, -malformed active and terminal run IDs, irrelevant terminal conclusions, -obsolete target cancellations, and remediation-action deduplication. The -focused queue-health suite is 80 passed; both queue-health production modules -are 100% statement and branch covered. No workflow permission, API scope, -queue-age threshold, cancellation behavior, or merge policy changes. The -full exact-tree suite is 3,982 passed, 28 skipped, and 40 subtests passed; -uncovered statements fell from 249 to 163 and partial branches from 26 to 19. -The only remaining uncovered production owners are the Noema document reader -successor and Rust materializer `ContextualWisdomLab/.github#2360`. Hosted-run -identity and conclusions remain PR-body authority. - -The Noema document-reader continuation executes the existing fail-closed trust -boundaries without changing production policy: unsupported and oversized -input, bounded DOCX archive and XML structure, empty content, visible Word -controls, ragged and escaped tables, local HWP reader configuration and process -failure, bounded/UTF-8/non-empty adapter output, code-point-safe prompt -truncation, and the smoke-test CLI. The focused suite is 11 passed and 2 -optional real-fixture skips; `noema_review_document.py` is 144/144 statements -and 52/52 branches. The warnings-as-errors full exact-tree suite is 3,988 -passed, 28 skipped, and 40 subtests passed. Repository coverage stays rounded -to 99% because the separately owned Rust materializer on -`ContextualWisdomLab/.github#2360` retains 128 uncovered statements and one -partial branch. That owner boundary is preserved: this PR does not duplicate -the Rust repair. The 100% gate therefore remains RED, the PR remains Draft, -and current hosted-run identity and conclusions remain PR-body authority after -the next ordinary-forward update. - -The coverage successor integrates the canonical Rust materializer owner by an -ordinary two-parent merge rather than copying its source or tests. The owner -branch contributes the full foundation ancestry, deterministic multi-root -`cargo vendor --sync --locked` closure, confinement of synthesized Cargo target -paths to each manifest root, real-Cargo integration contracts, and -toolchain-independent Git/mock/error/CLI coverage. Focused evidence is 26 -passed and 3 real-Cargo skips with -`materialize_base_rust_dependencies.py` at 155/155 statements and 60/60 -branches. The full merged tree is 4,030 passed, 8 skipped, and 40 subtests -passed; all 17,144 production statements and 6,982 branches are covered. This -closes the repository coverage Gap but is not merge authorization: the release -stack remains Draft/Proposed until fresh exact-head hosted Checks reach terminal -success and a qualifying independent review approves the unchanged head. - -The subsequent native-inspection continuation exposed a new exact-tree -coverage Gap rather than inheriting predecessor evidence. Runtime wheels and -build-interpreter snapshots now pass every admitted native member through the -pinned `llvm-readobj-18` boundary, but the first full run on that source left -six prescreener statements/four partial branches and one release-gate -statement/one partial branch uncovered. The RED suite still passed 4,033 tests, -8 skips, and 40 subtests, while `coverage report --fail-under=100` correctly -failed at 99%. The repair adds fail-closed cases for directory members, -analyzer reuse/failure, oversized native files, receipt omissions, unknown -runtime dynamic links, and malformed static-link records. The exact repaired -tree is 4,037 passed, 8 skipped, and 40 subtests passed with all 17,186 -production statements and 7,000 branches covered. Context Map ownership stays -in the central release-control gate; consumer repositories receive only its -immutable released workflow contract. Status remains Proposed/Draft and release -admission remains HOLD until fresh exact-head hosted Checks and a qualifying -independent approval complete. - -The next ordinary integration closes a distinct native-link review Gap. The -pinned analyzer previously proved which dynamic libraries each wheel needed, -but the sealed report did not bind why those external names were admissible on -the declared Linux, macOS, or Windows target. The central release-control -bounded context remains the single owner: it now classifies only explicit -operating-system runtimes, the wheel-tag-matched CPython DLL, the inspected -extension's own macOS install name, and the named Visual C++ runtimes. Unknown -names fail before verdict sealing, while every accepted name and review basis -is carried in `cwl.release-native-links/2`; consumers receive only the released -workflow contract. The native-link continuation and the coverage repair were -combined by an ordinary two-parent merge, preserving both histories without a -force update. The concurrent Maturin asset verifier initially reproduced a 99% -coverage failure with 22 missing statements and 10 partial branches; its -bounded-download, archive-shape, executable-identity, reviewed-link, CLI, and -prescreen failure paths are now executable contracts. Fresh current-tree -evidence is 4,049 passed, 8 skipped, and 40 subtests passed, with all 17,302 -production statements and 7,058 branches covered. Ruff E9/F/I, compileall, and -diff checks pass after import-order repair. Status is Proposed/Draft and -release admission remains HOLD because hosted exact-head Checks and a -qualifying independent approval are not yet complete. - -The Intel macOS continuation closes one part of the universal2 runtime Gap. -Three additional same-run artifacts contain x86_64 install receipts and exact -dependency wheel archives. The central verifier authenticates each ZIP, -source SHA, selected distribution row, x86_64 interpreter, and archive member; -the licence prescreen includes distinct x86_64 archive bytes in the Strix -fixture matrix, and the final verdict seals their artifact IDs and digests. -The thirteen publishable distributions remain the only release outputs. -The changed verifier, prescreen, and verdict collector have 100% statement -and branch coverage in the focused suite; the full local suite is 4,063 passed, -4 skipped, and 40 subtests passed. The fast-mlsirm admission consumer has not -yet accepted this verdict shape, and hosted exact-head checks are still -required. Release remains HOLD. - -An architecture-binding review then found that the Intel receipt's -`machine=x86_64` claim did not reach the bytes of native dependency wheels. -The common universal2 inspector deliberately permits an architecture subset, -but the prescreener discarded that subset and deduplicated package/hash pairs -before applying any Intel-specific constraint. An aarch64-only Mach-O wheel -could therefore satisfy the Intel continuation. A RED integration contract at -parent `51db1d0c00c2eab36d051b5307541558bbc735c2` reproduces that acceptance. -The repair requires x86_64 in every native member of each Intel variant before -deduplication; universal2 binaries containing both architectures remain valid, -and pure-Python wheels are unchanged. Local exact-tree evidence and hosted -current-head run identities remain PR-body authority. The local exact tree is -4,061 passed, 8 skipped, and 40 subtests passed, with all 17,383 production -statements and 7,098 branches covered. Status stays Proposed/Draft and release -admission remains HOLD pending terminal GREEN hosted Checks, downstream -verdict-shape acceptance, and qualifying independent approval. - -## 2026-09-27 Strix AnyIO security-lock carryover - -**Status:** Proposed on `ContextualWisdomLab/.github#2386`; fresh exact-head hosted Checks and qualifying independent approval remain mandatory. - -**Context Map / owner.** The central `.github` security/review bounded context owns the hash-locked Strix CI runtime. PyPI packages and the vulnerability advisory service are upstream evidence; product repositories consume only the released central workflow contract. - -**Gap / RCA.** Exact-head Python Security run [36236245577](https://github.com/ContextualWisdomLab/.github/actions/runs/36236245577), job `108402877544`, found AnyIO `4.14.0` vulnerable to `CVE-2026-63374`, `CVE-2026-64847`, and `CVE-2026-63349`; all three list `4.14.2` as fixed. The generated lock had no explicit AnyIO source constraint, so unrelated PR #2386 inherited a known-vulnerable transitive selection. - -**RED → GREEN / carryover.** RED `761be5b0f63422505b37e28a367a4c5170f302ba` imports #2385's source↔lock contract and fails `1 failed, 1 passed` because the source input lacks `anyio==4.14.2`. GREEN `c59ef9aed32ab4c5138c2b7770ddcc10d7ee8393` adds that exact source constraint; `a895dc5aec775076c3819679eadf0b50a563aa2e` adopts #2385's generated lock blob `eb83beda177c9d2e4ca9b7e2888a1ccb55a123ac`, whose only predecessor differences are version line 143 and hash lines 144–145. Exact remote blobs pass the focused contract `2 passed`. This is complete three-file delta integration, not a claim that #2385 or #2386 is accepted. Completion still requires fresh exact-head pip-audit/other required Checks, no unresolved actionable review, qualifying independent approval, and ordinary protected-main integration. -## 2026-09-27 Git blob protocol-hash SAST authority - -**Status:** Proposed on `ContextualWisdomLab/.github#2396`; fresh exact-head hosted Checks and qualifying independent approval remain mandatory. - -**Context Map / owner.** The central `.github` Pingora policy owns exact-head changed-file evidence admission. GitHub's Git blob API remains the upstream object-identity authority; Semgrep remains the independent static-analysis gate. - -**Gap / RCA.** Exact-head SAST run [36243375994](https://github.com/ContextualWisdomLab/.github/actions/runs/36243375994), job `108407968534`, reported `python.lang.security.insecure-hash-algorithms.insecure-hash-algorithm-sha1` at `scripts/ci/pingora_edge_policy.py:602`. The call recomputes Git's protocol-defined `blob \\0` object ID with `usedforsecurity=False`; it is equality evidence for the exact GitHub blob, not a cryptographic signature. Replacing it with SHA-256 would contradict the upstream 40-hex blob identifier and remove tamper detection. - -**Action / evidence.** RED is the exact hosted failure above. Commit `53f447f73f0ef33eb708bf44202ec4d5954ade66`, formatted by `d00cdff974f5ac665a5f7481620d550735bd26c8`, adds one rule-scoped `nosemgrep` annotation plus the protocol rationale without changing the hash input, comparison, download bound, or failure behavior. Existing executable cases still require exact byte count and reject altered bytes by Git blob-ID mismatch. Completion requires fresh exact-head SAST GREEN, the remaining protected checks, no unresolved actionable review thread, qualifying independent approval, and ordinary merge. - -## 2026-09-27 CodeQL terminal-proof fallback run identity - -**Status:** Proposed on `ContextualWisdomLab/.github#2405`; direct repair parent `5a77a8c711bc93330c24a4821dff7439f600a264`, tree `5ce8ba7448cb878a5b130ed1acaba1578e4940fd`. This documentation-only successor preserves that executable tree; the PR body is the authority for the current exact head and hosted-run IDs. Merge and required-workflow admission remain HOLD. - -**Context Map / owner.** The central `.github` CodeQL required-workflow and dispatch bounded context owns dispatch identity, terminal evidence, and exact job recovery. Product repositories consume the protected workflow contract; they do not copy the producer or manufacture success receipts. - -**Gap / failure scene.** The v2 handler names a run with `head/base/required-run/producer-source`, but its required-workflow fallback looked up only `head/base/required-run`. When authenticated status publication is unavailable, a completed clean handler job could not be found and a rerun ended false RED. Omitting the producer source would also allow a regenerated live merge revision to reuse predecessor evidence. - -**Action / evidence.** Correct the fallback lookup to include the live merge source and retain fail-closed base, head, required-run, workflow-path, job-name, GHAS-identity, and SARIF checks. The test-first repair reproduced two failures, then passed 96 focused workflow-contract tests; the new edge case rejects a stale merge-source title. Ruff E9/F/I on the changed dispatch-contract file and `git diff --check` pass. Fresh hosted Checks and a qualifying independent approval are still required on the unchanged executable delta before merge. diff --git a/opencode.jsonc b/opencode.jsonc index 3b5f34e2a6..8946175a13 100644 --- a/opencode.jsonc +++ b/opencode.jsonc @@ -294,15 +294,12 @@ // routes prioritized by scripts/ci/zdr_policy.py. Requires // CONTEXTUAL_ORCHESTRATOR_BASE_URL and CONTEXTUAL_ORCHESTRATOR_TOKEN, // which scripts/ci/contextual_orchestrator_review_sidecar.sh provisions on - // each runner before OpenCode starts. The sidecar exports a bare - // scheme://host:port, while the OpenAI-compatible provider appends only - // `/chat/completions`, so the `/v1` prefix belongs here. Without it the - // gateway answers route_not_found and OpenCode prints `Error: not found`. + // each runner before OpenCode starts. "contextual-orchestrator": { "npm": "@ai-sdk/openai-compatible", "name": "Contextual Orchestrator", "options": { - "baseURL": "{env:CONTEXTUAL_ORCHESTRATOR_BASE_URL}/v1", + "baseURL": "{env:CONTEXTUAL_ORCHESTRATOR_BASE_URL}", "apiKey": "{env:CONTEXTUAL_ORCHESTRATOR_TOKEN}" }, "models": { diff --git a/requirements-strix-ci-hashes.txt b/requirements-strix-ci-hashes.txt index eb83beda17..9e705850b5 100644 --- a/requirements-strix-ci-hashes.txt +++ b/requirements-strix-ci-hashes.txt @@ -140,9 +140,9 @@ annotated-types==0.7.0 \ --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \ --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89 # via pydantic -anyio==4.14.2 \ - --hash=sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494 \ - --hash=sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f +anyio==4.14.0 \ + --hash=sha256:b47c1f9ccf73e67021df785332508f99379c68fa7d0684e8e3492cb1d4b23f89 \ + --hash=sha256:dd9b7a2a9799ed6552fde617b2c5df02b7fdd7d88392fc48101e51bae46164d9 # via # google-genai # gql diff --git a/requirements-strix-ci.txt b/requirements-strix-ci.txt index 50e8a05f9b..19093441e9 100644 --- a/requirements-strix-ci.txt +++ b/requirements-strix-ci.txt @@ -1,5 +1,4 @@ strix-agent==1.5.3 -anyio==4.14.2 openai[httpx2]==2.54.0 aiohttp==3.14.3 google-cloud-aiplatform==1.133.0 diff --git a/scripts/ci/actions_queue_health.py b/scripts/ci/actions_queue_health.py index 7b1cc5e49f..bb73698551 100644 --- a/scripts/ci/actions_queue_health.py +++ b/scripts/ci/actions_queue_health.py @@ -1,19 +1,18 @@ #!/usr/bin/env python3 """Queue-health CLI with stable identity and audit-provenance guarantees. -Shared parsing and reporting primitives live in ``actions_queue_health_core.py``. -This entrypoint owns collection and the consistency boundary that binds active-run evidence to +The shared collector implementation lives in ``actions_queue_health_core.py``. +This entrypoint owns the consistency boundary that binds active-run evidence to a stable pull-request view, carries stable workflow identity, and exports the exact timestamp used for queue-age calculations. """ from __future__ import annotations -import importlib.util -import sys -import time from datetime import datetime, timezone +import importlib.util from pathlib import Path +import sys from urllib.parse import quote _CORE_MODULE_PATH = Path(__file__).with_name("actions_queue_health_core.py") @@ -164,7 +163,16 @@ def collect_snapshot( ), ) for workflow_run in workflow_runs: - active_snapshot[workflow_run["id"]] = workflow_run + workflow_run_id = workflow_run.get("id") + if ( + isinstance(workflow_run_id, bool) + or not isinstance(workflow_run_id, int) + or workflow_run_id <= 0 + ): + raise QueueHealthError( + "workflow run id must be a positive integer" + ) + active_snapshot[workflow_run_id] = workflow_run active_snapshots.append(active_snapshot) first_snapshot, second_snapshot = active_snapshots @@ -234,7 +242,16 @@ def collect_snapshot( TERMINAL_DIAGNOSTIC_STATUSES ): continue - terminal_diagnostic_snapshot[workflow_run["id"]] = workflow_run + workflow_run_id = workflow_run.get("id") + if ( + isinstance(workflow_run_id, bool) + or not isinstance(workflow_run_id, int) + or workflow_run_id <= 0 + ): + raise QueueHealthError( + "workflow run id must be a positive integer" + ) + terminal_diagnostic_snapshot[workflow_run_id] = workflow_run for terminal_status in TARGET_TERMINAL_DIAGNOSTIC_STATUSES: target_workflow_runs = _list_payload( diff --git a/scripts/ci/actions_queue_health_core.py b/scripts/ci/actions_queue_health_core.py index ab600efdbc..db3e5570ba 100644 --- a/scripts/ci/actions_queue_health_core.py +++ b/scripts/ci/actions_queue_health_core.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """Produce a read-only, exact-head GitHub Actions queue-health report. The collector intentionally treats queued, cancelled, skipped, missing, and @@ -8,14 +9,16 @@ from __future__ import annotations import argparse +from datetime import datetime, timezone import html import json +from pathlib import Path import re import subprocess -from collections.abc import Callable, Sequence -from datetime import datetime, timezone -from pathlib import Path -from typing import Any +import sys +import time +from typing import Any, Callable, Sequence, TextIO + REPOSITORY_PATTERN = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") QUEUE_STATES = {"QUEUED", "IN_PROGRESS", "PENDING", "REQUESTED"} @@ -109,13 +112,6 @@ def _list_payload( declared_total_counts.append(payload["total_count"]) if not isinstance(values, list) or not all(isinstance(value, dict) for value in values): raise QueueHealthError(f"GitHub response field {key!r} must be an array of objects") - if key == "workflow_runs" and any( - isinstance(value.get("id"), bool) - or not isinstance(value.get("id"), int) - or value["id"] <= 0 - for value in values - ): - raise QueueHealthError("workflow run id must be a positive integer") if isinstance(payload, dict) and PAGINATED_PAGES_KEY in payload: record_identities: list[tuple[str, int]] = [] for value in values: @@ -364,6 +360,133 @@ def _normalise_run(repository: str, run: dict[str, Any], jobs: list[dict[str, An } +def collect_snapshot( + repositories: Sequence[str], + *, + runner: Runner = subprocess.run, + generated_at: str | None = None, +) -> dict[str, Any]: + """Collect bounded queued/in-progress run and job data using read-only API calls.""" + validated = sorted({_repository_name(repository) for repository in repositories}) + if len(validated) != len(repositories): + raise QueueHealthError("collection repository list contains duplicates") + collected_repositories: list[dict[str, Any]] = [] + collection_errors: list[dict[str, str]] = [] + for repository in validated: + try: + metadata = github_json(f"repos/{repository}", runner=runner) + if not isinstance(metadata, dict): + raise QueueHealthError(f"repository metadata for {repository} is not an object") + pulls_endpoint = f"repos/{repository}/pulls?state=open&per_page={MAX_API_PAGE_SIZE}" + pull_requests = _list_payload( + github_json(pulls_endpoint, paginate=True, runner=runner), + "pulls", + max_items=MAX_API_PAGE_SIZE * MAX_API_PAGES, + ) + normalized_pull_requests = sorted( + (_normalise_pull_request(item) for item in pull_requests), + key=lambda item: item["number"], + ) + except IncompletePullRequestIdentity: + time.sleep(PULL_REQUEST_RETRY_DELAY_SECONDS) + try: + retry_pull_requests = _list_payload( + github_json(pulls_endpoint, paginate=True, runner=runner), + "pulls", + max_items=MAX_API_PAGE_SIZE * MAX_API_PAGES, + ) + normalized_pull_requests = sorted( + (_normalise_pull_request(item) for item in retry_pull_requests), + key=lambda item: item["number"], + ) + except QueueHealthError as retry_exc: + collection_errors.append( + { + "repository": repository, + "error": f"pull-request identity validation failed: {retry_exc}", + } + ) + continue + except QueueHealthError as exc: + collection_errors.append({"repository": repository, "error": str(exc)}) + continue + pull_requests_by_number = {item["number"]: item for item in normalized_pull_requests} + runs_by_id: dict[int, dict[str, Any]] = {} + try: + active_statuses = ("in_progress", "pending", "queued", "requested", "waiting") + snapshots: list[dict[int, dict[str, Any]]] = [] + for status_order in (active_statuses, tuple(reversed(active_statuses))): + snapshot: dict[int, dict[str, Any]] = {} + for status in status_order: + runs = _list_payload( + github_json( + f"repos/{repository}/actions/runs?status={status}" + f"&per_page={WORKFLOW_RUN_PAGE_SIZE}", + paginate=True, + max_pages=ACTIVE_RUN_MAX_API_PAGES, + runner=runner, + ), + "workflow_runs", + max_items=WORKFLOW_RUN_PAGE_SIZE * ACTIVE_RUN_MAX_API_PAGES, + ) + for run in runs: + run_id = run.get("id") + if isinstance(run_id, bool) or not isinstance(run_id, int) or run_id <= 0: + raise QueueHealthError("workflow run id must be a positive integer") + snapshot[run_id] = run + snapshots.append(snapshot) + first_snapshot, second_snapshot = snapshots + first_states = { + run_id: str(run.get("status") or "").upper() + for run_id, run in first_snapshot.items() + } + second_states = { + run_id: str(run.get("status") or "").upper() + for run_id, run in second_snapshot.items() + } + if first_states != second_states: + raise QueueHealthError("active workflow run snapshot changed during collection") + for run_id, run in second_snapshot.items(): + run_id = run.get("id") + candidate = _normalise_run(repository, run, []) + identity, _ = _run_identity(candidate, pull_requests_by_number) + if identity != "current_head" or candidate["status"] not in { + "IN_PROGRESS", + "WAITING", + }: + runs_by_id[run_id] = candidate + continue + jobs_payload = github_json( + f"repos/{repository}/actions/runs/{run_id}/jobs?per_page={MAX_API_PAGE_SIZE}", + paginate=True, + runner=runner, + ) + jobs = _list_payload( + jobs_payload, + "jobs", + max_items=MAX_API_PAGE_SIZE * MAX_API_PAGES, + ) + runs_by_id[run_id] = _normalise_run(repository, run, jobs) + except QueueHealthError as exc: + collection_errors.append({"repository": repository, "error": str(exc)}) + continue + collected_repositories.append( + { + "full_name": repository, + "default_branch": str(metadata.get("default_branch") or ""), + "pull_requests": normalized_pull_requests, + "runs": sorted(runs_by_id.values(), key=lambda item: item["id"]), + } + ) + timestamp = generated_at or datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + parse_timestamp(timestamp) + return { + "generated_at": timestamp, + "repositories": collected_repositories, + "collection_errors": collection_errors, + } + + def load_snapshot(path: Path) -> dict[str, Any]: """Load a JSON snapshot for offline, deterministic report generation.""" try: @@ -699,3 +822,34 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.add_argument("--queue-age-slo-seconds", type=int, default=DEFAULT_QUEUE_AGE_SLO_SECONDS) parser.add_argument("--now", help="Explicit timezone-aware evaluation time for deterministic reports") return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None, *, stderr: TextIO = sys.stderr) -> int: + """Collect or load a snapshot, write reports, and return a stable CLI status.""" + args = parse_args(argv) + try: + snapshot = load_snapshot(args.snapshot) if args.snapshot else collect_snapshot(load_allowlist(args.allowlist)) + now = parse_timestamp(args.now) if args.now else datetime.now(timezone.utc) + report = build_report( + snapshot, + now=now, + queue_age_slo_seconds=args.queue_age_slo_seconds, + ) + write_reports(report, args.output_json, args.output_html) + except (OSError, QueueHealthError, ValueError) as exc: + print(f"ERROR: queue-health report failed: {exc}", file=stderr) + return 2 + breaches = report["summary"]["unassigned_slo_breached_count"] + if breaches: + print(f"::warning::Actions queue-health found {breaches} unassigned current-head SLO breach(es).") + print( + "QUEUE_HEALTH_RESULT=" + f"observed={report['summary']['observed_job_count']} " + f"pending={report['summary']['pending_job_count']} " + f"slo_breaches={breaches}" + ) + return 0 + + +if __name__ == "__main__": # pragma: no cover - exercised through the CLI tests. + raise SystemExit(main()) diff --git a/scripts/ci/agent_mention_router.py b/scripts/ci/agent_mention_router.py index 68e544a6cf..59d55280c2 100755 --- a/scripts/ci/agent_mention_router.py +++ b/scripts/ci/agent_mention_router.py @@ -10,6 +10,7 @@ import os import re import subprocess +import threading import time from dataclasses import dataclass from typing import Any, Sequence @@ -150,6 +151,7 @@ def request( args: Sequence[str], *, input_payload: dict[str, Any] | None = None, + cancellation_event: threading.Event | None = None, ) -> Any: """Execute one bounded ``gh api`` request and decode optional JSON. @@ -176,6 +178,8 @@ def request( payload = None if input_payload is None else json.dumps(input_payload) attempt = 0 while True: + if cancellation_event is not None and cancellation_event.is_set(): + raise RuntimeError("gh api request cancelled") attempt += 1 try: completed = subprocess.run( @@ -189,10 +193,14 @@ def request( timeout=GITHUB_API_TIMEOUT_SECONDS, ) except subprocess.TimeoutExpired as exc: + if cancellation_event is not None and cancellation_event.is_set(): + raise RuntimeError("gh api request cancelled") from exc raise RuntimeError( "gh api timed out after " f"{GITHUB_API_TIMEOUT_SECONDS} seconds" ) from exc + if cancellation_event is not None and cancellation_event.is_set(): + raise RuntimeError("gh api request cancelled") return_code = int(getattr(completed, "returncode", 0)) if not return_code: output = completed.stdout.strip() @@ -202,7 +210,11 @@ def request( diagnostic = "no stderr output" retryable = RATE_LIMIT_DIAGNOSTIC_RE.search(diagnostic) is not None if retryable and attempt < GITHUB_API_MAX_ATTEMPTS: - time.sleep(attempt * 5) + backoff_seconds = attempt * 5 + if cancellation_event is None: + time.sleep(backoff_seconds) + elif cancellation_event.wait(backoff_seconds): + raise RuntimeError("gh api request cancelled") continue suffix = f" after {attempt} attempts" if attempt > 1 else "" raise RuntimeError( diff --git a/scripts/ci/agent_mention_sweep.py b/scripts/ci/agent_mention_sweep.py index 5b56fdcf4f..c969d16442 100755 --- a/scripts/ci/agent_mention_sweep.py +++ b/scripts/ci/agent_mention_sweep.py @@ -31,16 +31,12 @@ # log tail and metrics. Stop dispatching new work with margin to spare so # the sweep exits cleanly and reports what it completed. # -# Returning early only stops NEW work: list_recent_pull_requests' generator -# cleanup still blocks (executor.shutdown(wait=True)) until every currently -# RUNNING repository fetch finishes on its own. GitHubClient's rate-limit -# retry costs up to ~255s worst case for one repository (six attempts, each -# up to the 30s subprocess timeout, plus ~75s of backoff between them), and -# up to max_workers of those can be running concurrently at the moment the -# deadline trips (bounded by that ceiling, not multiplied by it, since they -# run in parallel). Budget = 900s job timeout - ~60s setup/checkout -# overhead - ~255s worst-case cleanup wait, with a further margin still -# unspent. +# Returning early sets one cancellation event shared by repository fetches. +# That event interrupts retry backoff immediately; an already-running gh +# subprocess retains its existing 30s timeout. Cleanup then waits for those +# bounded workers before returning, so no worker can keep using the shared +# client after the sweep reports completion. The 480s dispatch budget leaves +# the 30s worker tail plus setup and reporting margin inside the 900s job. DEFAULT_TIME_BUDGET_SECONDS = 480.0 @@ -204,7 +200,8 @@ def fetch(repository: str) -> list[dict[str, Any]]: "per_page=100", "-f", f"page={page}", - ] + ], + cancellation_event=stop_event, ) pull_requests = flatten_pages(response) if not pull_requests: diff --git a/scripts/ci/collect_release_strix_bindings.py b/scripts/ci/collect_release_strix_bindings.py deleted file mode 100644 index c4478fdeff..0000000000 --- a/scripts/ci/collect_release_strix_bindings.py +++ /dev/null @@ -1,377 +0,0 @@ -#!/usr/bin/env python3 -"""Collect one current-attempt Strix binding per licensed dependency.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import sys -import tempfile -from pathlib import Path -from typing import Any, BinaryIO, Callable, Iterable, Mapping - -try: - from scripts.ci import release_dependency_gate as gate - from scripts.ci.scan_release_native_links import _reader, scan - from scripts.ci.verify_release_distribution_set import ( - DIGEST_RE, - MAX_CONTROL_BYTES, - DistributionSetError, - _archive, - _artifact, - _digest, - _json_bytes, - _members, - _timestamp, - fetch_artifact, - verify_distribution_set, - ) -except ImportError: # pragma: no cover - trusted direct `python3 -I` invocation - sys.path.insert(0, str(Path(__file__).resolve().parent)) - import release_dependency_gate as gate - from scan_release_native_links import _reader, scan - from verify_release_distribution_set import ( - DIGEST_RE, - MAX_CONTROL_BYTES, - DistributionSetError, - _archive, - _artifact, - _digest, - _json_bytes, - _members, - _timestamp, - fetch_artifact, - verify_distribution_set, - ) - - -def collect_bindings( - capture_root: Path, - license_report: Path, - plan_path: Path, - artifacts: Iterable[Any], - attempt: Any, - *, - repository: str, - source_sha: str, - control_sha: str, - run_id: int, - run_attempt: int, - fetch: Callable[[str, int, BinaryIO], None], - report_path: Path, - verified_distributions: list[dict[str, Any]], - verdict_path: Path, - record_artifact_id: int, - record_artifact_digest: str, - archive_report_path: Path | None = None, - verified_scope_path: Path | None = None, - native_report_path: Path | None = None, - source_root: Path | None = None, -) -> gate.GateReport: - """Accept the exact matrix result set, then rerun the full gate unchanged.""" - - if (not isinstance(attempt, Mapping) or type(attempt.get("id")) is not int - or attempt["id"] != run_id or type(attempt.get("run_attempt")) is not int - or attempt["run_attempt"] != run_attempt or attempt.get("head_sha") != control_sha): - raise gate.GateError(gate.STRIX_BINDING_UNBOUND, "workflow attempt differs from collector") - started = _timestamp(attempt.get("run_started_at")) - expected = gate.strix_fanout_plan( - capture_root, license_report, control_sha, run_id, run_attempt, archive_report_path - ) - plan = gate.load_json(plan_path) - if plan != expected or plan["source_repository"] != repository or plan["source_sha"] != source_sha: - raise gate.GateError(gate.STRIX_BINDING_UNBOUND, "fanout plan differs from trusted capture") - listed: dict[str, Mapping[str, Any]] = {} - for item in artifacts: - if not isinstance(item, Mapping) or not isinstance(item.get("name"), str): - raise gate.GateError(gate.STRIX_BINDING_MALFORMED, "artifact metadata is invalid") - if item["name"] in listed: - raise gate.GateError(gate.STRIX_BINDING_UNBOUND, "duplicate artifact name in run") - listed[item["name"]] = item - prefix = f"release-strix-binding-a{run_attempt}-" - expected_names = {row["artifact_name"] for row in plan["dependencies"]} - if {name for name in listed if name.startswith(prefix)} != expected_names: - raise gate.GateError(gate.STRIX_BINDING_MISSING, "matrix binding artifact set is incomplete or has extras") - bindings = capture_root / "strix" / "bindings" - if (bindings.exists() or bindings.is_symlink() or report_path.exists() - or report_path.is_symlink() or verdict_path.exists() or verdict_path.is_symlink()): - raise gate.GateError(gate.STRIX_BINDING_UNBOUND, "collector destination already exists") - if not verified_distributions or type(record_artifact_id) is not int or record_artifact_id <= 0: - raise gate.GateError(gate.STRIX_BINDING_UNBOUND, "verified distribution set is unavailable") - _artifact(listed, "reproducibility-record", record_artifact_id, - _digest(record_artifact_digest), run_id, control_sha, started) - if any(not isinstance(row, Mapping) for row in verified_distributions): - raise gate.GateError( - gate.STRIX_BINDING_UNBOUND, - "verified distribution report contains a malformed row", - ) - wheel_filenames = [ - row.get("file") for row in verified_distributions - if row.get("leg") != "sdist" and isinstance(row.get("file"), str) - ] - sdist_filenames = [ - row.get("file") for row in verified_distributions - if row.get("leg") == "sdist" and isinstance(row.get("file"), str) - ] - if not wheel_filenames or len(sdist_filenames) != 1: - raise gate.GateError( - gate.STRIX_BINDING_UNBOUND, - "verified distribution report lacks wheel/sdist coverage", - ) - native_report_sha256 = None - native_link_analyzer = None - try: - with tempfile.TemporaryDirectory( - prefix=".release-distributions-", dir=bindings.parent - ) as distribution_scratch: - canonical_distributions = verify_distribution_set( - artifacts, - attempt, - repository=repository, - source_sha=source_sha, - control_sha=control_sha, - run_id=run_id, - run_attempt=run_attempt, - record_artifact_id=record_artifact_id, - record_artifact_digest=record_artifact_digest, - wheel_filename=wheel_filenames[0], - sdist_filename=sdist_filenames[0], - fetch=fetch, - output_dir=Path(distribution_scratch) / "verified", - ) - if native_report_path is not None: - if (native_report_path.is_symlink() or not native_report_path.is_file() - or native_report_path.stat().st_size > MAX_CONTROL_BYTES): - raise gate.GateError(gate.SCOPE_UNVERIFIABLE, "native link report is missing or oversized") - native_bytes = native_report_path.read_bytes() - native_payload = _json_bytes(native_bytes) - if native_payload != scan( - {"verified_distributions": canonical_distributions}, - Path(distribution_scratch) / "verified", source_sha, _reader() - ): - raise gate.GateError(gate.SOURCE_HASH_MISMATCH, - "native links differ from immutable distribution bytes") - native_report_sha256 = hashlib.sha256(native_bytes).hexdigest() - native_link_analyzer = native_payload["analyzer"] - except DistributionSetError as error: - raise gate.GateError( - gate.STRIX_BINDING_UNBOUND, - "distribution set failed immutable artifact verification", - ) from error - if verified_distributions != canonical_distributions: - raise gate.GateError( - gate.STRIX_BINDING_UNBOUND, - "verified distribution report differs from immutable artifacts", - ) - seen_ids: set[int] = {record_artifact_id} - with tempfile.TemporaryDirectory(prefix=".strix-bindings-", dir=bindings.parent) as scratch: - staging = Path(scratch) - for row in plan["dependencies"]: - name = row["artifact_name"] - item = listed[name] - artifact_id = item.get("id") - digest = item.get("digest") - if (type(artifact_id) is not int or artifact_id in seen_ids - or not isinstance(digest, str)): - raise gate.GateError(gate.STRIX_BINDING_UNBOUND, "binding artifact ID or digest is invalid") - _artifact(listed, name, artifact_id, digest, run_id, control_sha, started) - seen_ids.add(artifact_id) - member_name = f"{row['slug']}.json" - with _archive(repository, artifact_id, digest, fetch) as archive: - member = _members(archive, {member_name})[member_name] - if member.file_size > MAX_CONTROL_BYTES: - raise gate.GateError(gate.STRIX_BINDING_MALFORMED, "binding JSON exceeds size limit") - raw = archive.read(member) - payload = _json_bytes(raw) - if (not isinstance(payload, Mapping) or payload.get("schema") != gate.BINDING_SCHEMA - or payload.get("source_sha") != source_sha - or payload.get("control_sha") != control_sha - or type(payload.get("run_id")) is not int or payload["run_id"] != run_id - or type(payload.get("run_attempt")) is not int - or payload["run_attempt"] != run_attempt - or not isinstance(payload.get("fixture"), Mapping) - or payload["fixture"].get("id") != row["key"] - or payload["fixture"].get("sha256") != row["fixture_sha256"]): - raise gate.GateError(gate.STRIX_BINDING_UNBOUND, f"{row['key']}: binding differs from plan") - (staging / member_name).write_bytes(raw) - staging.rename(bindings) - scope_identities: list[dict[str, Any]] | None = None - variant_identities: list[dict[str, Any]] | None = None - if verified_scope_path is not None: - scope = gate.load_json(verified_scope_path) - rows = scope.get("verified_scope_evidence") if isinstance(scope, Mapping) else None - variants = scope.get("verified_runtime_variants") if isinstance(scope, Mapping) else None - if (not isinstance(rows, list) or len(rows) != 13 - or not all(isinstance(row, Mapping) for row in rows) - or not isinstance(variants, list) or len(variants) != 3 - or not all(isinstance(row, Mapping) for row in variants)): - raise gate.GateError(gate.SCOPE_UNVERIFIABLE, "verified scope set is incomplete") - scope_identities = [{key: row.get(key) for key in - ("leg", "artifact_id", "artifact_name", "artifact_digest")} - for row in rows] - if (any(not isinstance(row["leg"], str) - or row["artifact_name"] != f"repro-digest-{row['leg']}" - or type(row["artifact_id"]) is not int or row["artifact_id"] <= 0 - or not isinstance(row["artifact_digest"], str) - or DIGEST_RE.fullmatch(row["artifact_digest"]) is None - for row in scope_identities) - or len({row["leg"] for row in scope_identities}) != 13 - or len({row["artifact_id"] for row in scope_identities}) != 13 - or sum(row["leg"] == "sdist" for row in scope_identities) != 1): - raise gate.GateError(gate.SCOPE_UNVERIFIABLE, "verified scope identities are malformed") - used_ids = seen_ids | {row.get("artifact_id") for row in verified_distributions} - if any(row["artifact_id"] in used_ids for row in scope_identities): - raise gate.GateError(gate.SCOPE_UNVERIFIABLE, "scope artifact ID overlaps distribution set") - for row in scope_identities: - _artifact(listed, row["artifact_name"], row["artifact_id"], - row["artifact_digest"], run_id, control_sha, started) - variant_identities = [{key: row.get(key) for key in - ("leg", "arch", "artifact_id", "artifact_name", "artifact_digest")} - for row in variants] - if (any(not isinstance(row["leg"], str) or row["arch"] != "x86_64" - or row["artifact_name"] != f"repro-macos-x86-{row['leg']}" - or type(row["artifact_id"]) is not int or row["artifact_id"] <= 0 - or not isinstance(row["artifact_digest"], str) - or DIGEST_RE.fullmatch(row["artifact_digest"]) is None - for row in variant_identities) - or len({row["leg"] for row in variant_identities}) != 3 - or len({row["artifact_id"] for row in variant_identities}) != 3 - or {row["leg"] for row in variant_identities} - != {f"universal2-apple-darwin-py{version}" for version in ("3.12", "3.13", "3.14")} - or any(row["artifact_id"] in used_ids | {item["artifact_id"] for item in scope_identities} - for row in variant_identities)): - raise gate.GateError(gate.SCOPE_UNVERIFIABLE, "Intel runtime identities are malformed") - for row in variant_identities: - _artifact(listed, row["artifact_name"], row["artifact_id"], - row["artifact_digest"], run_id, control_sha, started) - for row in scope_identities: - _artifact(listed, row["artifact_name"], row["artifact_id"], - row["artifact_digest"], run_id, control_sha, started) - report = gate.gate(capture_root, stage=gate.FULL_STAGE, - **({"source_root": source_root} if source_root is not None else {})) - archive_reviews = [] - build_reviews = [] - tool_reviews = [] - if archive_report_path is not None and report.passed: - archive_payload = gate.load_json(archive_report_path) - by_key = {row["key"]: row for row in archive_payload["archives"]} - build_by_key = {row["key"]: row for row in archive_payload["build_packages"]} - tool_by_key = {row["key"]: row for row in archive_payload["build_tools"]} - for row in plan["dependencies"]: - if not {"runtime_archive", "build_package", "build_tool"} & row.keys(): - continue - build = "build_package" in row - tool = "build_tool" in row - approved = (tool_by_key if tool else build_by_key if build else by_key)[row["key"]] - dependency = gate.Dependency("github-release" if tool else "pypi", - approved["name"], approved["version"]) - failures = gate.validate_strix_binding( - bindings / f"{row['slug']}.json", dependency, - {"source_sha256": approved["source_sha256"]}, row["fixture_sha256"], - source_sha, fixture_key=row["key"], - ) - report.failures.extend(failures) - review = {"key": row["key"], "package_key": approved["package_key"], - "source_sha256": approved["source_sha256"], - "license": approved["license"], - "fixture_sha256": row["fixture_sha256"], - "legs": approved["legs"]} - (tool_reviews if tool else build_reviews if build else archive_reviews).append(review) - report_payload = report.to_json() - if archive_report_path is not None: - report_payload["runtime_archive_reviews"] = sorted(archive_reviews, key=lambda row: row["key"]) - report_payload["build_package_reviews"] = sorted(build_reviews, key=lambda row: row["key"]) - report_payload["build_tool_reviews"] = sorted(tool_reviews, key=lambda row: row["key"]) - report_path.write_text(json.dumps(report_payload, indent=2, sort_keys=True) + "\n") - if not report.passed: - raise gate.GateError(gate.STRIX_FINDINGS_OPEN, "full gate refused collected bindings") - binding_artifacts = [ - {"key": row["key"], "name": row["artifact_name"], - "id": listed[row["artifact_name"]]["id"], - "digest": listed[row["artifact_name"]]["digest"]} - for row in plan["dependencies"] if not {"runtime_archive", "build_package", "build_tool"} & row.keys() - ] - archive_binding_artifacts = [ - {"key": row["key"], "name": row["artifact_name"], - "id": listed[row["artifact_name"]]["id"], - "digest": listed[row["artifact_name"]]["digest"]} - for row in plan["dependencies"] if "runtime_archive" in row - ] - build_binding_artifacts = [ - {"key": row["key"], "name": row["artifact_name"], - "id": listed[row["artifact_name"]]["id"], - "digest": listed[row["artifact_name"]]["digest"]} - for row in plan["dependencies"] if "build_package" in row - ] - tool_binding_artifacts = [ - {"key": row["key"], "name": row["artifact_name"], - "id": listed[row["artifact_name"]]["id"], - "digest": listed[row["artifact_name"]]["digest"]} - for row in plan["dependencies"] if "build_tool" in row - ] - verdict = { - "schema": "cwl.release-full-set-verdict/1", "result": "PASS", - "source_repository": repository, "source_sha": source_sha, - "control_sha": control_sha, "run_id": run_id, "run_attempt": run_attempt, - "record_artifact_id": record_artifact_id, - "record_artifact_digest": record_artifact_digest, - "distributions": canonical_distributions, - "binding_artifacts": binding_artifacts, - "license_report_sha256": hashlib.sha256(license_report.read_bytes()).hexdigest(), - "gate_report_sha256": hashlib.sha256(report_path.read_bytes()).hexdigest(), - } - if archive_report_path is not None: - verdict["runtime_archive_binding_artifacts"] = archive_binding_artifacts - verdict["build_package_binding_artifacts"] = build_binding_artifacts - verdict["build_tool_binding_artifacts"] = tool_binding_artifacts - verdict["runtime_archive_license_sha256"] = expected["runtime_archive_license_sha256"] - if scope_identities is not None: - verdict["scope_evidence"] = sorted(scope_identities, key=lambda row: row["leg"]) - verdict["runtime_variants"] = sorted(variant_identities, key=lambda row: row["leg"]) - if native_report_sha256 is not None: - verdict["native_links_sha256"] = native_report_sha256 - verdict["native_link_analyzer"] = native_link_analyzer - verdict_path.write_text(json.dumps(verdict, indent=2, sort_keys=True) + "\n") - return report - - -def main() -> None: - parser = argparse.ArgumentParser() - for name in ( - "capture", "license-report", "plan", "metadata", "attempt", "repository", - "source-sha", "control-sha", "run-id", "run-attempt", "report", - "verified-distributions", "verdict", "record-artifact-id", "record-artifact-digest", - ): - parser.add_argument(f"--{name}", required=True) - parser.add_argument("--runtime-archive-license-report") - parser.add_argument("--verified-scope") - parser.add_argument("--native-report") - parser.add_argument("--source") - args = parser.parse_args() - artifacts = [_json_bytes(line.encode("utf-8")) for line in Path(args.metadata).read_text().splitlines()] - attempt = _json_bytes(Path(args.attempt).read_bytes()) - verified = _json_bytes(Path(args.verified_distributions).read_bytes()) - if not isinstance(verified, Mapping) or not isinstance(verified.get("verified_distributions"), list): - raise gate.GateError(gate.STRIX_BINDING_UNBOUND, "verified distribution report is malformed") - report = collect_bindings( - Path(args.capture), Path(args.license_report), Path(args.plan), - artifacts, attempt, repository=args.repository, source_sha=args.source_sha, - control_sha=args.control_sha, run_id=int(args.run_id), - run_attempt=int(args.run_attempt), fetch=fetch_artifact, - report_path=Path(args.report), - verified_distributions=verified["verified_distributions"], - verdict_path=Path(args.verdict), - record_artifact_id=int(args.record_artifact_id), - record_artifact_digest=args.record_artifact_digest, - archive_report_path=(Path(args.runtime_archive_license_report) - if args.runtime_archive_license_report else None), - verified_scope_path=Path(args.verified_scope) if args.verified_scope else None, - native_report_path=Path(args.native_report) if args.native_report else None, - source_root=Path(args.source) if args.source else None, - ) - print(json.dumps(report.to_json(), sort_keys=True)) - - -if __name__ == "__main__": # pragma: no cover - main() owns the tested CLI contract - main() diff --git a/scripts/ci/contextual_orchestrator_review_launcher.py b/scripts/ci/contextual_orchestrator_review_launcher.py index 811dc7d3f8..e8c462abcc 100644 --- a/scripts/ci/contextual_orchestrator_review_launcher.py +++ b/scripts/ci/contextual_orchestrator_review_launcher.py @@ -28,8 +28,6 @@ import logging import os import re -import queue -import threading import sys from pathlib import Path from typing import Any, Callable @@ -459,8 +457,7 @@ def _response_has_reasoning_without_content(response: object) -> bool: def _preflight_review_agents( - agents: list[object], *, client: Any, escalations_used: int = 0, - claim_escalation: Callable[[], bool] | None = None + agents: list[object], *, client: Any, escalations_used: int = 0 ) -> tuple[list[object], dict[str, object]]: """Probe each route with the runtime request contract and keep ready routes. @@ -527,7 +524,6 @@ def _preflight_review_agents( Args: agents: Selected zero-cost model agents. client: Vendored ``ModelClient``-compatible transport. - claim_escalation: Optional atomic reservation shared by concurrent probes. escalations_used: Escalations already spent earlier in this same preflight run (e.g. by a prior stage), so the shared budget is honored across calls rather than restarted at zero. @@ -661,10 +657,7 @@ def _preflight_review_agents( if ( not budget_signature or second_pass - or ( - not claim_escalation() if claim_escalation is not None - else escalations_used >= REVIEW_PREFLIGHT_MAX_ESCALATIONS - ) + or escalations_used >= REVIEW_PREFLIGHT_MAX_ESCALATIONS ): row["status"] = "rejected" if not budget_signature: @@ -760,133 +753,8 @@ def _preflight_review_agents( return [*viable, *deferred], report -def _preflight_review_agents_concurrently( - agents: list[object], *, client: Any, escalations_used: int = 0 -) -> tuple[list[object], dict[str, object]]: - """Fill the validated pool without waiting for one pending inference. - - Reuse the serial route validator; share the existing probe and escalation - budgets across at most MAX_PROBES outstanding calls. Pending calls are - neither cancelled nor classified as unavailable. Only completed ready - routes and explicitly retryable responses enter the serving pool. - """ - completed: queue.Queue = queue.Queue() - budget_lock = threading.Lock() - sealed = False - rows: list[dict[str, object]] = [] - probed: list[object] = [] - ready: list[object] = [] - streaks: dict[str, int] = {} - postponed: list[object] = [] - postponed_probed = 0 - walk = iter(agents) - second_pass = False - outstanding = 0 - exhausted = object() - - def claim() -> bool: - """Atomically reserve one of the shared escalated attempts.""" - nonlocal escalations_used - with budget_lock: - if sealed or escalations_used >= REVIEW_PREFLIGHT_MAX_ESCALATIONS: - return False - escalations_used += 1 - return True - - def probe(index: int, agent: object, postponed_probe: bool) -> None: - """Publish a completed sanitized route result or an unexpected exception.""" - try: - try: - _, report = _preflight_review_agents( - [agent], client=client, - escalations_used=(REVIEW_PREFLIGHT_MAX_ESCALATIONS if postponed_probe else 0), - claim_escalation=(None if postponed_probe else claim), - ) - except ReviewPreflightError as exc: - report = exc.report - row = report["routes"][0] - if postponed_probe and row.get("error_type") == "escalation_budget_exhausted": - row["error_type"] = "escalation_reserved_for_first_pass" - completed.put((index, row)) - except BaseException as exc: # propagate worker faults, never a review verdict - completed.put((index, exc)) - - try: - while len(ready) < REVIEW_PREFLIGHT_TARGET_READY: - try: - index, outcome = completed.get_nowait() - except queue.Empty: - agent = next(walk, exhausted) if len(probed) < REVIEW_PREFLIGHT_MAX_PROBES else exhausted - if agent is exhausted and not second_pass and postponed: - walk = iter(postponed) - second_pass = True - continue - if agent is not exhausted: - account = provider_account(str(getattr(agent, "provider_name", "") or "unknown")) - if second_pass: - postponed_probed += 1 - elif streaks.get(account, 0) >= REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429: - postponed.append(agent) - continue - index = len(probed) - probed.append(agent) - rows.append({ - "agent_id": str(getattr(agent, "id", "")), - "provider": str(getattr(agent, "provider_name", "") or "unknown"), - "model": str(getattr(agent, "model", "")), - "status": "pending", - }) - outstanding += 1 - # Lifecycle cancellation is owned by the sidecar process. - # Daemons avoid an interpreter exit joining a pending model. - threading.Thread(target=probe, args=(index, agent, second_pass), daemon=True).start() - continue - if not outstanding: - break - index, outcome = completed.get() - outstanding -= 1 - if isinstance(outcome, BaseException): - raise outcome - rows[index] = outcome - agent = probed[index] - account = provider_account(str(getattr(agent, "provider_name", "") or "unknown")) - streaks[account] = streaks.get(account, 0) + 1 if outcome.get("http_status") == 429 else 0 - if outcome.get("status") == "ready": - ready.append(agent) - finally: - with budget_lock: - sealed = True - - ready = [agent for agent, row in zip(probed, rows) if row["status"] == "ready"] - deferred: list[object] = [] - if ready: - for agent, row in zip(probed, rows): - if row.get("status") == "rejected" and row.get("http_status") in REVIEW_PREFLIGHT_DEFERRABLE_HTTP_STATUS: - row["status"] = "deferred" - deferred.append(_demote_agent(agent, REVIEW_PREFLIGHT_DEFERRED_PRIORITY_PENALTY)) - report = { - "contract": "strix-plain-chat-preflight-v2", - "candidate_count": len(agents), "probed_count": len(probed), - "ready_count": len(ready), "deferred_count": len(deferred), - "rejected_count": sum(row["status"] == "rejected" for row in rows), - "pending_count": sum(row["status"] == "pending" for row in rows), - "skipped_count": len(postponed) - postponed_probed, - "postponed_probed_count": postponed_probed, - "target_ready": REVIEW_PREFLIGHT_TARGET_READY, - "probe_budget": REVIEW_PREFLIGHT_MAX_PROBES, - "account_skip_after_429": REVIEW_PREFLIGHT_ACCOUNT_SKIP_AFTER_429, - "escalations_used": escalations_used, - "escalation_budget": REVIEW_PREFLIGHT_MAX_ESCALATIONS, - "routes": rows, - } - if not ready: - raise ReviewPreflightError("no provider route passed the Strix plain-chat preflight", report) - return [*ready, *deferred], report - - def _preflight_with_fallback( - primary_agents: list[object], fallback_agents: list[object], *, client: Any, - preflight: Callable | None = None + primary_agents: list[object], fallback_agents: list[object], *, client: Any ) -> tuple[list[object], dict[str, object], bool]: """Use the priced catalog only after every primary route rejects. @@ -903,16 +771,15 @@ def _preflight_with_fallback( ``primary_attempt`` nests the primary stage's own report -- including its own ``escalations_used`` -- whenever a fallback stage ran at all. """ - preflight = preflight or _preflight_review_agents try: - viable, report = preflight(primary_agents, client=client) + viable, report = _preflight_review_agents(primary_agents, client=client) return viable, report, False except ReviewPreflightError as primary_error: if not fallback_agents: raise escalations_used = int(primary_error.report.get("escalations_used", 0)) try: - viable, report = preflight( + viable, report = _preflight_review_agents( fallback_agents, client=client, escalations_used=escalations_used ) except ReviewPreflightError as fallback_error: @@ -1353,8 +1220,7 @@ def main(argv: list[str] | None = None) -> int: ) try: agents, preflight_report, fallback_used = _preflight_with_fallback( - agents, fallback_agents, client=client, - preflight=_preflight_review_agents_concurrently, + agents, fallback_agents, client=client ) except ReviewPreflightError as exc: _write_json(args.preflight_out, exc.report) diff --git a/scripts/ci/contextual_orchestrator_review_sidecar.sh b/scripts/ci/contextual_orchestrator_review_sidecar.sh index a08e26297b..3c2a1b51b9 100755 --- a/scripts/ci/contextual_orchestrator_review_sidecar.sh +++ b/scripts/ci/contextual_orchestrator_review_sidecar.sh @@ -14,7 +14,7 @@ # (fail-closed zero-cost) pool. set -euo pipefail -ORCHESTRATOR_PIN_SHA="${ORCHESTRATOR_PIN_SHA:-01bf92a3ec67a0e1f9b68978eb16b60301e985fd}" +ORCHESTRATOR_PIN_SHA="${ORCHESTRATOR_PIN_SHA:-767e67fbc6b881a452761f32abb69b9971b9b03b}" ORCHESTRATOR_GIT_URL="${ORCHESTRATOR_GIT_URL:-https://github.com/ContextualWisdomLab/contextual-orchestrator.git}" # The Strix gate and Noema SSRF guard accept this one process-local origin. # Keep it fixed so an environment override cannot create an unvalidated sidecar. @@ -43,13 +43,7 @@ CATALOG_LIMIT="${ORCHESTRATOR_CATALOG_LIMIT:-24}" # equivalence relation. CATALOG_ACCOUNT_CAP="${ORCHESTRATOR_CATALOG_ACCOUNT_CAP:-8}" ORCHESTRATOR_GITHUB_ENV="${GITHUB_ENV:-}" -sidecar_python="${SIDECAR_PYTHON:-$(command -v python3)}" -# setup-python with update-environment=false leaves the consumer's library path. -# Bind this process to the selected interpreter's matching shared runtime. -sidecar_python_lib="$(dirname "$(dirname "$(realpath "$(command -v "$sidecar_python")")")")/lib" -if [ -f "$sidecar_python_lib/libpython3.12.so.1.0" ]; then - export LD_LIBRARY_PATH="$sidecar_python_lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" -fi +sidecar_python="$(command -v python3)" log() { printf '[contextual-orchestrator-sidecar] %s\n' "$*"; } @@ -107,10 +101,6 @@ requirements_lock="$ORCHESTRATOR_SOURCE/requirements.lock" if [ ! -f "$requirements_lock" ]; then fail "vendored orchestrator is missing its hash-pinned requirements.lock" fi -# The pinned lock includes CPython 3.12 wheels; isolate them from consumer runtimes. -"$sidecar_python" -c 'import sys; sys.exit(0 if sys.version_info[:2] == (3, 12) else "sidecar requires Python 3.12 for its pinned wheel hashes")' -"$sidecar_python" -m venv "$ORCHESTRATOR_WORK/.venv" -sidecar_python="$ORCHESTRATOR_WORK/.venv/bin/python" log "installing hash-pinned orchestrator dependencies at ${checked_out}" "$sidecar_python" -m pip install --quiet --disable-pip-version-check --no-cache-dir \ --require-hashes \ @@ -119,15 +109,10 @@ log "installing hash-pinned orchestrator dependencies at ${checked_out}" PYTHONPATH="$ORCHESTRATOR_SOURCE:$ORG_REPO_ROOT" "$sidecar_python" -c \ 'from contextual_orchestrator.credentials import get_credential; from contextual_orchestrator.model_discovery import discover_all_models, free_discovered_models; from contextual_orchestrator.orchestrator import ModelClient, TaskOrchestrator, load_agents; from contextual_orchestrator.review_gateway import register_review_credentials; from contextual_orchestrator.server import SecurityConfig, serve' PYTHONPATH="$ORCHESTRATOR_SOURCE:$ORG_REPO_ROOT" "$sidecar_python" - <<'PY' -import faulthandler - -# Fatal startup diagnostics contain stack locations, never frame locals. -faulthandler.enable() - +import contextlib import http.client import io import json -import logging import threading from contextual_orchestrator.orchestrator import ModelAgent, ModelClient, TaskOrchestrator @@ -166,10 +151,7 @@ thread.start() try: connection = http.client.HTTPConnection("127.0.0.1", server.server_address[1], timeout=5) expected_rejection_log = io.StringIO() - capture = logging.StreamHandler(expected_rejection_log) - server_logger = logging.getLogger("contextual_orchestrator.server") - server_logger.addHandler(capture) - try: + with contextlib.redirect_stderr(expected_rejection_log): connection.request( "POST", "/v1/chat/completions", @@ -183,8 +165,6 @@ try: response = connection.getresponse() assert response.status == 413, response.status response.read() - finally: - server_logger.removeHandler(capture) assert ( "request_failed status=413 code=request_too_large" in expected_rejection_log.getvalue() @@ -392,10 +372,6 @@ until curl -fsSL "http://${ORCHESTRATOR_HOST}:${ORCHESTRATOR_PORT}/healthz" >/de fail "sidecar exited before healthz (status ${sidecar_status}); stderr: $(sed -n '1,20p' "$sidecar_stderr")" fi i=$((i + 1)) - if [ "$((i % 60))" -eq 0 ]; then - # Only report file presence; provider content stays in sanitized artifacts. - log "startup pending: polls=${i} discovery=$([ -s "$discovery_report" ] && echo present || echo absent) catalog=$([ -s "$catalog_file" ] && echo present || echo absent) policy=$([ -s "$policy_report" ] && echo present || echo absent) preflight=$([ -s "$preflight_report" ] && echo present || echo absent)" - fi sleep 1 done if [ ! -s "$preflight_report" ]; then diff --git a/scripts/ci/materialize_base_rust_dependencies.py b/scripts/ci/materialize_base_rust_dependencies.py index 4a46f232c8..6feec9cedf 100644 --- a/scripts/ci/materialize_base_rust_dependencies.py +++ b/scripts/ci/materialize_base_rust_dependencies.py @@ -19,12 +19,6 @@ manifests are read (never the pull request's), and vendoring itself uses Cargo's own built-in per-package checksum verification (every ``[[package]]`` entry in a lock file carries a ``checksum``), so no separate hash-pin parser is needed the way ``requirements*.txt`` needed one. - -An explicit ``--head-sha`` opts into a narrower lock-repair intake: every Cargo manifest -remains byte-identical to base, every registry record (including resolved dependency edges) -must already occur in the base lock union, and local identities must already be base-pinned. -Only lock bytes are overlaid; Cargo vendor --locked still verifies the unchanged manifests -and package checksums. Separate provenance records the base manifests and head lock blobs. """ from __future__ import annotations @@ -100,54 +94,49 @@ def _is_workspace_manifest(content: bytes) -> bool: return "workspace" in parsed -def _select_vendor_roots( +def _select_vendor_root( repo_root: pathlib.Path, base_sha: str, cargo_paths: list[str] -) -> list[str]: - """Return every directory whose base lock must be vendored, primary root first. - - A base tree may legitimately hold several lock roots: the standard cargo-fuzz - layout declares ``[workspace]`` in both the repository root and ``fuzz/`` so the - fuzz crate opts out of the parent workspace, and the two locks resolve *different* - crate sets. Selecting one root and dropping the rest would silently vendor an - incomplete closure, so every root is vendored into one shared directory via - ``cargo vendor --sync`` and every lock is asserted with ``--locked``. - - What still fails closed is a root that cannot be reconciled at all: a manifest - declaring a workspace with no sibling ``Cargo.lock``, or a lock with no sibling - ``Cargo.toml``. Those are unresolvable rather than merely plural. +) -> str | None: + """Return the single directory ``cargo vendor`` should be invoked from, or ``None``. + + Only one topology is supported: a single Cargo workspace root, or a single standalone + crate with no workspace. Any other shape (independent multi-root layouts) fails closed + rather than guess which root's lock file is authoritative -- the same restraint + ``materialize_base_python_requirements.py`` takes with uv workspaces. """ - manifests = { - (path.rsplit("/", 1)[0] if "/" in path else ".") - for path in cargo_paths - if path.endswith("Cargo.toml") - } - locks = { - (path.rsplit("/", 1)[0] if "/" in path else ".") - for path in cargo_paths - if path.endswith("Cargo.lock") - } - for manifest_path in sorted( - path for path in cargo_paths if path.endswith("Cargo.toml") - ): + manifests = [path for path in cargo_paths if path.endswith("Cargo.toml")] + locks = {path.rsplit("/", 1)[0] if "/" in path else "." for path in cargo_paths if path.endswith("Cargo.lock")} + workspace_dirs: list[str] = [] + for manifest_path in manifests: content = _git(repo_root, "show", f"{base_sha}:{manifest_path}") - if not _is_workspace_manifest(content): - continue - manifest_dir = manifest_path.rsplit("/", 1)[0] if "/" in manifest_path else "." - if manifest_dir not in locks: - raise RuntimeError( - f"base Cargo workspace root {manifest_dir} has no sibling Cargo.lock" - ) - for lock_dir in sorted(locks): - if lock_dir not in manifests: - raise RuntimeError( - f"base Cargo.lock at {lock_dir} has no sibling Cargo.toml" - ) - if not locks: - return [] - # Deterministic order with the repository root first when it is one of the roots, - # so the primary --manifest-path is stable across runs and hosts. - ordered = sorted(locks, key=lambda root: (root != ".", root)) - return ordered + if _is_workspace_manifest(content): + manifest_dir = manifest_path.rsplit("/", 1)[0] if "/" in manifest_path else "." + workspace_dirs.append(manifest_dir) + + if len(workspace_dirs) == 1: + (root,) = workspace_dirs + if root in locks: + return root + raise RuntimeError( + f"base Cargo workspace root {root} has no sibling Cargo.lock" + ) + if len(workspace_dirs) > 1: + raise RuntimeError( + "base tree declares more than one Cargo workspace root; " + "Rust dependency vendoring needs exactly one" + ) + if len(locks) == 1: + (root,) = locks + manifest_path = "Cargo.toml" if root == "." else f"{root}/Cargo.toml" + if manifest_path in manifests: + return root + raise RuntimeError(f"base Cargo.lock at {root} has no sibling Cargo.toml") + if len(locks) > 1: + raise RuntimeError( + "base tree has more than one Cargo.lock with no single workspace root; " + "Rust dependency vendoring needs exactly one" + ) + return None def _placeholder_target_paths(manifest_content: bytes) -> list[str]: @@ -166,16 +155,10 @@ def _placeholder_target_paths(manifest_content: bytes) -> list[str]: if "package" not in parsed: return [] paths = {"src/lib.rs", "src/main.rs"} - lib_path = ( - parsed.get("lib", {}).get("path") - if isinstance(parsed.get("lib"), dict) - else None - ) + lib_path = parsed.get("lib", {}).get("path") if isinstance(parsed.get("lib"), dict) else None if isinstance(lib_path, str): paths.add(lib_path) - for bin_target in ( - parsed.get("bin", []) if isinstance(parsed.get("bin"), list) else [] - ): + for bin_target in parsed.get("bin", []) if isinstance(parsed.get("bin"), list) else []: bin_path = bin_target.get("path") if isinstance(bin_target, dict) else None if isinstance(bin_path, str): paths.add(bin_path) @@ -183,10 +166,7 @@ def _placeholder_target_paths(manifest_content: bytes) -> list[str]: def _reconstruct_base_tree( - repo_root: pathlib.Path, - base_sha: str, - cargo_paths: list[str], - work_dir: pathlib.Path, + repo_root: pathlib.Path, base_sha: str, cargo_paths: list[str], work_dir: pathlib.Path ) -> None: """Write every tracked base Cargo manifest into ``work_dir`` at its repository path. @@ -200,17 +180,8 @@ def _reconstruct_base_tree( destination.write_bytes(content) if destination.name == "Cargo.toml": for target_path in _placeholder_target_paths(content): - target_relative_path = pathlib.PurePosixPath(target_path) - if ( - target_relative_path.is_absolute() - or ".." in target_relative_path.parts - ): - raise RuntimeError( - "Cargo target path must stay inside its manifest root: " - f"{target_path}" - ) target_destination = destination.parent / pathlib.Path( - *target_relative_path.parts + *pathlib.PurePosixPath(target_path).parts ) target_destination.parent.mkdir(parents=True, exist_ok=True) if not target_destination.exists(): @@ -218,30 +189,18 @@ def _reconstruct_base_tree( def _run_cargo_vendor( - manifest_path: pathlib.Path, - vendor_dir: pathlib.Path, - sync_manifests: list[pathlib.Path] | None = None, + manifest_path: pathlib.Path, vendor_dir: pathlib.Path ) -> subprocess.CompletedProcess[bytes]: - """Vendor the union of the base manifests, asserting every lock stays unchanged. - - ``--sync`` adds each further root's manifest to the same vendor directory, so no - root's dependencies are dropped. ``--locked`` makes cargo refuse to re-resolve: - without it a lock that disagrees with its manifest would be quietly updated and - the vendored set would no longer be the committed closure. - """ - command = [ - "cargo", - "vendor", - "--locked", - "--manifest-path", - str(manifest_path), - "--versioned-dirs", - ] - for sync_manifest in sync_manifests or []: - command.extend(["--sync", str(sync_manifest)]) - command.append(str(vendor_dir)) + """Run ``cargo vendor`` for one reconstructed base manifest and return the result.""" return subprocess.run( - command, + [ + "cargo", + "vendor", + "--manifest-path", + str(manifest_path), + "--versioned-dirs", + str(vendor_dir), + ], check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -249,151 +208,14 @@ def _run_cargo_vendor( ) -def _normalized_lock_records(content: bytes) -> list[dict]: - """Compare bounded lock records with uniquely resolved dependency identities.""" - parsed = tomllib.loads(content.decode("utf-8")) - packages = parsed.get("package") - if ( - set(parsed) - {"version", "package"} - or parsed.get("version") not in {3, 4} - or not isinstance(packages, list) - or len(packages) > 10000 - ): - raise ValueError("head intake requires a bounded version 3/4 Cargo lock") - identities = {} - by_name = {} - for package in packages: - if not isinstance(package, dict): - raise ValueError("invalid Cargo lock package") - source = package.get("source") - allowed = {"name", "version", "dependencies"} - if source is not None: - allowed |= {"source", "checksum"} - if ( - source != "registry+https://github.com/rust-lang/crates.io-index" - or not isinstance(package.get("checksum"), str) - or not re.fullmatch(r"[0-9a-f]{64}", package["checksum"]) - ): - raise ValueError( - "head intake requires existing crates.io checksum pins" - ) - if set(package) - allowed or any( - not isinstance(package.get(k), str) or not package[k] - for k in ("name", "version") - ): - raise ValueError("unsupported Cargo lock package fields") - identity = (package["name"], package["version"], source) - if identity in identities: - raise ValueError("duplicate Cargo lock package identity") - identities[identity] = package - by_name.setdefault(identity[0], []).append(identity) - records = [] - for package in packages: - dependencies = package.get("dependencies", []) - if not isinstance(dependencies, list) or len(dependencies) > 10000: - raise ValueError("invalid Cargo lock dependencies") - edges = [] - for dependency in dependencies: - if not isinstance(dependency, str): - raise ValueError("invalid Cargo lock dependency identity") - parts = dependency.split() - if not 1 <= len(parts) <= 3: - raise ValueError("unsupported Cargo lock dependency identity") - candidates = [ - identity - for identity in by_name.get(parts[0], []) - if (len(parts) < 2 or identity[1] == parts[1]) - and (len(parts) < 3 or f"({identity[2]})" == parts[2]) - ] - if len(candidates) != 1: - raise ValueError("missing or ambiguous Cargo lock dependency identity") - edges.append(candidates[0]) - if len(edges) != len(set(edges)): - raise ValueError("duplicate Cargo lock dependency edge") - records.append({**package, "dependencies": sorted(edges, key=repr)}) - return records - - -def _validated_head_locks( - repo_root: pathlib.Path, base_sha: str, head_sha: str, cargo_paths: list[str] -) -> tuple[dict[str, bytes], dict]: - """Allow head locks only to recombine records pinned in the base lock union. - - Manifest bytes remain from base. Local-package closure still requires the - unchanged base manifests to pass Cargo vendor --locked; this function does - not authorize new registry records, git sources or package checksums. - """ - if not SHA_RE.fullmatch(head_sha): - raise ValueError("head SHA must be exactly 40 hexadecimal characters") - head_paths = _regular_cargo_blob_paths(repo_root, head_sha) - if head_paths != cargo_paths: - raise ValueError("head Cargo manifest/lock paths must equal base") - changed = _git(repo_root, "diff", "--name-only", "-z", base_sha, head_sha).split( - b"\0" - ) - if any( - path and pathlib.PurePosixPath(path.decode()).name == "Cargo.toml" - for path in changed - ): - raise ValueError("head Cargo manifests must remain byte-identical to base") - - def lock_bytes(revision: str, path: str) -> bytes: - """Read a revision-pinned lock after checking its bounded blob size.""" - size = int(_git(repo_root, "cat-file", "-s", f"{revision}:{path}")) - if size > 16 * 1024 * 1024: - raise ValueError("Cargo lock exceeds bounded size") - return _git(repo_root, "show", f"{revision}:{path}") - - paths = [path for path in cargo_paths if path.endswith("Cargo.lock")] - base_records = [] - for path in paths: - base_records.extend(_normalized_lock_records(lock_bytes(base_sha, path))) - registry_records = { - json.dumps(row, sort_keys=True) for row in base_records if row.get("source") - } - local_identities = { - (row["name"], row["version"]) for row in base_records if not row.get("source") - } - locks = {} - receipts = [] - for path in paths: - content = lock_bytes(head_sha, path) - for row in _normalized_lock_records(content): - if row.get("source"): - if json.dumps(row, sort_keys=True) not in registry_records: - raise ValueError( - "head registry record differs from base lock union" - ) - elif (row["name"], row["version"]) not in local_identities: - raise ValueError("head local identity differs from base lock union") - locks[path] = content - receipts.append( - { - "path": path, - "base_lock_blob": _git(repo_root, "rev-parse", f"{base_sha}:{path}") - .decode() - .strip(), - "lock_blob": _git(repo_root, "rev-parse", f"{head_sha}:{path}") - .decode() - .strip(), - } - ) - return locks, { - "manifest_revision": base_sha.lower(), - "lock_revision": head_sha.lower(), - "locks": receipts, - } - - def materialize( repo_root: pathlib.Path, base_sha: str, output_dir: pathlib.Path, *, vendor_dir_for_config: str | None = None, - head_sha: str | None = None, ) -> list[str]: - """Vendor base manifests with base locks or explicitly bounded head locks. + """Vendor the base commit's Cargo dependency closure into ``output_dir``. Returns the list of source-tree-relative ``Cargo.lock`` paths that were vendored. An empty list means no Rust project (or no lock file) exists at the base commit, which is not an @@ -413,39 +235,19 @@ def materialize( resolved_repo = repo_root.resolve() cargo_paths = _regular_cargo_blob_paths(resolved_repo, base_sha) - vendor_roots = _select_vendor_roots(resolved_repo, base_sha, cargo_paths) + vendor_root = _select_vendor_root(resolved_repo, base_sha, cargo_paths) manifest: list[str] = [] - head_locks, provenance = ( - _validated_head_locks(resolved_repo, base_sha, head_sha, cargo_paths) - if head_sha is not None - else ({}, None) - ) - if vendor_roots: - primary_root, *additional_roots = vendor_roots - - def _manifest_for(root: str, base: pathlib.Path) -> pathlib.Path: - """Return the reconstructed manifest path for one validated root.""" - return base / ("Cargo.toml" if root == "." else f"{root}/Cargo.toml") - - def _lock_for(root: str) -> str: - """Return the repository-relative lock path for one validated root.""" - return "Cargo.lock" if root == "." else f"{root}/Cargo.lock" - + if vendor_root is not None: with tempfile.TemporaryDirectory() as work_dir: work_path = pathlib.Path(work_dir) _reconstruct_base_tree(resolved_repo, base_sha, cargo_paths, work_path) - for path, content in head_locks.items(): - (work_path / path).write_bytes(content) - manifest_path = _manifest_for(primary_root, work_path) - sync_manifests = [ - _manifest_for(root, work_path) for root in additional_roots - ] - # Every root's lock is reported, so a failure names the whole vendored set - # rather than only the primary root. - lock_path = ", ".join(_lock_for(root) for root in vendor_roots) + manifest_path = work_path / ( + "Cargo.toml" if vendor_root == "." else f"{vendor_root}/Cargo.toml" + ) + lock_path = "Cargo.lock" if vendor_root == "." else f"{vendor_root}/Cargo.lock" vendor_dir = output_dir / "vendor" try: - completed = _run_cargo_vendor(manifest_path, vendor_dir, sync_manifests) + completed = _run_cargo_vendor(manifest_path, vendor_dir) except (OSError, subprocess.TimeoutExpired) as exc: raise RuntimeError( f"could not run trusted cargo vendor for base manifest {lock_path}: " @@ -454,14 +256,10 @@ def _lock_for(root: str) -> str: if completed.returncode != 0: stderr = completed.stderr.decode("utf-8", errors="replace") normalized_stderr = " ".join(stderr.split()) - detail = ( - normalized_stderr[:500] - if normalized_stderr - else (f"exit status {completed.returncode}") - ) - raise RuntimeError( - f"cargo vendor failed for base lock {lock_path}: {detail}" + detail = normalized_stderr[:500] if normalized_stderr else ( + f"exit status {completed.returncode}" ) + raise RuntimeError(f"cargo vendor failed for base lock {lock_path}: {detail}") config_text = completed.stdout if vendor_dir_for_config is not None: config_text = config_text.replace( @@ -469,12 +267,8 @@ def _lock_for(root: str) -> str: vendor_dir_for_config.encode("utf-8"), ) (output_dir / "cargo-config.toml").write_bytes(config_text) - manifest = [_lock_for(root) for root in vendor_roots] + manifest = [lock_path] - if provenance is not None: - (output_dir / "lock-provenance.json").write_text( - json.dumps(provenance, indent=2) + "\n" - ) (output_dir / "manifest.json").write_text( json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8", @@ -487,7 +281,6 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser() parser.add_argument("--repo-root", required=True, type=pathlib.Path) parser.add_argument("--base-sha", required=True) - parser.add_argument("--head-sha", default=None) parser.add_argument("--output-dir", required=True, type=pathlib.Path) parser.add_argument("--vendor-dir-for-config", default=None) args = parser.parse_args(argv) @@ -498,28 +291,17 @@ def main(argv: list[str] | None = None) -> int: args.base_sha, args.output_dir, vendor_dir_for_config=args.vendor_dir_for_config, - head_sha=args.head_sha, ) except (OSError, RuntimeError, ValueError) as exc: print( - f"::error::Could not materialize base Rust dependencies: {exc}", - file=sys.stderr, + f"::error::Could not materialize base Rust dependencies: {exc}", file=sys.stderr ) return 1 if manifest: - if args.head_sha: - print( - f"Materialized Cargo vendor directory from base manifests and bounded head locks: {manifest[0]}." - ) - else: - print( - f"Materialized trusted base Cargo vendor directory from {manifest[0]}." - ) + print(f"Materialized trusted base Cargo vendor directory from {manifest[0]}.") else: - print( - "No tracked Cargo.lock exists at the validated base SHA; Rust vendoring skipped." - ) + print("No tracked Cargo.lock exists at the validated base SHA; Rust vendoring skipped.") return 0 diff --git a/scripts/ci/noema_preflight_capacity.py b/scripts/ci/noema_preflight_capacity.py deleted file mode 100644 index b9bed218b1..0000000000 --- a/scripts/ci/noema_preflight_capacity.py +++ /dev/null @@ -1,163 +0,0 @@ -"""Classify an all-429 review-sidecar preflight as provider capacity (#2148). - -The sidecar launcher writes ``strix_runs/contextual-orchestrator-preflight.json`` -(contract ``strix-plain-chat-preflight-v2``) before it exits on a failed -preflight. When every probed route was refused with HTTP 429 and none is ready, -the private-target ZDR pool is rate-limited rather than broken, which is the same -``provider_capacity_unavailable`` class ADR-0031 already re-dispatches after a -gateway failure. This module emits the same step outputs as -``two_phase._emit_transport_capacity_outputs`` (via the stdlib-only -``noema_transport_redispatch`` helpers) so the existing bounded -re-dispatch step can consume them. It never changes the job result: the -provisioning step has already failed and review remains required. -""" - -from __future__ import annotations - -import argparse -import json -import os -import stat -import re -import sys -from pathlib import Path -from typing import Any - -if __package__ in (None, ""): # pragma: no cover - executed as a workflow script - sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -# Stdlib-only on purpose: this runs on the runner's bare python3 after the -# sidecar step failed, before the HWP reader step installs defusedxml, so it -# must not import noema_review_gate (whose document import needs it). -from scripts.ci import noema_transport_redispatch as gate # noqa: E402 - -PREFLIGHT_CONTRACT = "strix-plain-chat-preflight-v2" -PREFLIGHT_CAPACITY_HTTP_STATUS = 429 -PREFLIGHT_CAPACITY_ROUTE_STATUSES = frozenset({"rejected", "deferred"}) -MAX_PREFLIGHT_REPORT_BYTES = 256 * 1024 - - -def _exact_int(value: Any) -> int | None: - """Return ``value`` only when it is a real ``int`` (``bool`` is rejected).""" - return value if type(value) is int else None - - -def _stage_retry_after(report: Any) -> list[int] | None: - """Return one all-429 stage's in-cap ``retry_after_s`` values, or None if not all-429. - - A stage qualifies only when ``ready_count`` is 0, ``probed_count`` is at - least 1, ``routes`` holds exactly ``probed_count`` rows, and every row is a - rejected or deferred route whose ``http_status`` is the integer 429. - """ - if not isinstance(report, dict) or report.get("contract") != PREFLIGHT_CONTRACT: - return None - probed = _exact_int(report.get("probed_count")) - routes = report.get("routes") - if _exact_int(report.get("ready_count")) != 0 or probed is None or probed < 1: - return None - if not isinstance(routes, list) or len(routes) != probed: - return None - waits: list[int] = [] - for row in routes: - if not isinstance(row, dict): - return None - if row.get("status") not in PREFLIGHT_CAPACITY_ROUTE_STATUSES: - return None - if _exact_int(row.get("http_status")) != PREFLIGHT_CAPACITY_HTTP_STATUS: - return None - wait = _exact_int(row.get("retry_after_s")) - if wait is not None and 1 <= wait <= gate.TRANSPORT_REDISPATCH_RETRY_AFTER_MAX_SECONDS: - waits.append(wait) - return waits - - -def classify_preflight_report(report: Any) -> tuple[int, int | None] | None: - """Return ``(probed_count, retry_after_seconds)`` for an all-429 report, else None. - - A nested ``primary_attempt`` (a fallback stage also ran) must itself be - all-429. ``retry_after_seconds`` is the longest provider-stated wait inside - ADR-0031's existing cap, or None so the deterministic jitter applies. - """ - waits = _stage_retry_after(report) - if waits is None: - return None - probed = report["probed_count"] - if "primary_attempt" in report: - primary_waits = _stage_retry_after(report["primary_attempt"]) - if primary_waits is None: - return None - waits.extend(primary_waits) - probed += report["primary_attempt"]["probed_count"] - return probed, (max(waits) if waits else None) - - -def load_preflight_report(path: Path) -> Any: - """Return the parsed report, or None when it is missing, oversized, or not JSON.""" - try: - if path.parent.is_symlink(): - return None - flags = os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0) - with os.fdopen(os.open(path, flags), "rb") as handle: - metadata = os.fstat(handle.fileno()) - if not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1: - return None - raw = handle.read(MAX_PREFLIGHT_REPORT_BYTES + 1) - if len(raw) > MAX_PREFLIGHT_REPORT_BYTES: - return None - return json.loads(raw.decode("utf-8")) - except (OSError, UnicodeDecodeError, ValueError, RecursionError): - return None - - -def emit_preflight_capacity_outputs(path: Path, *, expected_head: str) -> dict[str, str]: - """Write the ADR-0031 transport outputs for one failed sidecar preflight.""" - classified = classify_preflight_report(load_preflight_report(path)) - if classified is None: - outputs = {"transport_capacity_unavailable": "false", "transport_retry_eligible": "false"} - gate.append_github_output(outputs) - return outputs - probed, retry_after = classified - retry_attempt = gate.current_transport_retry_attempt() - delay = gate.transport_redispatch_delay_seconds( - transport_retry_attempt=retry_attempt, - head_sha=expected_head, - retry_after_seconds=retry_after, - ) - outputs = { - "transport_capacity_unavailable": "true", - "transport_retry_eligible": "true" if delay is not None else "false", - "transport_http_status": str(PREFLIGHT_CAPACITY_HTTP_STATUS), - "provider_attempt_count": str(probed), - } - if delay is not None: - outputs["transport_retry_delay_seconds"] = str(delay) - outputs["transport_retry_next_attempt"] = str(retry_attempt + 1) - print( - "::notice::Noema sidecar preflight was all-429 (provider capacity unavailable); " - f"bounded continuation re-dispatch is eligible in {delay}s " - f"(attempt {retry_attempt + 1}/{gate.MAX_TRANSPORT_REDISPATCH_ATTEMPTS})." - ) - else: - print( - "::error::Noema sidecar preflight was all-429 (provider capacity unavailable); " - "automatic re-dispatch budget is exhausted. Review remains required." - ) - gate.append_github_output(outputs) - return outputs - - -def main(argv: list[str]) -> int: - """Classify one preflight report; always exit 0 because the job already failed.""" - parser = argparse.ArgumentParser() - parser.add_argument("--preflight-report", required=True, type=Path) - parser.add_argument("--expected-head", required=True) - args = parser.parse_args(argv) - if not re.fullmatch(r"[0-9a-f]{40}", args.expected_head): - print("::error::--expected-head must be a canonical lowercase 40-character Git SHA.") - return 0 - emit_preflight_capacity_outputs(args.preflight_report, expected_head=args.expected_head) - return 0 - - -if __name__ == "__main__": # pragma: no cover - raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index 380ee22675..c8709304fc 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -26,15 +26,6 @@ from scripts.ci.opencode_review_normalize_output import changed_file_is_material from scripts.ci.noema_review_document import DocumentReadError, extract_review_document -from scripts.ci.noema_transport_redispatch import ( # noqa: F401 - MAX_TRANSPORT_REDISPATCH_ATTEMPTS, - TRANSPORT_REDISPATCH_JITTER_MAX_SECONDS, - TRANSPORT_REDISPATCH_JITTER_MIN_SECONDS, - TRANSPORT_REDISPATCH_RETRY_AFTER_MAX_SECONDS, - append_github_output, - current_transport_retry_attempt, - transport_redispatch_delay_seconds, -) PRIMARY_REVIEW_AUTHORS = { @@ -74,6 +65,10 @@ MAX_HTTP_ERROR_BODY_BYTES = 16 * 1024 # ADR-0031: transport-capacity class after gateway failover (not caller retries). TRANSPORT_CAPACITY_HTTP_STATUSES = frozenset({429, 500, 502, 503, 504}) +MAX_TRANSPORT_REDISPATCH_ATTEMPTS = 2 +TRANSPORT_REDISPATCH_JITTER_MIN_SECONDS = 60 +TRANSPORT_REDISPATCH_JITTER_MAX_SECONDS = 180 +TRANSPORT_REDISPATCH_RETRY_AFTER_MAX_SECONDS = 300 DIFF_HUNK_RE = re.compile(r"^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@") SAFE_MODEL_IDENTIFIER_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,199}$") @@ -263,6 +258,62 @@ def parse_http_retry_after_seconds(headers: Any) -> int | None: return seconds +def transport_redispatch_delay_seconds( + *, + transport_retry_attempt: int, + head_sha: str, + retry_after_seconds: int | None = None, +) -> int | None: + """Return the post-failure scheduling delay, or None when the re-dispatch bound is spent. + + ``transport_retry_attempt`` is the number of automatic capacity re-dispatches + already performed for this head (0 on the first failure). Prefer a capped + gateway ``Retry-After`` when present; otherwise use deterministic jitter in + ``[TRANSPORT_REDISPATCH_JITTER_MIN_SECONDS, TRANSPORT_REDISPATCH_JITTER_MAX_SECONDS]`` + keyed by head SHA and attempt so concurrent failures do not stampede. + """ + if transport_retry_attempt < 0 or transport_retry_attempt >= MAX_TRANSPORT_REDISPATCH_ATTEMPTS: + return None + if retry_after_seconds is not None: + if ( + type(retry_after_seconds) is int + and 1 <= retry_after_seconds <= TRANSPORT_REDISPATCH_RETRY_AFTER_MAX_SECONDS + ): + return retry_after_seconds + return None + digest = hashlib.sha256( + f"{head_sha.strip().lower()}:{transport_retry_attempt}".encode("utf-8") + ).digest() + span = ( + TRANSPORT_REDISPATCH_JITTER_MAX_SECONDS - TRANSPORT_REDISPATCH_JITTER_MIN_SECONDS + 1 + ) + offset = int.from_bytes(digest[:4], "big") % span + return TRANSPORT_REDISPATCH_JITTER_MIN_SECONDS + offset + + +def current_transport_retry_attempt() -> int: + """Parse the workflow-supplied automatic re-dispatch counter, failing closed to 0.""" + raw = (os.environ.get("NOEMA_TRANSPORT_RETRY_ATTEMPT") or "0").strip() + if not raw.isdecimal(): + return 0 + value = int(raw) + return value if value <= 64 else 0 + + +def append_github_output(values: dict[str, str]) -> None: + """Append allowlisted step outputs when running under GitHub Actions.""" + path = (os.environ.get("GITHUB_OUTPUT") or "").strip() + if not path or not values: + return + with open(path, "a", encoding="utf-8") as handle: + for key, value in values.items(): + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + continue + if any(ch in value for ch in ("\n", "\r", "\0")): + continue + handle.write(f"{key}={value}\n") + + def _stable_failure_diagnostic(exc: BaseException) -> str: """Return actionable trusted diagnostics without reflecting model values.""" message = scrub_sensitive_data(str(exc)) or type(exc).__name__ @@ -799,32 +850,17 @@ def fetch_file_content_at_ref(repo: str, path: str, ref: str) -> str: "gh", "api", f"repos/{repo}/contents/{encoded_path}?ref={encoded_ref}", - "--header", - "Accept: application/vnd.github.object+json", + "--jq", + ".content // empty", ] ) - try: - response = json.loads(content) - except json.JSONDecodeError as exc: - raise RuntimeError("GitHub content response was malformed") from exc - if ( - not isinstance(response, dict) - or type(response.get("size")) is not int - or response["size"] < 0 - or not isinstance(response.get("content"), str) - ): - raise RuntimeError("GitHub content response was malformed") - if response.get("encoding") != "base64": - raise RuntimeError("GitHub file content unavailable: API omitted the encoded body") - compact = "".join(response["content"].split()) - if not compact and response["size"]: - raise RuntimeError("GitHub file content unavailable: nonempty file has no encoded body") + compact = "".join(content.split()) + if not compact: + return "" try: raw = base64.b64decode(compact, validate=True) except (binascii.Error, ValueError) as exc: raise RuntimeError("GitHub content response contained malformed base64") from exc - if len(raw) != response["size"]: - raise RuntimeError("GitHub content response size did not match the decoded body") suffix = PurePosixPath(path).suffix.lower() if suffix in {".docx", ".hwp", ".hwpx"}: try: diff --git a/scripts/ci/noema_transport_redispatch.py b/scripts/ci/noema_transport_redispatch.py deleted file mode 100644 index c2b8e9c177..0000000000 --- a/scripts/ci/noema_transport_redispatch.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Stdlib-only Noema continuation helpers for startup and model failures.""" - -from __future__ import annotations - -import hashlib -import os -import re - -MAX_TRANSPORT_REDISPATCH_ATTEMPTS = 2 -TRANSPORT_REDISPATCH_JITTER_MIN_SECONDS = 60 -TRANSPORT_REDISPATCH_JITTER_MAX_SECONDS = 180 -TRANSPORT_REDISPATCH_RETRY_AFTER_MAX_SECONDS = 300 - - -def transport_redispatch_delay_seconds( - *, - transport_retry_attempt: int, - head_sha: str, - retry_after_seconds: int | None = None, -) -> int | None: - """Return the post-failure scheduling delay, or None when the re-dispatch bound is spent. - - ``transport_retry_attempt`` is the number of automatic capacity re-dispatches - already performed for this head (0 on the first failure). Prefer a capped - gateway ``Retry-After`` when present; otherwise use deterministic jitter in - ``[TRANSPORT_REDISPATCH_JITTER_MIN_SECONDS, TRANSPORT_REDISPATCH_JITTER_MAX_SECONDS]`` - keyed by head SHA and attempt so concurrent failures do not stampede. - """ - if transport_retry_attempt < 0 or transport_retry_attempt >= MAX_TRANSPORT_REDISPATCH_ATTEMPTS: - return None - if retry_after_seconds is not None: - if ( - type(retry_after_seconds) is int - and 1 <= retry_after_seconds <= TRANSPORT_REDISPATCH_RETRY_AFTER_MAX_SECONDS - ): - return retry_after_seconds - return None - digest = hashlib.sha256( - f"{head_sha.strip().lower()}:{transport_retry_attempt}".encode("utf-8") - ).digest() - span = ( - TRANSPORT_REDISPATCH_JITTER_MAX_SECONDS - TRANSPORT_REDISPATCH_JITTER_MIN_SECONDS + 1 - ) - offset = int.from_bytes(digest[:4], "big") % span - return TRANSPORT_REDISPATCH_JITTER_MIN_SECONDS + offset - - -def current_transport_retry_attempt() -> int: - """Parse the retry counter; invalid values exhaust the automatic budget.""" - raw = os.environ.get("NOEMA_TRANSPORT_RETRY_ATTEMPT") - if raw is None or raw == "null": - return 0 - if not re.fullmatch(r"[0-9]{1,2}", raw): - return MAX_TRANSPORT_REDISPATCH_ATTEMPTS - value = int(raw) - return min(value, MAX_TRANSPORT_REDISPATCH_ATTEMPTS) - - -def append_github_output(values: dict[str, str]) -> None: - """Append allowlisted step outputs when running under GitHub Actions.""" - path = (os.environ.get("GITHUB_OUTPUT") or "").strip() - if not path or not values: - return - with open(path, "a", encoding="utf-8") as handle: - for key, value in values.items(): - if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): - continue - if any(ch in value for ch in ("\n", "\r", "\0")): - continue - handle.write(f"{key}={value}\n") diff --git a/scripts/ci/pingora_edge_policy.py b/scripts/ci/pingora_edge_policy.py index 0d3a2c0948..b53a68c6ae 100644 --- a/scripts/ci/pingora_edge_policy.py +++ b/scripts/ci/pingora_edge_policy.py @@ -24,14 +24,14 @@ ``.npz``, ...) have no entry in ``BINARY_DOCUMENT_MAGIC``, which only knows ``.hwpx``/``.pdf``/``.png``. Rather than grow that registry for every such format, a file under a declared prefix whose suffix has no magic entry is -admitted only when no diff patch is available, the fetched bytes fail to decode -as UTF-8, and their replacement-decoded text contains no prohibited runtime -pattern. That keeps the module's central guarantee honest -- a file that -decodes as valid UTF-8 is never treated as a binary artifact, and one stray -invalid byte cannot conceal a readable runtime command -- while still -admitting genuinely opaque research binaries without maintaining an open-ended -magic-byte catalog. A suffix that *does* have a magic entry keeps that entry's -existing structural evidence check +admitted on the stricter complement of the UTF-8 decode this module already +performs for every ordinarily-scanned file: no diff patch available, *and* the +fetched bytes fail to decode as UTF-8. That keeps the module's central +guarantee honest -- a file that decodes as valid UTF-8 is never treated as a +binary artifact, since scanning exactly that content is what this module +exists to do -- while still admitting genuinely opaque research binaries +without maintaining an open-ended magic-byte catalog. A suffix that *does* +have a magic entry keeps that entry's existing structural evidence check (``_is_complete_png``, ``_is_complete_hwpx``, or the raw magic-prefix check for ``.pdf``) even under a declared prefix. """ @@ -40,7 +40,6 @@ import argparse import base64 -import hashlib import io import json import os @@ -57,7 +56,6 @@ MAX_FILE_BYTES = 1_048_576 MAX_RESPONSE_BYTES = 16_777_216 -MAX_BLOB_BYTES = 100_000_000 REPOSITORY_RE = re.compile(r"^(?!.*(?:\.\.|\.$))[A-Za-z0-9_.-]+/(?!.*(?:\.\.|\.$))[A-Za-z0-9_.-]+$") SHA_RE = re.compile(r"^[0-9a-f]{40}$") # A base ref threaded into evaluate_pull_request may be either a branch name @@ -73,8 +71,8 @@ # reference). Without this, any such file placed under a documentation # directory still falls through to `_needs_content_scan` -> `True` (binary # files never carry a GitHub diff `patch`), and then `_load_file_content` -# fails closed with a `PolicyError` for any instance over the Git blob API's -# 100 MB ceiling -- rejecting a legitimate research-paper citation +# fails closed with a `PolicyError` for any instance over the Contents API's +# 1 MiB base64 ceiling -- rejecting a legitimate research-paper citation # (this org's own "attach the relevant paper PDF" convention) for a reason # that has nothing to do with the Nginx runtime policy this module enforces. BINARY_DOCUMENT_MAGIC = { @@ -189,10 +187,15 @@ class PolicyError(RuntimeError): class ContentSizeExceededError(PolicyError): - """Signal a well-formed file above 100 MB for the narrow PDF convention. - - Malformed, truncated, or tampered evidence raises ``PolicyError`` instead - and cannot use that convention. + """Raised when a well-formed Contents API response exceeds MAX_FILE_BYTES. + + Distinct from every other ``PolicyError`` cause (a malformed response, a + non-file/non-base64 entry, corrupt base64, a declared size that does not + match the decoded bytes) so a caller can choose to trust a narrow, + path-scoped convention -- a genuinely oversized documentation PDF, the + one case this module cannot verify by content at all -- instead of + failing the whole check closed. Every other content-evidence failure + still fails closed exactly as before. """ @@ -210,7 +213,6 @@ class ArtifactDeclarationNotFoundError(PolicyError): OpenJson = Callable[[str, str], object] -OpenBytes = Callable[[str, str, int], bytes] class NoRedirectHandler(HTTPRedirectHandler): @@ -476,29 +478,6 @@ def _github_open_json(url: str, token: str) -> object: raise PolicyError("GitHub API returned malformed JSON policy evidence") from exc -def _github_open_raw_bytes(url: str, token: str, max_bytes: int) -> bytes: - """Read a Git blob with a strict byte limit and no redirect or body logging.""" - - _validate_github_api_url(url) - request = Request( # noqa: S310 - URL is validated immediately above - url, - headers={ - "Accept": "application/vnd.github.raw+json", - "Authorization": f"Bearer {token}", - "X-GitHub-Api-Version": "2022-11-28", - "User-Agent": "cwl-pingora-edge-policy/1", - }, - ) - try: - with github_opener.open(request, timeout=30) as response: - raw = response.read(max_bytes + 1) - except (HTTPError, URLError, TimeoutError) as exc: - raise PolicyError(f"GitHub raw blob request failed: {type(exc).__name__}") from exc - if len(raw) > max_bytes: - raise PolicyError("GitHub raw blob exceeded the bounded response size") - return raw - - def _load_changed_files(api_url: str, repository: str, pull_request: int, token: str, opener: OpenJson) -> tuple[ChangedFile, ...]: """Load every changed-file page while enforcing shape and pagination bounds.""" @@ -550,23 +529,21 @@ def _load_changed_files(api_url: str, repository: str, pull_request: int, token: raise PolicyError("GitHub changed-file pagination exceeded 3,000 files") # pragma: no cover -def _load_raw_file_bytes( - api_url: str, repository: str, path: str, head_sha: str, token: str, - opener: OpenJson, raw_opener: OpenBytes = _github_open_raw_bytes, -) -> bytes: +def _load_raw_file_bytes(api_url: str, repository: str, path: str, head_sha: str, token: str, opener: OpenJson) -> bytes: """Load one final head file's raw decoded bytes from the Contents API. - Files above the inline ceiling are fetched by the exact blob SHA named - by the Contents response at *head_sha*. The bounded raw response must - match both the declared size and the Git blob hash before use. + Raises ``ContentSizeExceededError`` specifically when the declared size + is a well-formed positive integer over ``MAX_FILE_BYTES`` -- a signal a + caller may treat differently from every other, genuinely malformed + response shape, which always raises the base ``PolicyError`` instead. GitHub's Contents API returns two distinct shapes for a file it cannot inline: some responses still report ``encoding: "base64"`` with a ``size`` over the inline-content ceiling and empty/absent ``content``; for files whose blob exceeds that ceiling, GitHub instead reports ``encoding: "none"`` with an accurate ``size`` and no ``content`` at - all. Both shapes use the same verified blob path. Only files above the - Git API's 100 MB blob limit retain ``ContentSizeExceededError``. + all. Both are treated as the same size-exceeded evidence; every other + response shape still fails closed. *head_sha* is also reused, unchanged, to fetch a base-ref-scoped file (the issue #2193 artifact-path declaration): any git ref -- a commit SHA @@ -584,37 +561,17 @@ def _load_raw_file_bytes( raise PolicyError(f"GitHub content evidence for {path} is not a regular file") encoding = payload.get("encoding") declared_size = payload.get("size") - if isinstance(declared_size, bool) or not isinstance(declared_size, int) or declared_size < 0: - raise PolicyError(f"GitHub content evidence for {path} has a malformed size or content field") - if encoding not in {"none", "base64"}: - raise PolicyError(f"GitHub content evidence for {path} has an invalid encoding") - if declared_size > MAX_FILE_BYTES: - blob_sha = payload.get("sha") - if not isinstance(blob_sha, str) or not SHA_RE.fullmatch(blob_sha): - raise PolicyError(f"GitHub content evidence for {path} has no valid blob SHA") - if declared_size > MAX_BLOB_BYTES: - if payload.get("content", "") != "": - raise PolicyError(f"GitHub content evidence for {path} has contradictory oversized content") - raise ContentSizeExceededError(f"GitHub content evidence for {path} exceeds the size contract") - raw = raw_opener(f"{api_url}/repos/{repository}/git/blobs/{blob_sha}", token, declared_size) - if len(raw) != declared_size: - raise PolicyError(f"GitHub raw blob evidence for {path} has a size mismatch") - # Git blob IDs are protocol SHA-1 object IDs, not security signatures. - digest = hashlib.sha1( # nosemgrep: python.lang.security.insecure-hash-algorithms.insecure-hash-algorithm-sha1 - f"blob {len(raw)}\0".encode(), usedforsecurity=False - ) - digest.update(raw) - actual_sha = digest.hexdigest() - if actual_sha != blob_sha: - raise PolicyError(f"GitHub raw blob evidence for {path} has a SHA mismatch") - return raw if encoding == "none": + if isinstance(declared_size, int) and declared_size > MAX_FILE_BYTES: + raise ContentSizeExceededError(f"GitHub content evidence for {path} exceeds the size contract") raise PolicyError(f"GitHub content evidence for {path} has no inline content and no verifiable oversized size") if encoding != "base64": raise PolicyError(f"GitHub content evidence for {path} is not a regular base64 file") encoded = payload.get("content") - if not isinstance(encoded, str): + if not isinstance(encoded, str) or not isinstance(declared_size, int) or declared_size < 0: raise PolicyError(f"GitHub content evidence for {path} has a malformed size or content field") + if declared_size > MAX_FILE_BYTES: + raise ContentSizeExceededError(f"GitHub content evidence for {path} exceeds the size contract") try: raw = base64.b64decode("".join(encoded.split()), validate=True) except (ValueError, TypeError) as exc: @@ -624,13 +581,10 @@ def _load_raw_file_bytes( return raw -def _load_file_content( - api_url: str, repository: str, path: str, head_sha: str, token: str, - opener: OpenJson, raw_opener: OpenBytes = _github_open_raw_bytes, -) -> str: +def _load_file_content(api_url: str, repository: str, path: str, head_sha: str, token: str, opener: OpenJson) -> str: """Load one final head file as bounded UTF-8 text from the Contents API.""" - raw = _load_raw_file_bytes(api_url, repository, path, head_sha, token, opener, raw_opener) + raw = _load_raw_file_bytes(api_url, repository, path, head_sha, token, opener) try: return raw.decode("utf-8") except UnicodeDecodeError as exc: @@ -645,19 +599,18 @@ def _binary_documentation_evidence_confirms( head_sha: str, token: str, opener: OpenJson, - raw_opener: OpenBytes = _github_open_raw_bytes, ) -> bool: """Return whether a claimed binary documentation asset is genuine. A missing diff ``patch`` alone is not proof of binary content: GitHub also omits a patch for a textual diff that exceeds its own rendering - limit, well under this module's ``MAX_BLOB_BYTES`` content-fetch + limit, well under this module's ``MAX_FILE_BYTES`` content-fetch ceiling. Whenever the file's raw bytes can be fetched at all, this verifies the declared format's magic prefix instead of trusting patch-presence alone. Only a file whose content evidently exceeds the - Git blob API's size ceiling -- the exact case ``_is_binary_documentation_asset`` + Contents API's size ceiling -- the exact case ``_is_binary_documentation_asset`` exists for, a cited, large research paper -- falls back to trusting the - path+suffix convention for PDFs over 100 MB only; every other + path+suffix convention for oversized PDFs only; every other content-evidence failure (a malformed API response, corrupt base64, a declared size that does not match the decoded bytes) propagates and fails the whole check closed, @@ -672,12 +625,10 @@ def _binary_documentation_evidence_confirms( bytes that decode cleanly are never admitted this way, so a valid-UTF-8 file cannot be mistaken for a binary artifact merely by sitting under a declared prefix -- it still reaches the normal content scan instead. - Inspect readable text even when other bytes are invalid UTF-8, so a stray - binary byte cannot conceal an active runtime command. """ try: - raw = _load_raw_file_bytes(api_url, repository, changed.path, head_sha, token, opener, raw_opener) + raw = _load_raw_file_bytes(api_url, repository, changed.path, head_sha, token, opener) except ContentSizeExceededError: return PurePosixPath(changed.path).suffix.lower() == ".pdf" suffix = PurePosixPath(changed.path).suffix.lower() @@ -689,8 +640,7 @@ def _binary_documentation_evidence_confirms( try: raw.decode("utf-8") except UnicodeDecodeError: - readable = raw.decode("utf-8", errors="replace") - return not any(pattern.search(readable) for _, pattern in CONTENT_RULES) + return True return False return raw.startswith(BINARY_DOCUMENT_MAGIC[suffix]) @@ -913,7 +863,6 @@ def evaluate_pull_request( token: str, base_ref: str | None = None, opener: OpenJson = _github_open_json, - raw_opener: OpenBytes = _github_open_raw_bytes, ) -> tuple[Violation, ...]: """Evaluate one pull request without checking out or executing its content. @@ -955,7 +904,7 @@ def evaluate_pull_request( # also omits one for an oversized textual diff), so this confirms # the format's magic prefix whenever the bytes can be fetched at # all, falling back to the path+suffix convention only when the - # content genuinely exceeds the Git blob API's size ceiling. A + # content genuinely exceeds the Contents API's size ceiling. A # removed file has no head content to fetch at all -- _needs_content_scan # already special-cases this the same way for every other file. if changed.status != "removed" and _is_binary_documentation_asset(changed, declared_prefixes): @@ -966,7 +915,6 @@ def evaluate_pull_request( head_sha=head_sha, token=token, opener=opener, - raw_opener=raw_opener, ): if declared_prefix is not None: # Names the reviewed declaration this admission relied @@ -975,7 +923,7 @@ def evaluate_pull_request( continue elif not _needs_content_scan(changed, declared_prefixes): continue - content = _load_file_content(resolved_api_url, repository, changed.path, head_sha, token, opener, raw_opener) + content = _load_file_content(resolved_api_url, repository, changed.path, head_sha, token, opener) violations.extend(scan_content(changed.path, content)) return tuple(violations) diff --git a/scripts/ci/pr_review_merge_scheduler_core.py b/scripts/ci/pr_review_merge_scheduler_core.py index 5a86bd24c8..4489ee62a3 100644 --- a/scripts/ci/pr_review_merge_scheduler_core.py +++ b/scripts/ci/pr_review_merge_scheduler_core.py @@ -3905,8 +3905,14 @@ def is_strix_scan_check_run(node: dict[str, Any]) -> bool: def dispatch_strix_evidence(repo: str, workflow: str, pr: dict[str, Any], *, dry_run: bool) -> str: """Dispatch same-head Strix workflow evidence before OpenCode reviews.""" - # A job rerun retains its original trusted workflow revision. Fresh dispatch - # selects the default-branch runtime and still enforces admission and live head. + job_id = matching_actions_job_id(pr, is_strix_scan_check_run) + if job_id: + if not dry_run and not review_dispatch_admitted("strix", repo, pr): + return "admission_deferred" + if not dry_run and not live_dispatch_head_matches(repo, pr): + return "stale_head" + rerun_actions_job(repo, job_id, dry_run=dry_run, action="rerun-strix-evidence") + return "rerun" if not dry_run else "dry_run" if dry_run: return "dry_run" require_github_actions_control_actor("inspect-active-strix-evidence") diff --git a/scripts/ci/prescreen_release_runtime_archives.py b/scripts/ci/prescreen_release_runtime_archives.py deleted file mode 100644 index 7631a38a06..0000000000 --- a/scripts/ci/prescreen_release_runtime_archives.py +++ /dev/null @@ -1,419 +0,0 @@ -#!/usr/bin/env python3 -"""Prescreen exact transported runtime wheels before Strix credentials exist.""" - -from __future__ import annotations - -import argparse -import email.parser -import hashlib -import io -import json -import re -import subprocess -import sys -import zipfile -from pathlib import Path, PurePosixPath -from typing import Any, Mapping - -try: - from scripts.ci import release_dependency_gate as gate - from scripts.ci.scan_release_native_links import NATIVE_MAGIC, TARGET_ARCHES, _links, _reader - from scripts.ci.verify_release_distribution_set import _json_bytes, DistributionSetError - from scripts.ci.verify_release_scope_evidence_set import _runtime_target_architecture -except ImportError: # pragma: no cover - trusted direct `python3 -I` invocation - sys.path.insert(0, str(Path(__file__).resolve().parent)) - import release_dependency_gate as gate - from scan_release_native_links import NATIVE_MAGIC, TARGET_ARCHES, _links, _reader - from verify_release_distribution_set import _json_bytes, DistributionSetError - from verify_release_scope_evidence_set import _runtime_target_architecture - - -def _native_wheel_libraries( - raw: bytes, - target: str, - *, - required_architecture: str | None = None, -) -> list[dict[str, Any]]: - """Inspect native members and require a runtime architecture when supplied.""" - libraries = [] - reader = None - with zipfile.ZipFile(io.BytesIO(raw)) as archive: - for entry in archive.infolist(): - if entry.is_dir(): - continue - with archive.open(entry) as stream: - magic = stream.read(8) - name = entry.filename.lower() - if not (magic.startswith(NATIVE_MAGIC) or name.endswith( - (".so", ".pyd", ".dll", ".dylib", ".a", ".lib", ".exe", ".wasm"))): - continue - if magic.startswith((b"!\n", b"\x00asm")) or name.endswith((".a", ".lib", ".wasm")): - raise gate.GateError(gate.NATIVE_LINK_UNKNOWN, f"{entry.filename}: static or wasm native member needs separate review") - if entry.file_size > 128 * 1024 * 1024: - raise gate.GateError(gate.CAPTURE_INCOMPLETE, f"{entry.filename}: native member exceeds inspection limit") - binary = archive.read(entry) - try: - reader = reader or _reader()["path"] - links = _links(binary, target, reader, allow_subset=True) - except (ValueError, OSError, subprocess.SubprocessError) as error: - raise gate.GateError(gate.NATIVE_LINK_UNKNOWN, f"{entry.filename}: native links could not be inspected") from error - link_architectures = {row["arch"] for row in links} - if (required_architecture is not None - and required_architecture not in link_architectures): - raise gate.GateError( - gate.NATIVE_LINK_UNKNOWN, - f"{entry.filename}: runtime variant requires {required_architecture} architecture", - ) - libraries.append({"path": entry.filename, - "needed": sorted({name for row in links for name in row["needed"]}), - "static_archives": []}) - return libraries - - -def _build_packages(item: Mapping[str, Any], folder: Path) -> list[dict[str, Any]]: - """Review the exact installed files recorded by one build interpreter.""" - leg = item["leg"] - receipt = _json_bytes((folder / f"{leg}.build-first.json").read_bytes()) - if not isinstance(receipt, Mapping) or receipt.get("leg") != leg: - raise gate.GateError(gate.SCOPE_UNVERIFIABLE, f"{leg}: build receipt is malformed") - snapshot = folder / f"{leg}.build-python.zip" - snapshot_bytes = gate.read_archive_snapshot(snapshot) - raw_sha = hashlib.sha256(snapshot_bytes).hexdigest() - if receipt.get("python_snapshot_sha256") != raw_sha or item.get("members", {}).get(snapshot.name) != raw_sha: - raise gate.GateError(gate.SOURCE_HASH_MISMATCH, f"{leg}: build snapshot changed after transport") - packages = receipt.get("python_packages") - if not isinstance(packages, list) or not packages: - raise gate.GateError(gate.SCOPE_UNVERIFIABLE, f"{leg}: build packages are missing") - target = "x86_64-unknown-linux-gnu" if leg == "sdist" else leg.rsplit("-py", 1)[0] - if target == "universal2-apple-darwin": - build_env = receipt.get("build_env") - architecture = {"ARM64": "aarch64", "X64": "x86_64"}.get( - build_env.rsplit("/", 1)[-1] if isinstance(build_env, str) else "") - if architecture is None: - raise gate.GateError(gate.SCOPE_UNVERIFIABLE, f"{leg}: build interpreter architecture is missing") - else: - architecture = next(iter(TARGET_ARCHES[target])) - native_rows = _native_wheel_libraries(snapshot_bytes, target, required_architecture=architecture) - listed_native = {f"{package['name']}/{file['path']}" for package in packages - for file in package["files"]} - if any(row["path"] not in listed_native for row in native_rows): - raise gate.GateError(gate.SCOPE_SET_MISMATCH, f"{leg}: unlisted native build file") - result = [] - with zipfile.ZipFile(snapshot) as archive: - members = {entry.filename: entry for entry in archive.infolist()} - for package in packages: - name, version = package["name"], package["version"] - files = package["files"] - metadata = [file["path"] for file in files if file["path"].endswith(".dist-info/METADATA")] - if len(metadata) != 1: - raise gate.GateError(gate.CAPTURE_INCOMPLETE, f"{leg}: {name} metadata is ambiguous") - metadata_root = PurePosixPath(metadata[0]).parent - def read_file(path: str) -> bytes: - entry = members.get(f"{name}/{path}") - if entry is None or entry.file_size > 4 * 1024 * 1024: - raise gate.GateError(gate.CAPTURE_INCOMPLETE, f"{leg}: {name} text file is missing or oversized") - with archive.open(entry) as stream: - return stream.read(4 * 1024 * 1024 + 1) - try: - message = email.parser.BytesParser().parsebytes(read_file(metadata[0])) - except (UnicodeError, ValueError) as error: - raise gate.GateError(gate.CAPTURE_INCOMPLETE, f"{leg}: {name} metadata is unreadable") from error - names, versions = message.get_all("Name", []), message.get_all("Version", []) - if (len(names) != 1 or len(versions) != 1 - or gate.normalize_project_name(names[0]) != name - or versions[0] != version): - raise gate.GateError(gate.CAPTURE_INCOMPLETE, f"{leg}: {name} metadata differs from installed identity") - candidates = {file["path"] for file in files if PurePosixPath(file["path"]).name.upper().startswith( - ("LICENSE", "LICENCE", "COPYING", "NOTICE", "UNLICENSE"))} - for declared in message.get_all("License-File", []): - path = PurePosixPath(declared) - if path.is_absolute() or ".." in path.parts or "\\" in declared: - raise gate.GateError(gate.ARCHIVE_PATH_ESCAPE, f"{leg}: {name} license path is unsafe") - matches = {str(metadata_root / path), str(metadata_root / "licenses" / path)} - present = matches & {file["path"] for file in files} - if not present: - raise gate.GateError(gate.CAPTURE_INCOMPLETE, f"{leg}: {name} declared license is missing") - candidates.update(present) - texts = {} - hashes = {} - total = 0 - for path in sorted(candidates): - data = read_file(path) - total += len(data) - if total > 16 * 1024 * 1024: - raise gate.GateError(gate.CAPTURE_INCOMPLETE, f"{leg}: {name} license text set is oversized") - try: - texts[path] = data.decode("utf-8") - except UnicodeError as error: - raise gate.GateError(gate.CAPTURE_INCOMPLETE, f"{leg}: {name} license text is undecodable") from error - hashes[path] = hashlib.sha256(data).hexdigest() - source_sha = hashlib.sha256(gate.canonical_json({"name": name, "version": version, - "files": files})).hexdigest() - key = f"pypi/{name}@{version}" - evidence = {"source_sha256": source_sha, - "license_expression": message.get("License-Expression", ""), - "license": message.get("License", ""), - "classifiers": message.get_all("Classifier", []), - "license_texts": texts, "license_member_sha256": hashes, - "archive_members": [{"type": "file", "name": file["path"], "linkname": ""} - for file in files], - "install_hook_sources": {}, - "parsed_inputs": [file["path"] for file in files if file["path"].endswith(".py")], - "native_libraries": [{**row, "path": row["path"].removeprefix(f"{name}/")} - for row in native_rows if row["path"].startswith(f"{name}/")], - "known_vulnerabilities": []} - failures, decision, source = gate.evaluate_dependency_license(evidence, key, None) - if failures: - raise gate.GateError(failures[0].code, f"{leg}: {key}: {failures[0].detail}") - native_failures, native_properties = gate.evaluate_native_links( - evidence, key, target=target, leg=leg) - if native_failures: - raise gate.GateError(native_failures[0].code, f"{leg}: {key}: {native_failures[0].detail}") - fixture_key = f"{key}/sha256/{source_sha}" - fixture = gate.build_fixture(gate.Dependency("pypi", name, version), evidence) - fixture["id"] = fixture_key - result.append({"key": fixture_key, "package_key": key, "name": name, - "version": version, "source_sha256": source_sha, - "license": decision.selected, "license_source": source, - "license_member_sha256": hashes, "fixture": fixture, - "native_properties": native_properties, - "fixture_sha256": gate.fixture_digest(fixture), - "legs": [leg], "snapshots": {leg: raw_sha}}) - return result - - -def _maturin_tool(item: Mapping[str, Any], folder: Path) -> dict[str, Any]: - """Bind the actual build executable to reviewed v1.15.0 release assets.""" - leg = item["leg"] - receipt_bytes = (folder / f"{leg}.build-first.json").read_bytes() - second_bytes = (folder / f"{leg}.build-second.json").read_bytes() - members = item.get("members", {}) - if (not isinstance(members, Mapping) - or any(members.get(f"{leg}.build-{name}.json") != hashlib.sha256(raw).hexdigest() - for name, raw in (("first", receipt_bytes), ("second", second_bytes)))): - raise gate.GateError(gate.SOURCE_HASH_MISMATCH, f"{leg}: maturin receipts changed after transport") - receipt = _json_bytes(receipt_bytes) - second = _json_bytes(second_bytes) - data = _json_bytes(Path(__file__).with_name("release_maturin_tool_evidence.json").read_bytes()) - if (not isinstance(data, Mapping) - or data.get("source_repository") != "PyO3/maturin" - or data.get("tag") != "v1.15.0" - or not isinstance(data.get("tag_commit"), str) - or re.fullmatch(r"[0-9a-f]{40}", data["tag_commit"]) is None - or not isinstance(data.get("source_archive_sha256"), str) - or re.fullmatch(r"[0-9a-f]{64}", data["source_archive_sha256"]) is None): - raise gate.GateError( - gate.SOURCE_HASH_MISMATCH, "maturin source provenance is malformed" - ) - target = "x86_64-unknown-linux-gnu" if leg == "sdist" else leg.rsplit("-py", 1)[0] - build_env = receipt.get("build_env") if isinstance(receipt, Mapping) else None - if not isinstance(build_env, str): - raise gate.GateError(gate.SCOPE_UNVERIFIABLE, f"{leg}: maturin build environment is missing") - if target == "universal2-apple-darwin": - key = f"{target}/{build_env.rsplit('/', 1)[-1]}" - valid_env = build_env.startswith("runner:") - elif leg == "sdist": - key = target - valid_env = build_env.startswith("runner:") and build_env.endswith("/X64") - elif target == "x86_64-pc-windows-msvc": - key = target - valid_env = build_env.startswith("runner:") and build_env.endswith("/X64") - else: - key = target - valid_env = build_env.startswith("container:") - asset = data.get("assets", {}).get(key) if isinstance(data, Mapping) else None - if (not valid_env or not isinstance(asset, Mapping) or not isinstance(second, Mapping) - or data.get("schema") != "cwl.release-maturin-tool/1" - or data.get("version") != "1.15.0" - or receipt.get("maturin_version") != "maturin 1.15.0" - or receipt.get("maturin_binary_sha256") != asset.get("binary_sha256") - or any(second.get(field) != receipt.get(field) for field in - ("build_env", "maturin_version", "maturin_binary_sha256")) - or not isinstance(asset.get("asset_sha256"), str) - or not re.fullmatch(r"[0-9a-f]{64}", asset["asset_sha256"])): - raise gate.GateError(gate.SOURCE_HASH_MISMATCH, f"{leg}: maturin executable differs from reviewed asset") - link_rows = asset.get("native_links") - if (not isinstance(link_rows, list) or len(link_rows) != 1 - or not isinstance(link_rows[0], Mapping) - or not isinstance(link_rows[0].get("needed"), list)): - raise gate.GateError(gate.NATIVE_LINK_UNKNOWN, f"{leg}: maturin native links are missing") - sha = asset["binary_sha256"] - texts = data["license_texts"] - evidence = {"source_sha256": sha, "license_expression": data["license_expression"], - "license_texts": texts, - "license_member_sha256": {name: hashlib.sha256(text.encode()).hexdigest() - for name, text in texts.items()}, - "archive_members": [{"type": "file", "name": "maturin", "linkname": ""}], - "install_hook_sources": {}, "parsed_inputs": [], - "native_libraries": [{"path": "maturin", "needed": link_rows[0]["needed"], - "static_archives": []}], "known_vulnerabilities": []} - package_key = "github-release/maturin@1.15.0" - failures, decision, source = gate.evaluate_dependency_license( - evidence, package_key, - {"chosen": data["license_choice"], "rationale": data["license_rationale"]}, - ) - if failures: - raise gate.GateError(failures[0].code, f"{leg}: maturin licence: {failures[0].detail}") - native_failures, native_properties = gate.evaluate_native_links( - evidence, package_key, target=target, leg=leg) - if native_failures: - raise gate.GateError(native_failures[0].code, - f"{leg}: maturin native links: {native_failures[0].detail}") - fixture_key = f"{package_key}/sha256/{sha}" - fixture = gate.build_fixture(gate.Dependency("github-release", "maturin", "1.15.0"), evidence) - fixture["id"] = fixture_key - return {"key": fixture_key, "package_key": package_key, "name": "maturin", - "version": "1.15.0", "source_sha256": sha, - "license": decision.selected, "license_source": source, - "license_member_sha256": evidence["license_member_sha256"], - "native_properties": native_properties, - "fixture": fixture, "fixture_sha256": gate.fixture_digest(fixture), - "source_tag_commit": data["tag_commit"], - "source_archive_sha256": data["source_archive_sha256"], - "asset_archive_sha256": asset["asset_sha256"], - "legs": [leg], "build_envs": {leg: build_env}} - - -def prescreen(scope: Any, root: Path) -> dict[str, list[dict[str, Any]]]: - """Rebind every wheel byte and apply the existing licence decision path.""" - variants = scope.get("verified_runtime_variants") if isinstance(scope, Mapping) else None - if (not isinstance(scope, Mapping) - or not isinstance(scope.get("verified_scope_evidence"), list) - or len(scope["verified_scope_evidence"]) != 13 - or not isinstance(variants, list) or len(variants) != 3): - raise gate.GateError(gate.SCOPE_UNVERIFIABLE, "verified scope evidence is incomplete") - rows: dict[tuple[str, str], dict[str, Any]] = {} - build_rows: dict[str, dict[str, Any]] = {} - tool_rows: dict[str, dict[str, Any]] = {} - seen_legs: set[str] = set() - seen_variants: set[str] = set() - for index, item in enumerate([*scope["verified_scope_evidence"], *variants]): - variant = index >= 13 - if (not isinstance(item, Mapping) or not isinstance(item.get("leg"), str) - or not re.fullmatch(r"[A-Za-z0-9_.+-]+", item["leg"]) - or item["leg"] in {".", ".."} - or (item["leg"] in (seen_variants if variant else seen_legs)) - or item.get("artifact_name") != ( - f"repro-macos-x86-{item['leg']}" if variant else f"repro-digest-{item['leg']}") - or variant and (item.get("arch") != "x86_64" - or not item["leg"].startswith("universal2-apple-darwin-py")) - or not isinstance(item.get("archives"), list)): - raise gate.GateError(gate.SCOPE_UNVERIFIABLE, "scope evidence row is malformed") - leg = item["leg"] - if leg != "sdist" and leg.rpartition("-py")[0] not in TARGET_ARCHES: - raise gate.GateError(gate.SCOPE_UNVERIFIABLE, "runtime archive coverage is incomplete") - runtime_architecture = None - if leg != "sdist": - runtime_name = f"{leg}.runtime.json" - runtime_path = gate._require_regular_file(root / item["artifact_name"] / runtime_name, - gate.SCOPE_UNVERIFIABLE) - if runtime_path.stat().st_size > 1024 * 1024: - raise gate.GateError(gate.SCOPE_UNVERIFIABLE, f"{leg}: runtime receipt is oversized") - runtime_bytes = runtime_path.read_bytes() - if item.get("members", {}).get(runtime_name) != hashlib.sha256(runtime_bytes).hexdigest(): - raise gate.GateError(gate.SOURCE_HASH_MISMATCH, f"{leg}: runtime receipt changed after transport") - try: - runtime_architecture = _runtime_target_architecture(_json_bytes(runtime_bytes), leg, intel=variant) - except DistributionSetError as error: - raise gate.GateError(gate.SCOPE_UNVERIFIABLE, str(error)) from error - if variant: - seen_variants.add(leg) - else: - seen_legs.add(leg) - for package in _build_packages(item, root / item["artifact_name"]): - if package["key"] in build_rows: - build_rows[package["key"]]["legs"].append(leg) - build_rows[package["key"]]["snapshots"][leg] = package["snapshots"][leg] - else: - build_rows[package["key"]] = package - tool = _maturin_tool(item, root / item["artifact_name"]) - if tool["key"] in tool_rows: - tool_rows[tool["key"]]["legs"].append(leg) - tool_rows[tool["key"]]["build_envs"][leg] = tool["build_envs"][leg] - else: - tool_rows[tool["key"]] = tool - if (leg == "sdist" and item["archives"] - or leg != "sdist" and not item["archives"]): - raise gate.GateError(gate.SCOPE_UNVERIFIABLE, f"{leg}: runtime archive set is incomplete") - for archive in item["archives"]: - if (not isinstance(archive, Mapping) - or set(archive) != {"file", "size", "sha256", "name", "version"} - or not isinstance(archive["file"], str) - or not re.fullmatch(r"[A-Za-z0-9_.+-]+\.whl", archive["file"]) - or not isinstance(archive["name"], str) - or not isinstance(archive["version"], str) - or not isinstance(archive["sha256"], str) - or not re.fullmatch(r"[0-9a-f]{64}", archive["sha256"]) - or type(archive["size"]) is not int or archive["size"] <= 0): - raise gate.GateError(gate.SCOPE_UNVERIFIABLE, f"{leg}: archive identity is malformed") - path = root / item["artifact_name"] / archive["file"] - raw = gate.read_archive_snapshot(path) - sha = hashlib.sha256(raw).hexdigest() - if sha != archive["sha256"] or len(raw) != archive["size"]: - raise gate.GateError(gate.SOURCE_HASH_MISMATCH, f"{leg}: archive bytes changed after transport") - key = f"pypi/{archive['name']}@{archive['version']}" - identity = (key, sha) - bound = gate.archive_license_evidence(raw, "pypi") - if bound["source_sha256"] != sha: - raise gate.GateError(gate.SOURCE_HASH_MISMATCH, f"{key}: licence evidence changed") - native_libraries = _native_wheel_libraries( - raw, leg.rsplit("-py", 1)[0], required_architecture=runtime_architecture, - ) - if identity in rows: - if leg not in rows[identity]["legs"]: - rows[identity]["legs"].append(leg) - continue - declared = gate.distribution_declared_metadata(path, archive["name"], archive["version"]) - member_names = [member["name"] for member in bound["archive_members"] - if member["type"] == "file"] - evidence = {**declared, **bound, - "install_hook_sources": {name: "" for name in member_names - if name.endswith(("/setup.py", "/build.rs"))}, - "parsed_inputs": [name for name in member_names if name.endswith(".py")], - "native_libraries": native_libraries, - "known_vulnerabilities": []} - failures, decision, source = gate.evaluate_dependency_license( - evidence, key, None, - ) - if failures: - raise gate.GateError(failures[0].code, f"{key}: {failures[0].detail}") - native_failures, native_properties = gate.evaluate_native_links( - evidence, key, target=leg.rsplit("-py", 1)[0], leg=leg) - if native_failures: - raise gate.GateError(native_failures[0].code, f"{key}: {native_failures[0].detail}") - fixture_key = f"{key}/sha256/{sha}" - fixture = gate.build_fixture(gate.Dependency("pypi", archive["name"], archive["version"]), evidence) - fixture["id"] = fixture_key - rows[identity] = {"key": fixture_key, "package_key": key, "name": archive["name"], - "version": archive["version"], "source_sha256": sha, - "license": decision.selected, "license_source": source, - "license_member_sha256": bound["license_member_sha256"], - "native_properties": native_properties, - "fixture": fixture, "fixture_sha256": gate.fixture_digest(fixture), - "legs": [leg]} - if (len(seen_legs) != 13 or "sdist" not in seen_legs - or seen_variants != {f"universal2-apple-darwin-py{version}" - for version in ("3.12", "3.13", "3.14")} - or not rows): - raise gate.GateError(gate.SCOPE_UNVERIFIABLE, "runtime archive coverage is incomplete") - return {"archives": sorted(rows.values(), key=lambda row: (row["key"], row["source_sha256"])), - "build_packages": sorted(build_rows.values(), key=lambda row: row["key"]), - "build_tools": sorted(tool_rows.values(), key=lambda row: row["key"])} - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--verified-scope", required=True) - parser.add_argument("--scope-root", required=True) - parser.add_argument("--output", required=True) - args = parser.parse_args() - output = Path(args.output) - if output.exists() or output.is_symlink(): - raise gate.GateError(gate.CAPTURE_INCOMPLETE, "archive license output already exists") - result = prescreen(_json_bytes(Path(args.verified_scope).read_bytes()), Path(args.scope_root)) - output.write_text(json.dumps({"schema": "cwl.release-runtime-archive-licenses/3", - **result}, indent=2, sort_keys=True) + "\n") - - -if __name__ == "__main__": - main() diff --git a/scripts/ci/release_dependency_capture_raw.sh b/scripts/ci/release_dependency_capture_raw.sh deleted file mode 100755 index 3fbb65f048..0000000000 --- a/scripts/ci/release_dependency_capture_raw.sh +++ /dev/null @@ -1,410 +0,0 @@ -#!/usr/bin/env bash -# Collect raw pre-publish dependency evidence for the central release gate (#2342). -# -# This script only *runs tools and writes their output verbatim*. Every decision -# — license policy, lock/environment reconciliation, archive-escape and -# install-hook detection, Strix binding validation — lives in the unit-tested -# scripts/ci/release_dependency_gate.py, which reads what this writes. Keeping -# the split that way means no untested shell ever decides whether a release may -# publish. -# -# It requires a runner: pip, cargo, readelf, and network access to the indexes. -# It is therefore exercised in GitHub Actions only; see -# .github/workflows/release-dependency-license-strix-gate.yml. -# -# Output layout (consumed by `release_dependency_gate.py capture` and `gate`): -# -# /python/lock.txt the hash-pinned lock that was collected -# /python/installed.json declared identity/licence per fetched -# distribution, in `pip inspect` shape -# /cargo/Cargo.lock the committed Cargo lock -# /cargo/metadata.json cargo metadata --format-version 1 --locked -# //metadata.json declared identity + license fields -# //source.sha256 sha256 of the distribution as fetched -# //members.txt "\t\t" per member -# //licenses/* bundled LICENSE/COPYING/NOTICE verbatim -# //hooks/* setup.py / build.rs sources verbatim -# //native.json dynamic/static link targets per shipped .so -# //parsed_inputs.txt file names the dependency parses - -set -euo pipefail - -RAW_ROOT="" -CAPTURE_ROOT="" -ECOSYSTEMS="" -PYTHON_LOCK="" -PYTHON_INTERPRETER="" -CARGO_MANIFEST="" -CARGO_DEV_MANIFEST="" -DOWNLOAD_ROOT="" -LICENSE_REPORT="" -MODE="capture" - -while [ "$#" -gt 0 ]; do - case "$1" in - --raw-root) RAW_ROOT="$2"; shift 2 ;; - --capture-root) CAPTURE_ROOT="$2"; shift 2 ;; - --ecosystems) ECOSYSTEMS="$2"; shift 2 ;; - --python-lock) PYTHON_LOCK="$2"; shift 2 ;; - --python-interpreter) PYTHON_INTERPRETER="$2"; shift 2 ;; - --cargo-manifest) CARGO_MANIFEST="$2"; shift 2 ;; - --cargo-dev-manifest) CARGO_DEV_MANIFEST="$2"; shift 2 ;; - --download-root) DOWNLOAD_ROOT="$2"; shift 2 ;; - --license-report) LICENSE_REPORT="$2"; shift 2 ;; - --install-gated) MODE="install"; shift ;; - *) echo "ERROR: unknown argument $1" >&2; exit 2 ;; - esac -done - -if [ "$MODE" = "install" ]; then - if [ -z "$PYTHON_LOCK" ] || [ ! -f "$PYTHON_LOCK" ] || [ -z "$DOWNLOAD_ROOT" ]; then - echo "ERROR: --install-gated requires --python-lock and --download-root." >&2 - exit 2 - fi - if [ -z "$LICENSE_REPORT" ] || [ -z "$CAPTURE_ROOT" ]; then - echo "ERROR: --install-gated requires --license-report and --capture-root." >&2 - exit 2 - fi -else - if [ -z "$RAW_ROOT" ] || [ -z "$CAPTURE_ROOT" ] || [ -z "$ECOSYSTEMS" ]; then - echo "ERROR: --raw-root, --capture-root and --ecosystems are required." >&2 - exit 2 - fi - mkdir -p "$RAW_ROOT" "$CAPTURE_ROOT" -fi - -# The pip entry point is a variable only so the wiring can be regression-tested -# without a network: a test points RELEASE_GATE_PIP at a recorder and asserts -# which pip invocations happened, and in what order, for a refused release. -PIP=(python3 -m pip) -if [ -n "${RELEASE_GATE_PIP:-}" ]; then - PIP=("${RELEASE_GATE_PIP}") -fi - -# Resolved from this script's own directory, never from the caller's cwd or an -# environment variable, so the trusted gate cannot be swapped by a PR. -GATE_SCRIPT="$(cd -- "$(dirname -- "$0")" && pwd)/release_dependency_gate.py" - -# pip's global --python re-executes pip against another interpreter, which is how -# the lock-only virtual environment is installed into by the gated install mode. -PIP_TARGET_ARGS=() -if [ -n "$PYTHON_INTERPRETER" ]; then - # `python3 -m venv` uses symlinks by default on POSIX, so a normal virtual - # environment's bin/python *is* a symlink; refusing symlinks outright rejected - # every real venv and made this path unreachable. What must be refused is a - # target that is not a regular executable file, or a dangling link, so the link - # is resolved and the resolved target is checked. - resolved_interpreter="$(cd -- "$(dirname -- "$PYTHON_INTERPRETER")" 2>/dev/null && pwd -P)/$(basename -- "$PYTHON_INTERPRETER")" - while [ -L "$resolved_interpreter" ]; do - link_target="$(readlink -- "$resolved_interpreter")" - case "$link_target" in - /*) resolved_interpreter="$link_target" ;; - *) resolved_interpreter="$(dirname -- "$resolved_interpreter")/$link_target" ;; - esac - done - if [ ! -f "$resolved_interpreter" ] || [ ! -x "$resolved_interpreter" ]; then - echo "ERROR: --python-interpreter must resolve to a regular executable interpreter." >&2 - exit 2 - fi - PIP_TARGET_ARGS=(--python "$PYTHON_INTERPRETER") -fi - -# Record one archive's members as "\t\t". Symlink and -# hardlink targets are preserved verbatim so the gate can detect escapes. -record_members() { - local archive="$1" destination="$2" - case "$archive" in - *.whl | *.zip) - unzip -Z1 "$archive" | while IFS= read -r member; do - printf 'file\t%s\t\n' "$member" - done - ;; - *) - tar -tvf "$archive" | while IFS= read -r line; do - local permissions name link type - permissions="${line%% *}" - name="$(printf '%s' "$line" | sed -E 's/^.* [0-9]{2}:[0-9]{2} //')" - link="" - type="file" - case "$permissions" in - l*) type="symlink"; link="${name#* -> }"; name="${name%% -> *}" ;; - h*) type="hardlink"; link="${name#* link to }"; name="${name%% link to *}" ;; - d*) type="directory" ;; - esac - printf '%s\t%s\t%s\n' "$type" "$name" "$link" - done - ;; - esac >"$destination" -} - -# Record every bundled license-like file verbatim, flattened into one directory. -record_license_files() { - local root="$1" destination="$2" - mkdir -p "$destination" - find "$root" -maxdepth 4 -type f \ - \( -iname 'LICENSE*' -o -iname 'COPYING*' -o -iname 'NOTICE*' \) -print0 | - while IFS= read -r -d '' found; do - cp -- "$found" "$destination/$(printf '%s' "${found#"$root"/}" | tr '/' '_')" - done -} - -# Record install/build hook sources verbatim so the gate can inspect them. -record_hook_sources() { - local root="$1" destination="$2" - mkdir -p "$destination" - find "$root" -maxdepth 3 -type f \ - \( -name 'setup.py' -o -name 'build.rs' -o -name 'conanfile.py' \) -print0 | - while IFS= read -r -d '' found; do - cp -- "$found" "$destination/$(printf '%s' "${found#"$root"/}" | tr '/' '_')" - done -} - -# Record dynamic NEEDED entries and shipped static archives for native libraries. -record_native_libraries() { - local root="$1" destination="$2" - local entries="[]" - while IFS= read -r library; do - local needed - needed="$(readelf -d "$library" 2>/dev/null | - sed -n 's/.*(NEEDED).*\[\(.*\)\]/\1/p' | - jq -R . | jq -s .)" - entries="$(jq --arg path "${library#"$root"/}" --argjson needed "${needed:-[]}" \ - '. + [{"path": $path, "needed": $needed, "static_archives": []}]' <<<"$entries")" - done < <(find "$root" -type f \( -name '*.so' -o -name '*.so.*' -o -name '*.pyd' \)) - printf '%s\n' "$entries" >"$destination" -} - -capture_python() { - local lock="$1" - mkdir -p "$CAPTURE_ROOT/python" "$DOWNLOAD_ROOT" - cp -- "$lock" "$CAPTURE_ROOT/python/lock.txt" - - local plain_requirements - plain_requirements="$DOWNLOAD_ROOT/pins-without-hashes.txt" - # Fetch by exact pin with hash checking deliberately disabled, then hash the - # bytes here and compare against the lock in the gate. Downloading *with* - # --require-hashes would make pip itself reject a tampered distribution, so - # the gate could never observe SOURCE_HASH_MISMATCH. The install of these same - # bytes happens later, offline and *with* --require-hashes, in install_gated. - sed -E 's/\\$//' "$lock" | grep -oE '^[A-Za-z0-9._-]+==[^ ;]+' \ - >"$plain_requirements" - # The real lock may carry --index-url, --extra-index-url or --find-links, - # while this reconstructed plain file has none of them. Dropping them silently - # made collection resolve from a different source than install. The trusted - # gate therefore parses and *validates* those directives — allowed HTTPS - # origin, no userinfo, bounded relative path — and emits them one per line; - # anything unsupported or untrusted fails here rather than being dropped. - # mapfile keeps each value a single argv element, so no lock content is ever - # word-split or re-interpreted by this shell. This runs before the first - # network action, so a refused directive means nothing was ever fetched. - local -a source_options=() - if [ ! -f "$GATE_SCRIPT" ] || [ -L "$GATE_SCRIPT" ]; then - echo "ERROR: trusted gate script is missing beside this script." >&2 - exit 2 - fi - # Deliberately not `mapfile < <(python3 ...)`: inside process substitution the - # validator's exit status is discarded by set -e, so a refusal would be read as - # "no options" and collection would continue from the default index — the same - # silent drop this fix exists to remove. The status is checked explicitly. - local options_file="$DOWNLOAD_ROOT/validated-source-options.txt" - if ! python3 -I "$GATE_SCRIPT" lock-source-options \ - --lock "$lock" --permitted-root "$(dirname -- "$lock")" >"$options_file"; then - echo "ERROR: lock source directives failed validation; refusing to collect." >&2 - exit 2 - fi - mapfile -t source_options <"$options_file" - # --only-binary=:all: is not only a build-hook guard for the gate environment: - # `pip download` executes an sdist's build backend to get its metadata even - # with --no-deps, so a wheel-only collection is what keeps unadjudicated - # dependency code from running before the licence stage. - "${PIP[@]}" download --no-deps --only-binary=:all: \ - "${source_options[@]}" \ - --dest "$DOWNLOAD_ROOT" -r "$plain_requirements" >/dev/null - - # The enumeration and the licence fields both come from the *fetched - # distributions*, never from `pip inspect` of an installed environment: the - # closure is not installed yet at this point, and must not be until the - # licence stage has passed. The file keeps the `pip inspect` shape the gate - # already reconciles against the lock. - local installed="$CAPTURE_ROOT/python/installed.json" - printf '{"installed": []}\n' >"$installed" - while IFS= read -r pin; do - [ -n "$pin" ] || continue - local name version slug target distribution extracted - name="${pin%%==*}" - version="${pin#*==}" - slug="pypi__$(printf '%s' "$name" | tr '[:upper:]' '[:lower:]' | tr '._' '--')__$version" - target="$RAW_ROOT/$slug" - mkdir -p "$target" - distribution="$(find "$DOWNLOAD_ROOT" -maxdepth 1 -type f \ - -iname "$(printf '%s' "$name" | tr '.-' '__')-${version}*" | head -n 1)" - if [ -z "$distribution" ]; then - echo "ERROR: no fetched distribution for ${name}==${version}" >&2 - exit 2 - fi - cp -- "$distribution" "$target/source.archive" - sha256sum "$target/source.archive" | cut -d' ' -f1 >"$target/source.sha256" - record_members "$distribution" "$target/members.txt" - extracted="$(mktemp -d)" - case "$distribution" in - *.whl) unzip -qq -o "$distribution" -d "$extracted" ;; - *) tar -xf "$distribution" -C "$extracted" ;; - esac - record_license_files "$extracted" "$target/licenses" - record_hook_sources "$extracted" "$target/hooks" - record_native_libraries "$extracted" "$target/native.json" - find "$extracted" -maxdepth 3 -type f -name '*.py' -printf '%P\n' | - LC_ALL=C sort >"$target/parsed_inputs.txt" - printf '{}\n' >"$target/bundled_library_licenses.json" - # Licence metadata is read out of the distribution's own METADATA/PKG-INFO - # by the trusted gate, which also re-checks that the archive declares the - # pinned name and version. A file whose metadata names another project - # fails here instead of being adjudicated under the wrong identity. - python3 -I "$GATE_SCRIPT" distribution-metadata \ - --distribution "$distribution" --name "$name" --version "$version" \ - >"$target/metadata.json" - jq --slurpfile declared "$target/metadata.json" \ - '.installed += [{"metadata": $declared[0]}]' "$installed" \ - >"$installed.next" - mv -- "$installed.next" "$installed" - rm -rf "${extracted:?}" - done <"$plain_requirements" -} - -# Install exactly the distributions the licence stage already judged: offline, -# from the collected bytes, with --require-hashes so pip itself proves each file -# matches the lock. No index is consulted and nothing is re-resolved or -# re-downloaded, so the installed bytes are the inspected bytes by construction — -# which a second hash-less download could not establish for a multi-hash lock. -install_gated() { - local lock="$1" - if [ -z "$LICENSE_REPORT" ]; then - echo "ERROR: --install-gated requires --license-report." >&2 - exit 2 - fi - if [ ! -f "$GATE_SCRIPT" ] || [ -L "$GATE_SCRIPT" ]; then - echo "ERROR: trusted gate script is missing beside this script." >&2 - exit 2 - fi - if [ ! -d "$DOWNLOAD_ROOT" ]; then - echo "ERROR: no collected distributions to install from: $DOWNLOAD_ROOT" >&2 - exit 2 - fi - if [ -z "$CAPTURE_ROOT" ]; then - echo "ERROR: --install-gated requires --capture-root to bind the judged lock." >&2 - exit 2 - fi - # The report alone is not permission: bind-install refuses unless the lock still - # digests to what the verdict read, every judged artifact is present in the - # collected root by digest, and the root holds nothing else. It then pins each - # project to the single judged digest, so a lock recording several hashes for one - # project cannot admit an artifact whose licence and contents were never judged. - local bound_requirements="$DOWNLOAD_ROOT/gated-requirements.txt" - if ! python3 -I "$GATE_SCRIPT" bind-install \ - --report "$LICENSE_REPORT" \ - --capture "$CAPTURE_ROOT" \ - --download-root "$DOWNLOAD_ROOT" \ - --output "$bound_requirements" >/dev/null; then - echo "ERROR: the licence verdict does not authorize installing these bytes." >&2 - exit 2 - fi - "${PIP[@]}" "${PIP_TARGET_ARGS[@]}" install \ - --require-hashes --only-binary=:all: --no-index \ - --find-links "$DOWNLOAD_ROOT" \ - -r "$bound_requirements" -} - -capture_cargo() { - local manifest="$1" workspace_root - local cargo_root="$CAPTURE_ROOT/${2:-cargo}" - mkdir -p "$cargo_root" - cargo metadata --format-version 1 --locked --manifest-path "$manifest" \ - >"$cargo_root/metadata.json" - workspace_root="$(jq -er '.workspace_root | select(type == "string" and startswith("/"))' \ - "$cargo_root/metadata.json")" - cp -- "$workspace_root/Cargo.lock" "$cargo_root/Cargo.lock" - cargo fetch --locked --manifest-path "$manifest" >/dev/null - - while IFS=$'\t' read -r name version license; do - local slug target crate extracted - slug="cargo__${name}__${version}" - target="$RAW_ROOT/$slug" - mkdir -p "$target" - crate="$(find "${CARGO_HOME:-$HOME/.cargo}/registry/cache" -type f \ - -name "${name}-${version}.crate" | head -n 1)" - if [ -z "$crate" ]; then - echo "ERROR: no fetched crate for ${name} ${version}" >&2 - exit 2 - fi - cp -- "$crate" "$target/source.archive" - sha256sum "$target/source.archive" | cut -d' ' -f1 >"$target/source.sha256" - record_members "$crate" "$target/members.txt" - extracted="$(mktemp -d)" - tar -xf "$crate" -C "$extracted" - record_license_files "$extracted" "$target/licenses" - record_hook_sources "$extracted" "$target/hooks" - printf '[]\n' >"$target/native.json" - printf '{}\n' >"$target/bundled_library_licenses.json" - find "$extracted" -maxdepth 3 -type f -name '*.rs' -printf '%P\n' | - LC_ALL=C sort >"$target/parsed_inputs.txt" - local inclusion='["wheel"]' - if [ "${2:-cargo}" = "cargo-dev" ]; then - inclusion='["dev"]' - if [ -f "$target/metadata.json" ]; then - inclusion="$(jq -c '(.distribution_inclusion + ["dev"]) | unique' "$target/metadata.json")" - fi - fi - jq -n --arg name "$name" --arg version "$version" --arg license "$license" \ - --argjson inclusion "$inclusion" '{ - ecosystem: "cargo", - name: $name, - version: $version, - license_expression: $license, - license: "", - classifiers: [], - distribution_inclusion: $inclusion, - known_vulnerabilities: [] - }' >"$target/metadata.json" - rm -rf "${extracted:?}" - done < <(jq -r '.packages[] | select(.source != null) | [.name, .version, (.license // "")] | @tsv' \ - "$cargo_root/metadata.json") -} - -if [ "$MODE" = "install" ]; then - install_gated "$PYTHON_LOCK" - echo "Installed the prescreened release closure from collected bytes." - exit 0 -fi - -case ",${ECOSYSTEMS}," in -*,python,*) - if [ -z "$PYTHON_LOCK" ] || [ ! -f "$PYTHON_LOCK" ]; then - echo "ERROR: --python-lock must name the hash-pinned release lock." >&2 - exit 2 - fi - if [ -z "$DOWNLOAD_ROOT" ]; then - echo "ERROR: --download-root is required so the gated install reuses these bytes." >&2 - exit 2 - fi - capture_python "$PYTHON_LOCK" - ;; -esac - -case ",${ECOSYSTEMS}," in -*,cargo,*) - if [ -z "$CARGO_MANIFEST" ] || [ ! -f "$CARGO_MANIFEST" ]; then - echo "ERROR: --cargo-manifest must name the release Cargo.toml." >&2 - exit 2 - fi - capture_cargo "$CARGO_MANIFEST" - if [ -n "$CARGO_DEV_MANIFEST" ]; then - if [ ! -f "$CARGO_DEV_MANIFEST" ]; then - echo "ERROR: development Cargo manifest is absent." >&2 - exit 2 - fi - capture_cargo "$CARGO_DEV_MANIFEST" cargo-dev - fi - ;; -esac - -echo "Raw dependency capture complete: $(find "$RAW_ROOT" -mindepth 1 -maxdepth 1 -type d | wc -l) dependencies." diff --git a/scripts/ci/release_dependency_gate.py b/scripts/ci/release_dependency_gate.py deleted file mode 100644 index 96347278f6..0000000000 --- a/scripts/ci/release_dependency_gate.py +++ /dev/null @@ -1,3082 +0,0 @@ -#!/usr/bin/env python3 -"""Fail-closed pre-publish dependency gate for org releases (issue #2342). - -``scripts/ci/sbom_inventory_aggregator.py`` is a *scheduled, informational* org -SBOM roll-up: it flags GPL/AGPL/NOASSERTION components for governance, but it -is not per-dependency, not fail-closed, and not bound to a release head. This -module is the missing gate. It runs in -``.github/workflows/release-dependency-license-strix-gate.yml`` **before** a -release workflow publishes anything, and it either exits ``0`` or refuses the -release. There is no neutral outcome, no allow-failure, and no bypass. - -Design: the gate is a pure function over *captured* inputs. Workflow steps run -``pip inspect``, ``cargo metadata --locked``, archive listing, ``readelf -d``, -and Strix; each writes a file into a capture directory. This module only reads -files. That split keeps every deterministic decision unit-testable without a -runner and makes the Actions-only parts explicit instead of simulated. - -Capture layout (produced by the workflow, consumed here):: - - / - release.json source repository/SHA + artifact names - python/lock.txt the hash-pinned lock that was installed - python/installed.json `pip inspect` of the build environment - cargo/Cargo.lock the committed Cargo lock - cargo/metadata.json `cargo metadata --format-version 1 --locked` - evidence/.json per-dependency captured evidence - strix/bindings/.json per-dependency Strix structured binding - license-selections.json optional dual-license selections - -Every resolved dependency of both ecosystems must appear in the lock *and* in -the environment/build graph; any asymmetry fails ``LOCK_ENV_MISMATCH`` or -``CARGO_LOCK_GRAPH_MISMATCH``. No dependency is exempt: bootstrap tools such as -``pip`` are pinned in this organization's own ``*-hashes.txt`` files, so a lock -that omits an installed distribution is a defect, not a special case. - -Strix evidence is accepted **only** as a machine-readable binding. A textual -"0 findings" or "No exploitable vulnerabilities detected" is rejected -(``STRIX_TEXTUAL_PASS_REJECTED``), and a missing or malformed binding is a -failure rather than a neutral result. The binding's fail-closed shape and error -type follow ``scripts/ci/strix_evidence_binding.py``, which is imported from -this script's **own** directory so the gate behaves identically wherever the -trusted verifier is materialized. -""" - -from __future__ import annotations - -import argparse -import ast -import email.parser -import hashlib -import io -import json -import os -import re -import stat -import subprocess -import sys -import tarfile -import urllib.parse -import uuid -import zipfile -from dataclasses import dataclass, field -from pathlib import Path, PurePosixPath -from typing import Any, Iterable, Mapping, Sequence - -try: - import tomllib -except ModuleNotFoundError: # Python 3.10; already declared in the dev group. - import tomli as tomllib - -try: - from scripts.ci.spdx_license_policy import ( - LICENSE_MISSING, - LICENSE_SELECTION_INVALID, - LICENSE_TEXT_MISSING, - LICENSE_TEXT_UNVERIFIED, - LicenseDecision, - evaluate_license_expression, - recognize_license_text, - scan_license_text, - spdx_from_classifiers, - ) -except ImportError: # pragma: no cover - direct `python3 -I