From 4d3061261ae46dc850d1141a94a635c94daf017c Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Thu, 15 Oct 2026 23:16:54 +0200 Subject: [PATCH] ci(models): alert on repeated maintenance failures --- .github/scripts/maintenance-alert.mjs | 102 ++++++++++++++++++++ .github/workflows/maintenance-alert.yml | 36 ++++++++ .github/workflows/model-metadata.yml | 16 ++-- CHANGELOG.md | 2 + RELEASE.md | 16 ++++ package.json | 2 +- tests/test-maintenance-alert.mjs | 118 ++++++++++++++++++++++++ 7 files changed, 284 insertions(+), 8 deletions(-) create mode 100644 .github/scripts/maintenance-alert.mjs create mode 100644 .github/workflows/maintenance-alert.yml create mode 100644 tests/test-maintenance-alert.mjs diff --git a/.github/scripts/maintenance-alert.mjs b/.github/scripts/maintenance-alert.mjs new file mode 100644 index 0000000..d2a5f27 --- /dev/null +++ b/.github/scripts/maintenance-alert.mjs @@ -0,0 +1,102 @@ +const tracked = new Map([ + [ + "Command Code catalog sync", + { + key: "catalog", + title: "Maintenance: repeated catalog sync failures", + action: + "Inspect the catalog/parser and pricing reports separately. Reproduce with npm run check:commandcode-catalog and npm run check:commandcode-pricing. Review CLI schema changes and official pricing; never automatically copy or zero unknown prices.", + }, + ], + [ + "Latest host compatibility", + { + key: "hosts", + title: "Maintenance: repeated latest-host compatibility failures", + action: + "Inspect the recorded Pi/OMP versions and failing mock suite. Reproduce with those exact hosts, compare the pinned CI baseline, and add a regression test before changing the supported pins.", + }, + ], +]) + +/** Uses only trusted workflow identity/conclusions, never logs or artifact text. */ +export async function maintainAlert({ github, context }) { + const run = context.payload.workflow_run + const repo = context.repo + const defaultBranch = context.payload.repository.default_branch + const config = tracked.get(run?.name) + if ( + !config || + run.status !== "completed" || + run.head_branch !== defaultBranch || + run.repository?.full_name !== `${repo.owner}/${repo.repo}` || + run.head_repository?.full_name !== `${repo.owner}/${repo.repo}` || + !["schedule", "workflow_dispatch"].includes(run.event) + ) + return "ignored" + const marker = `` + const recent = await github.paginate(github.rest.actions.listWorkflowRuns, { + ...repo, + workflow_id: run.workflow_id, + branch: defaultBranch, + status: "completed", + per_page: 100, + }) + const eligible = recent + .filter( + (item) => + item.run_number <= run.run_number && + ["schedule", "workflow_dispatch"].includes(item.event) && + item.head_repository?.full_name === `${repo.owner}/${repo.repo}`, + ) + .sort((a, b) => b.run_number - a.run_number) + // An older completion/rerun must not overwrite a newer incident or recovery. + if ( + recent.some( + (item) => + item.run_number > run.run_number && ["schedule", "workflow_dispatch"].includes(item.event), + ) + ) + return "stale" + const issues = await github.paginate(github.rest.issues.listForRepo, { + ...repo, + state: "all", + creator: "github-actions[bot]", + per_page: 100, + }) + const issue = issues.find( + (item) => + !item.pull_request && + item.body?.startsWith(marker) && + item.user?.login === "github-actions[bot]", + ) + if (run.conclusion === "success") { + if (issue?.state === "open") { + await github.rest.issues.update({ + ...repo, + issue_number: issue.number, + state: "closed", + state_reason: "completed", + body: `${marker}\n\nRecovered in [run ${run.run_number}](https://github.com/${repo.owner}/${repo.repo}/actions/runs/${run.id}).\n\n${config.action}`, + }) + return "closed" + } + return "healthy" + } + const failed = (item) => ["failure", "timed_out"].includes(item?.conclusion) + if (!failed(run) || eligible.length < 2 || !failed(eligible[0]) || !failed(eligible[1])) + return "below-threshold" + const body = `${marker}\n\nTwo consecutive scheduled/manual default-branch runs failed.\n\n- Latest: [run ${run.run_number}](https://github.com/${repo.owner}/${repo.repo}/actions/runs/${run.id})\n- Previous: [run ${eligible[1].run_number}](https://github.com/${repo.owner}/${repo.repo}/actions/runs/${eligible[1].id})\n\n${config.action}\n\nThis issue is updated rather than duplicated and closes after a successful run. No logs, payloads or credentials are copied here. This alert does not authorize a release.` + if (issue) { + await github.rest.issues.update({ + ...repo, + issue_number: issue.number, + state: "open", + title: config.title, + body, + }) + return "updated" + } + await github.rest.issues.create({ ...repo, title: config.title, body, assignees: [repo.owner] }) + return "created" +} diff --git a/.github/workflows/maintenance-alert.yml b/.github/workflows/maintenance-alert.yml new file mode 100644 index 0000000..196d014 --- /dev/null +++ b/.github/workflows/maintenance-alert.yml @@ -0,0 +1,36 @@ +name: Maintenance alerts + +on: + workflow_run: + workflows: [Command Code catalog sync, Latest host compatibility] + types: [completed] + +permissions: + contents: read + actions: read + issues: write + +concurrency: + group: commandcode-maintenance-alerts + cancel-in-progress: false + +jobs: + alert: + if: >- + github.repository == 'patlux/pi-commandcode-provider' && + github.event.workflow_run.head_branch == github.event.repository.default_branch && + github.event.workflow_run.head_repository.full_name == github.repository && + (github.event.workflow_run.event == 'schedule' || github.event.workflow_run.event == 'workflow_dispatch') + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + # Always load trusted default-branch code, never the triggering run's ref/artifacts. + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - uses: actions/github-script@v7 + with: + script: | + const { maintainAlert } = await import(`${process.env.GITHUB_WORKSPACE}/.github/scripts/maintenance-alert.mjs`) + core.info(await maintainAlert({ github, context })) diff --git a/.github/workflows/model-metadata.yml b/.github/workflows/model-metadata.yml index 855f5d1..d47b56f 100644 --- a/.github/workflows/model-metadata.yml +++ b/.github/workflows/model-metadata.yml @@ -45,14 +45,15 @@ jobs: - run: npm ci - name: Compare with the latest Command Code CLI shell: bash - run: npm run check:commandcode-catalog | tee commandcode-catalog-report.md + run: npm run check:commandcode-catalog 2>&1 | tee commandcode-catalog-report.md - name: Publish catalog report if: always() - run: cat commandcode-catalog-report.md >> "$GITHUB_STEP_SUMMARY" + run: | + if [ -f commandcode-catalog-report.md ]; then cat commandcode-catalog-report.md >> "$GITHUB_STEP_SUMMARY"; fi - name: Compare official pricing if: always() shell: bash - run: npm run check:commandcode-pricing | tee commandcode-pricing-report.md + run: npm run check:commandcode-pricing 2>&1 | tee commandcode-pricing-report.md - name: Publish pricing report if: always() run: | @@ -75,17 +76,18 @@ jobs: - run: npm ci - name: Synchronize with the latest Command Code CLI shell: bash - run: npm run sync:commandcode-catalog | tee commandcode-catalog-report.md + run: npm run sync:commandcode-catalog 2>&1 | tee commandcode-catalog-report.md - name: Format and verify synchronized files run: | - npm run format -- src/commandcode-catalog.ts src/commandcode-catalog-overrides.ts README.md + npx --no-install prettier --write src/commandcode-catalog.ts src/commandcode-catalog-overrides.ts README.md npm run typecheck npm run test:models npm run format:check git diff --check - name: Publish catalog report if: always() - run: cat commandcode-catalog-report.md >> "$GITHUB_STEP_SUMMARY" + run: | + if [ -f commandcode-catalog-report.md ]; then cat commandcode-catalog-report.md >> "$GITHUB_STEP_SUMMARY"; fi - name: Create or update synchronization PR uses: peter-evans/create-pull-request@v8 with: @@ -112,7 +114,7 @@ jobs: - name: Compare official pricing if: always() shell: bash - run: npm run check:commandcode-pricing | tee commandcode-pricing-report.md + run: npm run check:commandcode-pricing 2>&1 | tee commandcode-pricing-report.md - name: Publish pricing report if: always() run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index bd4468d..252fc8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Open one actionable maintenance issue after repeated scheduled catalog/latest-host failures, update it without comment spam and close it after recovery. Keep report stderr, avoid broad formatting in catalog automation and document weekly release triage without automating publication. + - Refresh the static capability snapshot to `command-code@1.82.4`: add image input and reasoning efforts for `google/gemini-4-argon`, `meta/muse-wonder-1.0`, and `Qwen/Qwen3.8-Max-Prime`; replace the retired free Ling 3.1 Flash with `inclusionai/ling-3.1-flash`, preserving its reasoning efforts and 32,768-token output limit. Max Prime has a 131,072-token output limit (#153). - Fix catalog synchronization with the CLI's new model-object layout. Read explicit input modalities instead of the removed global text-only set, and project the CLI's additional `file` modality onto the host's supported text/image inputs (#153). - Add reviewed display pricing for Gemini 4 Argon, Muse Wonder, Qwen 3.8 Max Prime, paid Ling 3.1 Flash, Nemotron 3.5 Super VL, Muse Spark Open, and the free Cat Jam preview. Remove the retired free Ling entry. Prices and context tiers were verified against the official pricing page on 2026-10-15; the snapshot covers all 93 advertised models. Cat Jam is free only while its stealth preview lasts, with no published expiry. Models not yet present in CLI 1.82.4 keep conservative capability defaults until upstream metadata is available (#153). diff --git a/RELEASE.md b/RELEASE.md index fc2e369..9bc7420 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -56,6 +56,22 @@ GitHub collaborators can release through this workflow without personal npm publ These checks prevent accidental releases from the wrong commit; they are not a sandbox against a trusted writer who can modify workflows. Repository, tag and environment permissions remain the trust boundary. +## Maintenance review and release cadence + +Review open maintenance alerts and unreleased fixes at least weekly. A calendar reminder or a green catalog PR is not release authorization. Prioritize broken authentication/transport, missing paid-model pricing, and host compatibility regressions; propose a patch release when a verified user-facing fix is ready rather than waiting for unrelated features. + +The daily catalog workflow reports capability-parser drift separately from official-pricing drift. Two consecutive failed scheduled/manual default-branch runs create or reopen one bot-owned maintenance issue. Further failures update that issue; a successful run closes it. The same policy covers weekly latest-host checks when that workflow is installed. Fork/PR runs and canceled runs do not trigger alerts. Investigate the linked reports; do not silence alerts by accepting guessed prices or loosening parser checks. + +For every proposed release, record: + +- Which merged PRs and issues the release includes, with contributor credit. +- Whether live catalog/pricing checks pass; distinguish expected upstream drift from code failures. Review unknown prices and expired promotions explicitly. +- Host versions actually tested, local versus packed-package results, and any separately authorized live checks. Mock tests do not prove upstream performance or plan access. +- Any migration requirements and known limitations. Breaking entitlement/routing changes belong in a separately approved minor-version release while the package is pre-1.0, with a clearly labeled migration notice; do not slip them into a routine patch. +- The requested version/channel and release authorization before executing the existing tag/OIDC process. + +After publication, use the completion checklist below before reporting an issue as released. Keep “merged”, “tested from source” and “available on npm” distinct. + ## Prepare a release PR Confirm the requested version and channel, a clean/current checkout, the canonical repository and the maintainer's own GitHub identity. Record the current npm dist-tags and latest stable GitHub Release so prerelease verification can prove they were preserved: diff --git a/package.json b/package.json index 5b30347..0656867 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "LICENSE" ], "scripts": { - "test": "npm run typecheck && node --test tests/test-release.mjs tests/test-pi-child-cleanup.mjs && tsx tests/test-package-manifest.ts && tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-pricing-check.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-usage.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", + "test": "npm run typecheck && node --test tests/test-release.mjs tests/test-pi-child-cleanup.mjs tests/test-maintenance-alert.mjs && tsx tests/test-package-manifest.ts && tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-pricing-check.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-usage.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", "typecheck": "tsc --noEmit", "format:check": "prettier --check '**/*.{ts,mjs,json,md}'", "format": "prettier --write '**/*.{ts,mjs,json,md}'", diff --git a/tests/test-maintenance-alert.mjs b/tests/test-maintenance-alert.mjs new file mode 100644 index 0000000..1c8d2d4 --- /dev/null +++ b/tests/test-maintenance-alert.mjs @@ -0,0 +1,118 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { maintainAlert } from "../.github/scripts/maintenance-alert.mjs" + +function fixture({ + conclusion = "failure", + previous = "failure", + issues = [], + event = "schedule", + branch = "main", + fork = false, + extra = [], +} = {}) { + const writes = [] + const repo = { full_name: "patlux/pi-commandcode-provider" } + const run = { + name: "Command Code catalog sync", + status: "completed", + conclusion, + head_branch: branch, + repository: repo, + head_repository: fork ? { full_name: "fork/repo" } : repo, + event, + workflow_id: 7, + run_number: 12, + id: 120, + } + const methods = { + actions: { listWorkflowRuns: "runs" }, + issues: { + listForRepo: "issues", + create: async (value) => writes.push(value), + update: async (value) => writes.push(value), + }, + } + const github = { + rest: methods, + paginate: async (method) => + method === "runs" + ? [run, { ...run, run_number: 11, id: 110, conclusion: previous }, ...extra] + : issues, + } + const context = { + repo: { owner: "patlux", repo: "pi-commandcode-provider" }, + payload: { workflow_run: run, repository: { default_branch: "main" } }, + } + return { github, context, writes } +} +const incident = { + number: 42, + state: "open", + body: "\nold", + user: { login: "github-actions[bot]" }, +} + +test("creates an actionable issue after two failures without including raw logs", async () => { + const f = fixture() + assert.equal(await maintainAlert(f), "created") + assert.equal(f.writes.length, 1) + assert.match(f.writes[0].body, /actions\/runs\/120/) + assert.match(f.writes[0].body, /official pricing/) + assert.deepEqual(f.writes[0].assignees, ["patlux"]) +}) +test("updates/reopens the same bot-owned incident instead of duplicating", async () => { + for (const state of ["open", "closed"]) { + const f = fixture({ issues: [{ ...incident, state }] }) + assert.equal(await maintainAlert(f), "updated") + assert.equal(f.writes[0].issue_number, 42) + assert.equal(f.writes[0].state, "open") + } +}) +test("closes an incident after recovery", async () => { + const f = fixture({ conclusion: "success", issues: [incident] }) + assert.equal(await maintainAlert(f), "closed") + assert.equal(f.writes[0].state, "closed") +}) +test("does not notify on a single failure, canceled run, or missing predecessor", async () => { + for (const settings of [ + { previous: "success" }, + { previous: "cancelled" }, + { conclusion: "cancelled" }, + ]) { + const f = fixture(settings) + assert.equal(await maintainAlert(f), "below-threshold") + assert.equal(f.writes.length, 0) + } + const f = fixture() + f.github.paginate = async () => [] + assert.equal(await maintainAlert(f), "below-threshold") +}) +test("ignores forks, PRs and non-default branches", async () => { + for (const settings of [{ fork: true }, { event: "pull_request" }, { branch: "feature" }]) { + const f = fixture(settings) + assert.equal(await maintainAlert(f), "ignored") + assert.equal(f.writes.length, 0) + } +}) +test("ignores stale reruns after a newer completed run", async () => { + const f = fixture({ extra: [{ run_number: 13, event: "schedule", conclusion: "success" }] }) + assert.equal(await maintainAlert(f), "stale") + assert.equal(f.writes.length, 0) +}) +test("does not edit user-authored matching issues or PRs", async () => { + const f = fixture({ + issues: [ + { ...incident, user: { login: "someone" } }, + { ...incident, pull_request: {} }, + ], + }) + assert.equal(await maintainAlert(f), "created") + assert.equal(f.writes[0].issue_number, undefined) +}) +test("tracks host failures independently", async () => { + const f = fixture({ issues: [incident] }) + f.context.payload.workflow_run.name = "Latest host compatibility" + assert.equal(await maintainAlert(f), "created") + assert.match(f.writes[0].body, /maintenance:hosts/) +})