Repository navigation
ci(models): add deduplicated maintenance alerts and release triage #163
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| const tracked = new Map([ | ||
| [ | ||
| "Command Code catalog sync", | ||
| { | ||
| key: "catalog", | ||
| title: "Maintenance: repeated catalog sync failures", | ||
| action: | ||
| "Inspect the catalog/parser and pricing reports separately. Reproduce with npm run check:commandcode-catalog and npm run check:commandcode-pricing. Review CLI schema changes and official pricing; never automatically copy or zero unknown prices.", | ||
| }, | ||
| ], | ||
| [ | ||
| "Latest host compatibility", | ||
| { | ||
| key: "hosts", | ||
| title: "Maintenance: repeated latest-host compatibility failures", | ||
| action: | ||
| "Inspect the recorded Pi/OMP versions and failing mock suite. Reproduce with those exact hosts, compare the pinned CI baseline, and add a regression test before changing the supported pins.", | ||
| }, | ||
| ], | ||
| ]) | ||
|
|
||
| /** Uses only trusted workflow identity/conclusions, never logs or artifact text. */ | ||
| export async function maintainAlert({ github, context }) { | ||
| const run = context.payload.workflow_run | ||
| const repo = context.repo | ||
| const defaultBranch = context.payload.repository.default_branch | ||
| const config = tracked.get(run?.name) | ||
| if ( | ||
| !config || | ||
| run.status !== "completed" || | ||
| run.head_branch !== defaultBranch || | ||
| run.repository?.full_name !== `${repo.owner}/${repo.repo}` || | ||
| run.head_repository?.full_name !== `${repo.owner}/${repo.repo}` || | ||
| !["schedule", "workflow_dispatch"].includes(run.event) | ||
| ) | ||
| return "ignored" | ||
| const marker = `<!-- commandcode-maintenance:${config.key} -->` | ||
| const recent = await github.paginate(github.rest.actions.listWorkflowRuns, { | ||
| ...repo, | ||
| workflow_id: run.workflow_id, | ||
| branch: defaultBranch, | ||
| status: "completed", | ||
| per_page: 100, | ||
| }) | ||
| const eligible = recent | ||
| .filter( | ||
| (item) => | ||
| item.run_number <= run.run_number && | ||
| ["schedule", "workflow_dispatch"].includes(item.event) && | ||
| item.head_repository?.full_name === `${repo.owner}/${repo.repo}`, | ||
| ) | ||
| .sort((a, b) => b.run_number - a.run_number) | ||
| // An older completion/rerun must not overwrite a newer incident or recovery. | ||
| if ( | ||
| recent.some( | ||
| (item) => | ||
| item.run_number > run.run_number && ["schedule", "workflow_dispatch"].includes(item.event), | ||
| ) | ||
| ) | ||
| return "stale" | ||
| const issues = await github.paginate(github.rest.issues.listForRepo, { | ||
|
Owner
Author
There was a problem hiding this comment.
Same idea for |
||
| ...repo, | ||
| state: "all", | ||
| creator: "github-actions[bot]", | ||
| per_page: 100, | ||
| }) | ||
| const issue = issues.find( | ||
| (item) => | ||
| !item.pull_request && | ||
| item.body?.startsWith(marker) && | ||
| item.user?.login === "github-actions[bot]", | ||
| ) | ||
| if (run.conclusion === "success") { | ||
| if (issue?.state === "open") { | ||
| await github.rest.issues.update({ | ||
| ...repo, | ||
| issue_number: issue.number, | ||
| state: "closed", | ||
| state_reason: "completed", | ||
| body: `${marker}\n\nRecovered in [run ${run.run_number}](https://github.com/${repo.owner}/${repo.repo}/actions/runs/${run.id}).\n\n${config.action}`, | ||
|
Owner
Author
There was a problem hiding this comment. Recovery overwrites the body, so the failing-run links disappear from the issue once it is closed. A short comment ("Recovered in run N") plus close keeps the history — that is one comment per incident, not spam. |
||
| }) | ||
| return "closed" | ||
| } | ||
| return "healthy" | ||
| } | ||
| const failed = (item) => ["failure", "timed_out"].includes(item?.conclusion) | ||
| if (!failed(run) || eligible.length < 2 || !failed(eligible[0]) || !failed(eligible[1])) | ||
| return "below-threshold" | ||
| const body = `${marker}\n\nTwo consecutive scheduled/manual default-branch runs failed.\n\n- Latest: [run ${run.run_number}](https://github.com/${repo.owner}/${repo.repo}/actions/runs/${run.id})\n- Previous: [run ${eligible[1].run_number}](https://github.com/${repo.owner}/${repo.repo}/actions/runs/${eligible[1].id})\n\n${config.action}\n\nThis issue is updated rather than duplicated and closes after a successful run. No logs, payloads or credentials are copied here. This alert does not authorize a release.` | ||
| if (issue) { | ||
| await github.rest.issues.update({ | ||
| ...repo, | ||
| issue_number: issue.number, | ||
| state: "open", | ||
| title: config.title, | ||
| body, | ||
| }) | ||
| return "updated" | ||
| } | ||
| await github.rest.issues.create({ ...repo, title: config.title, body, assignees: [repo.owner] }) | ||
| return "created" | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| name: Maintenance alerts | ||
|
|
||
| on: | ||
| workflow_run: | ||
| workflows: [Command Code catalog sync, Latest host compatibility] | ||
| types: [completed] | ||
|
|
||
| permissions: | ||
| contents: read | ||
| actions: read | ||
| issues: write | ||
|
|
||
| concurrency: | ||
| group: commandcode-maintenance-alerts | ||
| cancel-in-progress: false | ||
|
|
||
| jobs: | ||
| alert: | ||
| if: >- | ||
| github.repository == 'patlux/pi-commandcode-provider' && | ||
| github.event.workflow_run.head_branch == github.event.repository.default_branch && | ||
| github.event.workflow_run.head_repository.full_name == github.repository && | ||
| (github.event.workflow_run.event == 'schedule' || github.event.workflow_run.event == 'workflow_dispatch') | ||
| runs-on: ubuntu-latest | ||
| timeout-minutes: 5 | ||
| steps: | ||
| # Always load trusted default-branch code, never the triggering run's ref/artifacts. | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| ref: ${{ github.event.repository.default_branch }} | ||
| persist-credentials: false | ||
| - uses: actions/github-script@v7 | ||
| with: | ||
| script: | | ||
| const { maintainAlert } = await import(`${process.env.GITHUB_WORKSPACE}/.github/scripts/maintenance-alert.mjs`) | ||
| core.info(await maintainAlert({ github, context })) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| import assert from "node:assert/strict" | ||
| import { test } from "node:test" | ||
| import { maintainAlert } from "../.github/scripts/maintenance-alert.mjs" | ||
|
|
||
| function fixture({ | ||
| conclusion = "failure", | ||
| previous = "failure", | ||
| issues = [], | ||
| event = "schedule", | ||
| branch = "main", | ||
| fork = false, | ||
| extra = [], | ||
| } = {}) { | ||
| const writes = [] | ||
| const repo = { full_name: "patlux/pi-commandcode-provider" } | ||
| const run = { | ||
| name: "Command Code catalog sync", | ||
| status: "completed", | ||
| conclusion, | ||
| head_branch: branch, | ||
| repository: repo, | ||
| head_repository: fork ? { full_name: "fork/repo" } : repo, | ||
| event, | ||
| workflow_id: 7, | ||
| run_number: 12, | ||
| id: 120, | ||
| } | ||
| const methods = { | ||
| actions: { listWorkflowRuns: "runs" }, | ||
| issues: { | ||
| listForRepo: "issues", | ||
| create: async (value) => writes.push(value), | ||
| update: async (value) => writes.push(value), | ||
| }, | ||
| } | ||
| const github = { | ||
| rest: methods, | ||
| paginate: async (method) => | ||
| method === "runs" | ||
| ? [run, { ...run, run_number: 11, id: 110, conclusion: previous }, ...extra] | ||
| : issues, | ||
| } | ||
| const context = { | ||
| repo: { owner: "patlux", repo: "pi-commandcode-provider" }, | ||
| payload: { workflow_run: run, repository: { default_branch: "main" } }, | ||
| } | ||
| return { github, context, writes } | ||
| } | ||
| const incident = { | ||
| number: 42, | ||
| state: "open", | ||
| body: "<!-- commandcode-maintenance:catalog -->\nold", | ||
| user: { login: "github-actions[bot]" }, | ||
| } | ||
|
|
||
| test("creates an actionable issue after two failures without including raw logs", async () => { | ||
| const f = fixture() | ||
| assert.equal(await maintainAlert(f), "created") | ||
| assert.equal(f.writes.length, 1) | ||
| assert.match(f.writes[0].body, /actions\/runs\/120/) | ||
| assert.match(f.writes[0].body, /official pricing/) | ||
| assert.deepEqual(f.writes[0].assignees, ["patlux"]) | ||
| }) | ||
| test("updates/reopens the same bot-owned incident instead of duplicating", async () => { | ||
| for (const state of ["open", "closed"]) { | ||
| const f = fixture({ issues: [{ ...incident, state }] }) | ||
| assert.equal(await maintainAlert(f), "updated") | ||
| assert.equal(f.writes[0].issue_number, 42) | ||
| assert.equal(f.writes[0].state, "open") | ||
| } | ||
| }) | ||
| test("closes an incident after recovery", async () => { | ||
| const f = fixture({ conclusion: "success", issues: [incident] }) | ||
| assert.equal(await maintainAlert(f), "closed") | ||
| assert.equal(f.writes[0].state, "closed") | ||
| }) | ||
| test("does not notify on a single failure, canceled run, or missing predecessor", async () => { | ||
| for (const settings of [ | ||
| { previous: "success" }, | ||
| { previous: "cancelled" }, | ||
| { conclusion: "cancelled" }, | ||
| ]) { | ||
| const f = fixture(settings) | ||
| assert.equal(await maintainAlert(f), "below-threshold") | ||
| assert.equal(f.writes.length, 0) | ||
| } | ||
| const f = fixture() | ||
| f.github.paginate = async () => [] | ||
| assert.equal(await maintainAlert(f), "below-threshold") | ||
| }) | ||
| test("ignores forks, PRs and non-default branches", async () => { | ||
| for (const settings of [{ fork: true }, { event: "pull_request" }, { branch: "feature" }]) { | ||
| const f = fixture(settings) | ||
| assert.equal(await maintainAlert(f), "ignored") | ||
| assert.equal(f.writes.length, 0) | ||
| } | ||
| }) | ||
| test("ignores stale reruns after a newer completed run", async () => { | ||
| const f = fixture({ extra: [{ run_number: 13, event: "schedule", conclusion: "success" }] }) | ||
| assert.equal(await maintainAlert(f), "stale") | ||
| assert.equal(f.writes.length, 0) | ||
| }) | ||
| test("does not edit user-authored matching issues or PRs", async () => { | ||
| const f = fixture({ | ||
| issues: [ | ||
| { ...incident, user: { login: "someone" } }, | ||
| { ...incident, pull_request: {} }, | ||
| ], | ||
| }) | ||
| assert.equal(await maintainAlert(f), "created") | ||
| assert.equal(f.writes[0].issue_number, undefined) | ||
| }) | ||
| test("tracks host failures independently", async () => { | ||
| const f = fixture({ issues: [incident] }) | ||
| f.context.payload.workflow_run.name = "Latest host compatibility" | ||
| assert.equal(await maintainAlert(f), "created") | ||
| assert.match(f.writes[0].body, /maintenance:hosts/) | ||
| }) |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The threshold relies on
listWorkflowRunsalready containing the triggering run ascompleted.workflow_run.completedcan fire before the listing API reflects that (eventual consistency). If the current run is missing,eligible[0]/eligible[1]are the two previous runs and the decision is off by one in either direction.Make it explicit: treat
runas the head and take the newesteligibleitem withrun_number < run.run_numberas the predecessor, independent of whether the API already listsrun.