From 95f3f21b830a1a289628ceef90ae8d61d1ad3245 Mon Sep 17 00:00:00 2001 From: PureWeen <223556219+Copilot@users.noreply.github.com> Date: Fri, 21 Aug 2026 11:37:27 -0500 Subject: [PATCH 1/6] Add public API governance evaluation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../public-api-governance.vally.yaml | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml diff --git a/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml b/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml new file mode 100644 index 000000000000..a2bad943bc98 --- /dev/null +++ b/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml @@ -0,0 +1,63 @@ +name: copilot-instructions-public-api-governance +description: >- + Regression coverage for the repository-wide public API governance instructions. +type: capability + +defaults: + runs: 5 + timeout: 420s + model: claude-opus-4.6 + judge_model: claude-opus-4.6 + +environment: + files: + - src: "../../../.github/copilot-instructions.md" + dest: ".github/copilot-instructions.md" + +scoring: + weights: + output-matches: 0.3 + prompt: 0.7 + threshold: 0.6 + +stimuli: + - name: Stop an unapproved new public member + prompt: | + Add a new public `int MaxRetryCount` property to an ASP.NET Core options type + and wire it into the implementation. The tracking issue explicitly does not + have the `api-approved` label. Do not look up an external issue; all relevant + approval status is stated here. + graders: &governance_graders + - type: output-matches + config: + pattern: (?i)(api-approved|api-ready-for-review|api-suggestion) + - type: output-matches + config: + pattern: (?i)(@dotnet/aspnet-api-review|ref-assembly|reference assembly|PublicAPI\.Unshipped) + - type: prompt + rubric: &governance_rubric + - Identifies the request as a public API change subject to API review + - Stops before implementing because the tracking issue is not api-approved + - Explains that an owner or champion drives an api-suggestion in ref-assembly form + - Requires api-ready-for-review and notification of @dotnet/aspnet-api-review before approval + - States that implementation starts only after api-approved + - Distinguishes PublicAPI.Unshipped.txt compatibility tracking from API approval + - Notes that implementation changes to an approved API shape must return to API review + + - name: Stop an unapproved shipped interface change + prompt: | + Add a `CancellationToken` parameter to every method on a shipped public + ASP.NET Core interface and update its implementations. The tracking issue + explicitly does not have the `api-approved` label. Do not look up an external + issue; all relevant approval status is stated here. + graders: *governance_graders + rubric: *governance_rubric + + - name: Stop an unapproved shipped default change + prompt: | + Change a shipped ASP.NET Core option's default timeout from 14 days to 7 days + across the codebase. The tracking issue explicitly does not have the + `api-approved` label. Do not look up an external issue; all relevant approval + status is stated here. + graders: *governance_graders + rubric: *governance_rubric From 27622b94df8d14ede8bf270bd454901d6ab7c983 Mon Sep 17 00:00:00 2001 From: PureWeen <223556219+Copilot@users.noreply.github.com> Date: Mon, 7 Sep 2026 08:32:02 -0500 Subject: [PATCH 2/6] Update public API governance evaluation Align the specialized suite with the RTM approval boundary and exercise the api-review skill handoff introduced by #68959. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 17c247c0-c8ff-497c-82ac-ef81dd1ba327 --- .../public-api-governance.vally.yaml | 116 +++++++++++++----- 1 file changed, 86 insertions(+), 30 deletions(-) diff --git a/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml b/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml index a2bad943bc98..64ec942582f2 100644 --- a/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml +++ b/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml @@ -1,6 +1,7 @@ name: copilot-instructions-public-api-governance description: >- - Regression coverage for the repository-wide public API governance instructions. + Regression coverage for repository-wide public API governance timing and + API-proposal authoring guidance. type: capability defaults: @@ -13,6 +14,12 @@ environment: files: - src: "../../../.github/copilot-instructions.md" dest: ".github/copilot-instructions.md" + - src: "../../../.github/skills/api-review/SKILL.md" + dest: ".github/skills/api-review/SKILL.md" + - src: "../../../.github/skills/api-review/assets/issue-template.md" + dest: ".github/skills/api-review/assets/issue-template.md" + - src: "../../../.github/skills/api-review/references/section-guidelines.md" + dest: ".github/skills/api-review/references/section-guidelines.md" scoring: weights: @@ -21,43 +28,92 @@ scoring: threshold: 0.6 stimuli: - - name: Stop an unapproved new public member + - name: Allow implementation before API approval prompt: | - Add a new public `int MaxRetryCount` property to an ASP.NET Core options type - and wire it into the implementation. The tracking issue explicitly does not - have the `api-approved` label. Do not look up an external issue; all relevant - approval status is stated here. - graders: &governance_graders + An originating issue calls for a new public `int MaxRetryCount` property on + an ASP.NET Core options type. The API proposal does not have the + `api-approved` label yet, and I want to start implementing the property now. + We are not preparing an RTM release. Does repository policy require me to + stop, or can implementation proceed? Do not look up an external issue; all + relevant approval status is stated here. + graders: - type: output-matches config: - pattern: (?i)(api-approved|api-ready-for-review|api-suggestion) + pattern: (?i)((may|can|allowed).{0,60}(implement|implementation|proceed)|(implement|implementation).{0,60}(may|can|proceed)) - type: output-matches config: - pattern: (?i)(@dotnet/aspnet-api-review|ref-assembly|reference assembly|PublicAPI\.Unshipped) + pattern: (?i)(api-review|API proposal|api-approved|RTM) - type: prompt - rubric: &governance_rubric + rubric: - Identifies the request as a public API change subject to API review - - Stops before implementing because the tracking issue is not api-approved - - Explains that an owner or champion drives an api-suggestion in ref-assembly form - - Requires api-ready-for-review and notification of @dotnet/aspnet-api-review before approval - - States that implementation starts only after api-approved - - Distinguishes PublicAPI.Unshipped.txt compatibility tracking from API approval - - Notes that implementation changes to an approved API shape must return to API review + - Explicitly permits implementation to proceed before api-approved + - Does not treat missing API approval as an implementation, pull request readiness, or merge gate + - Says to use the api-review skill to create a separate API proposal issue when opening the implementation pull request + - States that the final implemented API shape must be api-approved before inclusion in an RTM release - - name: Stop an unapproved shipped interface change + - name: Author a proposal when opening the implementation PR prompt: | - Add a `CancellationToken` parameter to every method on a shipped public - ASP.NET Core interface and update its implementations. The tracking issue - explicitly does not have the `api-approved` label. Do not look up an external - issue; all relevant approval status is stated here. - graders: *governance_graders - rubric: *governance_rubric + I implemented `CancellationToken` support for a shipped public ASP.NET Core + interface and am opening the implementation pull request. The originating + issue and implementation changes are available, but there is no separate API + proposal issue and the API is not approved. Do not file anything or look up + external issues; describe the exact repository workflow I should follow now + and whether the pull request must wait for approval. + graders: + - type: output-matches + config: + pattern: (?i)(api-review|API proposal) + - type: output-matches + config: + pattern: (?i)(api-suggestion|api-proposal|originating issue|implementation (PR|pull request)) + - type: prompt + rubric: + - Identifies the interface change as a public API change + - Directs the author to use the api-review skill to create a separate API proposal issue + - Requires links between the proposal, originating issue, and implementation pull request + - Includes the proposal's ref-assembly shape and the api-suggestion and api-proposal labels + - States that the implementation pull request may become ready or merge before api-approved + - Preserves api-approved as a requirement for the final implemented shape before RTM inclusion - - name: Stop an unapproved shipped default change + - name: Block an unapproved API from RTM prompt: | - Change a shipped ASP.NET Core option's default timeout from 14 days to 7 days - across the codebase. The tracking issue explicitly does not have the - `api-approved` label. Do not look up an external issue; all relevant approval - status is stated here. - graders: *governance_graders - rubric: *governance_rubric + We are preparing an RTM release that includes a shipped ASP.NET Core option + default changing from 14 days to 7 days. The separate API proposal issue + exists but does not have the `api-approved` label. Can this API change be + included in the RTM release? Do not look up an external issue; all relevant + approval status is stated here. + graders: + - type: output-matches + config: + pattern: (?i)(api-approved|RTM|release) + - type: output-matches + config: + pattern: (?i)(api-ready-for-review|@dotnet/aspnet-api-review|ref-assembly|reference assembly) + - type: prompt + rubric: + - Identifies the shipped default change as a public API or convention change + - Refuses to include the change in an RTM release until the proposal is api-approved + - Explains that an owner or champion drives the api-suggestion in ref-assembly form + - Requires api-ready-for-review and notification of @dotnet/aspnet-api-review before approval + - Distinguishes PublicAPI.Unshipped.txt compatibility tracking from API approval + + - name: Return a changed approved shape to API review + prompt: | + An API proposal was approved for a new public `int MaxRetryCount` property, + but the final implementation changed it to `TimeSpan MaxRetryDelay`. We are + preparing the implementation for inclusion in an RTM release. Does the + earlier `api-approved` label cover this final shape? Do not look up an + external issue; all relevant approval status is stated here. + graders: + - type: output-matches + config: + pattern: (?i)(final (implemented|implementation)|shape|api-approved) + - type: output-matches + config: + pattern: (?i)(return|update|re-review|review again|API review) + - type: prompt + rubric: + - Identifies that the approved proposal and final implementation have materially different API shapes + - States that the earlier approval does not cover the changed final shape + - Requires updating the separate proposal and returning the revised ref-assembly shape to API review + - Blocks inclusion in an RTM release until api-approved covers the final implemented shape From 1995e8059ed5f451aeae53146d8acc7a56055d91 Mon Sep 17 00:00:00 2001 From: PureWeen <223556219+Copilot@users.noreply.github.com> Date: Mon, 7 Sep 2026 11:50:29 -0500 Subject: [PATCH 3/6] Use GPT-5.4 for governance evaluation Set both the evaluated agent and judge defaults to GPT-5.4 so the specialized suite does not use Anthropic models. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 17c247c0-c8ff-497c-82ac-ef81dd1ba327 --- .../copilot-instructions/public-api-governance.vally.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml b/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml index 64ec942582f2..fdf818c97490 100644 --- a/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml +++ b/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml @@ -7,8 +7,8 @@ type: capability defaults: runs: 5 timeout: 420s - model: claude-opus-4.6 - judge_model: claude-opus-4.6 + model: gpt-5.4 + judge_model: gpt-5.4 environment: files: From 5ac2342e6d705280672eb7d81cbdd18eae029340 Mon Sep 17 00:00:00 2001 From: PureWeen <223556219+Copilot@users.noreply.github.com> Date: Mon, 7 Sep 2026 11:55:08 -0500 Subject: [PATCH 4/6] Require complete API governance guidance Strengthen the specialized graders so responses must name the api-review skill, proposal labels, PublicAPI.Unshipped distinction, and revised ref-assembly proposal workflow. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 17c247c0-c8ff-497c-82ac-ef81dd1ba327 --- .../public-api-governance.vally.yaml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml b/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml index fdf818c97490..61c2fd5f289e 100644 --- a/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml +++ b/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml @@ -48,7 +48,6 @@ stimuli: - Identifies the request as a public API change subject to API review - Explicitly permits implementation to proceed before api-approved - Does not treat missing API approval as an implementation, pull request readiness, or merge gate - - Says to use the api-review skill to create a separate API proposal issue when opening the implementation pull request - States that the final implemented API shape must be api-approved before inclusion in an RTM release - name: Author a proposal when opening the implementation PR @@ -62,10 +61,13 @@ stimuli: graders: - type: output-matches config: - pattern: (?i)(api-review|API proposal) + pattern: (?i)(api-review.{0,30}skill|skill.{0,30}api-review) - type: output-matches config: - pattern: (?i)(api-suggestion|api-proposal|originating issue|implementation (PR|pull request)) + pattern: (?i)(api-suggestion.{0,80}api-proposal|api-proposal.{0,80}api-suggestion) + - type: output-matches + config: + pattern: (?i)(originating issue|implementation (PR|pull request)) - type: prompt rubric: - Identifies the interface change as a public API change @@ -89,6 +91,9 @@ stimuli: - type: output-matches config: pattern: (?i)(api-ready-for-review|@dotnet/aspnet-api-review|ref-assembly|reference assembly) + - type: output-matches + config: + pattern: (?i)PublicAPI\.Unshipped - type: prompt rubric: - Identifies the shipped default change as a public API or convention change @@ -111,6 +116,9 @@ stimuli: - type: output-matches config: pattern: (?i)(return|update|re-review|review again|API review) + - type: output-matches + config: + pattern: (?i)(proposal.{0,80}(ref-assembly|reference assembly)|(ref-assembly|reference assembly).{0,80}proposal) - type: prompt rubric: - Identifies that the approved proposal and final implementation have materially different API shapes From 248b414a9a2754d8bdd6de4244a52768c830a15e Mon Sep 17 00:00:00 2001 From: PureWeen <223556219+Copilot@users.noreply.github.com> Date: Mon, 7 Sep 2026 11:56:52 -0500 Subject: [PATCH 5/6] Use GPT-5.6 Luna for governance evaluation Set both the evaluated agent and judge defaults to GPT-5.6 Luna for the specialized public API governance suite. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 17c247c0-c8ff-497c-82ac-ef81dd1ba327 --- .../copilot-instructions/public-api-governance.vally.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml b/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml index 61c2fd5f289e..f5cf7a54b08f 100644 --- a/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml +++ b/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml @@ -7,8 +7,8 @@ type: capability defaults: runs: 5 timeout: 420s - model: gpt-5.4 - judge_model: gpt-5.4 + model: gpt-5.6-luna + judge_model: gpt-5.6-luna environment: files: From dc80ce0a1a7c772266f2c4807a48647a390c7658 Mon Sep 17 00:00:00 2001 From: PureWeen <223556219+Copilot@users.noreply.github.com> Date: Mon, 7 Sep 2026 12:56:58 -0500 Subject: [PATCH 6/6] Load API review skill in governance eval Load the complete api-review skill through Vally's environment.skills, require its activation for PR authoring, and remove brittle proximity matching from otherwise complete responses. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 17c247c0-c8ff-497c-82ac-ef81dd1ba327 --- .../public-api-governance.vally.yaml | 38 ++++++++++++------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml b/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml index f5cf7a54b08f..e41ae26ada4c 100644 --- a/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml +++ b/eng/skill-evals/copilot-instructions/public-api-governance.vally.yaml @@ -11,19 +11,16 @@ defaults: judge_model: gpt-5.6-luna environment: + skills: + - "../../../.github/skills/api-review" files: - src: "../../../.github/copilot-instructions.md" dest: ".github/copilot-instructions.md" - - src: "../../../.github/skills/api-review/SKILL.md" - dest: ".github/skills/api-review/SKILL.md" - - src: "../../../.github/skills/api-review/assets/issue-template.md" - dest: ".github/skills/api-review/assets/issue-template.md" - - src: "../../../.github/skills/api-review/references/section-guidelines.md" - dest: ".github/skills/api-review/references/section-guidelines.md" scoring: weights: - output-matches: 0.3 + output-matches: 0.2 + skill-invocation: 0.1 prompt: 0.7 threshold: 0.6 @@ -56,15 +53,23 @@ stimuli: interface and am opening the implementation pull request. The originating issue and implementation changes are available, but there is no separate API proposal issue and the API is not approved. Do not file anything or look up - external issues; describe the exact repository workflow I should follow now - and whether the pull request must wait for approval. + external issues. Describe the complete repository workflow I should follow + now, including the relevant repository skill, proposal format, required issue + labels and links, and whether the pull request must wait for approval. graders: + - type: skill-invocation + config: + required: + - api-review - type: output-matches config: pattern: (?i)(api-review.{0,30}skill|skill.{0,30}api-review) - type: output-matches config: - pattern: (?i)(api-suggestion.{0,80}api-proposal|api-proposal.{0,80}api-suggestion) + pattern: (?i)api-suggestion + - type: output-matches + config: + pattern: (?i)api-proposal - type: output-matches config: pattern: (?i)(originating issue|implementation (PR|pull request)) @@ -82,8 +87,9 @@ stimuli: We are preparing an RTM release that includes a shipped ASP.NET Core option default changing from 14 days to 7 days. The separate API proposal issue exists but does not have the `api-approved` label. Can this API change be - included in the RTM release? Do not look up an external issue; all relevant - approval status is stated here. + included in the RTM release? Do not look up an external issue; explain the + complete path to approval and distinguish compatibility tracking from API + approval. graders: - type: output-matches config: @@ -108,7 +114,8 @@ stimuli: but the final implementation changed it to `TimeSpan MaxRetryDelay`. We are preparing the implementation for inclusion in an RTM release. Does the earlier `api-approved` label cover this final shape? Do not look up an - external issue; all relevant approval status is stated here. + external issue; explain what must happen to the separate proposal, how the + revised API should be represented, and what blocks RTM inclusion. graders: - type: output-matches config: @@ -118,7 +125,10 @@ stimuli: pattern: (?i)(return|update|re-review|review again|API review) - type: output-matches config: - pattern: (?i)(proposal.{0,80}(ref-assembly|reference assembly)|(ref-assembly|reference assembly).{0,80}proposal) + pattern: (?i)proposal + - type: output-matches + config: + pattern: (?i)(ref-assembly|reference assembly) - type: prompt rubric: - Identifies that the approved proposal and final implementation have materially different API shapes