From 25f4c75c71218c8b03b8378877d3b6d4c9a2fd7e Mon Sep 17 00:00:00 2001 From: Jared Wein Date: Thu, 2 Jul 2026 12:55:52 -0400 Subject: [PATCH] Add product/component verification and severity assessment to frontend-triage agent Extend the frontend-triage agent with two new triage judgments driven by rules and prompt (no ML models or new tools). It now verifies whether the bug is filed against the correct Bugzilla Product :: Component by matching the file paths it localizes against module ownership in mots.yaml, and it assesses an appropriate Mozilla severity (S1-S4) from user impact. At high confidence the agent records bugzilla_update_bug proposals for a corrected component/product and/or severity for human review; at medium/low confidence it reports the assessment only. Adds two rule files (component-verification.md, severity-assessment.md), the corresponding job steps and structured-output keys in the system prompt, and ComponentAssessment/SeverityAssessment submodels parsed into FrontendTriageResult. --- .../hackbot_agents/frontend_triage/agent.py | 37 ++++++++++++++ .../frontend_triage/prompts/system.md | 25 ++++++++-- .../rules/component-verification.md | 50 +++++++++++++++++++ .../frontend_triage/rules/frontend-triage.md | 9 +++- .../rules/severity-assessment.md | 36 +++++++++++++ 5 files changed, 152 insertions(+), 5 deletions(-) create mode 100644 agents/frontend-triage/hackbot_agents/frontend_triage/rules/component-verification.md create mode 100644 agents/frontend-triage/hackbot_agents/frontend_triage/rules/severity-assessment.md diff --git a/agents/frontend-triage/hackbot_agents/frontend_triage/agent.py b/agents/frontend-triage/hackbot_agents/frontend_triage/agent.py index 4198f70afd..14f10ad550 100644 --- a/agents/frontend-triage/hackbot_agents/frontend_triage/agent.py +++ b/agents/frontend-triage/hackbot_agents/frontend_triage/agent.py @@ -32,6 +32,7 @@ from hackbot_runtime.actions import ACTIONS_SERVER_NAME from hackbot_runtime.actions.claude_sdk import actions_server_for, actions_to_tool_names from hackbot_runtime.claude import Reporter +from pydantic import BaseModel, ValidationError from searchfox import AsyncSearchfoxClient from .config import ( @@ -49,6 +50,25 @@ _JSON_BLOCK = re.compile(r"```json\s*(\{.*?\})\s*```", re.DOTALL) +class ComponentAssessment(BaseModel): + """Product/component verification result (see component-verification rules).""" + + current: str | None = None # the bug's current "Product :: Component" + correct: bool | None = None # whether the current component is right + suggested_product: str | None = None # correction, if correct is false + suggested_component: str | None = None + confidence: str | None = None # high | medium | low + rationale: str | None = None + + +class SeverityAssessment(BaseModel): + """Severity judgment (see severity-assessment rules).""" + + suggested: str | None = None # S1 | S2 | S3 | S4 + confidence: str | None = None # high | medium | low + rationale: str | None = None + + class FrontendTriageResult(HackbotAgentResult): bug_id: int # Structured plan (best-effort, parsed from the agent's final message). @@ -63,6 +83,9 @@ class FrontendTriageResult(HackbotAgentResult): relevant_tests: list[str] | None = ( None # existing tests covering the area (verify anchor) ) + # Triage judgments (best-effort, parsed from the agent's final message). + component_assessment: ComponentAssessment | None = None + severity_assessment: SeverityAssessment | None = None # The agent's full final message, always present as a fallback. result: str | None = None @@ -128,6 +151,14 @@ def _as_list(value): return [value] return value if isinstance(value, list) else None + def _as_model(model, value): + if not isinstance(value, dict): + return None + try: + return model.model_validate(value) + except ValidationError: + return None + actionable = data.get("actionable") if not isinstance(actionable, bool): actionable = None @@ -140,6 +171,12 @@ def _as_list(value): "actionable": actionable, "regressor_node": data.get("regressor_node"), "relevant_tests": _as_list(data.get("relevant_tests")), + "component_assessment": _as_model( + ComponentAssessment, data.get("component_assessment") + ), + "severity_assessment": _as_model( + SeverityAssessment, data.get("severity_assessment") + ), } diff --git a/agents/frontend-triage/hackbot_agents/frontend_triage/prompts/system.md b/agents/frontend-triage/hackbot_agents/frontend_triage/prompts/system.md index 56769876f7..f63d0481d9 100644 --- a/agents/frontend-triage/hackbot_agents/frontend_triage/prompts/system.md +++ b/agents/frontend-triage/hackbot_agents/frontend_triage/prompts/system.md @@ -8,7 +8,9 @@ You are given a bug ID. Your job is to triage it and produce a **proposed fix pl 2. **Read the relevant triage rules** from `{rules_dir}` — Glob the directory and Read only the rulesets that apply to this bug. Do not assume all rules apply to all bugs. 3. **Assess** what the rules say should happen, and whether the bug has open questions in its comments. 4. **Investigate** the source tree (read-only) to localize the cause — delegate deep searches to the `investigator` subagent (see below). -5. **Produce a fix plan**: the likely root cause, the specific files to change, and the approach. Record it as a brief Bugzilla comment. +5. **Verify the product/component** — using the localized file paths and `mots.yaml`, confirm the bug is filed against the right `Product :: Component` and propose a correction if not (see the `component-verification` rules). +6. **Assess severity** — determine an appropriate Mozilla severity (S1–S4) from the user impact (see the `severity-assessment` rules). +7. **Produce a fix plan**: the likely root cause, the specific files to change, and the approach. Record it as a brief Bugzilla comment. # This agent is READ-ONLY @@ -34,6 +36,8 @@ Your working directory is the Firefox source repository. You have Read, Grep, Gl When you reference a cause or a fix target, cite concrete paths (and ideally functions/selectors), e.g. `browser/components/tabbrowser/content/tabgroup.js`. +The tree also ships **`mots.yaml`** (module-ownership metadata; Glob for `**/mots.yaml`). It maps file-path globs to the owning module and that module's Bugzilla `Product :: Component`. It is your reference for verifying the bug's component — match the paths you localize to a module to find where the bug belongs. + # Code-search & history tools Your local checkout is **shallow** (no git history), so for anything beyond the current file contents use these network-backed tools. They query Mozilla's live infrastructure and reflect mozilla-central tip (which may differ slightly from the checkout — prefer them for symbol search and history, and local Read/Grep for the exact checked-out bytes). @@ -78,7 +82,7 @@ Before calling any action tool, state in your response: - **What** action you are recording and **why** (cite the specific rule) - **Your confidence**: high / medium / low -Record exactly one `bugzilla_add_comment` with your fix plan. Only record a `bugzilla_update_bug` (e.g. keyword/severity) when confidence is **high** and a specific triage rule directs it. Never record `status: RESOLVED`. +Record exactly one `bugzilla_add_comment` with your fix plan (which should also state the component-verification and severity conclusions). Only record a `bugzilla_update_bug` when confidence is **high** and a specific triage rule directs it — e.g. a corrected `component`/`product` (per the `component-verification` rules), a `severity` (per the `severity-assessment` rules), or an obvious keyword. You may combine several such fields into one `bugzilla_update_bug`, each justified in the `reasoning`. At medium/low confidence, state the assessment in the comment and structured output but do **not** record a field change. Never record `status: RESOLVED`. The `reasoning` parameter on every action tool is required and stored alongside the recorded action. Fill it properly. @@ -97,7 +101,20 @@ After recording your comment, end your final message with a fenced ```json block "confidence": "high | medium | low", "actionable": true, "regressor_node": "hg node of the introducing changeset, or null", - "relevant_tests": ["browser/.../tests/browser/browser_foo.js"] + "relevant_tests": ["browser/.../tests/browser/browser_foo.js"], + "component_assessment": {{ + "current": "Firefox :: New Tab Page", + "correct": true, + "suggested_product": null, + "suggested_component": null, + "confidence": "high | medium | low", + "rationale": "why, citing the mots.yaml module and path evidence" + }}, + "severity_assessment": {{ + "suggested": "S1 | S2 | S3 | S4", + "confidence": "high | medium | low", + "rationale": "user-impact reasoning" + }} }} ``` @@ -106,6 +123,8 @@ Field guidance for the handoff: - **`actionable`** — `false` when the bug is out of scope or skipped per the scoping rules (meta/tracking, intermittent/test-infra, enhancement/task), or when there is simply nothing to fix-plan; `true` when you produced a real fix plan. The executor uses this to decide whether to act. - **`regressor_node`** — when the bug is a regression and you identified/confirmed the introducing changeset (via the `mozilla_vcs` tools or `get_blame`), put its hg node here so the executor has a direct pointer; otherwise `null`. - **`relevant_tests`** — existing tests that cover the affected area (typically browser-chrome mochitests under a component's `tests/browser/` dir, or xpcshell tests). These are the executor's **verification anchor** — it can run them. Use `[]` if you searched and found none (a signal that the executor should add a test). +- **`component_assessment`** — your product/component verification (per the `component-verification` rules). Set `correct: true` and leave the suggestions null when the current component is right; otherwise set `correct: false` and fill `suggested_product` / `suggested_component`. Always give a `rationale`. Set to null only if you could not verify at all. +- **`severity_assessment`** — the severity you judged appropriate (per the `severity-assessment` rules), with `confidence` and a `rationale`. Set to null only if you could not assess it. If you could not localize a root cause, set `root_cause` to null, keep `confidence` low, set `actionable` accordingly, and have your comment ask the specific open questions that block triage. diff --git a/agents/frontend-triage/hackbot_agents/frontend_triage/rules/component-verification.md b/agents/frontend-triage/hackbot_agents/frontend_triage/rules/component-verification.md new file mode 100644 index 0000000000..013343fc8c --- /dev/null +++ b/agents/frontend-triage/hackbot_agents/frontend_triage/rules/component-verification.md @@ -0,0 +1,50 @@ +# Product / component verification + +Bugs are frequently filed against the wrong `Product :: Component`. Part of triage is +confirming the bug is filed where it belongs and, when it isn't, proposing the correct +`Product :: Component` so it reaches the right team. + +Do this **after** you have localized the affected code (you need concrete file paths to +verify ownership), and record the result in the `component_assessment` structured-output +object. + +## How to verify + +1. **Find the ownership map.** The Firefox source tree ships a module-ownership file, + `mots.yaml` (the maintainer/module metadata; commonly at the repo root or under + `source/`). Glob for it (`**/mots.yaml`) and Read it. Each module lists path globs it + owns (e.g. an `includes:` list) and its Bugzilla product/component (e.g. under a + `meta`/`components` mapping — field names may vary slightly, read the file to see the + shape). +2. **Match your localized paths to a module.** Take the file path(s) you identified as + the root cause and find the module whose path globs own them. That module's Bugzilla + product/component is the _expected_ component for a bug in that code. Use Searchfox to + confirm ownership when a path spans directories or the mapping is ambiguous. +3. **Compare with the bug.** Compare the expected `Product :: Component` against the + bug's current values. + +## What to record + +- If the current component matches, set `component_assessment.correct` to `true`, leave + the suggestions null, and give a one-line rationale. +- If it clearly belongs elsewhere, set `correct` to `false`, put the corrected values in + `suggested_product` / `suggested_component`, and in the rationale cite **both** the + `mots.yaml` module that owns the path **and** the path evidence. + +## Confidence and field changes + +- **High** — the localized path is unambiguously owned by one module and the mismatch is + clear. Only then may you record a `bugzilla_update_bug` proposing the corrected + `product` / `component` (see the system prompt's recording rules). Cite the mots.yaml + module and path in the `reasoning`. +- **Medium / low** — the code area is uncertain or the path is owned by multiple modules. + Report the assessment (and your best suggestion) in the comment and structured output, + but do **not** record a field change. + +## Interaction with scoping + +A wrong-component finding can mean the bug is out of scope for a _frontend fix_ — e.g. +the code you localized is Core (layout, DOM, graphics), not a Firefox frontend +component. In that case, still record the component assessment and the proposed +correction, but do not invent a frontend fix plan: set `actionable` per the scoping +rules and let the comment focus on the re-triage. diff --git a/agents/frontend-triage/hackbot_agents/frontend_triage/rules/frontend-triage.md b/agents/frontend-triage/hackbot_agents/frontend_triage/rules/frontend-triage.md index 93621bd150..39c64bcbd8 100644 --- a/agents/frontend-triage/hackbot_agents/frontend_triage/rules/frontend-triage.md +++ b/agents/frontend-triage/hackbot_agents/frontend_triage/rules/frontend-triage.md @@ -25,6 +25,10 @@ ruleset does not apply — note that and stop. 3. **Write a fix plan**: root cause, the specific files/functions/selectors to change, and the approach. Prefer a comprehensive fix at the right level over a spot fix. +4. **Verify the product/component and assess severity.** Using the file paths you + localized, apply the `component-verification` and `severity-assessment` rules to + confirm the bug is filed correctly and to judge its severity. Both belong in your + comment and structured output. ## Comment @@ -36,8 +40,9 @@ restate the whole bug. Do not claim the fix is verified — you did not run it. - **High** (you found the specific code and the cause is clear): record the plan comment. If a rule or convention clearly applies, you may also record a - `bugzilla_update_bug` for an obviously-correct field (e.g. adding a relevant - keyword). Do not change `status`/`resolution`. + `bugzilla_update_bug` for an obviously-correct field — e.g. adding a relevant + keyword, a corrected `component`/`product` (per `component-verification`), or a + `severity` (per `severity-assessment`). Do not change `status`/`resolution`. - **Medium** (plausible area, cause not pinned down): record the comment with your best hypothesis and the open questions that would confirm it. - **Low** (could not localize): record a comment stating what you checked and diff --git a/agents/frontend-triage/hackbot_agents/frontend_triage/rules/severity-assessment.md b/agents/frontend-triage/hackbot_agents/frontend_triage/rules/severity-assessment.md new file mode 100644 index 0000000000..3b6436b424 --- /dev/null +++ b/agents/frontend-triage/hackbot_agents/frontend_triage/rules/severity-assessment.md @@ -0,0 +1,36 @@ +# Severity assessment + +Assess an appropriate Mozilla severity for the bug and record it in the +`severity_assessment` structured-output object. Base the judgment on **user impact and +reach** as evidenced by the bug report and the code you investigated — how badly the +user is affected, how many users hit it, and whether a workaround exists. + +## Severity definitions + +- **S1 — catastrophic.** Crash, hang, data loss, security issue, or a bug that blocks + major functionality with **no workaround**. Affects a large number of users. +- **S2 — serious.** Major functionality is broken or a severe UX problem, and the + workaround (if any) is painful or non-obvious. Affects many users. +- **S3 — normal.** Blocks non-critical functionality, or a reasonable workaround exists. + **This is the default for most frontend papercuts.** +- **S4 — minor / trivial.** Cosmetic issues, small polish, or edge cases with negligible + impact. + +## Guidance + +- Frontend UI/UX papercuts are usually **S3** (or **S4** when purely cosmetic). Reserve + **S1 / S2** for genuine breakage: crashes, data/state loss, or a broken core workflow + with no easy workaround. +- Weigh: is it functional vs cosmetic? Is there a workaround? How frequently and how + broadly is it hit (mainline path vs rare configuration)? +- Do **not downgrade** an existing higher severity unless you have strong evidence the + impact is lower than currently recorded. + +## Confidence and field changes + +- **High** — impact is clear-cut (clearly cosmetic, or clearly a crash/data-loss). Only + then may you record a `bugzilla_update_bug` proposing the `severity` (see the system + prompt's recording rules), with a `reasoning` citing the impact evidence. Prefer not to + propose a change when the bug already carries a reasonable severity. +- **Medium / low** — suggest a severity in the comment and structured output, but do + **not** record a field change.