From ad6ddc31b6c65a24de88f5e892c52dbec3a2a955 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Tue, 25 Aug 2026 18:38:53 +0200 Subject: [PATCH] Replace degenerate agent output with a notice in Slack When capture flags a run's final message as degenerate, Slack now renders a short notice in place of the unusable fragments rather than pasting them into the thread. The PR link, the task context block, and the error line on the failed phase are all preserved, and the raw response is left untouched in the Task's results. The substitution happens before both the response blocks and the fallback text are built, so the fragments stay out of notifications and accessibility surfaces as well. Co-Authored-By: Claude Opus 5 (1M context) --- internal/reporting/slack.go | 18 +++- internal/reporting/slack_test.go | 167 +++++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+), 1 deletion(-) diff --git a/internal/reporting/slack.go b/internal/reporting/slack.go index dfcaced01..89ead2ada 100644 --- a/internal/reporting/slack.go +++ b/internal/reporting/slack.go @@ -117,6 +117,16 @@ func FormatProgressMessage(text, taskName string) SlackMessage { } } +// degenerateNoticeText returns the notice shown in place of an agent response +// that capture flagged as degenerate. The raw response is still preserved in +// the Task's results; only the Slack rendering is substituted. +func degenerateNoticeText(phase string) string { + if phase == "failed" { + return ":warning: The model returned an unusable reply. This ran twice and produced unusable output both times, so there is no answer to show." + } + return ":warning: The model returned an unusable reply, so there is no answer to show." +} + // FormatSlackTransitionMessage returns one or more rich Slack messages for a // task phase transition. When the agent response is short enough to fit in a // single message (≤ SlackBlockLimit blocks), a single SlackMessage is returned. @@ -133,9 +143,15 @@ func FormatSlackTransitionMessage(phase, taskName, message string, results map[s )) } - // Build the response blocks from the agent output. + // Build the response blocks from the agent output. When capture flagged + // the output as degenerate, the agent's final message is unusable, so a + // short notice is rendered in its place rather than the fragments. resp := results["response"] decoded := decodeResponse(resp) + if results["degenerate"] == "true" { + decoded = degenerateNoticeText(phase) + resp = decoded + } var responseBlocks []slack.Block if resp != "" { responseBlocks = responseToBlocks(decoded) diff --git a/internal/reporting/slack_test.go b/internal/reporting/slack_test.go index 00c4c75f6..aa19c0225 100644 --- a/internal/reporting/slack_test.go +++ b/internal/reporting/slack_test.go @@ -636,3 +636,170 @@ func assertContextContains(t *testing.T, block slack.Block, substr string) { } t.Errorf("context block does not contain %q", substr) } + +// garbledResponse is the real degenerate output posted to a customer thread on +// 2026-08-21: fragments of HTML with no answer in them. +const garbledResponse = "`stale=False\n\n" + +const ( + degenerateNotice = ":warning: The model returned an unusable reply, so there is no answer to show." + degenerateNoticeFailed = ":warning: The model returned an unusable reply. This ran twice and produced unusable output both times, so there is no answer to show." +) + +// allBlockText concatenates the text of every section and context block so a +// test can assert that a fragment appears nowhere in the rendered message. +func allBlockText(blocks []slack.Block) string { + var sb strings.Builder + for _, b := range blocks { + switch block := b.(type) { + case *slack.SectionBlock: + if block.Text != nil { + sb.WriteString(block.Text.Text) + sb.WriteString("\n") + } + for _, f := range block.Fields { + sb.WriteString(f.Text) + sb.WriteString("\n") + } + case *slack.HeaderBlock: + if block.Text != nil { + sb.WriteString(block.Text.Text) + sb.WriteString("\n") + } + case *slack.ContextBlock: + for _, elem := range block.ContextElements.Elements { + if txt, ok := elem.(*slack.TextBlockObject); ok { + sb.WriteString(txt.Text) + sb.WriteString("\n") + } + } + } + } + return sb.String() +} + +// assertNoFragments fails if any part of the garbled response leaked into the +// rendered blocks or the fallback text. +func assertNoFragments(t *testing.T, msg SlackMessage) { + t.Helper() + rendered := allBlockText(msg.Blocks) + for _, fragment := range []string{"stale=False", "", "") + assertNoFragments(t, got) + }) + + t.Run("failed keeps the error line", func(t *testing.T) { + results := map[string]string{ + "response": b64(garbledResponse), + "degenerate": "true", + } + got := firstMsg(t, FormatSlackTransitionMessage("failed", "spawner-1234567890.123456", "degenerate_output=turns=41,chars=58", results)) + want := degenerateNoticeFailed + "\nError: degenerate_output=turns=41,chars=58 (Task: spawner-1234567890.123456)" + if got.Text != want { + t.Errorf("fallback text = %q, want %q", got.Text, want) + } + assertBlockCount(t, got.Blocks, 4) // header + notice + error + context + assertSectionText(t, got.Blocks[0], ":x: *Something went wrong*") + assertSectionText(t, got.Blocks[1], degenerateNoticeFailed) + assertSectionText(t, got.Blocks[2], ":warning: *Error:* degenerate_output=turns=41,chars=58") + assertNoFragments(t, got) + }) + + t.Run("no response still renders the notice", func(t *testing.T) { + results := map[string]string{"degenerate": "true"} + got := firstMsg(t, FormatSlackTransitionMessage("failed", "spawner-1234567890.123456", "", results)) + if got.Text != degenerateNoticeFailed+" (Task: spawner-1234567890.123456)" { + t.Errorf("fallback text = %q", got.Text) + } + assertBlockCount(t, got.Blocks, 3) // header + notice + context + assertSectionText(t, got.Blocks[1], degenerateNoticeFailed) + }) + + t.Run("long degenerate output stays a single message", func(t *testing.T) { + var sb strings.Builder + for i := 0; i < 60; i++ { + if i > 0 { + sb.WriteString("\n\n") + } + sb.WriteString("### Header\n" + garbledResponse) + } + results := map[string]string{ + "response": b64(sb.String()), + "degenerate": "true", + } + msgs := FormatSlackTransitionMessage("succeeded", "spawner-1234567890.123456", "", results) + if len(msgs) != 1 { + t.Fatalf("message count = %d, want 1 (split path must not be entered)", len(msgs)) + } + assertBlockCount(t, msgs[0].Blocks, 2) // notice + context + if len(msgs[0].Blocks) > SlackBlockLimit { + t.Errorf("block count = %d, must be <= %d", len(msgs[0].Blocks), SlackBlockLimit) + } + assertNoFragments(t, msgs[0]) + }) + + t.Run("marker unset leaves the response untouched", func(t *testing.T) { + results := map[string]string{"response": b64("I need your GitHub username to proceed.")} + got := firstMsg(t, FormatSlackTransitionMessage("succeeded", "spawner-1234567890.123456", "", results)) + if got.Text != "I need your GitHub username to proceed. (Task: spawner-1234567890.123456)" { + t.Errorf("fallback text = %q", got.Text) + } + assertBlockCount(t, got.Blocks, 2) // response + context + assertSectionText(t, got.Blocks[0], "I need your GitHub username to proceed.") + }) + + t.Run("marker set to a non-true value leaves the response untouched", func(t *testing.T) { + results := map[string]string{ + "response": b64("I need your GitHub username to proceed."), + "degenerate": "false", + } + got := firstMsg(t, FormatSlackTransitionMessage("succeeded", "spawner-1234567890.123456", "", results)) + if got.Text != "I need your GitHub username to proceed. (Task: spawner-1234567890.123456)" { + t.Errorf("fallback text = %q", got.Text) + } + assertBlockCount(t, got.Blocks, 2) // response + context + assertSectionText(t, got.Blocks[0], "I need your GitHub username to proceed.") + }) +}