From 33b00ef841040130fdbc6171ec82719cf24d3493 Mon Sep 17 00:00:00 2001 From: jstet Date: Fri, 25 Sep 2026 14:51:32 +0200 Subject: [PATCH] wizard: drop generated questions whose name collides with a demographic (#47) The model is told `demographicsAddedSeparately: 'age, ...'` but still occasionally writes a birth-date / age / sex question and names it the same as a demographic. assembleSurvey then appends the qwac copy, sanitizeSurvey renames the duplicate (e.g. 'age' -> 'age2'), and the user ends up with two 'Wann sind Sie geboren?' rows under the same name family. Filter generated questions against the demographic name set inside `evaluate()` so both the first generation and the repair loop go through it. The qwac demographic is the canonical version and stays appended by assembleSurvey; the generated duplicate is dropped before the survey is built. New helper `dedupAgainstDemographics` exported for testing; new tests in lead.test.ts cover the end-to-end path (model returns a question named 'age' alongside the demographic) and the unit cases for the helper. --- src/lib/agents/lead.test.ts | 80 ++++++++++++++++++++++++++++++++++++- src/lib/agents/lead.ts | 25 +++++++++++- 2 files changed, 103 insertions(+), 2 deletions(-) diff --git a/src/lib/agents/lead.test.ts b/src/lib/agents/lead.test.ts index 171ad01..a05131f 100644 --- a/src/lib/agents/lead.test.ts +++ b/src/lib/agents/lead.test.ts @@ -1,6 +1,12 @@ import { describe, it, expect, vi } from 'vitest'; import { AxMockAIService, type AxChatRequest } from '@ax-llm/ax'; -import { LeadAgent, MAX_REPAIR_ATTEMPTS, assembleSurvey, qualityFeedback } from './lead.js'; +import { + LeadAgent, + MAX_REPAIR_ATTEMPTS, + assembleSurvey, + dedupAgainstDemographics, + qualityFeedback +} from './lead.js'; import { XLSFormValidator, type ValidationFinding } from './xlsform_validator.js'; import type { AgentInput, Question } from './types.js'; @@ -218,6 +224,78 @@ describe('LeadAgent.run', () => { // in its rendered prompt (system or user message). expect(rendered.some((s) => s.includes('Survey Language'))).toBe(true); }); + + /** The model is told `demographicsAddedSeparately`, but it still writes a + * birth-date question and names it `age` sometimes. Without dedup the + * survey ends up with two `age` rows and the same "Wann sind Sie + * geboren?" label (#47 follow-up). */ + it('drops a generated question whose name collides with a demographic (#47)', async () => { + const dup: Partial[] = [ + { name: 'age', label: 'Wann sind Sie geboren?', type: 'date', rationale: 'selbst' }, + ...good.slice(0, 7) + ]; + const { ai } = mockAI(dup, []); + const result = await new LeadAgent(ai).run({ + ...input, + selectedDemographics: ['age'], + demographicQuestions: [ + { + id: 'qwac_age', + name: 'age', + label: 'Wann sind Sie geboren?', + type: 'date', + required: true + } + ] + }); + // Exactly one `age` survives: the qwac copy appended by assembleSurvey. + const ages = result.survey.questions.filter((q) => q.name === 'age' || /^age\d/.test(q.name)); + expect(ages).toHaveLength(1); + expect(ages[0].id).toBe('qwac_age'); + }); +}); + +describe('dedupAgainstDemographics', () => { + const age: Question = { + id: 'qwac_age', + name: 'age', + label: 'Wann sind Sie geboren?', + type: 'date', + required: true + }; + const sex: Question = { + id: 'qwac_sex', + name: 'sex', + label: 'Geschlecht?', + type: 'select_one', + required: true, + choices: [] + }; + + it('returns the list unchanged when there are no demographics', () => { + const generated: Question[] = [ + { id: 'g1', name: 'age', label: 'l', type: 'text', required: true } + ]; + expect(dedupAgainstDemographics(generated, [])).toBe(generated); + }); + + it('drops a generated question whose name matches any demographic name', () => { + const generated: Question[] = [ + { id: 'g1', name: 'age', label: 'l', type: 'text', required: true, rationale: 'x' }, + { id: 'g2', name: 'satisfaction', label: 'l', type: 'text', required: true, rationale: 'x' } + ]; + expect(dedupAgainstDemographics(generated, [age, sex]).map((q) => q.name)).toEqual([ + 'satisfaction' + ]); + }); + + it('keeps generated questions whose name does not collide', () => { + const generated: Question[] = [ + { id: 'g1', name: 'satisfaction', label: 'l', type: 'text', required: true, rationale: 'x' }, + { id: 'g2', name: 'birthyear', label: 'l', type: 'text', required: true, rationale: 'x' } + ]; + expect(dedupAgainstDemographics(generated, [age, sex])).toEqual(generated); + }); }); describe('qualityFeedback', () => { diff --git a/src/lib/agents/lead.ts b/src/lib/agents/lead.ts index 518811c..edc1587 100644 --- a/src/lib/agents/lead.ts +++ b/src/lib/agents/lead.ts @@ -62,6 +62,19 @@ function uncovered(questions: Question[], count: number): number[] { return Array.from({ length: count }, (_, i) => i + 1).filter((n) => !served.has(n)); } +/** Drop generated questions whose `name` collides with a demographic; the + * qwac copy is the canonical one and will be appended by `assembleSurvey`. + * Same-label duplicates would survive sanitization as `age` + `age2`, both + * with the same German label, which is what users reported (#47). */ +export function dedupAgainstDemographics( + generated: Question[], + demographics: Question[] +): Question[] { + if (demographics.length === 0) return generated; + const names = new Set(demographics.map((d) => d.name)); + return generated.filter((q) => !names.has(q.name)); +} + /** With a single research question every question serves it, whether or not * the model said so; numbers beyond the list are dropped. */ function withResearchQuestions(questions: Question[], count: number): Question[] { @@ -184,7 +197,17 @@ export class LeadAgent { reasoning: generated.reasoning }; const evaluate = (parsed: Question[]): Evaluated => { - const questions = withResearchQuestions(parsed, rqCount); + // The model is told `demographicsAddedSeparately`, but it sometimes + // still writes a birth-date / age / sex question and names it the + // same as a demographic. assembleSurvey appends the qwac demographic + // alongside, sanitizeSurvey then renames the duplicate, and the + // user ends up with two questions sharing one label (#47). Drop + // the generated duplicate here so the demographic stays the only + // copy. + const questions = withResearchQuestions( + dedupAgainstDemographics(parsed, input.demographicQuestions), + rqCount + ); const survey = assembleSurvey(base, questions, input.demographicQuestions); const workbook = this.workbookGenerator.generate(survey); const findings = this.validate(workbook);