diff --git a/docs/production-picker-monitoring.md b/docs/production-picker-monitoring.md index dc00a16..f073df2 100644 --- a/docs/production-picker-monitoring.md +++ b/docs/production-picker-monitoring.md @@ -20,6 +20,12 @@ source, logs or chat. The workflow fails explicitly when reporting is missing or rejected; a green required-reporting run means both the picker check and App Health ingest succeeded. +The workflow's probe receipt also records allowlisted confidence and ranking +mode, whether the API marked the response as a fallback, and whether all three +perspectives are present. Strict failure criteria are unchanged. These fields +distinguish a genuine low-confidence selection from degraded routing without +including comments, provider bodies or request headers. + App Health's Overview feed must include production error logs and warning-level `*.degraded` logs to surface these outcomes independently of its 20-request aggregate-health threshold. Overview alerts remain retained occurrences rather diff --git a/scripts/probe-production-picker.mjs b/scripts/probe-production-picker.mjs index 887d753..f1a7531 100644 --- a/scripts/probe-production-picker.mjs +++ b/scripts/probe-production-picker.mjs @@ -21,8 +21,11 @@ export async function probePicker(origin='https://memes.significanthobbies.com', &&new Set(candidates.map(candidate=>candidate.id)).size===5 &&candidates.every(candidate=>typeof candidate.id==='string'&&candidate.id&&typeof candidate.media_url==='string'&&/^https:\/\//.test(candidate.media_url)&&Number.isFinite(candidate.score)); const perspectives=new Set((candidates??[]).map(candidate=>candidate.perspective)); - const degraded=body.degraded===true||body.confidence==='low'||(fixture.id==='perspectives'&&!['self','other','situation'].every(perspective=>perspectives.has(perspective))); - results.push({case:fixture.id,status:valid&&!degraded?'passed':'failed',status_code:response.status,degraded,duration_ms:Date.now()-started}); + const perspectives_complete=['self','other','situation'].every(perspective=>perspectives.has(perspective)); + const degraded=body.degraded===true||body.confidence==='low'||(fixture.id==='perspectives'&&!perspectives_complete); + const confidence=['high','medium','low'].includes(body.confidence)?body.confidence:null; + const ranking_mode=['general','perspective','general_fallback','retrieval_fallback'].includes(body.ranking_mode)?body.ranking_mode:null; + results.push({case:fixture.id,status:valid&&!degraded?'passed':'failed',status_code:response.status,degraded,fallback:body.degraded===true,confidence,ranking_mode,...(fixture.id==='perspectives'?{perspectives_complete}:{}),duration_ms:Date.now()-started}); } catch { results.push({case:fixture.id,status:'failed',reason:'request_or_response_failed',duration_ms:Date.now()-started}); } diff --git a/scripts/report-production-picker.mjs b/scripts/report-production-picker.mjs index ac39386..475d304 100644 --- a/scripts/report-production-picker.mjs +++ b/scripts/report-production-picker.mjs @@ -12,6 +12,10 @@ export async function reportPickerProbe(result,{key,fetchImpl=fetch,runUrl,relea if(Number.isInteger(fixture.status_code)&&fixture.status_code>=100&&fixture.status_code<=599) props[`${fixture.case}_status_code`]=fixture.status_code; if(typeof fixture.degraded==='boolean') props[`${fixture.case}_degraded`]=fixture.degraded; + if(typeof fixture.fallback==='boolean') props[`${fixture.case}_fallback`]=fixture.fallback; + if(['high','medium','low'].includes(fixture.confidence)) props[`${fixture.case}_confidence`]=fixture.confidence; + if(['general','perspective','general_fallback','retrieval_fallback'].includes(fixture.ranking_mode)) props[`${fixture.case}_ranking_mode`]=fixture.ranking_mode; + if(fixture.case==='perspectives'&&typeof fixture.perspectives_complete==='boolean') props.perspectives_complete=fixture.perspectives_complete; if(Number.isFinite(fixture.duration_ms)&&fixture.duration_ms>=0) props[`${fixture.case}_duration_ms`]=Math.min(Math.round(fixture.duration_ms),600000); } diff --git a/tests/production-picker-reporting.test.mjs b/tests/production-picker-reporting.test.mjs index 4168109..8693733 100644 --- a/tests/production-picker-reporting.test.mjs +++ b/tests/production-picker-reporting.test.mjs @@ -39,6 +39,19 @@ test('HTTP 200 degradation still produces an error-level failed probe event',asy assert.equal(body.logs[0].props.perspectives_degraded,true); }); +test('bounded cause fields reach App Health while unknown text is excluded',async()=>{ + let body; + await reportPickerProbe({status:'failed',results:[ + {case:'perspectives',status:'failed',degraded:true,fallback:false,confidence:'low',ranking_mode:'perspective',perspectives_complete:true}, + {case:'general',status:'failed',confidence:'private secret',ranking_mode:'private prompt',fallback:'private',perspectives_complete:'private'} + ]},{key:'test-key',fetchImpl:async(_url,init)=>{body=JSON.parse(init.body);return new Response(null,{status:202});}}); + const props=body.logs[0].props; + assert.equal(props.perspectives_fallback,false);assert.equal(props.perspectives_confidence,'low'); + assert.equal(props.perspectives_ranking_mode,'perspective');assert.equal(props.perspectives_complete,true); + assert.equal(Object.hasOwn(props,'general_confidence'),false);assert.equal(Object.hasOwn(props,'general_ranking_mode'),false); + assert.doesNotMatch(JSON.stringify(body),/private|test-key/); +}); + test('an unavailable public site still generates a failed probe log without an HTTP response',async()=>{ let body; await reportPickerProbe({status:'failed',results:[{case:'general',status:'failed',reason:'request_or_response_failed',duration_ms:100}]},{ diff --git a/tests/production-picker.test.mjs b/tests/production-picker.test.mjs index 813e1ab..efa124f 100644 --- a/tests/production-picker.test.mjs +++ b/tests/production-picker.test.mjs @@ -72,6 +72,20 @@ test('real perspective ranking accepts 30 candidates through the managed adapter assert.deepEqual(requests.map(request=>request.labels),[30,30,30,5]); }); +test('equally strong final perspective fits retain the five independently ranked memes',async()=>{ + let calls=0; + const fetchImpl=async(_url,init)=>{ + const {labels,inputs}=JSON.parse(init.body);calls++; + if(labels.length===5)return Response.json({results:inputs.map(()=>({label:labels[3],scores:Object.fromEntries(labels.map((label,index)=>[label,index===3?1:0]))}))}); + const winner=(calls-1)%3; + return Response.json({results:[{label:labels[winner],scores:Object.fromEntries(labels.map((label,index)=>[label,index===winner?0.9:0.01]))}]}); + }; + const ranked=await rankCandidatesByPerspective('I told my coworker the deadline was today and he started another coffee break.',catalogue.slice(0,30),{fetchImpl,limit:5}); + assert.equal(calls,4);assert.equal(ranked.length,5);assert.equal(new Set(ranked.map(x=>x.id)).size,5); + assert.deepEqual(new Set(ranked.map(x=>x.perspective)),new Set(['self','other','situation'])); + assert(ranked.every(x=>x.fit_label==='strong'&&x.classifier_score===.75)); +}); + const budget={idFromName:name=>name,get:()=>({fetch:async(url,options)=>{ const body=JSON.parse(options.body);const vector=url.endsWith('try-debit-vectorize');const amount=vector?body.dimensions:body.neurons; return Response.json({allowed:true,used:amount,remaining:(vector?45_000_000:9500)-amount,retryAfter:0,baselineVerified:true,[vector?'monthKey':'dayKey']:new Date().toISOString().slice(0,vector?7:10)}); @@ -157,6 +171,17 @@ test('functional probe rejects an HTTP 200 fallback and missing perspectives',as } }); +test('probe distinguishes low confidence from fallback and missing perspectives without private fields',async()=>{ + const candidates=Array.from({length:5},(_,index)=>({id:`candidate-${index}`,media_url:'https://example.test/image.png',score:80,perspective:['self','other','situation'][index%3]})); + const body={decision:'meme',confidence:'low',ranking_mode:'perspective',degraded:false,candidates,private:'private situation'}; + const result=await probePicker('https://example.test',async()=>Response.json(body)); + assert.equal(result.status,'failed');assert.equal(result.results[1].fallback,false); + assert.equal(result.results[1].confidence,'low');assert.equal(result.results[1].ranking_mode,'perspective'); + assert.equal(result.results[1].perspectives_complete,true);assert.doesNotMatch(JSON.stringify(result),/private situation/); + const unknown=await probePicker('https://example.test',async()=>Response.json({...body,confidence:'private secret',ranking_mode:'private account'})); + assert.equal(unknown.results[0].confidence,null);assert.equal(unknown.results[0].ranking_mode,null); +}); + test('browser reports caught picker errors without attaching comments or error text',()=>{ const app=readFileSync(new URL('../worker/public/app.js',import.meta.url),'utf8'); assert.match(app,/appHealthLog\?\.\('recommendation.failed'/); diff --git a/worker/src/classification.mjs b/worker/src/classification.mjs index 7df0411..bec74fb 100644 --- a/worker/src/classification.mjs +++ b/worker/src/classification.mjs @@ -133,7 +133,7 @@ function directOrdinalScore(answer) { return {classifier_score:score/(FIT_CRITERIA.length-1),fit_label:FIT_LABELS[fitIndex].key}; } -async function scoreDirectCandidates(comment,candidates,{apiKey,fetchImpl,timeoutMs,instructions=FIT_INSTRUCTIONS}={}) { +async function scoreDirectCandidates(comment,candidates,{apiKey,fetchImpl,timeoutMs,instructions=FIT_INSTRUCTIONS,allowTies=false}={}) { const state=directCandidateState(comment,candidates); state.task=instructions; const questions=Object.fromEntries(candidates.map((_,index)=>[`fit_${index}`,{ @@ -143,7 +143,7 @@ async function scoreDirectCandidates(comment,candidates,{apiKey,fetchImpl,timeou }])); const answers=await askJev({state,questions,apiKey,fetchImpl,timeoutMs}); const scored=candidates.map((record,index)=>({...record,...directOrdinalScore(answers[`fit_${index}`]),retrieval_rank:index+1})); - if(scored.length>1&&scored.every(record=>record.classifier_score===scored[0].classifier_score)) throw new Error('TypeSafe returned flat ordinal scores.'); + if(!allowTies&&scored.length>1&&scored.every(record=>record.classifier_score===scored[0].classifier_score)) throw new Error('TypeSafe returned flat ordinal scores.'); return scored; } @@ -162,12 +162,12 @@ function ordinalScore(result) { }; } -async function scoreOrdinalCandidates(comment,candidates,{fetchImpl,timeoutMs,instructions=FIT_INSTRUCTIONS,inputFor=candidateInput,apiKey}={}) { - if(apiKey) return scoreDirectCandidates(comment,candidates,{apiKey,fetchImpl,timeoutMs,instructions}); +async function scoreOrdinalCandidates(comment,candidates,{fetchImpl,timeoutMs,instructions=FIT_INSTRUCTIONS,inputFor=candidateInput,apiKey,allowTies=false}={}) { + if(apiKey) return scoreDirectCandidates(comment,candidates,{apiKey,fetchImpl,timeoutMs,instructions,allowTies}); const labels=FIT_LABELS.map(({label})=>label); const results=await classifyMany({inputs:candidates.map(record=>inputFor(comment,record)),labels,instructions,fetchImpl,timeoutMs}); const scored=candidates.map((record,index)=>({...record,...ordinalScore(results[index]),retrieval_rank:index+1})); - if(scored.length>1&&scored.every(record=>record.classifier_score===scored[0].classifier_score)) throw new Error('Classifier returned flat ordinal scores.'); + if(!allowTies&&scored.length>1&&scored.every(record=>record.classifier_score===scored[0].classifier_score)) throw new Error('Classifier returned flat ordinal scores.'); return scored; } @@ -290,7 +290,9 @@ export async function rankCandidatesByPerspective(comment,candidates,{fetchImpl= if(selected.lengthright.classifier_score-left.classifier_score||left.perspective.localeCompare(right.perspective)); const rescored=await scoreOrdinalCandidates(comment,selected.slice(0,limit),{ - fetchImpl,timeoutMs,apiKey, + // These distinct memes were already selected by non-flat perspective + // rankings. Equal final fit is valid; keep their viewpoints and scores. + fetchImpl,timeoutMs,apiKey,allowTies:true, instructions:`${FIT_INSTRUCTIONS} The input declares the intended viewpoint. Judge the candidate only for that viewpoint; do not silently switch to another participant or to the event itself.`, inputFor:(text,record)=>`COMMENT: ${text}\nVIEWPOINT: ${record.perspective_label}\nCANDIDATE: ${record.name}. Meaning: ${record.message} Social dynamic: ${record.relational_pattern} Example: ${record.example_context} Avoid: ${record.near_miss_context}` });