Keep valid perspective fit ties and explain strict probe failures

docs/production-picker-monitoring.md

@@ -20,6 +20,12 @@ source, logs or chat. The workflow fails explicitly when reporting is missing
 or rejected; a green required-reporting run means both the picker check and
 App Health ingest succeeded.
 
+The workflow's probe receipt also records allowlisted confidence and ranking
+mode, whether the API marked the response as a fallback, and whether all three
+perspectives are present. Strict failure criteria are unchanged. These fields
+distinguish a genuine low-confidence selection from degraded routing without
+including comments, provider bodies or request headers.
+
 App Health's Overview feed must include production error logs and warning-level
 `*.degraded` logs to surface these outcomes independently of its 20-request
 aggregate-health threshold. Overview alerts remain retained occurrences rather

scripts/probe-production-picker.mjs

@@ -21,8 +21,11 @@ export async function probePicker(origin='https://memes.significanthobbies.com',
         &&new Set(candidates.map(candidate=>candidate.id)).size===5
         &&candidates.every(candidate=>typeof candidate.id==='string'&&candidate.id&&typeof candidate.media_url==='string'&&/^https:\/\//.test(candidate.media_url)&&Number.isFinite(candidate.score));
       const perspectives=new Set((candidates??[]).map(candidate=>candidate.perspective));
-      const degraded=body.degraded===true||body.confidence==='low'||(fixture.id==='perspectives'&&!['self','other','situation'].every(perspective=>perspectives.has(perspective)));
-      results.push({case:fixture.id,status:valid&&!degraded?'passed':'failed',status_code:response.status,degraded,duration_ms:Date.now()-started});
+      const perspectives_complete=['self','other','situation'].every(perspective=>perspectives.has(perspective));
+      const degraded=body.degraded===true||body.confidence==='low'||(fixture.id==='perspectives'&&!perspectives_complete);
+      const confidence=['high','medium','low'].includes(body.confidence)?body.confidence:null;
+      const ranking_mode=['general','perspective','general_fallback','retrieval_fallback'].includes(body.ranking_mode)?body.ranking_mode:null;
+      results.push({case:fixture.id,status:valid&&!degraded?'passed':'failed',status_code:response.status,degraded,fallback:body.degraded===true,confidence,ranking_mode,...(fixture.id==='perspectives'?{perspectives_complete}:{}),duration_ms:Date.now()-started});
     } catch {
       results.push({case:fixture.id,status:'failed',reason:'request_or_response_failed',duration_ms:Date.now()-started});
     }

scripts/report-production-picker.mjs

@@ -12,6 +12,10 @@ export async function reportPickerProbe(result,{key,fetchImpl=fetch,runUrl,relea
     if(Number.isInteger(fixture.status_code)&&fixture.status_code>=100&&fixture.status_code<=599)
       props[`${fixture.case}_status_code`]=fixture.status_code;
     if(typeof fixture.degraded==='boolean') props[`${fixture.case}_degraded`]=fixture.degraded;
+    if(typeof fixture.fallback==='boolean') props[`${fixture.case}_fallback`]=fixture.fallback;
+    if(['high','medium','low'].includes(fixture.confidence)) props[`${fixture.case}_confidence`]=fixture.confidence;
+    if(['general','perspective','general_fallback','retrieval_fallback'].includes(fixture.ranking_mode)) props[`${fixture.case}_ranking_mode`]=fixture.ranking_mode;
+    if(fixture.case==='perspectives'&&typeof fixture.perspectives_complete==='boolean') props.perspectives_complete=fixture.perspectives_complete;
     if(Number.isFinite(fixture.duration_ms)&&fixture.duration_ms>=0)
       props[`${fixture.case}_duration_ms`]=Math.min(Math.round(fixture.duration_ms),600000);
   }

tests/production-picker-reporting.test.mjs

@@ -39,6 +39,19 @@ test('HTTP 200 degradation still produces an error-level failed probe event',asy
   assert.equal(body.logs[0].props.perspectives_degraded,true);
 });
 
+test('bounded cause fields reach App Health while unknown text is excluded',async()=>{
+  let body;
+  await reportPickerProbe({status:'failed',results:[
+    {case:'perspectives',status:'failed',degraded:true,fallback:false,confidence:'low',ranking_mode:'perspective',perspectives_complete:true},
+    {case:'general',status:'failed',confidence:'private secret',ranking_mode:'private prompt',fallback:'private',perspectives_complete:'private'}
+  ]},{key:'test-key',fetchImpl:async(_url,init)=>{body=JSON.parse(init.body);return new Response(null,{status:202});}});
+  const props=body.logs[0].props;
+  assert.equal(props.perspectives_fallback,false);assert.equal(props.perspectives_confidence,'low');
+  assert.equal(props.perspectives_ranking_mode,'perspective');assert.equal(props.perspectives_complete,true);
+  assert.equal(Object.hasOwn(props,'general_confidence'),false);assert.equal(Object.hasOwn(props,'general_ranking_mode'),false);
+  assert.doesNotMatch(JSON.stringify(body),/private|test-key/);
+});
+
 test('an unavailable public site still generates a failed probe log without an HTTP response',async()=>{
   let body;
   await reportPickerProbe({status:'failed',results:[{case:'general',status:'failed',reason:'request_or_response_failed',duration_ms:100}]},{

tests/production-picker.test.mjs

@@ -72,6 +72,20 @@ test('real perspective ranking accepts 30 candidates through the managed adapter
   assert.deepEqual(requests.map(request=>request.labels),[30,30,30,5]);
 });
 
+test('equally strong final perspective fits retain the five independently ranked memes',async()=>{
+  let calls=0;
+  const fetchImpl=async(_url,init)=>{
+    const {labels,inputs}=JSON.parse(init.body);calls++;
+    if(labels.length===5)return Response.json({results:inputs.map(()=>({label:labels[3],scores:Object.fromEntries(labels.map((label,index)=>[label,index===3?1:0]))}))});
+    const winner=(calls-1)%3;
+    return Response.json({results:[{label:labels[winner],scores:Object.fromEntries(labels.map((label,index)=>[label,index===winner?0.9:0.01]))}]});
+  };
+  const ranked=await rankCandidatesByPerspective('I told my coworker the deadline was today and he started another coffee break.',catalogue.slice(0,30),{fetchImpl,limit:5});
+  assert.equal(calls,4);assert.equal(ranked.length,5);assert.equal(new Set(ranked.map(x=>x.id)).size,5);
+  assert.deepEqual(new Set(ranked.map(x=>x.perspective)),new Set(['self','other','situation']));
+  assert(ranked.every(x=>x.fit_label==='strong'&&x.classifier_score===.75));
+});
+
 const budget={idFromName:name=>name,get:()=>({fetch:async(url,options)=>{
   const body=JSON.parse(options.body);const vector=url.endsWith('try-debit-vectorize');const amount=vector?body.dimensions:body.neurons;
   return Response.json({allowed:true,used:amount,remaining:(vector?45_000_000:9500)-amount,retryAfter:0,baselineVerified:true,[vector?'monthKey':'dayKey']:new Date().toISOString().slice(0,vector?7:10)});
@@ -157,6 +171,17 @@ test('functional probe rejects an HTTP 200 fallback and missing perspectives',as
   }
 });
 
+test('probe distinguishes low confidence from fallback and missing perspectives without private fields',async()=>{
+  const candidates=Array.from({length:5},(_,index)=>({id:`candidate-${index}`,media_url:'https://example.test/image.png',score:80,perspective:['self','other','situation'][index%3]}));
+  const body={decision:'meme',confidence:'low',ranking_mode:'perspective',degraded:false,candidates,private:'private situation'};
+  const result=await probePicker('https://example.test',async()=>Response.json(body));
+  assert.equal(result.status,'failed');assert.equal(result.results[1].fallback,false);
+  assert.equal(result.results[1].confidence,'low');assert.equal(result.results[1].ranking_mode,'perspective');
+  assert.equal(result.results[1].perspectives_complete,true);assert.doesNotMatch(JSON.stringify(result),/private situation/);
+  const unknown=await probePicker('https://example.test',async()=>Response.json({...body,confidence:'private secret',ranking_mode:'private account'}));
+  assert.equal(unknown.results[0].confidence,null);assert.equal(unknown.results[0].ranking_mode,null);
+});
+
 test('browser reports caught picker errors without attaching comments or error text',()=>{
   const app=readFileSync(new URL('../worker/public/app.js',import.meta.url),'utf8');
   assert.match(app,/appHealthLog\?\.\('recommendation.failed'/);

worker/src/classification.mjs

@@ -133,7 +133,7 @@ function directOrdinalScore(answer) {
   return {classifier_score:score/(FIT_CRITERIA.length-1),fit_label:FIT_LABELS[fitIndex].key};
 }
 
-async function scoreDirectCandidates(comment,candidates,{apiKey,fetchImpl,timeoutMs,instructions=FIT_INSTRUCTIONS}={}) {
+async function scoreDirectCandidates(comment,candidates,{apiKey,fetchImpl,timeoutMs,instructions=FIT_INSTRUCTIONS,allowTies=false}={}) {
   const state=directCandidateState(comment,candidates);
   state.task=instructions;
   const questions=Object.fromEntries(candidates.map((_,index)=>[`fit_${index}`,{
@@ -143,7 +143,7 @@ async function scoreDirectCandidates(comment,candidates,{apiKey,fetchImpl,timeou
   }]));
   const answers=await askJev({state,questions,apiKey,fetchImpl,timeoutMs});
   const scored=candidates.map((record,index)=>({...record,...directOrdinalScore(answers[`fit_${index}`]),retrieval_rank:index+1}));
-  if(scored.length>1&&scored.every(record=>record.classifier_score===scored[0].classifier_score)) throw new Error('TypeSafe returned flat ordinal scores.');
+  if(!allowTies&&scored.length>1&&scored.every(record=>record.classifier_score===scored[0].classifier_score)) throw new Error('TypeSafe returned flat ordinal scores.');
   return scored;
 }
 
@@ -162,12 +162,12 @@ function ordinalScore(result) {
   };
 }
 
-async function scoreOrdinalCandidates(comment,candidates,{fetchImpl,timeoutMs,instructions=FIT_INSTRUCTIONS,inputFor=candidateInput,apiKey}={}) {
-  if(apiKey) return scoreDirectCandidates(comment,candidates,{apiKey,fetchImpl,timeoutMs,instructions});
+async function scoreOrdinalCandidates(comment,candidates,{fetchImpl,timeoutMs,instructions=FIT_INSTRUCTIONS,inputFor=candidateInput,apiKey,allowTies=false}={}) {
+  if(apiKey) return scoreDirectCandidates(comment,candidates,{apiKey,fetchImpl,timeoutMs,instructions,allowTies});
   const labels=FIT_LABELS.map(({label})=>label);
   const results=await classifyMany({inputs:candidates.map(record=>inputFor(comment,record)),labels,instructions,fetchImpl,timeoutMs});
   const scored=candidates.map((record,index)=>({...record,...ordinalScore(results[index]),retrieval_rank:index+1}));
-  if(scored.length>1&&scored.every(record=>record.classifier_score===scored[0].classifier_score)) throw new Error('Classifier returned flat ordinal scores.');
+  if(!allowTies&&scored.length>1&&scored.every(record=>record.classifier_score===scored[0].classifier_score)) throw new Error('Classifier returned flat ordinal scores.');
   return scored;
 }
 
@@ -290,7 +290,9 @@ export async function rankCandidatesByPerspective(comment,candidates,{fetchImpl=
   if(selected.length<limit) throw new Error('Perspective ranking could not produce distinct candidates.');
   if(ordinalPerspectives&&!apiKey) return selected.slice(0,limit).sort((left,right)=>right.classifier_score-left.classifier_score||left.perspective.localeCompare(right.perspective));
   const rescored=await scoreOrdinalCandidates(comment,selected.slice(0,limit),{
-    fetchImpl,timeoutMs,apiKey,
+    // These distinct memes were already selected by non-flat perspective
+    // rankings. Equal final fit is valid; keep their viewpoints and scores.
+    fetchImpl,timeoutMs,apiKey,allowTies:true,
     instructions:`${FIT_INSTRUCTIONS} The input declares the intended viewpoint. Judge the candidate only for that viewpoint; do not silently switch to another participant or to the event itself.`,
     inputFor:(text,record)=>`COMMENT: ${text}\nVIEWPOINT: ${record.perspective_label}\nCANDIDATE: ${record.name}. Meaning: ${record.message} Social dynamic: ${record.relational_pattern} Example: ${record.example_context} Avoid: ${record.near_miss_context}`
   });