Skip to content

Commit 7919441

Browse files
committed
ownership-lab: semantic-coverage Stage C/D scripts and the witness generator (sequential mode, receiver probe, null-result, per-row TFM)
Scripts of the semantic-coverage track (Stage A1/A2, B, C, D, E1/E2/E3 census and scan) copied under corpus/ownership-lab/sc/scripts; the H-10 witness generator gains a sequential mode (dispose between calls), a receiver-ownership probe template, a null-result verdict, a per-row target framework, source-hash provenance and resumable partial output. Research corpus only; no production change. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Am9eQwzNfbugH72eVKetC2
1 parent 87feda1 commit 7919441

23 files changed

Lines changed: 1035 additions & 10 deletions
Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
"""A2: proof identities from RelDump relations. id_full(m) = H(il(m), sorted il of the transitive same-assembly callee
2+
closure, plus external callee closures resolved in the dependency dumps); id_typed(m) = the same over callees whose
3+
return type is disposable (the callees that can produce the returned object). Bodiless callees contribute a marker."""
4+
import json, hashlib, sys, collections, time
5+
S='/tmp/claude-0/-home-user/8a9da608-aa27-5449-8306-1fae9426f9b9/scratchpad'
6+
def load(path):
7+
d=json.load(open(path)); by=collections.defaultdict(list)
8+
for r in d['records']: by[r['metadata_name']].append(r)
9+
for t,hsh in d.get('cctors',{}).items(): by[t+'::.cctor'].append({'metadata_name':t+'::.cctor','il_hash':hsh,'callees':[],'external_callees':[],'ret_disposable':True,'ret_is_type_parameter':False,'static_field_types':[]})
10+
return d, by
11+
def closure(name, dumps, typed, seen=None):
12+
"""dumps: list of (by) dicts to resolve names; returns the set of (name, il_hash or 'NOBODY') reached"""
13+
seen=set() if seen is None else seen
14+
if name in seen: return seen
15+
seen.add(name)
16+
recs=None
17+
for by in dumps:
18+
if name in by: recs=by[name]; break
19+
if not recs: return seen
20+
for r in recs:
21+
for c in list(r['callees'])+list(r['external_callees'])+[t+'::.cctor' for t in r.get('static_field_types',[])]:
22+
if typed:
23+
crec=None
24+
for by in dumps:
25+
if c in by: crec=by[c]; break
26+
if crec is not None and not any(x['ret_disposable'] or x.get('ret_is_type_parameter') for x in crec): continue
27+
closure(c, dumps, typed, seen)
28+
return seen
29+
def ident(name, dumps, typed):
30+
cl=closure(name, dumps, typed); parts=[]
31+
for n in sorted(cl):
32+
recs=None
33+
for by in dumps:
34+
if n in by: recs=by[n]; break
35+
if recs is None: parts.append(n+'=EXTERNAL_UNRESOLVED'); continue
36+
parts.append(n+'='+'|'.join(sorted((x['il_hash'] or 'NOBODY') for x in recs)))
37+
return hashlib.sha256('\n'.join(parts).encode()).hexdigest()[:16], len(cl), sum(1 for p in parts if p.endswith('EXTERNAL_UNRESOLVED') or 'NOBODY' in p)
38+
out={}
39+
# --- A1 fixture sensitivity and no-op stability
40+
A=f'{S}/lab/sc/a1/out'
41+
_,cv1=load(f'{A}/rel-caller.v1.json'); _,hv1=load(f'{A}/rel-helper.v1.json'); _,hv1b=load(f'{A}/rel-helper.v1b.json'); _,hv2=load(f'{A}/rel-helper.v2.json')
42+
sens=[]
43+
for m in ['MakeDirect','MakeOneHop','MakeTwoHop','MakeVirtual','MakeViaProperty','MakeGeneric']:
44+
n='CallerLib.Caller::'+m
45+
f1,s1,u1=ident(n,[cv1,hv1],False); f2,s2,u2=ident(n,[cv1,hv2],False); f1b,_,_=ident(n,[cv1,hv1b],False)
46+
t1,ts1,_=ident(n,[cv1,hv1],True); t2,ts2,_=ident(n,[cv1,hv2],True)
47+
sens.append({'variant':m,'id_full_v1':f1,'id_full_v2':f2,'full_changes':f1!=f2,'id_full_size':s1,'unresolved_or_bodiless_in_slice':u1,'noop_rebuild_stable':f1==f1b,'id_typed_v1':t1,'id_typed_v2':t2,'typed_changes':t1!=t2,'id_typed_size':ts1})
48+
out['a1_sensitivity']=sens
49+
mv=lambda p: json.load(open(p))['mvid']
50+
out['a1_noop_rebuild']={'helper_v1_mvid':mv(f'{A}/rel-helper.v1.json'),'helper_v1b_mvid':mv(f'{A}/rel-helper.v1b.json'),'il_hashes_identical':all((hv1[k][0]['il_hash']==hv1b[k][0]['il_hash']) for k in hv1)}
51+
# --- SkiaSharp: identity size, TFM stability, fanout, time
52+
def skia(tag): return load(f'{S}/lab/ct/relh-SkiaSharp@{tag}.json')[1]
53+
t0=time.time(); tags=['3.119.4@net6.0','4.152.0@net6.0','4.153.1@net6.0','4.153.1@netstandard2.0']; ids={}; sizes={}
54+
for tag in tags:
55+
by=skia(tag); ids[tag]={}; sizes[tag]=[]
56+
for n in by:
57+
if any(r['ret_disposable'] for r in by[n]):
58+
f,sz,_=ident(n,[by],False); t,tsz,_=ident(n,[by],True); ids[tag][n]=(f,t); sizes[tag].append((sz,tsz))
59+
out['skia_derivation_seconds']=round(time.time()-t0,1)
60+
rows=['SkiaSharp.SKBitmap::Decode','SkiaSharp.SKBitmap::FromImage','SkiaSharp.SKData::AsStream','SkiaSharp.SKFont::GetTextPath','SkiaSharp.SKPath::ParseSvgPathData','SkiaSharp.SKTypeface::ToFont']
61+
def cmp(a,b):
62+
common=[n for n in ids[a] if n in ids[b]]
63+
return {'common':len(common),'full_changed':sum(1 for n in common if ids[a][n][0]!=ids[b][n][0]),'typed_changed':sum(1 for n in common if ids[a][n][1]!=ids[b][n][1]),
64+
'rows_full_changed':[n for n in rows if n in ids[a] and n in ids[b] and ids[a][n][0]!=ids[b][n][0]],'rows_typed_changed':[n for n in rows if n in ids[a] and n in ids[b] and ids[a][n][1]!=ids[b][n][1]]}
65+
out['skia_tfm_stability_4.153.1_net6_vs_ns20']=cmp('4.153.1@net6.0','4.153.1@netstandard2.0')
66+
out['skia_fanout_4.152.0_to_4.153.1']=cmp('4.152.0@net6.0','4.153.1@net6.0')
67+
out['skia_fanout_3.119.4_to_4.153.1']=cmp('3.119.4@net6.0','4.153.1@net6.0')
68+
import statistics
69+
sz=sizes['4.153.1@net6.0']; out['skia_identity_size_4.153.1']={'surface_callables':len(sz),'full_mean':round(statistics.mean(s for s,_ in sz),1),'full_max':max(s for s,_ in sz),'typed_mean':round(statistics.mean(t for _,t in sz),1),'typed_max':max(t for _,t in sz)}
70+
by=skia('4.153.1@net6.0'); out['skia_row_slices']={n:{'full_size':ident(n,[by],False)[1],'typed_size':ident(n,[by],True)[1],'typed_slice':sorted(closure(n,[by],True))[:12]} for n in rows if n in by}
71+
json.dump(out,open(f'{S}/lab/sc/a2-results.json','w'),indent=1); print(json.dumps(out,indent=1)[:6000])
Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
import json, hashlib, collections
2+
def load(path):
3+
d=json.load(open(path)); by=collections.defaultdict(list)
4+
for r in d['records']: by[r['metadata_name']].append(r)
5+
for t,hsh in d.get('cctors',{}).items(): by[t+'::.cctor'].append({'metadata_name':t+'::.cctor','il_hash':hsh,'callees':[],'external_callees':[],'ret_disposable':True,'ret_is_type_parameter':False,'static_field_types':[]})
6+
return d, by
7+
def meta_name(by, callable_):
8+
for k,v in by.items():
9+
if v and v[0].get('callable')==callable_: return k
10+
return None
11+
def closure(name, dumps, typed, seen=None):
12+
seen=set() if seen is None else seen
13+
if name in seen: return seen
14+
seen.add(name); recs=None
15+
for by in dumps:
16+
if name in by: recs=by[name]; break
17+
if not recs: return seen
18+
for r in recs:
19+
for c in list(r['callees'])+list(r['external_callees'])+[t+'::.cctor' for t in r.get('static_field_types',[])]:
20+
if typed:
21+
crec=None
22+
for by in dumps:
23+
if c in by: crec=by[c]; break
24+
if crec is not None and not any(x['ret_disposable'] or x.get('ret_is_type_parameter') for x in crec): continue
25+
closure(c, dumps, typed, seen)
26+
return seen
27+
def ident(name, dumps, typed):
28+
cl=closure(name, dumps, typed); parts=[]; unres=0
29+
for n in sorted(cl):
30+
recs=None
31+
for by in dumps:
32+
if n in by: recs=by[n]; break
33+
if recs is None: parts.append(n+'=EXTERNAL_UNRESOLVED'); unres+=1; continue
34+
hs='|'.join(sorted((x['il_hash'] or 'NOBODY') for x in recs)); parts.append(n+'='+hs); unres+= ('NOBODY' in hs)
35+
return hashlib.sha256('\n'.join(parts).encode()).hexdigest()[:16], len(cl), unres
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/usr/bin/env bash
2+
S=/tmp/claude-0/-home-user/8a9da608-aa27-5449-8306-1fae9426f9b9/scratchpad
3+
for i in $(seq 1 120); do grep -q WHOLESRC_DONE $S/lab/sc/wholesrc.log && break; sleep 10; done
4+
cd $S/lab/sc && python3 derive_whole.py
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/usr/bin/env bash
2+
S=/tmp/claude-0/-home-user/8a9da608-aa27-5449-8306-1fae9426f9b9/scratchpad
3+
for i in $(seq 1 240); do [ -f $S/lab/sc/usage-share.json ] && break; sleep 15; done
4+
cd $S/lab/sc && python3 e1pool.py
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/usr/bin/env bash
2+
# Stage E2: after the E1 pool census, freeze the pool (copy into paperwork), then run e2.py over the first 12 nominated
3+
# repositories in pool order; each clone is deleted after its census to keep the disk bounded. No OFF/ON here.
4+
S=/tmp/claude-0/-home-user/8a9da608-aa27-5449-8306-1fae9426f9b9/scratchpad; P=/home/user/Own.NET-paperwork
5+
export PATH=/root/.dotnet:$PATH DOTNET_CLI_TELEMETRY_OPTOUT=1 DOTNET_NOLOGO=1
6+
for i in $(seq 1 300); do grep -q E1POOL_DONE $S/lab/sc/e1pool.log && break; sleep 15; done
7+
for i in $(seq 1 300); do grep -q WITNESS_DONE $S/lab/sc/witness3.log && break; sleep 15; done
8+
cp $S/lab/sc/e1pool.json $P/paper-eval/semantic-coverage/e1-pool-frozen-v1.json
9+
python3 - <<'PY'
10+
import json; S='/tmp/claude-0/-home-user/8a9da608-aa27-5449-8306-1fae9426f9b9/scratchpad'
11+
pool=json.load(open(f'{S}/lab/sc/e1pool.json'))['frozen_pool_top30']; open(f'{S}/lab/sc/e2-order.txt','w').write('\n'.join(x['repo'].replace('github.com/','') for x in pool[:12]))
12+
PY
13+
while read -r repo; do [ -z "$repo" ] && continue; echo "=== $repo $(date -u +%H:%M:%S)"; timeout 3600 python3 $S/lab/sc/e2.py "$repo" 2>&1 | tail -3; rm -rf "$S/lab/sc/e2/${repo//\//__}"; df -h / | tail -1 | awk '{print "avail",$4}'; done < $S/lab/sc/e2-order.txt
14+
echo E2_DONE
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/usr/bin/env bash
2+
S=/tmp/claude-0/-home-user/8a9da608-aa27-5449-8306-1fae9426f9b9/scratchpad
3+
for i in $(seq 1 240); do grep -q E1POOL_DONE $S/lab/sc/e1pool.log && break; sleep 15; done
4+
cd $S/lab/sc && python3 usage_share.py > usage_share2.log 2>&1
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/usr/bin/env bash
2+
# witness pass 3 (sequential mode + repaired args, retries only rows without a fresh/cached verdict), then the
3+
# receiver-ownership probe over the fresh rows with receivers, then the Stage E2 census chain
4+
S=/tmp/claude-0/-home-user/8a9da608-aa27-5449-8306-1fae9426f9b9/scratchpad
5+
export PATH=/root/.dotnet:$PATH DOTNET_CLI_TELEMETRY_OPTOUT=1 DOTNET_NOLOGO=1
6+
python3 $S/lab/witness/gen.py $S/lab/sc/witness-rows.json $S/lab/sc/witness-results.json > $S/lab/sc/witness4.log 2>&1
7+
python3 $S/lab/witness/gen.py $S/lab/sc/recvprobe-rows.json $S/lab/sc/recvprobe-results.json > $S/lab/sc/recvprobe.log 2>&1
8+
echo OWNPROBE_DONE >> $S/lab/sc/recvprobe.log
9+
bash $S/lab/sc/chain_e2.sh > $S/lab/sc/e2.log 2>&1
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/usr/bin/env bash
2+
# pass 4: PostgreSQL moved to /tmp/pgsc (harness resets scratch-tree permissions); retry rows without fresh/cached verdicts,
3+
# retry receiver probes with errors, then the Stage E2 census chain
4+
S=/tmp/claude-0/-home-user/8a9da608-aa27-5449-8306-1fae9426f9b9/scratchpad
5+
export PATH=/root/.dotnet:$PATH DOTNET_CLI_TELEMETRY_OPTOUT=1 DOTNET_NOLOGO=1
6+
for i in $(seq 1 200); do grep -q WITNESS_DONE $S/lab/sc/recvprobe.log && break; sleep 10; done
7+
python3 $S/lab/witness/gen.py $S/lab/sc/witness-rows.json $S/lab/sc/witness-results.json > $S/lab/sc/witness5.log 2>&1
8+
python3 $S/lab/witness/gen.py $S/lab/sc/recvprobe-rows.json $S/lab/sc/recvprobe-results.json > $S/lab/sc/recvprobe2.log 2>&1
9+
echo OWNPROBE_DONE >> $S/lab/sc/recvprobe2.log
10+
bash $S/lab/sc/chain_e2.sh > $S/lab/sc/e2.log 2>&1
Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
"""Stage B: whole-source derivation with the frozen rules (E1/E2/E3 + H-20) over src-whole of each library; whole-set
2+
extractor run; the per-file union is used only if the whole-set run fails. Rows = public factory-surface callables
3+
proved fresh (E1/E3) + receiver releases proved by E2, pinned to the package assembly."""
4+
import json, os, sys, glob, subprocess, time, re
5+
S='/tmp/claude-0/-home-user/8a9da608-aa27-5449-8306-1fae9426f9b9/scratchpad'; D=f'{S}/lab/disc'; R='/home/user/Own.NET'
6+
LIBS=sys.argv[1:] or ['SkiaSharp','RabbitMQ.Client','MailKit','StackExchange.Redis','Npgsql','MQTTnet','SSH.NET','LibGit2Sharp']
7+
GL='// synthesised: SDK implicit usings\nglobal using System;\nglobal using System.Collections.Generic;\nglobal using System.IO;\nglobal using System.Linq;\nglobal using System.Net.Http;\nglobal using System.Threading;\nglobal using System.Threading.Tasks;\n'
8+
env={**os.environ,'PATH':'/root/.dotnet:'+os.environ['PATH'],'DOTNET_NOLOGO':'1','DOTNET_CLI_TELEMETRY_OPTOUT':'1','PYTHONPATH':R,'OWEN_RE_BODY':'1','OWEN_RE_MINTED_RETURN':'1','OWEN_RE_MIXED_RETURN':'1','OWEN_LAB_THROWEXIT':'1'}
9+
for k in ('OWEN_RE_ORACLE','OWEN_P037X_RELATIONAL','OWEN_LAB_NULLINIT','OWEN_LAB_NULLGUARD'): env.pop(k,None)
10+
DLL=f'{R}/frontend/roslyn/OwnSharp.Extractor/bin/Release/net8.0/ownsharp-extract.dll'
11+
for pid in LIBS:
12+
L=f'{D}/libs/{pid}'; d=f'{L}/src-whole'; acq=json.load(open(f'{L}/acquire.json')); api=json.load(open(f'{L}/api.json'))
13+
open(f'{d}/__GlobalUsings.cs','w').write(GL); src=sorted(glob.glob(f'{d}/*.cs')); t0=time.time()
14+
refdir=os.path.dirname(acq['main_assembly']); e=dict(env); e['OWEN_RE_DUMP_EFFECTS']=f'{L}/dump-whole.json'
15+
r=subprocess.run(['dotnet',DLL,'--flow-locals',*src,'-o',f'{L}/facts-whole.json','--ref-dir',refdir],cwd=R,capture_output=True,text=True,env=e)
16+
open(f'{L}/derive-whole.err','w').write(r.stderr[-20000:])
17+
ok=os.path.exists(f'{L}/facts-whole.json') and r.returncode==0
18+
summ=subprocess.run(['python3','-m','ownlang','summaries',f'{L}/facts-whole.json'],cwd=R,capture_output=True,text=True,env=env).stdout if ok else ''
19+
try: summaries=json.loads(summ)['summaries']
20+
except Exception: summaries=[]
21+
fresh={s['method'].split('(')[0] for s in summaries if s.get('returns',{}).get('owned')=='fresh'}
22+
try: dump=json.load(open(f'{L}/dump-whole.json'))
23+
except Exception: dump={}
24+
rel={x['callable'] for x in dump.get('receiver_release',[]) if x.get('releases')}
25+
surface={f['callable'] for f in api['factories']}; relsurf={x['callable'] for x in api['release_name_candidates']}
26+
asm=(acq.get('identity') or {}).get('name') or pid; mvid=(acq.get('identity') or {}).get('mvid')
27+
rows=[{'callable':c,'effect':'return_fresh_owned','provenance':'BODY_PROVED','assembly':{'name':asm,'mvid':mvid},'derived_from':f'whole-source {acq.get("repository_url")}@{acq.get("repository_commit")} via E1/E3'} for c in sorted(fresh&surface)]
28+
rows+=[{'callable':c,'effect':'receiver_terminal_release','provenance':'BODY_PROVED','assembly':{'name':asm,'mvid':mvid},'derived_from':'whole-source via E2'} for c in sorted(rel&relsurf)]
29+
census=re.search(r're-body:[^\n]*',r.stderr); stats=None
30+
try: stats=json.load(open(f'{L}/facts-whole.json')).get('stats')
31+
except Exception: pass
32+
out={'package':pid,'files':len(src),'whole_set_ok':ok,'returncode':r.returncode,'seconds':round(time.time()-t0,1),'stats':stats,'re_body_census':census.group(0) if census else None,'fresh_all':sorted(fresh),'fresh_on_surface':sorted(fresh&surface),'fresh_not_on_public_surface':sorted(fresh-surface),'release_all':sorted(rel),'release_on_surface':sorted(rel&relsurf),'surface_factories':len(surface),'surface_release_candidates':len(relsurf),'rows':rows}
33+
json.dump(out,open(f'{L}/rows-whole.json','w'),indent=1)
34+
print(json.dumps({k:out[k] for k in ('package','files','whole_set_ok','seconds','stats','re_body_census','surface_factories')}),'fresh_all',len(fresh),'rows',len(rows),flush=True)
35+
print('DERIVE_WHOLE_DONE')
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
"""Stage E1: public nomination pool. Per CURRENT trusted row (rows-applied of the 22 libraries, with rows-whole replacing
2+
the 8 Stage B libraries; production-convention rows excluded), Sourcegraph stream (count 1000) aggregated per repository;
3+
exclusions: the row libraries' own repositories, forks by name heuristics, mirrors/generated; the top 30 are frozen."""
4+
import json, os, glob, time, urllib.request, urllib.parse, collections, re, sys
5+
from concurrent.futures import ThreadPoolExecutor
6+
S='/tmp/claude-0/-home-user/8a9da608-aa27-5449-8306-1fae9426f9b9/scratchpad'; D=f'{S}/lab/disc'
7+
WHOLE={'SkiaSharp','RabbitMQ.Client','MailKit','StackExchange.Redis','Npgsql','MQTTnet','SSH.NET','LibGit2Sharp'}
8+
ADO={'CreateCommand','BeginTransaction','ExecuteReader','ExecuteReaderAsync'}; ADOPK={'Npgsql','MySqlConnector','Microsoft.Data.Sqlite.Core'}
9+
rows=[]; own=set()
10+
for acq in sorted(glob.glob(f'{D}/libs/*/acquire.json')):
11+
L=os.path.dirname(acq); pid=os.path.basename(L); a=json.load(open(acq))
12+
if not a.get('main_assembly'): continue
13+
src=f'{L}/rows-whole.json' if pid in WHOLE else f'{L}/rows-applied.json'
14+
if not os.path.exists(src): continue
15+
ents=json.load(open(src)); ents=ents['rows'] if 'rows' in ents else ents['entries']
16+
api=json.load(open(f'{L}/api.json')) if os.path.exists(f'{L}/api.json') else {}; st={f['callable']:f.get('is_static',False) for f in api.get('factories',[])}
17+
rep=(a.get('repository_url') or '').replace('https://github.com/','').replace('.git','').rstrip('/').lower()
18+
if rep and 'go.microsoft' not in rep: own.add(rep)
19+
for e in ents:
20+
c=e['callable']; parts=c.split('.'); m=parts[-1]
21+
if m in ADO and pid in ADOPK: continue
22+
rows.append({'package':pid,'callable':c,'effect':e['effect'],'method':m,'type':parts[-2],'namespace':'.'.join(parts[:-2]),'is_static':bool(st.get(c,False)),'source':'whole' if pid in WHOLE else 'subset'})
23+
own|={'mono/skiasharp','jstedfast/mailkit','k4os/k4os.compression.lz4'}
24+
def q_for(r): return (f'"{r["type"]}.{r["method"]}(" lang:C#' if r['is_static'] else f'".{r["method"]}(" "using {r["namespace"]}" lang:C#')+' patterntype:keyword count:1000'
25+
def stream(query):
26+
url='https://sourcegraph.com/.api/search/stream?'+urllib.parse.urlencode({'q':query,'display':'1000'}); txt=''
27+
for attempt in range(5):
28+
try:
29+
req=urllib.request.Request(url,headers={'Accept':'text/event-stream','User-Agent':'own.net-sc'})
30+
with urllib.request.urlopen(req,timeout=180) as resp: txt=resp.read().decode('utf-8','replace'); break
31+
except Exception: time.sleep(3*(attempt+1))
32+
per=collections.Counter(); ev=None
33+
for line in txt.splitlines():
34+
if line.startswith('event: '): ev=line[7:].strip()
35+
elif line.startswith('data: ') and ev=='matches':
36+
try:
37+
for m in json.loads(line[6:]):
38+
if m.get('type')=='content': per[m.get('repository','?')]+=len(m.get('lineMatches') or []) or len(m.get('chunkMatches') or []) or 1
39+
except Exception: pass
40+
return per
41+
def work(r): p=stream(q_for(r)); time.sleep(0.2); return r,p
42+
with ThreadPoolExecutor(3) as ex: done=list(ex.map(work,rows))
43+
tot=collections.Counter(); rowsets=collections.defaultdict(set); pkgsets=collections.defaultdict(set)
44+
for r,per in done:
45+
for repo,n in per.items(): tot[repo]+=n; rowsets[repo].add(r['callable']); pkgsets[repo].add(r['package'])
46+
FORK=re.compile(r'(fork|mirror|-copy|backup|archive|vendor)',re.I)
47+
def is_own(repo): rl=repo.lower().replace('github.com/',''); return any(rl==o or rl.startswith(o+'/') for o in own)
48+
reps=[]
49+
for k,v in tot.most_common(400):
50+
reps.append({'repo':k,'opportunities':v,'distinct_rows':len(rowsets[k]),'packages':sorted(pkgsets[k]),'excluded':'own' if is_own(k) else ('fork-or-mirror-by-name' if FORK.search(k) else None)})
51+
pool=[x for x in reps if not x['excluded']][:30]
52+
json.dump({'generated_utc':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()),'rows_queried':len(rows),'repositories':reps,'frozen_pool_top30':pool},open(f'{S}/lab/sc/e1pool.json','w'),indent=1)
53+
print('E1POOL_DONE rows',len(rows)); [print(' ',x['repo'],x['opportunities'],x['distinct_rows'],x['packages']) for x in pool[:30]]

0 commit comments

Comments
 (0)