[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8},"actionData","errors","root",{"_103":104,"_105":106},"routes/embed.x.$handle.$name.$",{"_9":10,"_11":12,"_13":14,"_15":16,"_17":18,"_19":20,"_21":-5},"handle","sebas1","name","cognitive-influence-shield-v7-benchmark","path","evaluate_open_campaigns_production.ts","project",{"_22":45,"_11":12,"_33":46,"_47":-5,"_48":42,"_49":50,"_51":50,"_52":53,"_54":-5,"_55":56,"_57":-5,"_58":42,"_59":-5,"_60":-5,"_61":62,"_63":64,"_65":-5,"_66":-5,"_67":-5,"_68":-5,"_69":70,"_71":70,"_72":73,"_74":73,"_75":73,"_76":70,"_77":70,"_78":79,"_80":81,"_82":83,"_84":42},"file",{"_22":23,"_11":14,"_24":25,"_26":-5,"_27":28,"_29":30,"_31":32,"_33":34,"_35":36,"_37":38,"_39":40,"_41":42,"_43":-5,"_13":44},"permalink","https://www.val.town/x/sebas1/cognitive-influence-shield-v7-benchmark/code/evaluate_open_campaigns_production.ts","extension","id","1f6dc790-141b-4b30-90a8-4924692aa7bf","fileId","a57e92aa-a407-11f1-8992-1607ee4eb77e","parentId","branchId","7a977325-5523-41d5-a402-9b80aef3f988","branchVersion",25,"latest",true,"createdAt",["D",1788049155856],"version",4,"fileType","script","content","import { analyzeProduction, candidatePairs, normalizeText, tokenSet, jaccard, type P } from \"./production_engine.ts\";\nconst BASE=\"https://zenodo.org/records/13912659/files/\";\nconst CAMPAIGNS=[\n {name:\"Honduras\",bad:\"honduras-bad-anonymized\",good:\"honduras-good-anonymized\"},\n {name:\"UAE\",bad:\"uae-bad-anonymized\",good:\"uae-good-anonymized\"}\n];\nconst WINDOWS=32,WINDOW_BYTES=262144,TARGET_ACCOUNTS=128,MAX_POSTS_PER_ACCOUNT=12,MIN_POSTS=2,SEED=\"cis-v7-open-production\";\nfunction fnv1a(s:string){let h=0x811c9dc5;for(let i=0;i>>0}return h>>>0}\nfunction parseLineLossless(line:string){return JSON.parse(line.replace(/\"tweetid\":(\\d+)/g,'\"tweetid\":\"$1\"').replace(/\"userid\":(\\d+)/g,'\"userid\":\"$1\"'))}\nfunction firstUrl(v:any):string|null{if(Array.isArray(v))return v.length?String(v[0]):null;const s=String(v??\"\").trim();if(!s)return null;try{const j=JSON.parse(s);if(Array.isArray(j)&&j.length)return String(j[0])}catch{}try{const m=s.match(/https?:\\/\\/[^\\s'\\\",\\]]+/);return m?m[0]:null}catch{return null}}\nasync function totalSize(url:string){const r=await fetch(url,{headers:{range:\"bytes=0-0\"}});const cr=r.headers.get(\"content-range\")||\"\";await r.arrayBuffer();const m=cr.match(/\\/(\\d+)$/);if(!m)throw Error(\"missing total size \"+url);return Number(m[1])}\nasync function readWindow(url:string,start:number,total:number){const end=Math.min(total-1,start+WINDOW_BYTES-1);const r=await fetch(url,{headers:{range:`bytes=${start}-${end}`}});if(r.status!==206)throw Error(`range status ${r.status} ${url}`);let txt=await r.text();if(start>0){const n=txt.indexOf(\"\\n\");txt=n>=0?txt.slice(n+1):\"\"}if(end=0?txt.slice(0,n):\"\"}return txt.split(\"\\n\").filter(Boolean)}\nasync function sampleSource(campaign:string,name:string,control:boolean){const url=BASE+name+\"?download=1\",total=await totalSize(url),maxStart=Math.max(0,total-WINDOW_BYTES),starts=Array.from({length:WINDOWS},(_,i)=>Math.floor(i*maxStart/(WINDOWS-1))),chunks=await Promise.all(starts.map(s=>readWindow(url,s,total)));const by=new Map(),seen=new Set();let parseErrors=0,unsafeIdPrevented=0;for(const lines of chunks)for(const line of lines){let x:any;try{x=parseLineLossless(line)}catch{parseErrors++;continue}const id=String(x.tweetid??\"\"),account=String(x.userid??\"\");if(!id||!account||seen.has(id))continue;seen.add(id);if(id.length>=16||account.length>=16)unsafeIdPrevented++;const good=Number(x.good);if((control&&good!==1)||(!control&&good!==0))continue;let arr=by.get(account);if(!arr){arr=[];by.set(account,arr)}if(arr.length>=MAX_POSTS_PER_ACCOUNT)continue;arr.push({id,account,text:String(x.tweet_text??\"\"),t:Number(x.tweet_time)/1000,url:firstUrl(x.urls),label:control?0:1,campaign})}const eligible=[...by.entries()].filter(([,v])=>v.length>=MIN_POSTS);eligible.sort((a,b)=>fnv1a(SEED+\"|\"+campaign+\"|\"+a[0])-fnv1a(SEED+\"|\"+campaign+\"|\"+b[0])||a[0].localeCompare(b[0]));const chosen=eligible.slice(0,TARGET_ACCOUNTS);return{posts:chosen.flatMap(([,v])=>v),meta:{file:name,url,totalBytes:total,windowBytes:WINDOW_BYTES,windows:WINDOWS,starts,parseErrors,observedPosts:seen.size,observedAccounts:by.size,eligibleAccounts:eligible.length,selectedAccounts:chosen.length,unsafeIntegerIdsHandled:unsafeIdPrevented}}}\nfunction wilson(success:number,total:number,z=1.96){if(total<=0)return[null,null];const p=success/total,d=1+z*z/total,c=(p+z*z/(2*total))/d,h=z*Math.sqrt((p*(1-p)+z*z/(4*total))/total)/d;return[+Math.max(0,c-h).toFixed(4),+Math.min(1,c+h).toFixed(4)]}\nfunction metrics(items:any[],thr:number){let tp=0,fp=0,tn=0,fn=0;for(const x of items){const pred=x.score>=thr;if(pred&&x.label)tp++;else if(pred)fp++;else if(!x.label)tn++;else fn++}const precision=tp/(tp+fp||1),recall=tp/(tp+fn||1),specificity=tn/(tn+fp||1),f1=(precision+recall)?2*precision*recall/(precision+recall):0;return{threshold:+thr.toFixed(3),tp,fp,tn,fn,precision:+precision.toFixed(4),recall:+recall.toFixed(4),specificity:+specificity.toFixed(4),f1:+f1.toFixed(4),balanced_accuracy:+((recall+specificity)/2).toFixed(4),precision_wilson95:wilson(tp,tp+fp),recall_wilson95:wilson(tp,tp+fn),specificity_wilson95:wilson(tn,tn+fp)}}\nfunction auc(items:any[]){const P=items.filter(x=>x.label),N=items.filter(x=>!x.label);if(!P.length||!N.length)return null;let w=0;for(const p of P)for(const n of N)w+=p.score>n.score?1:p.score===n.score?.5:0;return +(w/(P.length*N.length)).toFixed(4)}\nfunction ap(items:any[]){const o=[...items].sort((a,b)=>b.score-a.score),npos=o.filter(x=>x.label).length;if(!npos)return null;let tp=0,s=0;o.forEach((x,i)=>{if(x.label){tp++;s+=tp/(i+1)}});return +(s/npos).toFixed(4)}\nfunction chooseThreshold(cal:any[]){const cand=[...new Set([0,100.0001,...cal.map(x=>x.score)])].sort((a,b)=>a-b);let best:any=null;for(const t of cand){const m=metrics(cal,t),key=[m.f1,m.balanced_accuracy,m.precision,-t];if(!best||key[0]>best.key[0]||key[0]===best.key[0]&&(key[1]>best.key[1]||key[1]===best.key[1]&&(key[2]>best.key[2]||key[2]===best.key[2]&&key[3]>best.key[3])))best={t,m,key}}return best}\nfunction chooseThresholdBA(cal:any[]){const cand=[...new Set([0,100.0001,...cal.map(x=>x.score)])].sort((a,b)=>a-b);let best:any=null;for(const t of cand){const m=metrics(cal,t),key=[m.balanced_accuracy,m.f1,m.precision,-t];if(!best||key[0]>best.key[0]||key[0]===best.key[0]&&(key[1]>best.key[1]||key[1]===best.key[1]&&(key[2]>best.key[2]||key[2]===best.key[2]&&key[3]>best.key[3])))best={t,m,key}}return best}\nfunction fitness(m:any){const reasons=[] as string[];if(m.roc_auc===null||m.roc_auc<=.5)reasons.push(\"held-out ROC-AUC is not above chance\");if(m.balanced_accuracy<=.5)reasons.push(\"held-out balanced accuracy is not above chance\");if((m.tn===0&&m.fn===0)||(m.tp===0&&m.fp===0))reasons.push(\"threshold degenerates to single-class prediction\");return{status:reasons.length?\"REJECTED_ON_HELDOUT\":\"NOT_REJECTED_BY_MINIMUM_NONDISCRIMINATION_GATE\",reasons,auto_flip_forbidden:true}}\nfunction prevalenceScenarios(m:any){return [.001,.01,.05,.1,.25,.5].map(pi=>{const se=m.recall,sp=m.specificity,ppv=(se*pi)/((se*pi)+(1-sp)*(1-pi)||1),npv=(sp*(1-pi))/((sp*(1-pi))+(1-se)*pi||1);return{prevalence:pi,ppv:+ppv.toFixed(4),npv:+npv.toFixed(4)}})}\nfunction edgeBreakdown(edges:any[],posts:P[]){let io_io=0,control_control=0,cross=0;for(const e of edges){const a=posts[e[0]].label,b=posts[e[1]].label;if(a&&b)io_io++;else if(!a&&!b)control_control++;else cross++}return{io_io,control_control,cross}}\nfunction blockingAudit(posts:P[],limit=600){const subset=[...posts].sort((a,b)=>fnv1a(\"audit|\"+a.id)-fnv1a(\"audit|\"+b.id)||a.id.localeCompare(b.id)).slice(0,limit),{pairs,norms}=candidatePairs(subset),cand=new Set(pairs.map(([i,j])=>i+\",\"+j)),tok=norms.map(tokenSet);let qualifying=0,captured=0;const misses:any[]=[];for(let i=0;i=.82&&Math.min(Array.from(norms[i]).length,Array.from(norms[j]).length)>=20){qualifying++;if(cand.has(i+\",\"+j))captured++;else if(misses.length<5)misses.push({a:subset[i].id,b:subset[j].id,sim:+sim.toFixed(4)})}}return{subset_posts:subset.length,qualifying_exhaustive_pairs:qualifying,captured_by_production_candidates:captured,recall:qualifying?+(captured/qualifying).toFixed(4):1,miss_examples:misses}}\nasync function evaluateCampaign(c:any){const started=performance.now(),[bad,good]=await Promise.all([sampleSource(c.name,c.bad,false),sampleSource(c.name,c.good,true)]),posts=[...bad.posts,...good.posts],labels=new Map();for(const p of posts)labels.set(p.account,p.label as number);const ar=analyzeProduction(posts),scoreMap=new Map(ar.account_review.map((x:any)=>[x.account_id,x.review_score])),scored=[...labels.entries()].map(([account,label])=>({account,label,score:Number(scoreMap.get(account)||0),campaign:c.name})),io=scored.filter(x=>x.label),ctl=scored.filter(x=>!x.label);return{name:c.name,posts,scored,summary:{campaign:c.name,sampling:{bad:bad.meta,good:good.meta},post_count:posts.length,account_count:scored.length,class_counts:{io:io.length,control:ctl.length},production_graph:{candidate_pairs:ar.candidate_pair_count,duplicate_edges:ar.duplicate_pairs.length,duplicate_breakdown:edgeBreakdown(ar.duplicate_pairs,posts),sync_hits:ar.sync_hits.length,sync_breakdown:edgeBreakdown(ar.sync_hits,posts),coordination_groups:ar.coordination_groups.length},score_summary:{io:{min:Math.min(...io.map(x=>x.score)),max:Math.max(...io.map(x=>x.score)),mean:+(io.reduce((a,x)=>a+x.score,0)/io.length).toFixed(3)},control:{min:Math.min(...ctl.map(x=>x.score)),max:Math.max(...ctl.map(x=>x.score)),mean:+(ctl.reduce((a,x)=>a+x.score,0)/ctl.length).toFixed(3)}},within_campaign_auc:auc(scored),within_campaign_ap:ap(scored),blocking_audit:blockingAudit(posts),runtime_ms:Math.round(performance.now()-started)}}}\nconst evaluated=[] as any[];for(const c of CAMPAIGNS)evaluated.push(await evaluateCampaign(c));\nfunction transfer(cal:any,test:any){const best=chooseThreshold(cal.scored),m=metrics(test.scored,best.t);m.roc_auc=auc(test.scored);m.average_precision=ap(test.scored);return{objective:\"F1_FIRST_LEGACY\",calibration_campaign:cal.name,test_campaign:test.name,selected_threshold:best.t,calibration_metrics:best.m,test_metrics:m,deployment_fitness_gate:fitness(m),prevalence_scenarios:prevalenceScenarios(m)}}\nfunction transferBA(cal:any,test:any){const best=chooseThresholdBA(cal.scored),m=metrics(test.scored,best.t);m.roc_auc=auc(test.scored);m.average_precision=ap(test.scored);return{objective:\"BALANCED_ACCURACY_FIRST\",calibration_campaign:cal.name,test_campaign:test.name,selected_threshold:best.t,calibration_metrics:best.m,test_metrics:m,deployment_fitness_gate:fitness(m),prevalence_scenarios:prevalenceScenarios(m)}}\nconst result={version:\"V7_OPEN_PRODUCTION_EVIDENCE_1\",status:\"OK\",source_doi:\"10.5281/zenodo.13912659\",id_parser:\"LOSSLESS_QUOTE_TWEETID_USERID_BEFORE_JSON_PARSE\",engine:\"V6_PRODUCTION_RELEVANT_TEXT_PATH_PORT_CONFORMANCE_REQUIRED\",sampling_design:\"32_EQUAL_BYTE_WINDOWS_PER_FILE_THEN_DETERMINISTIC_ACCOUNT_HASH\",parameters:{windows:WINDOWS,window_bytes:WINDOW_BYTES,target_accounts_per_class:TARGET_ACCOUNTS,max_posts_per_account:MAX_POSTS_PER_ACCOUNT,min_observed_posts:MIN_POSTS},campaigns:evaluated.map(x=>x.summary),cross_campaign_transfer_legacy:[transfer(evaluated[0],evaluated[1]),transfer(evaluated[1],evaluated[0])],cross_campaign_transfer_balanced_accuracy:[transferBA(evaluated[0],evaluated[1]),transferBA(evaluated[1],evaluated[0])],claim_boundary:\"This is an open-data systematic-sample evaluation of the production-relevant text candidate path. It is not full-corpus deployment calibration, bot identity, intent, truth, authenticity, or attribution.\",auto_flip_forbidden:true};\nconsole.log(JSON.stringify(result)); console.log(\"BA_ONLY:\"+JSON.stringify(result.cross_campaign_transfer_balanced_accuracy));","deleted",false,"label",[14],"b1e371df-d8a4-4440-b746-e6e1cb82335e",["D",1788046289516],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","V7 open IO benchmark and conformance runner","imageUrl","userId","d3788f19-13c5-4356-87e8-dce639626bb9","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","01a04116-f28c-754c-9173-77bf5814fcd6","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_9":10,"_102":98},"org",{"_9":10,"_95":96,"_97":98,"_55":56,"_99":100,"_101":42},"branch",{"_35":85,"_11":86,"_22":28,"_87":45,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":-5,"_93":12,"_94":10},"isOwned",35,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","displayName","sebas","avatar","https://lh3.googleusercontent.com/a/ACg8ocJaC6uiXPmN9dvgA2XcSlzBSkPJO0GN5zKzPm3a8cbjmXC9HYA=s96-c","tier","free","canRestrictVals","profileImageUrl","ctx",{"_78":-5,"_136":137,"_138":139,"_140":141,"_142":42,"_143":42,"_144":42,"_145":32,"_146":-5,"_147":-7,"_148":-7,"_149":62,"_150":42,"_151":42,"_152":-5,"_153":42,"_154":42,"_155":42,"_156":-5},"ENV",{"_107":108,"_109":110,"_111":86,"_112":113,"_114":115,"_116":117,"_118":119,"_120":121,"_122":123,"_124":125,"_126":127,"_128":129,"_130":131,"_132":133,"_134":135},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town","orgs",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]