[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_415":416,"_417":418},"routes/_app.x.$handle.$name",{"_100":99,"_32":45,"_108":90,"_27":109,"_13":14,"_110":-5,"_11":111,"_112":113,"_114":115,"_116":117,"_118":119,"_20":-5,"_120":-5,"_121":-5,"_122":25,"_123":124,"_125":59,"_126":-5,"_127":-5,"_128":129},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_43":44,"_32":45,"_38":46,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":53,"_54":-5,"_18":55,"_56":57,"_58":59,"_60":-5,"_61":-5,"_62":63,"_64":65,"_66":67,"_68":69,"_70":71,"_72":-5,"_73":74,"_75":74,"_76":77,"_78":77,"_79":77,"_80":74,"_81":74,"_82":83,"_84":85,"_86":87,"_88":25},"version",54,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_33":34,"_35":37,"_38":39,"_40":14,"_41":42,"_24":25},"previous",{"_30":31,"_32":28,"_33":34,"_35":36},"deleted",false,"deletedHere","path","validate_export.ts","latestPath","fileId","f8379820-b2c6-11f1-98b8-1607ee4eb77e","name","fileType","script","content","import { createHash } from 'node:crypto';\nimport { sqlite } from 'https://esm.town/v/std/sqlite/main.ts';\nconst endpoint='https://omriluz--7645eebcb2c011f1a2dc1607ee4eb77e.web.val.run';\nconst expected:Record={all:33842,confirmed:27650,directories:142,excluded:2547,unresolved:3645};\nawait sqlite.execute('CREATE TABLE IF NOT EXISTS export_validation(dataset TEXT PRIMARY KEY,checked_at TEXT,rows INTEGER,bytes INTEGER,sha256 TEXT,result_json TEXT)');\nconst results=await Promise.all(Object.entries(expected).map(async([kind,expectedRows])=>{const start=Date.now();try{\n const r=await fetch(endpoint+'?kind='+kind+'&compressed=1',{headers:{authorization:'Bearer '+Deno.env.get('AUDIT_EXPORT_KEY')},signal:AbortSignal.timeout(55000)});\n if(!r.ok)throw Error('HTTP '+r.status+' '+await r.text());\n const hash=createHash('sha256');let bytes=0,lines=0;const reader=r.body!.pipeThrough(new DecompressionStream('gzip')).getReader();\n while(true){const part=await reader.read();if(part.done)break;hash.update(part.value);bytes+=part.value.length;for(const c of part.value)if(c===10)lines++;}\n const result={kind,checked_at:new Date().toISOString(),http_status:r.status,rows:lines-1,expected:expectedRows,complete:lines-1===expectedRows,uncompressed_bytes:bytes,sha256:hash.digest('hex'),elapsed_ms:Date.now()-start};\n if(!result.complete)throw Error('Row count mismatch '+JSON.stringify(result));\n await sqlite.execute({sql:'INSERT OR REPLACE INTO export_validation VALUES (?,?,?,?,?,?)',args:[kind,result.checked_at,result.rows,bytes,result.sha256,JSON.stringify(result)]});return result;\n }catch(e){return {kind,error:String(e)};}}));\nconsole.log(JSON.stringify({results,all_passed:results.every((x:any)=>x.complete===true)}));","import { createHash } from 'node:crypto';\nimport { sqlite } from 'https://esm.town/v/std/sqlite/main.ts';\nconst endpoint='https://omriluz--7645eebcb2c011f1a2dc1607ee4eb77e.web.val.run';\nconst expected:Record={confirmed:27650};\nawait sqlite.execute('CREATE TABLE IF NOT EXISTS export_validation(dataset TEXT PRIMARY KEY,checked_at TEXT,rows INTEGER,bytes INTEGER,sha256 TEXT,result_json TEXT)');\nconst results=await Promise.all(Object.entries(expected).map(async([kind,expectedRows])=>{const start=Date.now();try{\n const r=await fetch(endpoint+'?kind='+kind+'&compressed=1',{headers:{authorization:'Bearer '+Deno.env.get('AUDIT_EXPORT_KEY')},signal:AbortSignal.timeout(55000)});\n if(!r.ok)throw Error('HTTP '+r.status+' '+await r.text());\n const hash=createHash('sha256');let bytes=0,lines=0;const reader=r.body!.pipeThrough(new DecompressionStream('gzip')).getReader();\n while(true){const part=await reader.read();if(part.done)break;hash.update(part.value);bytes+=part.value.length;for(const c of part.value)if(c===10)lines++;}\n const result={kind,checked_at:new Date().toISOString(),http_status:r.status,rows:lines-1,expected:expectedRows,complete:lines-1===expectedRows,uncompressed_bytes:bytes,sha256:hash.digest('hex'),elapsed_ms:Date.now()-start};\n if(!result.complete)throw Error('Row count mismatch '+JSON.stringify(result));\n await sqlite.execute({sql:'INSERT OR REPLACE INTO export_validation VALUES (?,?,?,?,?,?)',args:[kind,result.checked_at,result.rows,bytes,result.sha256,JSON.stringify(result)]});return result;\n }catch(e){return {kind,error:String(e)};}}));\nconsole.log(JSON.stringify({results,all_passed:results.every((x:any)=>x.complete===true)}));","createdAt",["D",1789670678504],"branchVersion","branchId","cd292256-c961-4ceb-b973-27c236575749","id","7e58a892-b130-4895-9cbc-de82614ff56d","publisher-network-audit",["D",1789665566213],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Browse publishers, directory teardowns and ads.txt evidence.","imageUrl","f5be24e4-c798-4816-91e5-037b60b986f5","pinnedFileId","01a0b061-5e61-74b2-9c7e-7b033afc92fc","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","7f67379d-2d38-49bd-81ac-2c7f92a739e2","tursoName","7e58a892b13048959cbcde82614ff56d","tursoHostname","7e58a892b13048959cbcde82614ff56d-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789665692456],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_100":99,"_107":103},"org",{"_100":99,"_101":-5,"_102":103,"_18":55,"_104":105,"_106":25},"branch",{"_13":89,"_32":90,"_43":42,"_91":44,"_92":-5,"_93":-5,"_94":-5,"_95":-5,"_96":-5,"_97":45,"_98":99},"isOwned",59,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","omriluz","handle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18ycWxkOWxiOE00SkVtZENmRWM4YWxLWTRVUFMifQ","tier","free","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_43":44,"_32":45,"_38":410,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":53,"_54":-5,"_18":55,"_56":57,"_58":59,"_60":-5,"_61":-5,"_62":63,"_64":65,"_66":67,"_68":69,"_70":411,"_72":-5,"_73":74,"_75":74,"_76":77,"_78":77,"_79":77,"_80":74,"_81":74,"_82":412,"_84":413,"_86":414,"_88":25},"tree",[309,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331],"files",[309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331],"counts",{"_299":63,"_300":158,"_301":63,"_302":303,"_304":305},"contents",[130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_43":294,"_32":50,"_30":295,"_154":-5,"_41":42,"_40":158,"_156":59,"_38":296,"_13":63,"_33":297,"_35":-5,"_24":25,"_161":-5,"_27":298},{"_43":288,"_32":289,"_30":290,"_154":-5,"_41":42,"_40":158,"_156":59,"_38":291,"_13":63,"_33":20,"_35":292,"_24":25,"_161":-5,"_27":293},{"_43":281,"_32":282,"_30":283,"_154":-5,"_41":42,"_40":284,"_156":59,"_38":285,"_13":63,"_33":34,"_35":286,"_24":25,"_161":-5,"_27":287},{"_43":274,"_32":275,"_30":276,"_154":-5,"_41":42,"_40":277,"_156":59,"_38":278,"_13":63,"_33":34,"_35":279,"_24":25,"_161":-5,"_27":280},{"_43":266,"_32":267,"_30":268,"_154":-5,"_41":42,"_40":269,"_156":59,"_38":270,"_13":271,"_33":34,"_35":272,"_24":25,"_161":-5,"_27":273},{"_43":259,"_32":260,"_30":261,"_154":-5,"_41":42,"_40":262,"_156":59,"_38":263,"_13":63,"_33":34,"_35":264,"_24":25,"_161":-5,"_27":265},{"_43":252,"_32":253,"_30":254,"_154":-5,"_41":42,"_40":255,"_156":59,"_38":256,"_13":63,"_33":34,"_35":257,"_24":25,"_161":-5,"_27":258},{"_43":245,"_32":246,"_30":247,"_154":-5,"_41":42,"_40":248,"_156":59,"_38":249,"_13":165,"_33":159,"_35":250,"_24":25,"_161":-5,"_27":251},{"_43":238,"_32":239,"_30":240,"_154":-5,"_41":42,"_40":241,"_156":59,"_38":242,"_13":165,"_33":34,"_35":243,"_24":25,"_161":-5,"_27":244},{"_43":230,"_32":231,"_30":232,"_154":-5,"_41":42,"_40":233,"_156":59,"_38":234,"_13":235,"_33":34,"_35":236,"_24":25,"_161":-5,"_27":237},{"_43":223,"_32":224,"_30":225,"_154":-5,"_41":42,"_40":226,"_156":59,"_38":227,"_13":158,"_33":34,"_35":228,"_24":25,"_161":-5,"_27":229},{"_43":217,"_32":218,"_30":57,"_154":-5,"_41":42,"_40":158,"_156":25,"_38":219,"_13":63,"_33":159,"_35":220,"_24":25,"_161":221,"_27":222},{"_43":210,"_32":211,"_30":212,"_154":-5,"_41":42,"_40":213,"_156":59,"_38":214,"_13":63,"_33":34,"_35":215,"_24":25,"_161":-5,"_27":216},{"_43":203,"_32":204,"_30":205,"_154":-5,"_41":42,"_40":206,"_156":59,"_38":207,"_13":63,"_33":34,"_35":208,"_24":25,"_161":-5,"_27":209},{"_43":196,"_32":197,"_30":198,"_154":-5,"_41":42,"_40":199,"_156":59,"_38":200,"_13":63,"_33":34,"_35":201,"_24":25,"_161":-5,"_27":202},{"_43":189,"_32":190,"_30":191,"_154":-5,"_41":42,"_40":192,"_156":59,"_38":193,"_13":63,"_33":34,"_35":194,"_24":25,"_161":-5,"_27":195},{"_43":182,"_32":183,"_30":184,"_154":-5,"_41":42,"_40":185,"_156":59,"_38":186,"_13":170,"_33":34,"_35":187,"_24":25,"_161":-5,"_27":188},{"_43":174,"_32":175,"_30":176,"_154":-5,"_41":42,"_40":177,"_156":59,"_38":178,"_13":179,"_33":34,"_35":180,"_24":25,"_161":-5,"_27":181},{"_43":167,"_32":168,"_30":169,"_154":-5,"_41":42,"_40":170,"_156":59,"_38":171,"_13":165,"_33":34,"_35":172,"_24":25,"_161":-5,"_27":173},{"_43":163,"_32":28,"_30":31,"_154":-5,"_41":42,"_40":14,"_156":59,"_38":164,"_13":165,"_33":34,"_35":37,"_24":25,"_161":-5,"_27":166},{"_43":151,"_32":152,"_30":153,"_154":-5,"_41":42,"_40":155,"_156":59,"_38":157,"_13":158,"_33":159,"_35":160,"_24":25,"_161":-5,"_27":162},"5d55a250-cd87-4eeb-b054-01fd224810e9","worker.ts","0eb0b968-b2bf-11f1-9613-1607ee4eb77e","parentId",36,"latest",["D",1789668488514],1,"http","import { runScan } from './scan_core.ts';\nimport { runEnrich } from './enrich_core.ts';\nexport default async function(req:Request){\n const key=Deno.env.get('AUDIT_JOB_KEY');\n if(!key||req.headers.get('authorization')!=='Bearer '+key)return new Response('Unauthorized',{status:401});\n if(req.method!=='POST')return new Response('POST required',{status:405});\n try {const b=await req.json();const shard=Number(b.shard),shards=Number(b.shards);if(!Number.isInteger(shard)||!Number.isInteger(shards)||shards<1||shards>16||shard<0||shard>=shards)return new Response('Invalid shard',{status:400});\n const result=b.kind==='scan'?await runScan(shard,shards):b.kind==='retry'?await runScan(shard,shards,true):b.kind==='enrich'?await runEnrich(shard,shards):null;\n if(!result)return new Response('Unknown kind',{status:400});return Response.json({kind:b.kind,shard,...result});\n }catch(e){return Response.json({error:String(e)},{status:500});}\n}","label",[152],"e396f688-055e-41fc-9925-6fbd9901dbb0",["D",1789670678504],3,[28],"2b083803-4ca0-4fff-943d-cefb65802939","seed.ts","034e6604-b2bc-11f1-960c-1607ee4eb77e",5,["D",1789665652151],"import { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\nimport { gzipSync } from \"node:zlib\";\nimport { Buffer } from \"node:buffer\";\nconst sources = {raptive:\"https://ads.cafemedia.com/sellers.json\",mediavine:\"https://www.mediavine.com/sellers.json\",ezoic:\"https://ezoic.ai/sellers.json\",journey:\"https://sellers.journeymv.com/sellers.json\"};\nawait sqlite.batch([\n {sql:\"CREATE TABLE IF NOT EXISTS snapshots(network TEXT PRIMARY KEY,url TEXT,fetched_at TEXT,http_status INTEGER,sha256 TEXT,raw_json TEXT)\"},\n {sql:\"CREATE TABLE IF NOT EXISTS sellers(network TEXT,seller_id TEXT,domain TEXT,name TEXT,seller_type TEXT,confidential INTEGER,PRIMARY KEY(network,seller_id,domain))\"},\n {sql:\"CREATE INDEX IF NOT EXISTS sellers_domain ON sellers(domain)\"},\n {sql:\"CREATE TABLE IF NOT EXISTS checks(domain TEXT PRIMARY KEY,checked_at TEXT,status TEXT,final_url TEXT,http_status INTEGER,body_sha256 TEXT,ads_body TEXT,evidence_json TEXT,attempts_json TEXT)\"},\n {sql:\"CREATE TABLE IF NOT EXISTS enrichment(domain TEXT PRIMARY KEY,checked_at TEXT,status TEXT,sitemap_count INTEGER,sitemap_status TEXT,result_json TEXT)\"}\n]);\nfunction domainOf(value:unknown):string {try {const x=String(value||'').trim().toLowerCase(); if(!x || /\\s/.test(x)) return ''; const u=new URL(x.includes('://')?x:'https://'+x); const h=u.hostname.replace(/^www\\./,'').replace(/\\.$/,''); if(!h.includes('.')||/^\\d+(\\.\\d+){3}$/.test(h)||h==='localhost')return ''; return h;}catch{return '';}}\nconst summary=[];\nfor(const [network,url] of Object.entries(sources)){\n try {\n const existing=await sqlite.execute({sql:\"SELECT network FROM snapshots WHERE network=?\",args:[network]});\n if(existing.rows.length){summary.push({network,cached:true});continue;}\n const r=await fetch(url,{signal:AbortSignal.timeout(20000),headers:{'User-Agent':'PublisherNetworkResearch/1.0'}}); const raw=await r.text();\n if(!r.ok) throw Error('HTTP '+r.status); const data=JSON.parse(raw);if(!Array.isArray(data.sellers))throw Error('missing sellers');\n const rows=data.sellers.map((x:any)=>[network,String(x.seller_id??''),domainOf(x.domain),String(x.name??''),String(x.seller_type??'').toUpperCase(),Number(!!x.is_confidential)]);\n const sha=Array.from(new Uint8Array(await crypto.subtle.digest('SHA-256',new TextEncoder().encode(raw))),x=>x.toString(16).padStart(2,'0')).join('');\n const packed=JSON.stringify({encoding:'gzip+base64',data:Buffer.from(gzipSync(raw)).toString('base64'),records:data.sellers.length,original_bytes:raw.length});\n for(let i=0;ix[2]).filter(Boolean)).size,missing_domain:rows.filter((x:any)=>!x[2]).length,bytes:raw.length,stored_bytes:packed.length,sha256:sha,examples:rows.slice(0,3)};summary.push(item);console.log(JSON.stringify(item));\n }catch(e){const item={network,error:String(e)};summary.push(item);console.log(JSON.stringify(item));}\n}\nconsole.log(JSON.stringify({summary}));",[168],"0b4e22bc-abdb-41be-ba07-762fb791a2a4","scan_core.ts","01a0b074-c8b8-73d7-bd15-f8089b2fc32b",35,["D",1789668450576],6,"import { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\nimport { gzipSync } from \"node:zlib\";\nimport { Buffer } from \"node:buffer\";\nconst SYSTEMS:Record={raptive:['cafemedia.com','ads.cafemedia.com','adthrive.com','raptive.com'],mediavine:['mediavine.com'],journey:['journeymv.com','sellers.journeymv.com'],ezoic:['ezoic.ai','ezoic.com']};\nexport async function runScan(shard=0,shards=1,retryMode=false){\nconst started=Date.now(), LIMIT=2200, WORKERS=retryMode?45:70, BUDGET=18000;\nconst condition=retryMode?\"c.status='unverified' AND c.checked_at < (SELECT value FROM audit_settings WHERE key='retry_cutoff')\":\"c.domain IS NULL\";\nconst selected=await sqlite.execute({sql:\"SELECT DISTINCT s.domain FROM sellers s LEFT JOIN checks c ON s.domain=c.domain WHERE s.domain<>'' AND (\"+condition+\") AND ((unicode(substr(s.domain,1,1))*31+unicode(substr(s.domain,2,1))+length(s.domain)) % ?) = ? ORDER BY CASE WHEN s.domain LIKE '%directory%' OR s.domain LIKE '%locator%' OR s.domain LIKE '%finder%' THEN 0 ELSE 1 END,s.domain LIMIT ?\",args:[shards,shard,LIMIT]});\nconst domains=selected.rows.map(x=>String(x.domain));\nconst records=await sqlite.execute({sql:\"SELECT domain,network,seller_id FROM sellers WHERE domain IN (SELECT value FROM json_each(?))\",args:[JSON.stringify(domains)]});\nconst map=new Map();for(const r of records.rows){const a=map.get(String(r.domain))??[];a.push(r);map.set(String(r.domain),a);}\nasync function limitedText(r:Response,max=1000000){const rd=r.body?.getReader();if(!rd)return '';let n=0;const arr:Uint8Array[]=[];while(true){const a=await rd.read();if(a.done)break;n+=a.value.length;if(n>max){await rd.cancel();throw Error('body_too_large');}arr.push(a.value);}const b=new Uint8Array(n);let p=0;for(const a of arr){b.set(a,p);p+=a.length;}return new TextDecoder().decode(b);}\nasync function check(domain:string){const attempts:any[]=[];const end=Date.now()+(retryMode?25000:12000);\n for(const url of ['https://'+domain+'/ads.txt','https://www.'+domain+'/ads.txt','http://'+domain+'/ads.txt']){if(Date.now()>end-200)break;try{\n const r=await fetch(url,{redirect:'follow',signal:AbortSignal.timeout(Math.max(200,Math.min(retryMode?12000:6000,end-Date.now()))),headers:{'User-Agent':'PublisherNetworkResearch/1.0 (public ads.txt verification)','Accept':'text/plain,*/*;q=0.5'}});const body=await limitedText(r);const a:any={url,status:r.status,final_url:r.url,bytes:body.length};attempts.push(a);\n if(!r.ok)continue;const parsed=body.split(/\\r?\\n/).map((line,i)=>({raw:line.trim(),parts:line.split('#')[0].trim().split(',').map(x=>x.trim()),line_number:i+1})).filter(x=>x.parts.length>=3&&/\\./.test(x.parts[0])&&['DIRECT','RESELLER'].includes(x.parts[2].toUpperCase()));\n if(/<(?:!doctype\\s+html|html|head|body)(?:\\s|>)/i.test(body.slice(0,2000))||(!parsed.length&&!/(text\\/plain|application\\/octet-stream)/i.test(r.headers.get('content-type')??'')&&body.trim()!=='')){a.invalid_body=true;continue;}\n const evidence:any[]=[];for(const record of map.get(domain)??[])for(const line of parsed)if(SYSTEMS[String(record.network)]?.includes(line.parts[0].toLowerCase())&&line.parts[1]===String(record.seller_id))evidence.push({network:record.network,seller_id:record.seller_id,system:line.parts[0].toLowerCase(),relationship:line.parts[2].toUpperCase(),line:line.raw,line_number:line.line_number});\n const sha=Array.from(new Uint8Array(await crypto.subtle.digest('SHA-256',new TextEncoder().encode(body))),x=>x.toString(16).padStart(2,'0')).join('');a.network_lines=parsed.filter(x=>Object.values(SYSTEMS).flat().includes(x.parts[0].toLowerCase())).map(x=>x.raw);a.parsed_lines=parsed.length;\n return [domain,new Date().toISOString(),evidence.length?'confirmed':'no_matching_line',r.url,r.status,sha,'gzip+base64:'+Buffer.from(gzipSync(body)).toString('base64'),JSON.stringify(evidence),JSON.stringify(attempts)];\n }catch(e){attempts.push({url,error:String(e)});}}\n const missing=attempts.length>0&&attempts.every(x=>[404,410].includes(x.status));return [domain,new Date().toISOString(),missing?'ads_txt_missing':'unverified',attempts.at(-1)?.final_url??'',attempts.at(-1)?.status??null,'','','[]',JSON.stringify(attempts)];}\nconst INSERT=\"INSERT OR REPLACE INTO checks SELECT json_extract(value,'$[0]'),json_extract(value,'$[1]'),json_extract(value,'$[2]'),json_extract(value,'$[3]'),json_extract(value,'$[4]'),json_extract(value,'$[5]'),json_extract(value,'$[6]'),json_extract(value,'$[7]'),json_extract(value,'$[8]') FROM json_each(?)\";\nlet next=0,saved=0,failedWrites=0;const counts:Record={};let buf:any[]=[];let bufSize=0;let chain=Promise.resolve();\nfunction enqueue(){if(!buf.length)return;const batch=buf;buf=[];bufSize=0;chain=chain.then(async()=>{for(let attempt=0;attempt<4;attempt++){try{await sqlite.execute({sql:INSERT,args:[JSON.stringify(batch)]});saved+=batch.length;return;}catch(e){if(attempt===3){failedWrites+=batch.length;console.log(JSON.stringify({write_error:String(e),unsaved_domains:batch.map(r=>r[0])}));}else await new Promise(r=>setTimeout(r,400*(attempt+1)));}}});}\nawait Promise.all(Array.from({length:WORKERS},async()=>{while(next120000)enqueue();buf.push(row);bufSize+=size;if(buf.length>=35)enqueue();}}));enqueue();await chain;\nreturn {attempted:next,saved,failedWrites,duration_ms:Date.now()-started,batch_counts:counts};\n}",[175],"594ad361-19be-443c-b029-86a91f941bb9","scan.ts","5a6159ba-b2bc-11f1-b240-1607ee4eb77e",14,["D",1789666546661],"import { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\nimport { gzipSync } from \"node:zlib\";\nimport { Buffer } from \"node:buffer\";\nconst SYSTEMS:Record={raptive:['cafemedia.com','ads.cafemedia.com','adthrive.com','raptive.com'],mediavine:['mediavine.com'],journey:['journeymv.com','sellers.journeymv.com'],ezoic:['ezoic.ai','ezoic.com']};\nconst started=Date.now(), LIMIT=7000, WORKERS=300, BUDGET=40000;\nconst selected=await sqlite.execute({sql:\"SELECT DISTINCT s.domain FROM sellers s LEFT JOIN checks c ON s.domain=c.domain WHERE s.domain<>'' AND c.domain IS NULL ORDER BY CASE WHEN s.domain LIKE '%directory%' OR s.domain LIKE '%locator%' OR s.domain LIKE '%finder%' THEN 0 ELSE 1 END,s.domain LIMIT ?\",args:[LIMIT]});\nconst domains=selected.rows.map(x=>String(x.domain));\nconst records=await sqlite.execute({sql:\"SELECT domain,network,seller_id FROM sellers WHERE domain IN (SELECT value FROM json_each(?))\",args:[JSON.stringify(domains)]});\nconst map=new Map();for(const r of records.rows){const a=map.get(String(r.domain))??[];a.push(r);map.set(String(r.domain),a);}\nasync function limitedText(r:Response,max=1000000){const rd=r.body?.getReader();if(!rd)return '';let n=0;const arr:Uint8Array[]=[];while(true){const a=await rd.read();if(a.done)break;n+=a.value.length;if(n>max){await rd.cancel();throw Error('body_too_large');}arr.push(a.value);}const b=new Uint8Array(n);let p=0;for(const a of arr){b.set(a,p);p+=a.length;}return new TextDecoder().decode(b);}\nasync function check(domain:string){const attempts:any[]=[];const end=Date.now()+12000;\n for(const url of ['https://'+domain+'/ads.txt','https://www.'+domain+'/ads.txt','http://'+domain+'/ads.txt']){if(Date.now()>end-200)break;try{\n const r=await fetch(url,{redirect:'follow',signal:AbortSignal.timeout(Math.max(200,Math.min(6000,end-Date.now()))),headers:{'User-Agent':'PublisherNetworkResearch/1.0 (public ads.txt verification)','Accept':'text/plain,*/*;q=0.5'}});const body=await limitedText(r);const a:any={url,status:r.status,final_url:r.url,bytes:body.length};attempts.push(a);\n if(!r.ok)continue;const parsed=body.split(/\\r?\\n/).map((line,i)=>({raw:line.trim(),parts:line.split('#')[0].trim().split(',').map(x=>x.trim()),line_number:i+1})).filter(x=>x.parts.length>=3&&/\\./.test(x.parts[0])&&['DIRECT','RESELLER'].includes(x.parts[2].toUpperCase()));\n if(/<(?:!doctype\\s+html|html|head|body)(?:\\s|>)/i.test(body.slice(0,2000))||(!parsed.length&&!/(text\\/plain|application\\/octet-stream)/i.test(r.headers.get('content-type')??'')&&body.trim()!=='')){a.invalid_body=true;continue;}\n const evidence:any[]=[];for(const record of map.get(domain)??[])for(const line of parsed)if(SYSTEMS[String(record.network)]?.includes(line.parts[0].toLowerCase())&&line.parts[1]===String(record.seller_id))evidence.push({network:record.network,seller_id:record.seller_id,system:line.parts[0].toLowerCase(),relationship:line.parts[2].toUpperCase(),line:line.raw,line_number:line.line_number});\n const sha=Array.from(new Uint8Array(await crypto.subtle.digest('SHA-256',new TextEncoder().encode(body))),x=>x.toString(16).padStart(2,'0')).join('');a.network_lines=parsed.filter(x=>Object.values(SYSTEMS).flat().includes(x.parts[0].toLowerCase())).map(x=>x.raw);a.parsed_lines=parsed.length;\n return [domain,new Date().toISOString(),evidence.length?'confirmed':'no_matching_line',r.url,r.status,sha,'gzip+base64:'+Buffer.from(gzipSync(body)).toString('base64'),JSON.stringify(evidence),JSON.stringify(attempts)];\n }catch(e){attempts.push({url,error:String(e)});}}\n const missing=attempts.length>0&&attempts.every(x=>[404,410].includes(x.status));return [domain,new Date().toISOString(),missing?'ads_txt_missing':'unverified',attempts.at(-1)?.final_url??'',attempts.at(-1)?.status??null,'','','[]',JSON.stringify(attempts)];}\nconst INSERT=\"INSERT OR REPLACE INTO checks SELECT json_extract(value,'$[0]'),json_extract(value,'$[1]'),json_extract(value,'$[2]'),json_extract(value,'$[3]'),json_extract(value,'$[4]'),json_extract(value,'$[5]'),json_extract(value,'$[6]'),json_extract(value,'$[7]'),json_extract(value,'$[8]') FROM json_each(?)\";\nlet next=0,saved=0,failedWrites=0;const counts:Record={};let buf:any[]=[];let bufSize=0;let chain=Promise.resolve();\nfunction enqueue(){if(!buf.length)return;const batch=buf;buf=[];bufSize=0;chain=chain.then(async()=>{for(let attempt=0;attempt<4;attempt++){try{await sqlite.execute({sql:INSERT,args:[JSON.stringify(batch)]});saved+=batch.length;return;}catch(e){if(attempt===3){failedWrites+=batch.length;console.log(JSON.stringify({write_error:String(e),unsaved_domains:batch.map(r=>r[0])}));}else await new Promise(r=>setTimeout(r,400*(attempt+1)));}}});}\nawait Promise.all(Array.from({length:WORKERS},async()=>{while(next120000)enqueue();buf.push(row);bufSize+=size;if(buf.length>=35)enqueue();}}));enqueue();await chain;\nconsole.log(JSON.stringify({attempted:next,saved,failedWrites,duration_ms:Date.now()-started,batch_counts:counts,totals:(await sqlite.execute('SELECT status,COUNT(*) n FROM checks GROUP BY status')).rows}));",[183],"c4e04489-1554-4ff6-bcd3-7534a8e161ce","save_reviews.ts","6d4dc742-b2c7-11f1-9fd6-1607ee4eb77e",51,["D",1789670502709],"import { sqlite } from 'https://esm.town/v/std/sqlite/main.ts';\n// Analyst review of the captured homepage, sitemap examples, and source-page excerpts.\n// A publisher disclosure is not independent validation of its collection pipeline.\nconst reviews:any[]=[];\nfunction add(domain:string,classification:string,niche:string,patterns:any[],sources:any[],fields:string[],notes:string){reviews.push({domain,classification,niche,patterns:patterns.map(([pattern,role,example])=>({pattern,role,example,review:'observed_sample_url_structure'})),sources:sources.map(([source,url,note,status])=>({source,url,note,status:status??'publisher_disclosed'})),fields,notes});}\nconst unknown=(url:string,note='No named upstream provider was established in the checked pages.')=>[[null,url,note,'not_established']];\nadd('surgerycenterdirectory.com','directory','US ambulatory surgery centers and surgeons',[\n ['/surgery-center/{state}/{city}/{facility-slug}/','Facility profile','https://www.surgerycenterdirectory.com/surgery-center/ohio/wooster/wasc-llc-dba-wooster-ambulatory-surgery-center-wooster-oh/'],\n ['/surgery-center/{state}/{city}/','City landing page','https://www.surgerycenterdirectory.com/surgery-center/new-york/port-jefferson-statio/'],\n ['/surgeons/{name}-{identifier}/','Surgeon profile','https://www.surgerycenterdirectory.com/surgeons/lynn-mcmahan-1669475695/']\n],[['CMS and Google Places','https://www.surgerycenterdirectory.com/','Footer explicitly names both for facility data and claims quarterly updates. This does not establish the separate surgeon-data feed.']],['facility name','address','specialty','ratings and quality information','surgeon identifier'],'25,956 sitemap URLs include 18,161 surgeon-profile paths and 5,705 facility-profile paths; page count is not the number of surgery centers.');\nadd('mpgbuddy.com','database_reference','Vehicle fuel economy',[\n ['/cars/{make}/{model}/{year}','Model-year data page','https://mpgbuddy.com/cars/lamborghini/aventador-roadster/2022'],\n ['/cars/{make}/{model}','Model aggregation','https://mpgbuddy.com/cars/acura/2-2cl-3-0cl'],\n ['/cars/{make}','Make aggregation','https://mpgbuddy.com/cars/acura']\n],[['EPA / US Department of Energy fuel-economy dataset','https://mpgbuddy.com/about','The About page identifies fueleconomy.gov as the dataset source.']],['make','model','year','MPG','fuel costs','emissions','transmission','drivetrain'],'Model names may be numeric; the generic crawler occasionally labels them IDs. Semantic template corrected here.');\nadd('daycarealert.com','directory','Daycare licensing and inspections',[\n ['/daycare/{license-id}/{facility-slug}','Licensed provider profile','https://daycarealert.com/daycare/MO-000550201/albany-r-iii-school-district'],\n ['/daycares/{state-or-province}/{city}','City directory','https://daycarealert.com/daycares/tx/brownsville'],\n ['/daycare-costs/{state}','Cost landing page','https://daycarealert.com/daycare-costs/alabama']\n],[['Government childcare licensing agencies','https://daycarealert.com/data-sources','The site names public licensing, inspection and violation records from 48 US states, DC and three Canadian provinces.']],['license identifier','provider name','location','licensing','inspection and violation records'],'The sitemap traversal was partial. The homepage listing total must not substitute for a sitemap count.');\nadd('allinclusivedining.com','directory','All-inclusive resort dining',[\n ['/{destination}/{resort}/restaurants','Resort restaurant collection','https://www.allinclusivedining.com/cancun/atelier-de-hoteles-playa-mujeres/restaurants'],\n ['/cuisines/{cuisine}','Cuisine collection','https://www.allinclusivedining.com/cuisines/american']\n],[['Official resort websites, traveler reports and AI-assisted research','https://www.allinclusivedining.com/about','The About page describes review and cross-checking before publishing, with reader corrections.']],['resort','restaurant','cuisine','access rules','reservations','dress code'],'The repeating third segment includes restaurants and other resort detail pages. Not every URL is a separate restaurant.');\nadd('aandewaittimes.uk','directory','UK emergency-department waiting times',[\n ['/hospital/{hospital-slug}','Hospital page','https://www.aandewaittimes.uk/hospital/ysbyty-ystrad-fawr'],\n ['/a-e-wait-times/{region}','Regional collection','https://www.aandewaittimes.uk/a-e-wait-times/wales']\n],[['NHS Trust public live pages','https://www.aandewaittimes.uk/','The homepage identifies NHS Trust live pages as the source.']],['hospital','location','wait time','opening hours','update time'],'99 hospital paths were observed within 119 sitemap URLs. No medical or triage recommendation is implied by this audit.');\nadd('aewaittime.co.uk','directory','UK emergency-department waiting times',[\n ['/hospitals/{hospital-slug}','Hospital page','https://www.aewaittime.co.uk/hospitals/alexandra-hospital-redditch'],\n ['/trusts/{trust-slug}','NHS trust page','https://www.aewaittime.co.uk/trusts/aneurin-bevan-university'],\n ['/regions/{region}','Regional collection','https://www.aewaittime.co.uk/regions/birmingham']\n],[['NHS Trust public waiting-times pages','https://www.aewaittime.co.uk/','Homepage says the values are collected directly from each trust\\'s public page.']],['hospital','trust','region','wait time','update time'],'121 hospital paths and 42 trust paths occur within 218 sitemap URLs.');\nadd('erstat.ca','directory','Canadian emergency departments and clinics',[\n ['/hospitals/{province}/{place-or-hospital}','Hospital or city collection','https://erstat.ca/hospitals/ab/athabasca'],\n ['/clinics/{province}/{city}','Clinic collection','https://erstat.ca/clinics/bc/surrey'],\n ['/hopitaux/{province}/{hospital}','French-language hospital page','https://erstat.ca/hopitaux/nb/hopital-regional-chaleur']\n],[['Provincial health-authority feeds','https://erstat.ca/about','Reviewed excerpts name NS Health and Quebec CPU; the Alberta sample explicitly names Alberta Health Services.']],['province','hospital','wait time','forecast where available','closure notices'],'Partial sitemap count. A city collection is not necessarily a single hospital profile.');\nadd('nailsalondirect.com','directory','UK nail salons',[\n ['/salon/{business-location-slug}/','Salon profile','https://nailsalondirect.com/salon/aesthete-beauty-dundee-dundee/'],\n ['/services/{service}/{city}/','Service-by-city landing page','https://nailsalondirect.com/services/gel-nails/london/'],\n ['/nail-salons/{city}/{filter}/','Filtered city landing page','https://nailsalondirect.com/nail-salons/london/open-sunday/']\n],[['Business websites and publicly available business listings','https://nailsalondirect.com/about/','About page discloses public business information; the reviewed salon page names business websites and public listings. No specific aggregator is identified.']],['name','address','phone','website','opening hours','ratings','amenities'],'14,205 sitemap URLs include 10,492 salon paths. City, service and filter pages expand the URL count beyond the entity count.');\nadd('freeclinicdirectory.org','directory','Free and low-cost clinics',[\n ['/detail/{clinic-location-slug}.html','Clinic profile','https://freeclinicdirectory.org/detail/anniston_quality_health_care_anniston_al.html'],\n ['/{state}_care/{county}_{state-code}_county.html','County collection','https://freeclinicdirectory.org/california_care/los_angeles_ca_county.html']\n],[['Clinic and community submissions with editorial review','https://freeclinicdirectory.org/','Homepage names clinic staff, patients, social workers and community organizations as contributors; reviewed add/update and claim workflows are described.','observed_submission_mechanism'],[null,'https://freeclinicdirectory.org/','The initial bulk listing source was not established.','not_established']],['clinic name','address','phone','services','website'],'18,987 sitemap URLs include 12,019 clinic-detail paths. Submission workflows do not prove that all original records were submitted.');\nadd('findmyguitar.com','database_reference','Guitar specifications and comparisons',[\n ['/reviews/{guitar}-review.php','Individual model page','https://findmyguitar.com/reviews/Gibson-70s-Explorer-review.php'],\n ['/compare/{model-a}-vs-{model-b}-comparison.php','Pairwise comparison','https://findmyguitar.com/compare/Fender-PO-220E-Orchestra-vs-Fender-PS-220E-Parlor-comparison.php'],\n ['/top-lists/{filter-combination}.php','Filtered best-of collection','https://findmyguitar.com/top-lists/best-12-string-ibanez-acoustic-guitars.php']\n],[['Manufacturer announcements/websites and online marketplaces','https://findmyguitar.com/about.php','About page describes adding newly announced models, algorithmic scoring and marketplace scanning for price and availability.']],['brand','model','specifications','score','price','availability'],'Capped at 100,000 collected URLs. Of those, 93,254 were comparison paths; this is a lower bound, not the complete site size.');\nadd('householdrobot.net','database_reference','Household robots and technical comparisons',[\n ['/robots/{model}.html','Robot profile','https://householdrobot.net/robots/3itech-p10-ultra.html'],\n ['/vs/{model-a}-vs-{model-b}.html','Pairwise comparison','https://householdrobot.net/vs/aiper-scuba-v3-ultra-vs-wybot-s3.html'],\n ['/best/{use-case}.html','Use-case collection','https://householdrobot.net/best/lawn-mower-budget.html']\n],[['Manufacturer-verified specifications','https://householdrobot.net/','Homepage describes collecting objective specifications and manufacturer verification. Exact collection tools are not disclosed.']],['model','suction','battery life','navigation','dimensions','noise'],'195 sitemap URLs include model, comparison, best-for, troubleshooting and guide pages.');\nadd('worldpopulationreview.com','database_reference','Population and geographic statistics',[\n ['/us-cities/{state}/{city}','US city statistics','https://worldpopulationreview.com/us-cities/oregon/cottage-grove'],\n ['/cities/{country}/{city}','International city statistics','https://worldpopulationreview.com/cities/afghanistan/aibak'],\n ['/us-counties/{state}/{county}','US county statistics','https://worldpopulationreview.com/us-counties/alabama/autauga-county']\n],[['World Bank, United Nations, US Census/ACS, Pew and selected secondary sources','https://worldpopulationreview.com/methodology','Methodology identifies primary and secondary sources and explains estimates when direct counts are unavailable.']],['geography','population','year','indicator','source'],'100,000 collected URLs is the crawl cap. Different metrics and geographies create multiple page families.');\nadd('hometownlocator.com','database_reference','US gazetteers and local demographic data',[\n ['{state}.hometownlocator.com','State-level subdomain architecture','https://www.hometownlocator.com/']\n],[['USGS GNIS, US Census Bureau and other official sources','https://www.hometownlocator.com/','Homepage explicitly describes aggregation and normalization of these datasets.']],['place','geographic boundaries','demographics'],'No usable complete sitemap tree was obtained. State-subdomain architecture is visible, but no full path-level teardown or site-wide count is claimed.');\nadd('blackhealth.org','directory_hybrid','Health editorial content and care-provider directories',[\n ['/care/{care-type}/{state-code}/{city}/','Care-type-by-city collection','https://www.blackhealth.org/care/hospice/wa/spokane/'],\n ['/clinic/{name-location-id}/','Clinic profile','https://www.blackhealth.org/clinic/10225-grand-avenue-franklin-park-il-60131-franklin-park-il-2e99c384/'],\n ['/hospitals/{state}/{hospital-id}/','Hospital profile','https://www.blackhealth.org/hospitals/alaska/alaska-native-medical-center-020026/']\n],[['CMS provider information; official datasets and research for editorial content','https://www.blackhealth.org/about/methodology/','Methodology names CDC, NIH, HRSA, FDA, CMS and other primary sources. The hospice example explicitly uses CMS. This is not proof that every directory record shares one feed.']],['provider','location','care type','rating where available','identifier'],'Partial sitemap traversal. Large geographic care collections coexist with health articles and tools.');\nadd('flyindex.org','directory_hybrid','Airport facilities and travel information',[\n ['/airports/{iata}/','Airport hub','https://flyindex.org/airports/abe/'],\n ['/airports/{iata}/{topic}/','Airport topic page','https://flyindex.org/airports/abe/arrival-guide/'],\n ['/airports/{iata}/restaurants/{restaurant-slug}/','Airport restaurant profile','https://flyindex.org/airports/jfk/restaurants/alidoro-near-gate-41/']\n],[['Public TSA/CBP data and official airport sources','https://flyindex.org/','Homepage identifies the sources. About page describes a separate editorial workflow and disclosed AI-assisted drafting.']],['airport','terminal','restaurant','shopping','security timing'],'Airport hubs multiply into topic pages, business pages and category collections.');\nadd('attorneygeneral.net','directory_hybrid','US government contacts and consumer-protection guides',[\n ['/states/{state}/attorney-general/','State office profile','https://attorneygeneral.net/states/alabama/attorney-general/'],\n ['/scams/{topic}/','Editorial guidance','https://attorneygeneral.net/scams/ai-impersonation-scam/']\n],[['Official state Attorney General and federal agency websites','https://attorneygeneral.net/about/','Editorial policy names state AG offices, FTC, CFPB, FBI, DOJ, SEC and statutes.']],['state','office','contact information','complaint process'],'50 state-office URLs sit alongside topical guidance. Independent website, not an official agency.');\nadd('courtsentences.co.uk','database_reference','UK court-sentencing reports',[\n ['/{court-slug}/sentences','Court result collection','https://courtsentences.co.uk/colchester-magistrates-court/sentences']\n],[['Media reports, court attendance and community submissions','https://courtsentences.co.uk/','Homepage explicitly describes these sources and review before publication.']],['court','sentencing report','source'],'The dominant URL template aggregates results by court rather than assigning every case an observed separate sitemap URL.');\nadd('aniimoguide.com','database_reference','Aniimo game database and guides',[\n ['/aniidex/{creature}','Creature profile','https://aniimoguide.com/aniidex/celestis'],\n ['/best-teams/{creature}','Creature-based team page','https://aniimoguide.com/best-teams/celestis'],\n ['/runes/{item}','Rune profile','https://aniimoguide.com/runes/advanced-atk-rune']\n],[['Official Aniimo wiki and official game website','https://aniimoguide.com/about','About page identifies the wiki for stats, skills, traits, evolution, habitat and artwork, and the official site for release/platform information.']],['creature','stats','skills','traits','evolution','habitat'],'Entity data is reused for profiles, team pages and item databases; editorial news also exists.');\nadd('myigboname.com','database_reference','Igbo names and meanings',[\n ['/entries/{name}','Name entry','https://myigboname.com/entries/abah'],\n ['/meanings/{meaning-group}','Meaning collection','https://myigboname.com/meanings/gift-blessing']\n],[['Moderated community contributions and Igbo etymology','https://myigboname.com/','Homepage says entries are reviewed by moderators before publication.']],['name','meaning','origin','pronunciation'],'The entry table also supports meaning groups, quizzes and games.');\nadd('ngobase.org','directory','NGOs and charities',[\n ['/list_ngos_by_sub_work_area/{sector-code}','Sector collection','https://ngobase.org/list_ngos_by_sub_work_area/ANM.WL'],\n ['/list_ngos_by_country_sub_work_area/{country-code}/{sector-code}','Country-by-sector collection','https://ngobase.org/list_ngos_by_country_sub_work_area/PK/ANM.WL'],\n ['/list_ngos_by_city/{place-code}','City collection','https://ngobase.org/list_ngos_by_city/PK.PB.MUL']\n],[['Organization-submitted and claimed profiles','https://ngobase.org/faq','FAQ describes NGOs submitting details, claiming profiles, and team verification.','observed_submission_mechanism']],['organization','country','city','work area','description'],'Sitemap examples emphasize coded geographic and sector filters. Original bulk source was not established.');\nadd('nailsalondirect.com','directory','UK nail salons',[\n ['/salon/{business-location-slug}/','Salon profile','https://nailsalondirect.com/salon/aesthete-beauty-dundee-dundee/'],\n ['/services/{service}/{city}/','Service-by-city collection','https://nailsalondirect.com/services/gel-nails/london/'],\n ['/nail-salons/{city}/{filter}/','Filtered city collection','https://nailsalondirect.com/nail-salons/london/open-sunday/']\n],[['Business websites and public listings','https://nailsalondirect.com/about/','Site identifies public business information; sample salon credits business websites and public listings. Exact aggregator undisclosed.']],['name','address','phone','website','hours','ratings','amenities'],'10,492 salon-profile paths among 14,205 sitemap URLs.');\nadd('consignmentshopslocator.com','directory','Consignment shops',[\n ['/states/{state}/{city}','City page containing multiple store cards','https://consignmentshopslocator.com/states/colorado/denver'],\n ['/states/{state}','State landing page','https://consignmentshopslocator.com/states/alabama']\n],unknown('https://consignmentshopslocator.com/states/colorado/denver','Google Maps directions are observed but do not establish a Google Places or scraping feed.'),['name','address','phone','website','hours','categories','acceptance rules','payout terms','consignment period'],'1,180 of 1,193 sitemap URLs are city pages. Several stores appear on each city page; page count is not store count.');\nadd('selfcarwashfinder.com','directory','Self-service car washes',[\n ['/states/{state}/{city}/{business}','Business profile','https://www.selfcarwashfinder.com/states/california/temecula/touch-free-automatic-car-wash'],\n ['/cities/{city-state}','City collection','https://www.selfcarwashfinder.com/cities/abbeville-louisiana'],\n ['/near/{place}','Near-location landing page','https://www.selfcarwashfinder.com/near/houston']\n],unknown('https://www.selfcarwashfinder.com/'),[],'8,264 business paths, 4,870 city paths and 3,855 near-location paths explain most of the 17,047 URLs. Classification reviewed from title and sitemap structure; underlying feed unknown.');\nadd('boatrampfinder.com','directory','Boat ramps and launch sites',[\n ['/ramps/{ramp-slug}/','Ramp profile','https://boatrampfinder.com/ramps/washunga-bay-boat-ramp/'],\n ['/boat-ramps/{state}/{city}/','City collection','https://boatrampfinder.com/boat-ramps/pennsylvania/sheffield-township/'],\n ['/boat-ramps/{state}/','State collection','https://boatrampfinder.com/boat-ramps/virginia/']\n],unknown('https://boatrampfinder.com/ramps/washunga-bay-boat-ramp/','Address, coordinates and Google Maps directions are visible; the original ramp dataset is not identified.'),['ramp name','waterbody','address','city','county','coordinates'],'Captured sitemap count may include a KML asset from the early parser version; treat it as provisional until recounted. 28,069 ramp-profile paths were observed.');\nadd('housingapartments.org','directory','Affordable and low-income housing',[\n ['/rental_detail/{numeric-id}','Property profile','https://housingapartments.org/rental_detail/10000'],\n ['/{state}_rentals/{city}_{state-code}_city','City collection','https://housingapartments.org/texas_rentals/austin_tx_city']\n],[['User/provider photo submissions','https://housingapartments.org/rental_detail/10000','The sample explicitly describes submitted and reviewed photos only.','observed_submission_mechanism'],...unknown('https://housingapartments.org/','The initial housing-record source was not established; photo credits do not identify the listing feed.')],['property','address','application information','amenities','utilities','photos'],'Four-digit numeric IDs are IDs, not years; the generic pattern label is corrected in this semantic template.');\nadd('thriftshopfinder.com','directory','Thrift and secondhand stores',[],[['Publicly available business data','https://thriftshopfinder.com/about','About page describes public data, verification and corrections, but does not name an upstream provider.']],['name','address','phone','hours','website'],'Sitemap unavailable, so no complete URL-pattern or page-count claim. The advertised number of stores is not a measured sitemap total.');\nadd('lakenearme.com','directory','Lakes by city',[\n ['/{state}/{city}/','City page with lake cards','https://lakenearme.com/california/la-mesa/']\n],[['Visitor corrections','https://lakenearme.com/california/la-mesa/','Sample page contains a Submit Lake Info correction workflow.','observed_submission_mechanism'],...unknown('https://lakenearme.com/','Map links and displayed ratings do not prove the upstream listing provider.')],['lake','location','coordinates','ratings','map link'],'Partial sitemap traversal. One city page can list multiple lakes.');\nadd('beautymasterlist.com','database_reference','Beauty products, ingredients and alternatives',[\n ['/products/{brand}/{product}','Product profile','https://beautymasterlist.com/products/avant/ultimate-hyaluronic-acid-resurfacing-duo-moisturiser'],\n ['/ingredients/{ingredient}','Ingredient page','https://beautymasterlist.com/ingredients/niacinamide'],\n ['/products/{brand}/{product}/dupes','Alternative-product page','https://beautymasterlist.com/products/avant/ultimate-hyaluronic-acid-resurfacing-duo-moisturiser/dupes'],\n ['/compare/{product-a}-vs-{product-b}','Comparison','https://beautymasterlist.com/compare/tula-skincare-24-7-weightless-moisture-day-night-gel-cream-vs-summer-fridays-rich-cushion-cream-ultra-plumping-moisturizer']\n],unknown('https://beautymasterlist.com/'),[],'84,200 sitemap URLs include products, ingredients, alternatives and comparisons. Some malformed search URLs were also published; this is a sitemap census, not a clean-indexed-page estimate.');\nadd('onlinedmv.com','directory_hybrid','DMV offices and driving guides',[\n ['/locations/{office-slug}','Office profile','https://www.onlinedmv.com/locations/tuscaloosa-county-tag-title'],\n ['/{state-code}-{state}/{city}.html','City landing page','https://www.onlinedmv.com/tx-texas/kountze.html']\n],unknown('https://www.onlinedmv.com/about-us.html','About describes a maintained directory and links a data-accuracy policy; that policy was not fully reviewed here.'),['office','location','contact information'],'7,173 office-profile paths were observed among 11,953 sitemap URLs.');\nadd('a-z-animals.com','directory_hybrid','Animal reference profiles and editorial articles',[\n ['/animals/{animal}/','Animal reference profile','https://a-z-animals.com/animals/staffordshire-bull-terrier/'],\n ['/blog/{article}/','Editorial blog','https://a-z-animals.com/blog/do-animals-have-a-sixth-sense/'],\n ['/animals/{animal}/{article}/','Related animal article','https://a-z-animals.com/animals/frog/what-happened-to-the-panamanian-golden-frog/']\n],[['Encyclopedias, biologists, researchers and cited publications','https://a-z-animals.com/about-us/','About describes editorial research and citations at the bottom of articles.']],[],'Reference/editorial hybrid, not a local business directory. Partial sitemap count.');\nfor(const [domain,niche,note] of [\n ['computingforgeeks.com','Linux and infrastructure tutorials','Database in the title refers to a tutorial topic, not a directory of entities.'],\n ['careeraddict.com','Career advice and services','Homepage and sitemap are primarily career articles, topics, authors and CV templates; auxiliary job search exists.'],\n ['craftering.com','Craft tutorials and idea roundups','Despite Crafting Directory branding, captured homepage shows dated idea-roundup posts.'],\n ['foodeau.com','Recipe and home-idea roundups','Despite Recipes Directory branding, captured homepage and sitemap show articles, tags and categories.'],\n ['frenchiewiki.com','French bulldog guides','138 of 144 sitemap URLs use /blog/; Wiki is branding rather than a structured entity directory.'],\n ['electriccarwiki.com','EV guides and product articles','Captured homepage shows dated posts, guides and reviews; Wiki branding is not sufficient evidence of a database.'],\n ['germanytravel.blog','Germany travel guides','Sitemap contains city-grouped travel articles; geography alone is not a directory. Homepage was unavailable.'],\n ['indiagardening.com','Gardening articles with shop and directory links','Primary homepage is an article publication with dated posts; secondary nursery directory and shop links are present.']])add(domain,'blog',niche,[],[],[],note);\nconst now=new Date().toISOString();let saved=0;\nfor(const r of reviews){await sqlite.execute({sql:'INSERT OR REPLACE INTO manual_reviews VALUES (?,?,?,?,?,?,?,?,?,?)',args:[r.domain,now,r.classification,r.niche,r.patterns.length?JSON.stringify(r.patterns):null,JSON.stringify(r.sources),r.notes,'reviewed_captured_samples',JSON.stringify([...new Set(r.sources.map((s:any)=>s.url).filter(Boolean))]),JSON.stringify(r.fields)]});saved++;}\nconsole.log(JSON.stringify({writes:saved,unique_reviewed:(await sqlite.execute('SELECT COUNT(*) n FROM manual_reviews')).rows,types:(await sqlite.execute('SELECT classification,COUNT(*) n FROM manual_reviews GROUP BY classification')).rows}));",[190],"55e7b401-95fe-4e33-bb3e-9c8911f12dea","reconcile_accounts.ts","7f9aac2e-b2c4-11f1-8461-1607ee4eb77e",40,["D",1789669244924],"import { sqlite } from 'https://esm.town/v/std/sqlite/main.ts';\nimport { gunzipSync } from 'node:zlib';\nimport { Buffer } from 'node:buffer';\nconst SYSTEMS:Record={'cafemedia.com':'raptive','ads.cafemedia.com':'raptive','adthrive.com':'raptive','raptive.com':'raptive','mediavine.com':'mediavine','journeymv.com':'journey','sellers.journeymv.com':'journey','ezoic.ai':'ezoic','ezoic.com':'ezoic'};\nawait sqlite.execute('CREATE TABLE IF NOT EXISTS account_reconciliation(domain TEXT PRIMARY KEY,reconciled_at TEXT,old_status TEXT,new_status TEXT,same_domain_matches INTEGER,other_domain_matches INTEGER,unreconciled_lines_json TEXT)');\nconst sellerRows=(await sqlite.execute('SELECT network,seller_id,domain FROM sellers')).rows;\nconst known=new Map>();for(const r of sellerRows){const k=r.network+':'+r.seller_id;const ds=known.get(k)??new Set();if(r.domain)ds.add(String(r.domain));known.set(k,ds);}\nlet after='',processed=0,recovered=0,cross=0,unresolved=0;\nwhile(true){const q=await sqlite.execute({sql:\"SELECT c.domain,c.checked_at,c.status,c.ads_body,c.evidence_json FROM checks c LEFT JOIN account_reconciliation a ON c.domain=a.domain WHERE c.domain>? AND c.status IN ('confirmed','no_matching_line') AND a.domain IS NULL ORDER BY c.domain LIMIT 250\",args:[after]});if(!q.rows.length)break;\n const changes:any[]=[],audit:any[]=[];\n for(const r of q.rows){let body='';try{const s=String(r.ads_body??'');if(s.startsWith('gzip+base64:'))body=gunzipSync(Buffer.from(s.slice(12),'base64')).toString('utf8');}catch{}\n const ev:any[]=[],other:any[]=[];let sameN=0,otherN=0;\n for(const [i,raw] of body.split(/\\r?\\n/).entries()){const p=raw.split('#')[0].split(',').map(x=>x.trim());const network=SYSTEMS[p[0]?.toLowerCase()];if(!network||p.length<3||p[2].toUpperCase()!=='DIRECT')continue;\n const ds=known.get(network+':'+p[1]);if(!ds){other.push({network,seller_id:p[1],line:raw.trim(),line_number:i+1,reason:'ID_not_found_in_captured_network_sellers_json'});continue;}\n const same=ds.has(String(r.domain));if(same)sameN++;else otherN++;\n ev.push({network,seller_id:p[1],system:p[0].toLowerCase(),relationship:'DIRECT',line:raw.trim(),line_number:i+1,match_scope:same?'same_domain_seller_record':'seller_account_registered_under_other_domain',seller_record_domains:[...ds].sort()});\n }\n const old=JSON.parse(String(r.evidence_json??'[]'));if(!ev.length&&r.status==='confirmed'&&old.length)ev.push(...old);\n const status=ev.length?'confirmed':other.length?'network_line_unreconciled':'no_matching_line';\n changes.push([r.domain,status,JSON.stringify(ev)]);audit.push([r.domain,new Date().toISOString(),r.status,status,sameN,otherN,JSON.stringify(other)]);\n if(r.status!=='confirmed'&&status==='confirmed')recovered++;if(otherN)cross++;if(status==='network_line_unreconciled')unresolved++;\n }\n await sqlite.execute({sql:\"UPDATE checks AS c SET status=json_extract(j.value,'$[1]'),evidence_json=json_extract(j.value,'$[2]') FROM json_each(?) j WHERE c.domain=json_extract(j.value,'$[0]')\",args:[JSON.stringify(changes)]});\n await sqlite.execute({sql:\"INSERT OR REPLACE INTO account_reconciliation SELECT json_extract(value,'$[0]'),json_extract(value,'$[1]'),json_extract(value,'$[2]'),json_extract(value,'$[3]'),json_extract(value,'$[4]'),json_extract(value,'$[5]'),json_extract(value,'$[6]') FROM json_each(?)\",args:[JSON.stringify(audit)]});\n processed+=q.rows.length;after=String(q.rows.at(-1)!.domain);\n}\nconsole.log(JSON.stringify({processed,recovered,cross_domain_account_sites:cross,unreconciled:unresolved,totals:(await sqlite.execute('SELECT status,COUNT(*) n FROM checks GROUP BY status')).rows}));",[197],"f24a1098-71f3-4234-9251-0f37ad2d8fb6","reclassify.ts","baca6510-b2c3-11f1-9291-1607ee4eb77e",39,["D",1789668914727],"import { sqlite } from 'https://esm.town/v/std/sqlite/main.ts';\nimport { classifySite } from './classify.ts';\nlet after='',changed=0;\nwhile(true){const q=await sqlite.execute({sql:\"SELECT domain,json_extract(result_json,'$.homepage') homepage,json_extract(result_json,'$.url_patterns') patterns,json_extract(result_json,'$.sitemap.files') files,json_extract(result_json,'$.sample_urls') samples FROM enrichment WHERE domain>? AND COALESCE(json_extract(result_json,'$.classification.version'),0)<2 ORDER BY domain LIMIT 100\",args:[after]});if(!q.rows.length)break;\n const updates=[];for(const r of q.rows){const meta=JSON.parse(String(r.homepage??'{}')),patterns=JSON.parse(String(r.patterns??'[]')),files=JSON.parse(String(r.files??'[]'));const total=patterns.reduce((n:number,p:any)=>n+p.count,0);let sample:string[]=[];for(const p of patterns){const n=Math.max(1,Math.round(300*p.count/Math.max(total,1)));for(let i=0;i (\n \n \n {title}\n \n \n \n \n \n \n \n {children}\n \n \n);\n\nconst app = new Hono();\n\napp.get(\"/\", (c) =>\n c.html(\n \n

Hello from Hono JSX

\n A minimal Val Town starter.{\" \"}\n \n Remix this val.\n \n

\n Server count:{\" \"}\n {count}\n

\n \n Increment\n \n

\n Lives in memory on the server. Resets when the val restarts.\n

\n \n ,\n ));\n\napp.get(\"/about\", (c) =>\n c.html(\n \n

About

Server-rendered with Hono JSX on Val Town.

\n ,\n ));\n\n// API routes\napp.post(\"/api/increment\", (c) => c.json({ count: ++count }));\n\n// Serve static files\napp.get(\"/public/*\", (c) => serveFile(c.req.path));\n\n// View source on Val Town\napp.get(\"/source\", (c) => c.redirect(parseVal().links.self.val));\n\n// Unwrap and rethrow Hono errors as the original error\napp.onError((err) => Promise.reject(err));\n\nexport default app.fetch;","publisher-explorer-omri",[218],"6f31dca8-8873-4aac-bffd-8fbb5d7bca4d","export_core.ts","ae74786c-b2c5-11f1-bc1c-1607ee4eb77e",52,["D",1789670600733],"import { sqlite } from 'https://esm.town/v/std/sqlite/main.ts';\nexport const DATASETS=['all','confirmed','directories','excluded','unresolved'];\nconst encoder=new TextEncoder();\nconst parse=(v:unknown,def:any=[])=>{try{return JSON.parse(String(v??'null'))??def;}catch{return def;}};\nfunction cell(v:unknown){let s=v==null?'':String(v).replace(/[\\r\\n\\t]+/g,' ');if(/^[=+@-]/.test(s))s=\"'\"+s;return '\"'+s.replaceAll('\"','\"\"')+'\"';}\nconst HEAD=['domain','listed_networks','verification_status','confirmed_networks','verification_scope','matched_seller_ids','ads_txt_evidence_json','ads_txt_requested_url','ads_txt_final_url','ads_checked_at_utc','ads_http_status','ads_body_sha256','site_type','classification_confidence','classification_review','site_title','sitemap_page_urls','sitemap_count_status','sitemap_checked_at_utc','sitemap_roots_json','sitemap_files_fetched','niche','url_patterns_json','data_sources_json','data_source_status','source_pages_checked_json','entity_fields_json','notes'];\nexport async function* csvChunks(kind:string){\n if(!DATASETS.includes(kind))throw Error('Unknown dataset');\n yield encoder.encode('\\ufeff'+HEAD.map(cell).join(',')+'\\r\\n');let after='';\n const filter=kind==='confirmed'?\"AND domain IN (SELECT domain FROM checks WHERE status='confirmed')\":kind==='excluded'?\"AND domain IN (SELECT domain FROM checks WHERE status IN ('no_matching_line','ads_txt_missing'))\":kind==='unresolved'?\"AND domain IN (SELECT domain FROM checks WHERE status IN ('unverified','network_line_unreconciled'))\":kind==='directories'?\"AND domain IN (SELECT c.domain FROM checks c LEFT JOIN enrichment e ON e.domain=c.domain LEFT JOIN manual_reviews m ON m.domain=c.domain WHERE c.status='confirmed' AND COALESCE(m.classification,json_extract(e.result_json,'$.classification.type')) IN ('directory','directory_hybrid','database_reference'))\":\"\";\n while(true){const q=await sqlite.execute({sql:`WITH d AS (SELECT domain,group_concat(DISTINCT network) networks FROM sellers WHERE domain<>'' AND domain>? ${filter} GROUP BY domain ORDER BY domain LIMIT 700)\nSELECT d.domain,d.networks,c.status,c.final_url,c.checked_at,c.http_status,c.body_sha256,c.evidence_json,c.attempts_json,e.sitemap_count,e.sitemap_status,e.checked_at sitemap_checked_at,\njson_extract(e.result_json,'$.classification') auto_class,json_extract(e.result_json,'$.homepage.title') title,\njson_extract(e.result_json,'$.sitemap.roots') roots,json_array_length(json_extract(e.result_json,'$.sitemap.files')) files_fetched,json_extract(e.result_json,'$.url_patterns') patterns,\nm.classification reviewed_type,m.review_status,m.niche,m.patterns_json,m.data_sources_json,m.evidence_urls_json,m.entity_fields_json,m.notes,\njson_extract(dr.result_json,'$.observed_schema_fields') observed_fields,\n(SELECT json_group_array(json_extract(p.value,'$.final_url')) FROM json_each(dr.result_json,'$.pages') p WHERE json_extract(p.value,'$.status')=200) checked_pages\nFROM d LEFT JOIN checks c ON c.domain=d.domain LEFT JOIN enrichment e ON e.domain=d.domain LEFT JOIN manual_reviews m ON m.domain=d.domain LEFT JOIN directory_research dr ON dr.domain=d.domain ORDER BY d.domain`,args:[after]});\n if(!q.rows.length)break;const lines:string[]=[];\n for(const r of q.rows){const status=String(r.status??'not_checked'),auto=parse(r.auto_class,{}),type=String(r.reviewed_type??auto.type??'not_evaluated');const directory=['directory','directory_hybrid','database_reference'].includes(type);\n if(kind==='confirmed'&&status!=='confirmed')continue;\n if(kind==='directories'&&(status!=='confirmed'||!directory))continue;\n if(kind==='excluded'&&!['no_matching_line','ads_txt_missing'].includes(status))continue;\n if(kind==='unresolved'&&!['not_checked','unverified','network_line_unreconciled'].includes(status))continue;\n const evidence=parse(r.evidence_json),attempts=parse(r.attempts_json);const original=attempts.find((a:any)=>a.status===200&&!a.invalid_body&&a.parsed_lines!=null)?.url??'https://'+r.domain+'/ads.txt';\n const same=evidence.some((x:any)=>x.match_scope==='same_domain_seller_record');const scope=evidence.length?(same?'same_domain_seller_record':'related_seller_account_only'):'not_confirmed';\n const sources=parse(r.data_sources_json);const sourceStatus=sources.length?[...new Set(sources.map((x:any)=>x.status??'reviewed'))].join(';'):'not_established';\n const sourceRows=sources.length?sources:[{source:null,status:'not_established',note:'The upstream data origin was not established by this audit.'}];\n const patterns=parse(r.patterns_json??r.patterns);const checkedPages=[...new Set([...parse(r.evidence_urls_json),...parse(r.checked_pages)].filter(Boolean))];\n const count=(r.sitemap_count===0&&r.sitemap_status!=='complete_discovered_tree')?null:r.sitemap_count;\n const values=[r.domain,r.networks,status,[...new Set(evidence.map((x:any)=>x.network))].sort().join(';'),scope,[...new Set(evidence.map((x:any)=>x.network+':'+x.seller_id))].join(';'),JSON.stringify(evidence),original,r.final_url,r.checked_at,r.http_status,r.body_sha256,type,r.review_status?'reviewed':auto.confidence??'',r.review_status??auto.review_status??'not_evaluated',r.title,count,r.sitemap_status??'not_attempted',r.sitemap_checked_at,r.roots,r.files_fetched,r.niche,JSON.stringify(directory?patterns:patterns.slice(0,3)),directory?JSON.stringify(sourceRows):'',directory?sourceStatus:'not_applicable',directory?JSON.stringify(checkedPages):'',r.entity_fields_json??r.observed_fields??'',r.notes??''];\n lines.push(values.map(cell).join(','));\n }\n if(lines.length)yield encoder.encode(lines.join('\\r\\n')+'\\r\\n');after=String(q.rows.at(-1)!.domain);\n }\n}\nexport function csvStream(kind:string){const iterator=csvChunks(kind);return new ReadableStream({async pull(controller){try{const next=await iterator.next();if(next.done)controller.close();else controller.enqueue(next.value);}catch(e){controller.error(e);}},async cancel(){await iterator.return(undefined);}});}\n",[224],"69e1606f-3b30-4175-9675-7bd1fba3e94d","enrich_core.ts","01a0b075-429a-70af-87d6-fa93cd9ce0e4",43,["D",1789669472058],8,"import { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\nimport { gunzipSync } from \"node:zlib\";\nimport { XMLParser } from \"npm:fast-xml-parser@4.5.3\";\nimport { classifySite } from './classify.ts';\nexport async function runEnrich(shard=0,shards=1){\n const started=Date.now(),BUDGET=30000,WORKERS=100,LIMIT=3000,SITE_MS=18000;\n const rows=await sqlite.execute({sql:\"SELECT c.domain,c.attempts_json FROM checks c LEFT JOIN enrichment e ON c.domain=e.domain WHERE c.status='confirmed' AND e.domain IS NULL AND ((unicode(substr(c.domain,1,1))*31+unicode(substr(c.domain,2,1))+length(c.domain)) % ?) = ? ORDER BY CASE WHEN c.domain LIKE '%directory%' OR c.domain LIKE '%locator%' OR c.domain LIKE '%finder%' OR c.domain IN ('worldpopulationreview.com','campsitephotos.com') THEN 0 ELSE 1 END,c.domain LIMIT ?\",args:[shards,shard,LIMIT]});\n const domains=rows.rows.map(x=>String(x.domain));\n const baseByDomain=new Map();for(const r of rows.rows){let base='https://'+r.domain;try{const a=JSON.parse(String(r.attempts_json??'[]')).find((a:any)=>a.status===200&&!a.invalid_body&&a.parsed_lines!=null);if(a?.url)base=new URL(a.url).origin;}catch{}baseByDomain.set(String(r.domain),base);}\n const parser=new XMLParser({ignoreAttributes:true,removeNSPrefix:true,parseTagValue:false,processEntities:false,trimValues:true});\n const arr=(x:any)=>x==null?[]:Array.isArray(x)?x:[x];\n const unesc=(s:string)=>s.replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'\"').replace(/'/g,\"'\");\n const plain=(s:string)=>unesc(s.replace(/]*>[\\s\\S]*?<\\/script>/gi,' ').replace(/]*>[\\s\\S]*?<\\/style>/gi,' ').replace(/<[^>]+>/g,' ').replace(/\\s+/g,' ')).trim();\n async function get(url:string,end:number,max=7000000):Promise{try{const left=end-Date.now();if(left<300)return {url,error:'site_time_budget'};const r=await fetch(url,{signal:AbortSignal.timeout(Math.min(8000,left)),headers:{'User-Agent':'PublisherNetworkResearch/1.0 (sitemap and public-page research)'}});const rd=r.body?.getReader();const chunks:Uint8Array[]=[];let n=0;if(rd)while(true){const p=await rd.read();if(p.done)break;n+=p.value.length;if(n>max){await rd.cancel();return {url,final_url:r.url,status:r.status,error:'body_size_limit'};}chunks.push(p.value);}let b=new Uint8Array(n);let offset=0;for(const c of chunks){b.set(c,offset);offset+=c.length;}if(b[0]===31&&b[1]===139)b=new Uint8Array(gunzipSync(b,{maxOutputLength:20000000}));return {url,final_url:r.url,status:r.status,body:new TextDecoder().decode(b),bytes:b.length};}catch(e){return {url,error:String(e)};}}\n function links(html:string,base:string){const out:any[]=[];for(const m of html.matchAll(/]*href\\s*=\\s*[\"']([^\"']+)[\"'][^>]*>([\\s\\S]*?)<\\/a>/gi)){try{const u=new URL(unesc(m[1]),base);if(!/^https?:$/.test(u.protocol))continue;out.push({url:u.href,text:plain(m[2]).slice(0,100)});}catch{}}return out;}\n function metadata(body:string){const title=plain(body.match(/]*>([\\s\\S]*?)<\\/title>/i)?.[1]??'');const desc=body.match(/]*name\\s*=\\s*[\"']description[\"'][^>]*content\\s*=\\s*[\"']([^\"']*)/i)?.[1]??body.match(/]*content\\s*=\\s*[\"']([^\"']*)[\"'][^>]*name\\s*=\\s*[\"']description/i)?.[1]??'';const schema=[...body.matchAll(/\"@type\"\\s*:\\s*\"([^\"]+)\"/g)].map(x=>x[1]);return {title,description:unesc(desc),schema_types:[...new Set(schema)],wordpress:/wp-content|wp-includes|wp-json/i.test(body)};}\n function patterns(urls:string[]){const first:Record={};const paths=urls.map(x=>{try{return new URL(x).pathname;}catch{return '';}});for(const p of paths){const s=p.split('/').filter(Boolean);if(s[0])first[s[0]]=(first[s[0]]??0)+1;}const counts=new Map();for(let i=0;i/^\\d{4}$/.test(x)?'{year}':/^\\d+$/.test(x)?'{id}':j===0&&s.length>1&&(first[x]??0)>=3?x:x.endsWith('.html')?'{slug}.html':x.endsWith('.php')?'{slug}.php':'{slug}').join('/')+(p.endsWith('/')&&s.length?'/':'');const c=counts.get(k)??{count:0,examples:[]};c.count++;if(c.examples.length<3)c.examples.push(urls[i]);counts.set(k,c);}return [...counts.entries()].map(([pattern,v])=>({pattern,...v})).sort((a,b)=>b.count-a.count).slice(0,20);}\n async function enrich(domain:string){const end=Date.now()+SITE_MS,base=baseByDomain.get(domain)??'https://'+domain;const [home,robots]=await Promise.all([get(base+'/',end,1500000),get(base+'/robots.txt',end,500000)]);const origin=home.final_url?new URL(home.final_url).origin:base;const canonicalHost=new URL(origin).hostname.replace(/^www\\./,'');const meta=metadata(home.status===200?home.body??'':'');const homeText=plain(home.status===200?home.body??'':'');const declared=(robots.status===200&&robots.body)?[...robots.body.matchAll(/^\\s*sitemap\\s*:\\s*(\\S+)\\s*$/gim)].map((m:any)=>unesc(m[1])):[];\n const roots=[...new Set(declared)];const queue:string[]=[...roots],seen=new Set(),pageUrls=new Set(),files:any[]=[];let errors=0,valid=0,offsite=0,assets=0,capped=false;\n async function parseMap(url:string){if(seen.has(url))return;seen.add(url);const f=await get(url,end);const rec:any={url,final_url:f.final_url??'',http_status:f.status??null,bytes:f.bytes??0};files.push(rec);if(f.status!==200||f.error){rec.error=f.error??'HTTP '+f.status;errors++;return;}try{if(!/<(?:[\\w-]+:)?(?:sitemapindex|urlset)\\b/.test(f.body)){rec.error='not_xml_sitemap';errors++;return;}const doc=parser.parse(f.body);if(doc.sitemapindex){const children=arr(doc.sitemapindex.sitemap).map(x=>x?.loc).filter((x:any)=>typeof x==='string');rec.kind='index';rec.child_count=children.length;valid++;for(const x of children){try{const u=new URL(unesc(x),f.final_url||url).href;if(!/^https?:/.test(u))continue;if(!seen.has(u)&&!queue.includes(u))queue.push(u);}catch{errors++;}}}else if(doc.urlset){rec.kind='urlset';let n=0;valid++;for(const item of arr(doc.urlset.url)){if(typeof item?.loc!=='string')continue;let u:URL;try{u=new URL(unesc(item.loc));u.hash='';if(!/^https?:$/.test(u.protocol))continue;}catch{continue;}const h=u.hostname.replace(/^www\\./,'');if(h!==domain&&h!==canonicalHost&&!h.endsWith('.'+domain)){offsite++;continue;}if(/\\.(?:jpg|jpeg|png|gif|webp|svg|pdf|mp4|mp3|css|js|zip|kml|xml|json|csv|txt|xlsx|rss|gz)$/i.test(u.pathname)){assets++;continue;}if(pageUrls.size>=100000){capped=true;break;}pageUrls.add(u.href);n++;}rec.page_url_entries=n;}else{rec.error='unknown_xml_root';errors++;}}catch(e){rec.error=String(e);errors++;}}\n if(!queue.length){for(const name of ['/sitemap.xml','/sitemap_index.xml','/wp-sitemap.xml']){await parseMap(origin+name);if(valid){roots.push(origin+name);errors=0;break;}if(Date.now()>end-1500)break;}}\n while(queue.length&&Date.now()!seen.has(x));await Promise.all(batch.map(parseMap));}\n if(queue.length||capped)capped=true;\n const urls=[...pageUrls],cls=classifySite(domain,meta,homeText,urls,files),homeLinks=links(home.body??'',home.final_url??origin);\n const rank=(u:string)=>/\\/(?:methodology|data-sources?|sources)(?:\\/|$|[?#])/i.test(u)?3:/\\/(?:about(?:-us)?|aboutus)(?:[./]|$)/i.test(u)?2:1;\n const about=[...new Map(homeLinks.filter(x=>{try{return new URL(x.url).hostname.replace(/^www\\./,'')===canonicalHost&&/\\/(?:about(?:-us)?|aboutus|methodology|data-sources?|sources|faq)(?:[./?#]|$)/i.test(x.url);}catch{return false;}}).map(x=>[x.url,x])).values()].sort((a,b)=>rank(b.url)-rank(a.url)).slice(0,5);\n const sourcePages:any[]=[];if(['directory','database_reference'].includes(cls.type)&&about[0]&&Date.now()=10)break;}if(provenance.length>=10)break;}\n const status=valid?(errors||capped?'partial':'complete_discovered_tree'):'unavailable';const result={domain,checked_at:new Date().toISOString(),homepage:{url:home.final_url??origin,http_status:home.status??null,error:home.error??null,...meta,text_excerpt:homeText.slice(0,1800)},classification:cls,sitemap:{status,page_count:valid?urls.length:null,count_definition:'Unique non-asset page URLs in fetched sitemaps; not indexed pages or total site pages',roots,files,remaining_files:queue.length,offsite_urls_skipped:offsite,asset_urls_skipped:assets,limits:{max_files:80,max_unique_urls:100000,site_ms:SITE_MS},robots_status:robots.status??null},url_patterns:patterns(urls),sample_urls:urls.length>60?Array.from({length:60},(_,i)=>urls[Math.floor(i*urls.length/60)]):urls,about_links:about,source_evidence:provenance,data_source_status:provenance.length?'mentions_found_requires_review':home.status===200?'unknown_not_disclosed_in_checked_pages':'unknown_pages_unavailable',source_pages:sourcePages};\n return [domain,result.checked_at,'done',result.sitemap.page_count,status,JSON.stringify(result)];\n }\n const INSERT=\"INSERT OR REPLACE INTO enrichment SELECT json_extract(value,'$[0]'),json_extract(value,'$[1]'),json_extract(value,'$[2]'),json_extract(value,'$[3]'),json_extract(value,'$[4]'),json_extract(value,'$[5]') FROM json_each(?)\";\n let i=0,saved=0,failedWrites=0,buf:any[]=[],bufSize=0,chain=Promise.resolve();\n function enqueue(){if(!buf.length)return;const batch=buf;buf=[];bufSize=0;chain=chain.then(async()=>{for(let a=0;a<3;a++){try{await sqlite.execute({sql:INSERT,args:[JSON.stringify(batch)]});saved+=batch.length;return;}catch(e){if(a===2){failedWrites+=batch.length;console.log(JSON.stringify({write_error:String(e),domains:batch.map(r=>r[0])}));}else await new Promise(r=>setTimeout(r,400*(a+1)));}}});}\n await Promise.all(Array.from({length:WORKERS},async()=>{while(i110000)enqueue();buf.push(row);bufSize+=n;if(buf.length>=12)enqueue();}}));enqueue();await chain;\n return {attempted:i,enriched:saved,failedWrites,duration_ms:Date.now()-started};\n}",[231],"4b6d882c-9f29-4b36-9c98-1022fb8088dd","enrich.ts","11ddbb7e-b2bd-11f1-b912-1607ee4eb77e",15,["D",1789666680342],"import { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\nimport { gunzipSync } from \"node:zlib\";\nimport { XMLParser } from \"npm:fast-xml-parser@4.5.3\";\nconst started=Date.now(),BUDGET=50000,WORKERS=160,LIMIT=5000;\nconst rows=await sqlite.execute({sql:\"SELECT c.domain FROM checks c LEFT JOIN enrichment e ON c.domain=e.domain WHERE c.status='confirmed' AND e.domain IS NULL ORDER BY CASE WHEN c.domain LIKE '%directory%' OR c.domain LIKE '%locator%' OR c.domain LIKE '%finder%' OR c.domain IN ('worldpopulationreview.com','campsitephotos.com') THEN 0 ELSE 1 END,c.domain LIMIT ?\",args:[LIMIT]});\nconst domains=rows.rows.map(x=>String(x.domain));\nconst parser=new XMLParser({ignoreAttributes:true,removeNSPrefix:true,parseTagValue:false,processEntities:false,trimValues:true});\nconst arr=(x:any)=>x==null?[]:Array.isArray(x)?x:[x];\nconst unesc=(s:string)=>s.replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'\"').replace(/'/g,\"'\");\nconst plain=(s:string)=>unesc(s.replace(/]*>[\\s\\S]*?<\\/script>/gi,' ').replace(/]*>[\\s\\S]*?<\\/style>/gi,' ').replace(/<[^>]+>/g,' ').replace(/\\s+/g,' ')).trim();\nasync function get(url:string,end:number,max=7000000):Promise{try{const left=end-Date.now();if(left<300)return {url,error:'site_time_budget'};const r=await fetch(url,{signal:AbortSignal.timeout(Math.min(6500,left)),headers:{'User-Agent':'PublisherNetworkResearch/1.0 (sitemap and public-page research)'}});const rd=r.body?.getReader();const chunks:Uint8Array[]=[];let n=0;if(rd)while(true){const p=await rd.read();if(p.done)break;n+=p.value.length;if(n>max){await rd.cancel();return {url,final_url:r.url,status:r.status,error:'body_size_limit'};}chunks.push(p.value);}let b=new Uint8Array(n);let offset=0;for(const c of chunks){b.set(c,offset);offset+=c.length;}if(b[0]===31&&b[1]===139)b=new Uint8Array(gunzipSync(b,{maxOutputLength:20000000}));return {url,final_url:r.url,status:r.status,body:new TextDecoder().decode(b),bytes:b.length};}catch(e){return {url,error:String(e)};}}\nfunction links(html:string,base:string){const out:any[]=[];for(const m of html.matchAll(/]*href\\s*=\\s*[\"']([^\"']+)[\"'][^>]*>([\\s\\S]*?)<\\/a>/gi)){try{const u=new URL(unesc(m[1]),base);if(!/^https?:$/.test(u.protocol))continue;out.push({url:u.href,text:plain(m[2]).slice(0,100)});}catch{}}return out;}\nfunction metadata(body:string){const title=plain(body.match(/]*>([\\s\\S]*?)<\\/title>/i)?.[1]??'');const desc=body.match(/]*name\\s*=\\s*[\"']description[\"'][^>]*content\\s*=\\s*[\"']([^\"']*)/i)?.[1]??body.match(/]*content\\s*=\\s*[\"']([^\"']*)[\"'][^>]*name\\s*=\\s*[\"']description/i)?.[1]??'';const schema=[...body.matchAll(/\"@type\"\\s*:\\s*\"([^\"]+)\"/g)].map(x=>x[1]);return {title,description:unesc(desc),schema_types:[...new Set(schema)],wordpress:/wp-content|wp-includes|wp-json/i.test(body)};}\nfunction categorize(domain:string,meta:any,text:string,urls:string[],maps:any[]){const head=(meta.title+' '+meta.description+' '+text.slice(0,500)).toLowerCase();const d=/\\bdirector(?:y|ies)\\b|\\blocator\\b|find (?:a |the |local )?(?:business|clinic|campground|store|boat ramp|toilet|doctor|facility|facilities|dentist)/i.test(head)||/directory|locator/.test(domain);const places=meta.schema_types.some((x:string)=>/LocalBusiness|MedicalClinic|Place|CivicStructure|LodgingBusiness/.test(x));const tool=/calculator|converter|generator|checker|word finder|bpm finder|fare finder|ip location/.test(head);const reference=/population|census|dictionary|encyclopedia|database|surname|lyrics|postal code|zip code/.test(head);const forum=/forum|community forums|discussion board/.test(meta.title.toLowerCase())||urls.some(x=>/\\/(?:threads|forums)\\//.test(x));const posts=maps.some(x=>/post-sitemap|wp-sitemap-posts-post/.test(x.url));const blog=posts||meta.schema_types.some((x:string)=>/BlogPosting|Article|NewsArticle/.test(x))||/\\bblog\\b|recipes|latest articles|latest posts/.test(head);let type='unknown',confidence='low',reason='Insufficient distinctive evidence';if(d){type='directory';confidence=d?'medium':'low';reason='Directory/locator language or place/business schema; automated classification';}else if(tool){type='tool';confidence='medium';reason='Calculator/converter/generator/finder language';}else if(forum){type='forum';confidence='medium';reason='Forum language or thread URL structure';}else if(reference){type='database_reference';confidence='medium';reason='Structured reference/database language';}else if(blog){type='blog';confidence='medium';reason=posts?'Post sitemap discovered':'Article/blog/recipe language or schema';}return {type,confidence,reason,review_status:'automated_not_manually_reviewed',has_editorial_section:blog};}\nfunction patterns(urls:string[]){const first:Record={};const paths=urls.map(x=>{try{return new URL(x).pathname;}catch{return '';}});for(const p of paths){const s=p.split('/').filter(Boolean);if(s[0])first[s[0]]=(first[s[0]]??0)+1;}const counts=new Map();for(let i=0;i/^\\d{4}$/.test(x)?'{year}':/^\\d+$/.test(x)?'{id}':j===0&&s.length>1&&(first[x]??0)>=3?x:x.endsWith('.html')?'{slug}.html':x.endsWith('.php')?'{slug}.php':'{slug}').join('/')+(p.endsWith('/')&&s.length?'/':'');const c=counts.get(k)??{count:0,examples:[]};c.count++;if(c.examples.length<3)c.examples.push(urls[i]);counts.set(k,c);}return [...counts.entries()].map(([pattern,v])=>({pattern,...v})).sort((a,b)=>b.count-a.count).slice(0,15);}\nasync function enrich(domain:string){const end=Date.now()+18000;const [home,robots]=await Promise.all([get('https://'+domain+'/',end,1500000),get('https://'+domain+'/robots.txt',end,500000)]);const origin=home.final_url?new URL(home.final_url).origin:'https://'+domain;const meta=metadata(home.body??'');const homeText=plain(home.body??'');const declared=(robots.status===200&&robots.body)?[...robots.body.matchAll(/^\\s*sitemap\\s*:\\s*(\\S+)\\s*$/gim)].map((m:any)=>unesc(m[1])):[];\n const roots=[...new Set(declared)];const queue:string[]=[...roots],seen=new Set(),pageUrls=new Set(),files:any[]=[];let errors=0,valid=0,offsite=0,assets=0,capped=false;\n async function parseMap(url:string){if(seen.has(url))return;seen.add(url);const f=await get(url,end);const rec:any={url,final_url:f.final_url??'',http_status:f.status??null,bytes:f.bytes??0};files.push(rec);if(f.status!==200||f.error){rec.error=f.error??'HTTP '+f.status;errors++;return;}try{if(!/<(?:[\\w-]+:)?(?:sitemapindex|urlset)\\b/.test(f.body)){rec.error='not_xml_sitemap';errors++;return;}const doc=parser.parse(f.body);if(doc.sitemapindex){const children=arr(doc.sitemapindex.sitemap).map(x=>x?.loc).filter((x:any)=>typeof x==='string');rec.kind='index';rec.child_count=children.length;valid++;for(const x of children){try{const u=new URL(unesc(x),f.final_url||url).href;if(!seen.has(u)&&!queue.includes(u))queue.push(u);}catch{errors++;}}}else if(doc.urlset){rec.kind='urlset';let n=0;valid++;for(const item of arr(doc.urlset.url)){if(typeof item?.loc!=='string')continue;let u:URL;try{u=new URL(unesc(item.loc));u.hash='';}catch{continue;}if(u.hostname.replace(/^www\\./,'')!==domain&&!u.hostname.endsWith('.'+domain)){offsite++;continue;}if(/\\.(?:jpg|jpeg|png|gif|webp|svg|pdf|mp4|mp3|css|js|zip|kml|xml|json|csv|txt|xlsx|rss|gz)$/i.test(u.pathname)){assets++;continue;}pageUrls.add(u.href);n++;if(pageUrls.size>=100000){capped=true;break;}}rec.page_url_entries=n;}else{rec.error='unknown_xml_root';errors++;}}catch(e){rec.error=String(e);errors++;}}\n if(!queue.length){for(const name of ['/sitemap.xml','/sitemap_index.xml','/wp-sitemap.xml']){await parseMap(origin+name);if(valid){roots.push(origin+name);errors=0;break;}if(Date.now()>end-1500)break;}}\n while(queue.length&&Date.now()!seen.has(x));await Promise.all(batch.map(parseMap));}\n if(queue.length||seen.size>=80||capped)capped=true;\n const urls=[...pageUrls];const cls=categorize(domain,meta,homeText,urls,files);const homeLinks=links(home.body??'',home.final_url??origin);const about=homeLinks.filter(x=>{try{return new URL(x.url).hostname.replace(/^www\\./,'')===domain&&/about|methodology|data-source|sources|faq/i.test(x.url+' '+x.text);}catch{return false;}}).sort((a,b)=>Number(/methodology|sources/.test(b.url))-Number(/methodology|sources/.test(a.url))).slice(0,5);\n const sourcePages:any[]=[];if(cls.type==='directory'||cls.type==='database_reference'){if(about[0]&&Date.now()=12)break;}}\n const status=valid?(errors||capped?'partial':'complete_discovered_tree'):'unavailable';const result={domain,checked_at:new Date().toISOString(),homepage:{url:home.final_url??origin,http_status:home.status??null,error:home.error??null,...meta,text_excerpt:homeText.slice(0,1800)},classification:cls,sitemap:{status,page_count:valid?urls.length:null,count_definition:'Unique non-asset page URLs in fetched sitemaps; not indexed pages or total site pages',roots,files,remaining_files:queue.length,offsite_urls_skipped:offsite,asset_urls_skipped:assets,limits:{max_files:80,max_unique_urls:100000,site_ms:18000},robots_status:robots.status??null},url_patterns:patterns(urls),sample_urls:urls.slice(0,12),about_links:about,source_evidence:provenance,data_source_status:provenance.length?'mentions_found_requires_review':'unknown_not_disclosed_in_checked_pages',source_pages:sourcePages};\n return [domain,result.checked_at,'done',result.sitemap.page_count,status,JSON.stringify(result)];}\nlet i=0;const out:any[]=[];await Promise.all(Array.from({length:WORKERS},async()=>{while(i180000)await save();chunk.push(r);size+=n;}await save();console.log(JSON.stringify({enriched:out.length,duration_ms:Date.now()-started,totals:(await sqlite.execute(\"SELECT json_extract(result_json,'$.classification.type') type,sitemap_status,COUNT(*) n FROM enrichment GROUP BY 1,2\")).rows,examples:out.slice(0,4).map(r=>{const d=JSON.parse(r[5]);return {domain:r[0],count:r[3],status:r[4],classification:d.classification};})}));",[239],"e22921c3-81aa-46a0-b970-aa99c953c531","download.ts","7645eebc-b2c0-11f1-a2dc-1607ee4eb77e",50,["D",1789670391403],"import { DATASETS, csvChunks } from './export_core.ts';\nimport { gzipSync } from 'node:zlib';\nimport { Buffer } from 'node:buffer';\nexport default async function(req:Request){\n const u=new URL(req.url),expected=Deno.env.get('AUDIT_EXPORT_KEY');\n const supplied=req.headers.get('authorization')?.replace(/^Bearer /,'')??u.searchParams.get('key');\n if(!expected||supplied!==expected)return new Response('Unauthorized',{status:401});\n if(req.method!=='GET')return new Response('GET required',{status:405});\n const kind=u.searchParams.get('kind')??'confirmed';\n if(!DATASETS.includes(kind))return new Response('Invalid dataset',{status:400});\n const compressed=u.searchParams.get('compressed')!=='0';\n const parts:Uint8Array[]=[];for await(const part of csvChunks(kind))parts.push(part);\n const csv=Buffer.concat(parts),body=compressed?gzipSync(csv):csv;\n return new Response(body,{headers:{'content-type':compressed?'application/gzip':'text/csv; charset=utf-8','content-length':String(body.length),'content-disposition':'attachment; filename=\"publisher-audit-'+kind+'-2026-09-17.csv'+(compressed?'.gz':'')+'\"','cache-control':'private, no-store','referrer-policy':'no-referrer','x-content-type-options':'nosniff','access-control-allow-origin':'*'}});\n}",[246],"62756cac-b216-4a2e-a06d-e36b33990c22","directory_research.ts","43ca09aa-b2c5-11f1-8c69-1607ee4eb77e",44,["D",1789669574067],"import { sqlite } from 'https://esm.town/v/std/sqlite/main.ts';\nawait sqlite.execute('CREATE TABLE IF NOT EXISTS directory_research(domain TEXT PRIMARY KEY,checked_at TEXT,result_json TEXT)');\nconst start=Date.now(),BUDGET=26000,WORKERS=16;\nconst rows=(await sqlite.execute(\"SELECT e.domain,e.result_json FROM enrichment e LEFT JOIN directory_research d ON e.domain=d.domain WHERE json_extract(e.result_json,'$.classification.type') IN ('directory','database_reference') AND d.domain IS NULL ORDER BY e.sitemap_count DESC LIMIT 400\")).rows;\nconst plain=(s:string)=>s.replace(/]*>[\\s\\S]*?<\\/script>/gi,' ').replace(/]*>[\\s\\S]*?<\\/style>/gi,' ').replace(/<[^>]+>/g,' ').replace(/&/g,'&').replace(/ /g,' ').replace(/\\s+/g,' ').trim();\nasync function page(url:string){try{const r=await fetch(url,{signal:AbortSignal.timeout(11000),headers:{'User-Agent':'PublisherNetworkResearch/1.0 (directory source research)'}});if(!r.ok){await r.body?.cancel();return {url,final_url:r.url,status:r.status};}const reader=r.body?.getReader(),chunks:Uint8Array[]=[];let n=0;if(reader)while(true){const p=await reader.read();if(p.done)break;n+=p.value.length;if(n>1800000){await reader.cancel();return {url,final_url:r.url,status:r.status,error:'body_size_limit'};}chunks.push(p.value);}const b=new Uint8Array(n);let k=0;for(const c of chunks){b.set(c,k);k+=c.length;}const html=new TextDecoder().decode(b),text=plain(html),title=plain(html.match(/]*>([\\s\\S]*?)<\\/title>/i)?.[1]??'');\n const headings=[...html.matchAll(/]*>([\\s\\S]*?)<\\/h[1-3]>/gi)].map(x=>plain(x[1])).slice(0,30);\n const links:any[]=[];for(const m of html.matchAll(/]*href\\s*=\\s*[\"']([^\"']+)[\"'][^>]*>([\\s\\S]*?)<\\/a>/gi)){try{const u=new URL(m[1].replace(/&/g,'&'),r.url);if(!/^https?:$/.test(u.protocol))continue;const label=plain(m[2]).slice(0,100);if(u.hostname!==new URL(r.url).hostname||/about|methodology|sources|submit|add.?listing|claim|contribut|correct|contact/i.test(u.pathname+' '+label))links.push({url:u.href,label});}catch{}}\n const snippets:any[]=[];const re=/sourced from|provided by|obtained from|compiled from|collected from|data (?:is|are|comes)|our data|we (?:use|collect|gather|compile)|OpenStreetMap|Census Bureau|U\\.S\\. Census|USGS|Recreation\\.gov|Wikidata|MusicBrainz|Google Places|Google Maps|HRSA|Centers for Medicare|\\bCMS\\b|NHS|submitted by|community.contribut|publicly.available|official.sources|licensed.data|scrap(?:ing|ed)/gi;for(const m of text.matchAll(re)){const i=m.index??0;snippets.push(text.slice(Math.max(0,i-100),i+260));if(snippets.length>=14)break;}\n const schemaKeys=[...new Set([...html.matchAll(/\"(address|telephone|openingHours|openingHoursSpecification|geo|latitude|longitude|aggregateRating|review|priceRange|sameAs|url|name|description|medicalSpecialty|servesCuisine|amenityFeature|dateModified|datePublished|identifier)\"\\s*:/g)].map(m=>m[1]))];\n const signals={owner_submission:/submit (?:a |your )?(?:business|listing|location|clinic|church)|add (?:a |your )?(?:business|listing|location|clinic|church)|claim (?:this |your )?(?:listing|business|clinic)|suggest an update/i.test(text),reviews:/reviews|ratings|testimonials/i.test(text),google_maps:/maps\\.google|maps\\.googleapis|maps\\.app\\.goo|google\\.com\\/maps/.test(html),openstreetmap:/openstreetmap|osm\\.org/.test(html)};\n return {url,final_url:r.url,status:r.status,title,headings,summary_text:text.slice(0,2400),source_snippets:snippets,relevant_links:[...new Map(links.map(x=>[x.url,x])).values()].slice(0,50),schema_keys:schemaKeys,signals};\n}catch(e){return {url,error:String(e)};}}\nlet i=0,saved=0;\nawait Promise.all(Array.from({length:WORKERS},async()=>{while(ix.url),...(e.url_patterns??[]).filter((x:any)=>x.examples?.length).slice(0,2).map((x:any)=>x.examples[0])];const urls=[...new Set(picks)].slice(0,5);const pages=await Promise.all(urls.map(page));const result={domain:row.domain,checked_at:new Date().toISOString(),status:'automated_page_research_requires_review',pages,observed_schema_fields:[...new Set(pages.flatMap((p:any)=>p.schema_keys??[]))],source_status:'unknown_until_disclosure_is_reviewed',note:'Map and platform signals do not prove the listing database source.'};await sqlite.execute({sql:'INSERT OR REPLACE INTO directory_research VALUES (?,?,?)',args:[String(row.domain),result.checked_at,JSON.stringify(result)]});saved++;}}));\nconsole.log(JSON.stringify({researched:saved,elapsed_ms:Date.now()-start,remaining:(await sqlite.execute(\"SELECT COUNT(*) n FROM enrichment e LEFT JOIN directory_research d ON e.domain=d.domain WHERE json_extract(e.result_json,'$.classification.type') IN ('directory','database_reference') AND d.domain IS NULL\")).rows}));",[253],"558d7440-dd48-40ee-a096-168391b2a1da","classify.ts","aac5576c-b2c1-11f1-bee6-1607ee4eb77e",28,["D",1789668028848],"export function classifySite(domain:string,meta:any,text:string,urls:string[],maps:any[]){\n const title=String(meta.title??'').toLowerCase(),head=(title+' '+String(meta.description??'')).toLowerCase();\n const body=String(text??'').slice(0,1800).toLowerCase();\n const schemas=Array.isArray(meta.schema_types)?meta.schema_types:[];\n const sample=urls.length>300?Array.from({length:300},(_,i)=>urls[Math.floor(i*urls.length/300)]):urls;\n const paths=sample.map(s=>{try{return new URL(s).pathname.toLowerCase();}catch{return '';}});\n const ratio=(r:RegExp)=>paths.length?paths.filter(p=>r.test(p)).length/paths.length:0;\n const articlePaths=ratio(/^\\/(?:blog|blogs|article|articles|posts|news|recipes|recetas|rezepte|guides)\\//);\n const entityPaths=ratio(/^\\/(?:hospital|hospitals|surgery-center|ramps|campgrounds|businesses|clinics|prisons|dentists|nursing-homes|shelters)\\//);\n const geoPaths=ratio(/^\\/(?:states|locations|places|stores|cities)\\/[^/]+\\/[^/]+/);\n const forum=/\\bforums?\\b|\\bforo\\b|discussion board|community discussions/.test(head)||ratio(/^\\/(?:threads|forums|discussions)\\//)>.3;\n const tool=/calculator|converter|generator|word finder|word solver|bpm finder|fare finder|ip location|tracking tool|online tracker/.test(head)||schemas.includes('SoftwareApplication')&&/calculate|convert|generate|track/.test(body);\n const reference=/encyclopedia|encyclopaedia|dictionary|gazetteer|\\bdatabase\\b|\\bwiki\\b|world population|drop sources|postal codes|zip codes/.test(head);\n const directDirectory=/\\bdirector(?:y|ies)\\b|\\blocator\\b|business listings|\\bannuaire\\b|branchenbuch|find.{0,40}(?:near you|near me|business|clinic|campground|boat ramp|dentist|shelter)/.test(head);\n const nameDirectory=/directory|locator|annuaire/.test(domain);\n const postMap=maps.some(m=>/(?:^|\\/)post-sitemap|\\/wp-sitemap-posts-(?:post|recipe)|\\/sitemap-posts(?:[-_.?]|$)|\\/sitemap[._-]post(?:[-_.?]|$)/i.test(String(m.url??'')));\n const editorial=postMap||articlePaths>.35||schemas.some((s:string)=>['Blog','BlogPosting','Article','NewsArticle','Recipe'].includes(s))||/\\bblog\\b|\\brecipes?\\b|recetas|ricette|receitas|recettes|rezepte|przepisy|рецепт|cooking|lifestyle|travel tips|diy projects|gardening tips|latest articles|latest posts/.test(head+' '+body.slice(0,300));\n let type='unknown',confidence='low',reason='Insufficient accessible evidence to assign a primary site type';\n if(forum){type='forum';confidence='medium';reason='Forum title/description or repeated discussion-thread URLs';}\n else if(tool){type='tool';confidence='medium';reason='Title/description or application schema describes an interactive utility';}\n else if(reference&&!directDirectory){type='database_reference';confidence='medium';reason='Reference/database/encyclopedia language in title or description';}\n else if(directDirectory||entityPaths>.4||geoPaths>.5){type='directory';confidence='medium';reason=directDirectory?'Directory or entity-search language in title/description':'Repeated entity or geographic listing URL structure';}\n else if(nameDirectory&&!editorial){type='directory';confidence='low';reason='Domain-name hint only; manual review required';}\n else if(editorial){type='blog';confidence='medium';reason=postMap?'Editorial post sitemap':articlePaths>.35?'Predominantly article/blog/recipe URL structure':'Editorial/recipe language or article schema';}\n return {type,confidence,reason,review_status:'automated_v2_not_manually_reviewed',has_editorial_section:editorial,version:2,evidence:{article_url_ratio:Number(articlePaths.toFixed(3)),entity_url_ratio:Number(entityPaths.toFixed(3)),geographic_url_ratio:Number(geoPaths.toFixed(3)),post_sitemap:postMap}};\n}",[260],"75e758c0-6550-4a8e-a4bf-d244768ed56e","batch_scan.ts","19a948f8-b2bf-11f1-b5e0-1607ee4eb77e",29,["D",1789668108011],2,"import { sqlite } from 'https://esm.town/v/std/sqlite/main.ts';\nconst endpoint='https://omriluz--0eb0b968b2bf11f196131607ee4eb77e.web.val.run';\nconst start=Date.now(),shards=8;\nconst pending=(await sqlite.execute({sql:\"SELECT DISTINCT ((unicode(substr(s.domain,1,1))*31+unicode(substr(s.domain,2,1))+length(s.domain)) % ?) shard FROM sellers s LEFT JOIN checks c ON s.domain=c.domain WHERE s.domain<>'' AND c.domain IS NULL ORDER BY shard\",args:[shards]})).rows.map(r=>Number(r.shard));\nconst results=await Promise.all(pending.map(async shard=>{try{const r=await fetch(endpoint,{method:'POST',headers:{'content-type':'application/json',authorization:'Bearer '+Deno.env.get('AUDIT_JOB_KEY')},body:JSON.stringify({kind:'scan',shard,shards}),signal:AbortSignal.timeout(75000)});const text=await r.text();try{const value=JSON.parse(text);return {shard,status:r.status,...(value&&typeof value==='object'?value:{error:value})};}catch{return {shard,status:r.status,body:text.slice(0,600)};}}catch(e){return {shard,error:String(e)};}}));\nconsole.log(JSON.stringify({duration_ms:Date.now()-start,results,totals:(await sqlite.execute('SELECT status,COUNT(*) n FROM checks GROUP BY status')).rows,remaining:(await sqlite.execute(\"SELECT COUNT(DISTINCT s.domain) n FROM sellers s LEFT JOIN checks c ON s.domain=c.domain WHERE s.domain<>'' AND c.domain IS NULL\")).rows}));",[267],"487510d4-a261-4fb6-9156-e600336de14a","batch_retry.ts","c332e11a-b2c2-11f1-8885-1607ee4eb77e",37,["D",1789668499337],"import { sqlite } from 'https://esm.town/v/std/sqlite/main.ts';\nconst endpoint='https://omriluz--0eb0b968b2bf11f196131607ee4eb77e.web.val.run';\nconst start=Date.now(),shards=8;\nconst pending=(await sqlite.execute({sql:\"SELECT DISTINCT ((unicode(substr(c.domain,1,1))*31+unicode(substr(c.domain,2,1))+length(c.domain)) % ?) shard FROM checks c WHERE c.status='unverified' AND c.checked_at < (SELECT value FROM audit_settings WHERE key='retry_cutoff') ORDER BY shard\",args:[shards]})).rows.map(r=>Number(r.shard));\nconst results=await Promise.all(pending.map(async shard=>{try{const r=await fetch(endpoint,{method:'POST',headers:{'content-type':'application/json',authorization:'Bearer '+Deno.env.get('AUDIT_JOB_KEY')},body:JSON.stringify({kind:'retry',shard,shards}),signal:AbortSignal.timeout(75000)});const text=await r.text();try{const value=JSON.parse(text);return {shard,status:r.status,...(value&&typeof value==='object'?value:{error:value})};}catch{return {shard,status:r.status,body:text.slice(0,400)};}}catch(e){return {shard,error:String(e)};}}));\nconsole.log(JSON.stringify({duration_ms:Date.now()-start,results,totals:(await sqlite.execute('SELECT status,COUNT(*) n FROM checks GROUP BY status')).rows,remaining_retries:(await sqlite.execute(\"SELECT COUNT(*) n FROM checks c WHERE c.status='unverified' AND c.checked_at < (SELECT value FROM audit_settings WHERE key='retry_cutoff')\")).rows}));",[275],"614711a8-16a6-49a3-b849-5b107926bb2d","batch_enrich.ts","6381902c-b2c2-11f1-809f-1607ee4eb77e",31,["D",1789668338773],"import { sqlite } from 'https://esm.town/v/std/sqlite/main.ts';\nconst endpoint='https://omriluz--0eb0b968b2bf11f196131607ee4eb77e.web.val.run';\nconst start=Date.now(),shards=8;\nconst pending=(await sqlite.execute({sql:\"SELECT DISTINCT ((unicode(substr(c.domain,1,1))*31+unicode(substr(c.domain,2,1))+length(c.domain)) % ?) shard FROM checks c LEFT JOIN enrichment e ON c.domain=e.domain WHERE c.status='confirmed' AND e.domain IS NULL ORDER BY shard\",args:[shards]})).rows.map(r=>Number(r.shard));\nconst results=await Promise.all(pending.map(async shard=>{try{const r=await fetch(endpoint,{method:'POST',headers:{'content-type':'application/json',authorization:'Bearer '+Deno.env.get('AUDIT_JOB_KEY')},body:JSON.stringify({kind:'enrich',shard,shards}),signal:AbortSignal.timeout(75000)});const text=await r.text();try{const value=JSON.parse(text);return {shard,status:r.status,...(value&&typeof value==='object'?value:{error:value})};}catch{return {shard,status:r.status,body:text.slice(0,600)};}}catch(e){return {shard,error:String(e)};}}));\nconsole.log(JSON.stringify({duration_ms:Date.now()-start,results,totals:(await sqlite.execute('SELECT sitemap_status,COUNT(*) n FROM enrichment GROUP BY sitemap_status')).rows,remaining:(await sqlite.execute(\"SELECT COUNT(*) n FROM checks c LEFT JOIN enrichment e ON c.domain=e.domain WHERE c.status='confirmed' AND e.domain IS NULL\")).rows}));",[282],"af49aca0-800e-4cdb-a656-605cafe7a1cb","README.md","01a0b061-5e61-74b2-9c7e-77e87e2983ad",["D",1789665566213],"# hono-jsx-starter\n\nA minimal [Hono](https://hono.dev) + Hono JSX starter for Val Town.\nServer-rendered HTML, no build step, no React.\n\n## Picking a starter\n\n| Starter | When to pick it |\n| --- | --- |\n| [`basic-html-starter`](https://www.val.town/x/templates/basic-html-starter) | Static HTML pages. Sprinkle of vanilla JS. No server-side rendering. |\n| **[`hono-jsx-starter`](https://www.val.town/x/templates/hono-jsx-starter)** *(you're here)* | Dynamic server-rendered HTML with JSX. API routes. No React, no SPA. |\n| [`react-hono-starter`](https://www.val.town/x/templates/react-hono-starter) | Client-side React SPA with a Hono API backend. |\n\n## What's here\n\n```\nindex.tsx Hono app, Layout, routes\npublic/\n favicon.svg Static asset\n counter.ts Tiny client-side script\n```\n\nThe whole app fits in one file. As you grow it, the natural next moves are:\n\n- Pull `Layout` into its own `.tsx` file once you have 3+ pages.\n- Split routes with `app.route(\"/api\", api)` if `/api/*` grows.\n- Add more files under `public/` — they're served by a single\n `app.get(\"/public/*\", ...)` route.\n\n## Patterns worth knowing\n\n**JSX pragma.** Every `.tsx` file that uses JSX needs this at the top:\n\n```tsx\n/** @jsxImportSource npm:hono@4/jsx */\n```\n\n**Server state.** The counter on `/` is `let count = 0` at module scope. It\nlives in memory on the Val Town server and survives across requests — but resets\nwhenever the val restarts (deploys, idle timeouts, etc.). For persistent state,\nswap it for [`std/sqlite`](https://www.val.town/x/std/sqlite) or\n[`std/blob`](https://www.val.town/x/std/blob).\n\n**API routes.** The `Increment` button POSTs to `/api/increment`, which returns\nJSON. Mix server-rendered pages and JSON endpoints in the same Hono app:\n\n```ts\napp.post(\"/api/increment\", (c) => c.json({ count: ++count }));\n```\n\n**Static files.** `serveFile` reads the file by path and serves it with the\nright content-type. `.ts` files are transpiled to JS on request by Val Town — so\nyou can write `public/counter.ts` and reference it as\n`

Diff · v54