[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_233":234,"_235":236},"routes/_app.x.$handle.$name",{"_100":99,"_32":45,"_108":90,"_27":109,"_13":14,"_110":-5,"_11":111,"_112":113,"_114":115,"_116":117,"_118":119,"_20":-5,"_120":-5,"_121":-5,"_122":25,"_123":124,"_125":59,"_126":-5,"_127":-5,"_128":129},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_43":44,"_32":45,"_38":46,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":53,"_54":-5,"_18":55,"_56":57,"_58":59,"_60":-5,"_61":-5,"_62":63,"_64":65,"_66":67,"_68":69,"_70":71,"_72":-5,"_73":74,"_75":74,"_76":77,"_78":77,"_79":77,"_80":74,"_81":74,"_82":83,"_84":85,"_86":87,"_88":25},"version",15,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_33":34,"_35":37,"_38":39,"_40":14,"_41":42,"_24":25},"previous",{"_30":31,"_32":28,"_33":34,"_35":36},"deleted",false,"deletedHere","path","enrich.ts","latestPath","fileId","11ddbb7e-b2bd-11f1-b912-1607ee4eb77e","name","fileType","script","content","import { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\nimport { gunzipSync } from \"node:zlib\";\nimport { XMLParser } from \"npm:fast-xml-parser@4.5.3\";\nconst started=Date.now(),BUDGET=20000,WORKERS=60,LIMIT=800;\nconst rows=await sqlite.execute({sql:\"SELECT c.domain FROM checks c LEFT JOIN enrichment e ON c.domain=e.domain WHERE c.status='confirmed' AND e.domain IS NULL ORDER BY CASE WHEN c.domain LIKE '%directory%' OR c.domain LIKE '%locator%' OR c.domain LIKE '%finder%' OR c.domain IN ('worldpopulationreview.com','campsitephotos.com') THEN 0 ELSE 1 END,c.domain LIMIT ?\",args:[LIMIT]});\nconst domains=rows.rows.map(x=>String(x.domain));\nconst parser=new XMLParser({ignoreAttributes:true,removeNSPrefix:true,parseTagValue:false,processEntities:false,trimValues:true});\nconst arr=(x:any)=>x==null?[]:Array.isArray(x)?x:[x];\nconst unesc=(s:string)=>s.replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'\"').replace(/'/g,\"'\");\nconst plain=(s:string)=>unesc(s.replace(/]*>[\\s\\S]*?<\\/script>/gi,' ').replace(/]*>[\\s\\S]*?<\\/style>/gi,' ').replace(/<[^>]+>/g,' ').replace(/\\s+/g,' ')).trim();\nasync function get(url:string,end:number,max=7000000):Promise{try{const left=end-Date.now();if(left<300)return {url,error:'site_time_budget'};const r=await fetch(url,{signal:AbortSignal.timeout(Math.min(6500,left)),headers:{'User-Agent':'PublisherNetworkResearch/1.0 (sitemap and public-page research)'}});const rd=r.body?.getReader();const chunks:Uint8Array[]=[];let n=0;if(rd)while(true){const p=await rd.read();if(p.done)break;n+=p.value.length;if(n>max){await rd.cancel();return {url,final_url:r.url,status:r.status,error:'body_size_limit'};}chunks.push(p.value);}let b=new Uint8Array(n);let offset=0;for(const c of chunks){b.set(c,offset);offset+=c.length;}if(b[0]===31&&b[1]===139)b=new Uint8Array(gunzipSync(b,{maxOutputLength:20000000}));return {url,final_url:r.url,status:r.status,body:new TextDecoder().decode(b),bytes:b.length};}catch(e){return {url,error:String(e)};}}\nfunction links(html:string,base:string){const out:any[]=[];for(const m of html.matchAll(/]*href\\s*=\\s*[\"']([^\"']+)[\"'][^>]*>([\\s\\S]*?)<\\/a>/gi)){try{const u=new URL(unesc(m[1]),base);if(!/^https?:$/.test(u.protocol))continue;out.push({url:u.href,text:plain(m[2]).slice(0,100)});}catch{}}return out;}\nfunction metadata(body:string){const title=plain(body.match(/]*>([\\s\\S]*?)<\\/title>/i)?.[1]??'');const desc=body.match(/]*name\\s*=\\s*[\"']description[\"'][^>]*content\\s*=\\s*[\"']([^\"']*)/i)?.[1]??body.match(/]*content\\s*=\\s*[\"']([^\"']*)[\"'][^>]*name\\s*=\\s*[\"']description/i)?.[1]??'';const schema=[...body.matchAll(/\"@type\"\\s*:\\s*\"([^\"]+)\"/g)].map(x=>x[1]);return {title,description:unesc(desc),schema_types:[...new Set(schema)],wordpress:/wp-content|wp-includes|wp-json/i.test(body)};}\nfunction categorize(domain:string,meta:any,text:string,urls:string[],maps:any[]){const head=(meta.title+' '+meta.description+' '+text.slice(0,500)).toLowerCase();const d=/\\bdirector(?:y|ies)\\b|\\blocator\\b|find (?:a |the |local )?(?:business|clinic|campground|store|boat ramp|toilet|doctor|facility|facilities|dentist)/i.test(head)||/directory|locator/.test(domain);const places=meta.schema_types.some((x:string)=>/LocalBusiness|MedicalClinic|Place|CivicStructure|LodgingBusiness/.test(x));const tool=/calculator|converter|generator|checker|word finder|bpm finder|fare finder|ip location/.test(head);const reference=/population|census|dictionary|encyclopedia|database|surname|lyrics|postal code|zip code/.test(head);const forum=/forum|community forums|discussion board/.test(meta.title.toLowerCase())||urls.some(x=>/\\/(?:threads|forums)\\//.test(x));const posts=maps.some(x=>/post-sitemap|wp-sitemap-posts-post/.test(x.url));const blog=posts||meta.schema_types.some((x:string)=>/BlogPosting|Article|NewsArticle/.test(x))||/\\bblog\\b|recipes|latest articles|latest posts/.test(head);let type='unknown',confidence='low',reason='Insufficient distinctive evidence';if(d){type='directory';confidence=d?'medium':'low';reason='Directory/locator language or place/business schema; automated classification';}else if(tool){type='tool';confidence='medium';reason='Calculator/converter/generator/finder language';}else if(forum){type='forum';confidence='medium';reason='Forum language or thread URL structure';}else if(reference){type='database_reference';confidence='medium';reason='Structured reference/database language';}else if(blog){type='blog';confidence='medium';reason=posts?'Post sitemap discovered':'Article/blog/recipe language or schema';}return {type,confidence,reason,review_status:'automated_not_manually_reviewed',has_editorial_section:blog};}\nfunction patterns(urls:string[]){const first:Record={};const paths=urls.map(x=>{try{return new URL(x).pathname;}catch{return '';}});for(const p of paths){const s=p.split('/').filter(Boolean);if(s[0])first[s[0]]=(first[s[0]]??0)+1;}const counts=new Map();for(let i=0;i/^\\d{4}$/.test(x)?'{year}':/^\\d+$/.test(x)?'{id}':j===0&&s.length>1&&(first[x]??0)>=3?x:x.endsWith('.html')?'{slug}.html':x.endsWith('.php')?'{slug}.php':'{slug}').join('/')+(p.endsWith('/')&&s.length?'/':'');const c=counts.get(k)??{count:0,examples:[]};c.count++;if(c.examples.length<3)c.examples.push(urls[i]);counts.set(k,c);}return [...counts.entries()].map(([pattern,v])=>({pattern,...v})).sort((a,b)=>b.count-a.count).slice(0,15);}\nasync function enrich(domain:string){const end=Date.now()+18000;const [home,robots]=await Promise.all([get('https://'+domain+'/',end,1500000),get('https://'+domain+'/robots.txt',end,500000)]);const origin=home.final_url?new URL(home.final_url).origin:'https://'+domain;const meta=metadata(home.body??'');const homeText=plain(home.body??'');const declared=(robots.status===200&&robots.body)?[...robots.body.matchAll(/^\\s*sitemap\\s*:\\s*(\\S+)\\s*$/gim)].map((m:any)=>unesc(m[1])):[];\n const roots=[...new Set(declared)];const queue:string[]=[...roots],seen=new Set(),pageUrls=new Set(),files:any[]=[];let errors=0,valid=0,offsite=0,assets=0,capped=false;\n async function parseMap(url:string){if(seen.has(url))return;seen.add(url);const f=await get(url,end);const rec:any={url,final_url:f.final_url??'',http_status:f.status??null,bytes:f.bytes??0};files.push(rec);if(f.status!==200||f.error){rec.error=f.error??'HTTP '+f.status;errors++;return;}try{if(!/<(?:[\\w-]+:)?(?:sitemapindex|urlset)\\b/.test(f.body)){rec.error='not_xml_sitemap';errors++;return;}const doc=parser.parse(f.body);if(doc.sitemapindex){const children=arr(doc.sitemapindex.sitemap).map(x=>x?.loc).filter((x:any)=>typeof x==='string');rec.kind='index';rec.child_count=children.length;valid++;for(const x of children){try{const u=new URL(unesc(x),f.final_url||url).href;if(!seen.has(u)&&!queue.includes(u))queue.push(u);}catch{errors++;}}}else if(doc.urlset){rec.kind='urlset';let n=0;valid++;for(const item of arr(doc.urlset.url)){if(typeof item?.loc!=='string')continue;let u:URL;try{u=new URL(unesc(item.loc));u.hash='';}catch{continue;}if(u.hostname.replace(/^www\\./,'')!==domain&&!u.hostname.endsWith('.'+domain)){offsite++;continue;}if(/\\.(?:jpg|jpeg|png|gif|webp|svg|pdf|mp4|mp3|css|js|zip|kml|xml|json|csv|txt|xlsx|rss|gz)$/i.test(u.pathname)){assets++;continue;}pageUrls.add(u.href);n++;if(pageUrls.size>=100000){capped=true;break;}}rec.page_url_entries=n;}else{rec.error='unknown_xml_root';errors++;}}catch(e){rec.error=String(e);errors++;}}\n if(!queue.length){for(const name of ['/sitemap.xml','/sitemap_index.xml','/wp-sitemap.xml']){await parseMap(origin+name);if(valid){roots.push(origin+name);errors=0;break;}if(Date.now()>end-1500)break;}}\n while(queue.length&&Date.now()!seen.has(x));await Promise.all(batch.map(parseMap));}\n if(queue.length||seen.size>=80||capped)capped=true;\n const urls=[...pageUrls];const cls=categorize(domain,meta,homeText,urls,files);const homeLinks=links(home.body??'',home.final_url??origin);const about=homeLinks.filter(x=>{try{return new URL(x.url).hostname.replace(/^www\\./,'')===domain&&/about|methodology|data-source|sources|faq/i.test(x.url+' '+x.text);}catch{return false;}}).sort((a,b)=>Number(/methodology|sources/.test(b.url))-Number(/methodology|sources/.test(a.url))).slice(0,5);\n const sourcePages:any[]=[];if(cls.type==='directory'||cls.type==='database_reference'){if(about[0]&&Date.now()=12)break;}}\n const status=valid?(errors||capped?'partial':'complete_discovered_tree'):'unavailable';const result={domain,checked_at:new Date().toISOString(),homepage:{url:home.final_url??origin,http_status:home.status??null,error:home.error??null,...meta,text_excerpt:homeText.slice(0,1800)},classification:cls,sitemap:{status,page_count:valid?urls.length:null,count_definition:'Unique non-asset page URLs in fetched sitemaps; not indexed pages or total site pages',roots,files,remaining_files:queue.length,offsite_urls_skipped:offsite,asset_urls_skipped:assets,limits:{max_files:80,max_unique_urls:100000,site_ms:18000},robots_status:robots.status??null},url_patterns:patterns(urls),sample_urls:urls.slice(0,12),about_links:about,source_evidence:provenance,data_source_status:provenance.length?'mentions_found_requires_review':'unknown_not_disclosed_in_checked_pages',source_pages:sourcePages};\n return [domain,result.checked_at,'done',result.sitemap.page_count,status,JSON.stringify(result)];}\nlet i=0;const out:any[]=[];await Promise.all(Array.from({length:WORKERS},async()=>{while(i180000)await save();chunk.push(r);size+=n;}await save();console.log(JSON.stringify({enriched:out.length,duration_ms:Date.now()-started,totals:(await sqlite.execute(\"SELECT json_extract(result_json,'$.classification.type') type,sitemap_status,COUNT(*) n FROM enrichment GROUP BY 1,2\")).rows,examples:out.slice(0,4).map(r=>{const d=JSON.parse(r[5]);return {domain:r[0],count:r[3],status:r[4],classification:d.classification};})}));","import { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\nimport { gunzipSync } from \"node:zlib\";\nimport { XMLParser } from \"npm:fast-xml-parser@4.5.3\";\nconst started=Date.now(),BUDGET=50000,WORKERS=160,LIMIT=5000;\nconst rows=await sqlite.execute({sql:\"SELECT c.domain FROM checks c LEFT JOIN enrichment e ON c.domain=e.domain WHERE c.status='confirmed' AND e.domain IS NULL ORDER BY CASE WHEN c.domain LIKE '%directory%' OR c.domain LIKE '%locator%' OR c.domain LIKE '%finder%' OR c.domain IN ('worldpopulationreview.com','campsitephotos.com') THEN 0 ELSE 1 END,c.domain LIMIT ?\",args:[LIMIT]});\nconst domains=rows.rows.map(x=>String(x.domain));\nconst parser=new XMLParser({ignoreAttributes:true,removeNSPrefix:true,parseTagValue:false,processEntities:false,trimValues:true});\nconst arr=(x:any)=>x==null?[]:Array.isArray(x)?x:[x];\nconst unesc=(s:string)=>s.replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'\"').replace(/'/g,\"'\");\nconst plain=(s:string)=>unesc(s.replace(/]*>[\\s\\S]*?<\\/script>/gi,' ').replace(/]*>[\\s\\S]*?<\\/style>/gi,' ').replace(/<[^>]+>/g,' ').replace(/\\s+/g,' ')).trim();\nasync function get(url:string,end:number,max=7000000):Promise{try{const left=end-Date.now();if(left<300)return {url,error:'site_time_budget'};const r=await fetch(url,{signal:AbortSignal.timeout(Math.min(6500,left)),headers:{'User-Agent':'PublisherNetworkResearch/1.0 (sitemap and public-page research)'}});const rd=r.body?.getReader();const chunks:Uint8Array[]=[];let n=0;if(rd)while(true){const p=await rd.read();if(p.done)break;n+=p.value.length;if(n>max){await rd.cancel();return {url,final_url:r.url,status:r.status,error:'body_size_limit'};}chunks.push(p.value);}let b=new Uint8Array(n);let offset=0;for(const c of chunks){b.set(c,offset);offset+=c.length;}if(b[0]===31&&b[1]===139)b=new Uint8Array(gunzipSync(b,{maxOutputLength:20000000}));return {url,final_url:r.url,status:r.status,body:new TextDecoder().decode(b),bytes:b.length};}catch(e){return {url,error:String(e)};}}\nfunction links(html:string,base:string){const out:any[]=[];for(const m of html.matchAll(/]*href\\s*=\\s*[\"']([^\"']+)[\"'][^>]*>([\\s\\S]*?)<\\/a>/gi)){try{const u=new URL(unesc(m[1]),base);if(!/^https?:$/.test(u.protocol))continue;out.push({url:u.href,text:plain(m[2]).slice(0,100)});}catch{}}return out;}\nfunction metadata(body:string){const title=plain(body.match(/]*>([\\s\\S]*?)<\\/title>/i)?.[1]??'');const desc=body.match(/]*name\\s*=\\s*[\"']description[\"'][^>]*content\\s*=\\s*[\"']([^\"']*)/i)?.[1]??body.match(/]*content\\s*=\\s*[\"']([^\"']*)[\"'][^>]*name\\s*=\\s*[\"']description/i)?.[1]??'';const schema=[...body.matchAll(/\"@type\"\\s*:\\s*\"([^\"]+)\"/g)].map(x=>x[1]);return {title,description:unesc(desc),schema_types:[...new Set(schema)],wordpress:/wp-content|wp-includes|wp-json/i.test(body)};}\nfunction categorize(domain:string,meta:any,text:string,urls:string[],maps:any[]){const head=(meta.title+' '+meta.description+' '+text.slice(0,500)).toLowerCase();const d=/\\bdirector(?:y|ies)\\b|\\blocator\\b|find (?:a |the |local )?(?:business|clinic|campground|store|boat ramp|toilet|doctor|facility|facilities|dentist)/i.test(head)||/directory|locator/.test(domain);const places=meta.schema_types.some((x:string)=>/LocalBusiness|MedicalClinic|Place|CivicStructure|LodgingBusiness/.test(x));const tool=/calculator|converter|generator|checker|word finder|bpm finder|fare finder|ip location/.test(head);const reference=/population|census|dictionary|encyclopedia|database|surname|lyrics|postal code|zip code/.test(head);const forum=/forum|community forums|discussion board/.test(meta.title.toLowerCase())||urls.some(x=>/\\/(?:threads|forums)\\//.test(x));const posts=maps.some(x=>/post-sitemap|wp-sitemap-posts-post/.test(x.url));const blog=posts||meta.schema_types.some((x:string)=>/BlogPosting|Article|NewsArticle/.test(x))||/\\bblog\\b|recipes|latest articles|latest posts/.test(head);let type='unknown',confidence='low',reason='Insufficient distinctive evidence';if(d){type='directory';confidence=d?'medium':'low';reason='Directory/locator language or place/business schema; automated classification';}else if(tool){type='tool';confidence='medium';reason='Calculator/converter/generator/finder language';}else if(forum){type='forum';confidence='medium';reason='Forum language or thread URL structure';}else if(reference){type='database_reference';confidence='medium';reason='Structured reference/database language';}else if(blog){type='blog';confidence='medium';reason=posts?'Post sitemap discovered':'Article/blog/recipe language or schema';}return {type,confidence,reason,review_status:'automated_not_manually_reviewed',has_editorial_section:blog};}\nfunction patterns(urls:string[]){const first:Record={};const paths=urls.map(x=>{try{return new URL(x).pathname;}catch{return '';}});for(const p of paths){const s=p.split('/').filter(Boolean);if(s[0])first[s[0]]=(first[s[0]]??0)+1;}const counts=new Map();for(let i=0;i/^\\d{4}$/.test(x)?'{year}':/^\\d+$/.test(x)?'{id}':j===0&&s.length>1&&(first[x]??0)>=3?x:x.endsWith('.html')?'{slug}.html':x.endsWith('.php')?'{slug}.php':'{slug}').join('/')+(p.endsWith('/')&&s.length?'/':'');const c=counts.get(k)??{count:0,examples:[]};c.count++;if(c.examples.length<3)c.examples.push(urls[i]);counts.set(k,c);}return [...counts.entries()].map(([pattern,v])=>({pattern,...v})).sort((a,b)=>b.count-a.count).slice(0,15);}\nasync function enrich(domain:string){const end=Date.now()+18000;const [home,robots]=await Promise.all([get('https://'+domain+'/',end,1500000),get('https://'+domain+'/robots.txt',end,500000)]);const origin=home.final_url?new URL(home.final_url).origin:'https://'+domain;const meta=metadata(home.body??'');const homeText=plain(home.body??'');const declared=(robots.status===200&&robots.body)?[...robots.body.matchAll(/^\\s*sitemap\\s*:\\s*(\\S+)\\s*$/gim)].map((m:any)=>unesc(m[1])):[];\n const roots=[...new Set(declared)];const queue:string[]=[...roots],seen=new Set(),pageUrls=new Set(),files:any[]=[];let errors=0,valid=0,offsite=0,assets=0,capped=false;\n async function parseMap(url:string){if(seen.has(url))return;seen.add(url);const f=await get(url,end);const rec:any={url,final_url:f.final_url??'',http_status:f.status??null,bytes:f.bytes??0};files.push(rec);if(f.status!==200||f.error){rec.error=f.error??'HTTP '+f.status;errors++;return;}try{if(!/<(?:[\\w-]+:)?(?:sitemapindex|urlset)\\b/.test(f.body)){rec.error='not_xml_sitemap';errors++;return;}const doc=parser.parse(f.body);if(doc.sitemapindex){const children=arr(doc.sitemapindex.sitemap).map(x=>x?.loc).filter((x:any)=>typeof x==='string');rec.kind='index';rec.child_count=children.length;valid++;for(const x of children){try{const u=new URL(unesc(x),f.final_url||url).href;if(!seen.has(u)&&!queue.includes(u))queue.push(u);}catch{errors++;}}}else if(doc.urlset){rec.kind='urlset';let n=0;valid++;for(const item of arr(doc.urlset.url)){if(typeof item?.loc!=='string')continue;let u:URL;try{u=new URL(unesc(item.loc));u.hash='';}catch{continue;}if(u.hostname.replace(/^www\\./,'')!==domain&&!u.hostname.endsWith('.'+domain)){offsite++;continue;}if(/\\.(?:jpg|jpeg|png|gif|webp|svg|pdf|mp4|mp3|css|js|zip|kml|xml|json|csv|txt|xlsx|rss|gz)$/i.test(u.pathname)){assets++;continue;}pageUrls.add(u.href);n++;if(pageUrls.size>=100000){capped=true;break;}}rec.page_url_entries=n;}else{rec.error='unknown_xml_root';errors++;}}catch(e){rec.error=String(e);errors++;}}\n if(!queue.length){for(const name of ['/sitemap.xml','/sitemap_index.xml','/wp-sitemap.xml']){await parseMap(origin+name);if(valid){roots.push(origin+name);errors=0;break;}if(Date.now()>end-1500)break;}}\n while(queue.length&&Date.now()!seen.has(x));await Promise.all(batch.map(parseMap));}\n if(queue.length||seen.size>=80||capped)capped=true;\n const urls=[...pageUrls];const cls=categorize(domain,meta,homeText,urls,files);const homeLinks=links(home.body??'',home.final_url??origin);const about=homeLinks.filter(x=>{try{return new URL(x.url).hostname.replace(/^www\\./,'')===domain&&/about|methodology|data-source|sources|faq/i.test(x.url+' '+x.text);}catch{return false;}}).sort((a,b)=>Number(/methodology|sources/.test(b.url))-Number(/methodology|sources/.test(a.url))).slice(0,5);\n const sourcePages:any[]=[];if(cls.type==='directory'||cls.type==='database_reference'){if(about[0]&&Date.now()=12)break;}}\n const status=valid?(errors||capped?'partial':'complete_discovered_tree'):'unavailable';const result={domain,checked_at:new Date().toISOString(),homepage:{url:home.final_url??origin,http_status:home.status??null,error:home.error??null,...meta,text_excerpt:homeText.slice(0,1800)},classification:cls,sitemap:{status,page_count:valid?urls.length:null,count_definition:'Unique non-asset page URLs in fetched sitemaps; not indexed pages or total site pages',roots,files,remaining_files:queue.length,offsite_urls_skipped:offsite,asset_urls_skipped:assets,limits:{max_files:80,max_unique_urls:100000,site_ms:18000},robots_status:robots.status??null},url_patterns:patterns(urls),sample_urls:urls.slice(0,12),about_links:about,source_evidence:provenance,data_source_status:provenance.length?'mentions_found_requires_review':'unknown_not_disclosed_in_checked_pages',source_pages:sourcePages};\n return [domain,result.checked_at,'done',result.sitemap.page_count,status,JSON.stringify(result)];}\nlet i=0;const out:any[]=[];await Promise.all(Array.from({length:WORKERS},async()=>{while(i180000)await save();chunk.push(r);size+=n;}await save();console.log(JSON.stringify({enriched:out.length,duration_ms:Date.now()-started,totals:(await sqlite.execute(\"SELECT json_extract(result_json,'$.classification.type') type,sitemap_status,COUNT(*) n FROM enrichment GROUP BY 1,2\")).rows,examples:out.slice(0,4).map(r=>{const d=JSON.parse(r[5]);return {domain:r[0],count:r[3],status:r[4],classification:d.classification};})}));","createdAt",["D",1789666680342],"branchVersion","branchId","cd292256-c961-4ceb-b973-27c236575749","id","7e58a892-b130-4895-9cbc-de82614ff56d","publisher-network-audit",["D",1789665566213],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Browse publishers, directory teardowns and ads.txt evidence.","imageUrl","f5be24e4-c798-4816-91e5-037b60b986f5","pinnedFileId","01a0b061-5e61-74b2-9c7e-7b033afc92fc","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","7f67379d-2d38-49bd-81ac-2c7f92a739e2","tursoName","7e58a892b13048959cbcde82614ff56d","tursoHostname","7e58a892b13048959cbcde82614ff56d-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789665692456],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_100":99,"_107":103},"org",{"_100":99,"_101":-5,"_102":103,"_18":55,"_104":105,"_106":25},"branch",{"_13":89,"_32":90,"_43":42,"_91":44,"_92":-5,"_93":-5,"_94":-5,"_95":-5,"_96":-5,"_97":45,"_98":99},"isOwned",59,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","omriluz","handle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18ycWxkOWxiOE00SkVtZENmRWM4YWxLWTRVUFMifQ","tier","free","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_43":44,"_32":45,"_38":228,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":53,"_54":-5,"_18":55,"_56":57,"_58":59,"_60":-5,"_61":-5,"_62":63,"_64":65,"_66":67,"_68":69,"_70":229,"_72":-5,"_73":74,"_75":74,"_76":77,"_78":77,"_79":77,"_80":74,"_81":74,"_82":230,"_84":231,"_86":232,"_88":25},"tree",[187,190,191,192,193,194],"files",[187,188,189,190,191,192,193,194],"counts",{"_176":63,"_177":156,"_178":63,"_179":180,"_181":182},"contents",[130,131,132,133,134,135],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_43":171,"_32":50,"_30":172,"_139":-5,"_41":42,"_40":156,"_141":59,"_38":173,"_13":63,"_33":174,"_35":-5,"_24":25,"_145":-5,"_27":175},{"_43":165,"_32":166,"_30":167,"_139":-5,"_41":42,"_40":156,"_141":59,"_38":168,"_13":63,"_33":20,"_35":169,"_24":25,"_145":-5,"_27":170},{"_43":162,"_32":28,"_30":31,"_139":-5,"_41":42,"_40":14,"_141":59,"_38":163,"_13":143,"_33":34,"_35":37,"_24":25,"_145":-5,"_27":164},{"_43":154,"_32":155,"_30":57,"_139":-5,"_41":42,"_40":156,"_141":25,"_38":157,"_13":63,"_33":158,"_35":159,"_24":25,"_145":160,"_27":161},{"_43":147,"_32":148,"_30":149,"_139":-5,"_41":42,"_40":150,"_141":59,"_38":151,"_13":140,"_33":34,"_35":152,"_24":25,"_145":-5,"_27":153},{"_43":136,"_32":137,"_30":138,"_139":-5,"_41":42,"_40":140,"_141":59,"_38":142,"_13":143,"_33":34,"_35":144,"_24":25,"_145":-5,"_27":146},"2b083803-4ca0-4fff-943d-cefb65802939","seed.ts","034e6604-b2bc-11f1-960c-1607ee4eb77e","parentId",5,"latest",["D",1789665652151],3,"import { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\nimport { gzipSync } from \"node:zlib\";\nimport { Buffer } from \"node:buffer\";\nconst sources = {raptive:\"https://ads.cafemedia.com/sellers.json\",mediavine:\"https://www.mediavine.com/sellers.json\",ezoic:\"https://ezoic.ai/sellers.json\",journey:\"https://sellers.journeymv.com/sellers.json\"};\nawait sqlite.batch([\n {sql:\"CREATE TABLE IF NOT EXISTS snapshots(network TEXT PRIMARY KEY,url TEXT,fetched_at TEXT,http_status INTEGER,sha256 TEXT,raw_json TEXT)\"},\n {sql:\"CREATE TABLE IF NOT EXISTS sellers(network TEXT,seller_id TEXT,domain TEXT,name TEXT,seller_type TEXT,confidential INTEGER,PRIMARY KEY(network,seller_id,domain))\"},\n {sql:\"CREATE INDEX IF NOT EXISTS sellers_domain ON sellers(domain)\"},\n {sql:\"CREATE TABLE IF NOT EXISTS checks(domain TEXT PRIMARY KEY,checked_at TEXT,status TEXT,final_url TEXT,http_status INTEGER,body_sha256 TEXT,ads_body TEXT,evidence_json TEXT,attempts_json TEXT)\"},\n {sql:\"CREATE TABLE IF NOT EXISTS enrichment(domain TEXT PRIMARY KEY,checked_at TEXT,status TEXT,sitemap_count INTEGER,sitemap_status TEXT,result_json TEXT)\"}\n]);\nfunction domainOf(value:unknown):string {try {const x=String(value||'').trim().toLowerCase(); if(!x || /\\s/.test(x)) return ''; const u=new URL(x.includes('://')?x:'https://'+x); const h=u.hostname.replace(/^www\\./,'').replace(/\\.$/,''); if(!h.includes('.')||/^\\d+(\\.\\d+){3}$/.test(h)||h==='localhost')return ''; return h;}catch{return '';}}\nconst summary=[];\nfor(const [network,url] of Object.entries(sources)){\n try {\n const existing=await sqlite.execute({sql:\"SELECT network FROM snapshots WHERE network=?\",args:[network]});\n if(existing.rows.length){summary.push({network,cached:true});continue;}\n const r=await fetch(url,{signal:AbortSignal.timeout(20000),headers:{'User-Agent':'PublisherNetworkResearch/1.0'}}); const raw=await r.text();\n if(!r.ok) throw Error('HTTP '+r.status); const data=JSON.parse(raw);if(!Array.isArray(data.sellers))throw Error('missing sellers');\n const rows=data.sellers.map((x:any)=>[network,String(x.seller_id??''),domainOf(x.domain),String(x.name??''),String(x.seller_type??'').toUpperCase(),Number(!!x.is_confidential)]);\n const sha=Array.from(new Uint8Array(await crypto.subtle.digest('SHA-256',new TextEncoder().encode(raw))),x=>x.toString(16).padStart(2,'0')).join('');\n const packed=JSON.stringify({encoding:'gzip+base64',data:Buffer.from(gzipSync(raw)).toString('base64'),records:data.sellers.length,original_bytes:raw.length});\n for(let i=0;ix[2]).filter(Boolean)).size,missing_domain:rows.filter((x:any)=>!x[2]).length,bytes:raw.length,stored_bytes:packed.length,sha256:sha,examples:rows.slice(0,3)};summary.push(item);console.log(JSON.stringify(item));\n }catch(e){const item={network,error:String(e)};summary.push(item);console.log(JSON.stringify(item));}\n}\nconsole.log(JSON.stringify({summary}));","label",[137],"594ad361-19be-443c-b029-86a91f941bb9","scan.ts","5a6159ba-b2bc-11f1-b240-1607ee4eb77e",14,["D",1789666546661],"import { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\nimport { gzipSync } from \"node:zlib\";\nimport { Buffer } from \"node:buffer\";\nconst SYSTEMS:Record={raptive:['cafemedia.com','ads.cafemedia.com','adthrive.com','raptive.com'],mediavine:['mediavine.com'],journey:['journeymv.com','sellers.journeymv.com'],ezoic:['ezoic.ai','ezoic.com']};\nconst started=Date.now(), LIMIT=7000, WORKERS=300, BUDGET=40000;\nconst selected=await sqlite.execute({sql:\"SELECT DISTINCT s.domain FROM sellers s LEFT JOIN checks c ON s.domain=c.domain WHERE s.domain<>'' AND c.domain IS NULL ORDER BY CASE WHEN s.domain LIKE '%directory%' OR s.domain LIKE '%locator%' OR s.domain LIKE '%finder%' THEN 0 ELSE 1 END,s.domain LIMIT ?\",args:[LIMIT]});\nconst domains=selected.rows.map(x=>String(x.domain));\nconst records=await sqlite.execute({sql:\"SELECT domain,network,seller_id FROM sellers WHERE domain IN (SELECT value FROM json_each(?))\",args:[JSON.stringify(domains)]});\nconst map=new Map();for(const r of records.rows){const a=map.get(String(r.domain))??[];a.push(r);map.set(String(r.domain),a);}\nasync function limitedText(r:Response,max=1000000){const rd=r.body?.getReader();if(!rd)return '';let n=0;const arr:Uint8Array[]=[];while(true){const a=await rd.read();if(a.done)break;n+=a.value.length;if(n>max){await rd.cancel();throw Error('body_too_large');}arr.push(a.value);}const b=new Uint8Array(n);let p=0;for(const a of arr){b.set(a,p);p+=a.length;}return new TextDecoder().decode(b);}\nasync function check(domain:string){const attempts:any[]=[];const end=Date.now()+12000;\n for(const url of ['https://'+domain+'/ads.txt','https://www.'+domain+'/ads.txt','http://'+domain+'/ads.txt']){if(Date.now()>end-200)break;try{\n const r=await fetch(url,{redirect:'follow',signal:AbortSignal.timeout(Math.max(200,Math.min(6000,end-Date.now()))),headers:{'User-Agent':'PublisherNetworkResearch/1.0 (public ads.txt verification)','Accept':'text/plain,*/*;q=0.5'}});const body=await limitedText(r);const a:any={url,status:r.status,final_url:r.url,bytes:body.length};attempts.push(a);\n if(!r.ok)continue;const parsed=body.split(/\\r?\\n/).map((line,i)=>({raw:line.trim(),parts:line.split('#')[0].trim().split(',').map(x=>x.trim()),line_number:i+1})).filter(x=>x.parts.length>=3&&/\\./.test(x.parts[0])&&['DIRECT','RESELLER'].includes(x.parts[2].toUpperCase()));\n if(/<(?:!doctype\\s+html|html|head|body)(?:\\s|>)/i.test(body.slice(0,2000))||(!parsed.length&&!/(text\\/plain|application\\/octet-stream)/i.test(r.headers.get('content-type')??'')&&body.trim()!=='')){a.invalid_body=true;continue;}\n const evidence:any[]=[];for(const record of map.get(domain)??[])for(const line of parsed)if(SYSTEMS[String(record.network)]?.includes(line.parts[0].toLowerCase())&&line.parts[1]===String(record.seller_id))evidence.push({network:record.network,seller_id:record.seller_id,system:line.parts[0].toLowerCase(),relationship:line.parts[2].toUpperCase(),line:line.raw,line_number:line.line_number});\n const sha=Array.from(new Uint8Array(await crypto.subtle.digest('SHA-256',new TextEncoder().encode(body))),x=>x.toString(16).padStart(2,'0')).join('');a.network_lines=parsed.filter(x=>Object.values(SYSTEMS).flat().includes(x.parts[0].toLowerCase())).map(x=>x.raw);a.parsed_lines=parsed.length;\n return [domain,new Date().toISOString(),evidence.length?'confirmed':'no_matching_line',r.url,r.status,sha,'gzip+base64:'+Buffer.from(gzipSync(body)).toString('base64'),JSON.stringify(evidence),JSON.stringify(attempts)];\n }catch(e){attempts.push({url,error:String(e)});}}\n const missing=attempts.length>0&&attempts.every(x=>[404,410].includes(x.status));return [domain,new Date().toISOString(),missing?'ads_txt_missing':'unverified',attempts.at(-1)?.final_url??'',attempts.at(-1)?.status??null,'','','[]',JSON.stringify(attempts)];}\nconst INSERT=\"INSERT OR REPLACE INTO checks SELECT json_extract(value,'$[0]'),json_extract(value,'$[1]'),json_extract(value,'$[2]'),json_extract(value,'$[3]'),json_extract(value,'$[4]'),json_extract(value,'$[5]'),json_extract(value,'$[6]'),json_extract(value,'$[7]'),json_extract(value,'$[8]') FROM json_each(?)\";\nlet next=0,saved=0,failedWrites=0;const counts:Record={};let buf:any[]=[];let bufSize=0;let chain=Promise.resolve();\nfunction enqueue(){if(!buf.length)return;const batch=buf;buf=[];bufSize=0;chain=chain.then(async()=>{for(let attempt=0;attempt<4;attempt++){try{await sqlite.execute({sql:INSERT,args:[JSON.stringify(batch)]});saved+=batch.length;return;}catch(e){if(attempt===3){failedWrites+=batch.length;console.log(JSON.stringify({write_error:String(e),unsaved_domains:batch.map(r=>r[0])}));}else await new Promise(r=>setTimeout(r,400*(attempt+1)));}}});}\nawait Promise.all(Array.from({length:WORKERS},async()=>{while(next120000)enqueue();buf.push(row);bufSize+=size;if(buf.length>=35)enqueue();}}));enqueue();await chain;\nconsole.log(JSON.stringify({attempted:next,saved,failedWrites,duration_ms:Date.now()-started,batch_counts:counts,totals:(await sqlite.execute('SELECT status,COUNT(*) n FROM checks GROUP BY status')).rows}));",[148],"c05c2462-155e-4a5e-9bc0-278c591951a7","index.tsx",1,["D",1789665566213],"http","/** @jsxImportSource npm:hono@4/jsx */\nimport { Hono } from \"npm:hono@4\";\nimport { parseVal, serveFile } from \"https://esm.town/v/std/utils/index.ts\";\n\n// In-memory state. Resets whenever the server restarts.\n// For persistence across restarts, use std/sqlite or std/blob.\nlet count = 0;\n\nconst Layout = ({ title, children }: { title: string; children: any }) => (\n \n \n {title}\n \n \n \n \n \n \n \n {children}\n \n \n);\n\nconst app = new Hono();\n\napp.get(\"/\", (c) =>\n c.html(\n \n

Hello from Hono JSX

\n A minimal Val Town starter.{\" \"}\n \n Remix this val.\n \n

\n Server count:{\" \"}\n {count}\n

\n \n Increment\n \n

\n Lives in memory on the server. Resets when the val restarts.\n

\n \n ,\n ));\n\napp.get(\"/about\", (c) =>\n c.html(\n \n

About

Server-rendered with Hono JSX on Val Town.

\n ,\n ));\n\n// API routes\napp.post(\"/api/increment\", (c) => c.json({ count: ++count }));\n\n// Serve static files\napp.get(\"/public/*\", (c) => serveFile(c.req.path));\n\n// View source on Val Town\napp.get(\"/source\", (c) => c.redirect(parseVal().links.self.val));\n\n// Unwrap and rethrow Hono errors as the original error\napp.onError((err) => Promise.reject(err));\n\nexport default app.fetch;","publisher-explorer-omri",[155],"4b6d882c-9f29-4b36-9c98-1022fb8088dd",["D",1789666680342],[28],"af49aca0-800e-4cdb-a656-605cafe7a1cb","README.md","01a0b061-5e61-74b2-9c7e-77e87e2983ad",["D",1789665566213],"# hono-jsx-starter\n\nA minimal [Hono](https://hono.dev) + Hono JSX starter for Val Town.\nServer-rendered HTML, no build step, no React.\n\n## Picking a starter\n\n| Starter | When to pick it |\n| --- | --- |\n| [`basic-html-starter`](https://www.val.town/x/templates/basic-html-starter) | Static HTML pages. Sprinkle of vanilla JS. No server-side rendering. |\n| **[`hono-jsx-starter`](https://www.val.town/x/templates/hono-jsx-starter)** *(you're here)* | Dynamic server-rendered HTML with JSX. API routes. No React, no SPA. |\n| [`react-hono-starter`](https://www.val.town/x/templates/react-hono-starter) | Client-side React SPA with a Hono API backend. |\n\n## What's here\n\n```\nindex.tsx Hono app, Layout, routes\npublic/\n favicon.svg Static asset\n counter.ts Tiny client-side script\n```\n\nThe whole app fits in one file. As you grow it, the natural next moves are:\n\n- Pull `Layout` into its own `.tsx` file once you have 3+ pages.\n- Split routes with `app.route(\"/api\", api)` if `/api/*` grows.\n- Add more files under `public/` — they're served by a single\n `app.get(\"/public/*\", ...)` route.\n\n## Patterns worth knowing\n\n**JSX pragma.** Every `.tsx` file that uses JSX needs this at the top:\n\n```tsx\n/** @jsxImportSource npm:hono@4/jsx */\n```\n\n**Server state.** The counter on `/` is `let count = 0` at module scope. It\nlives in memory on the Val Town server and survives across requests — but resets\nwhenever the val restarts (deploys, idle timeouts, etc.). For persistent state,\nswap it for [`std/sqlite`](https://www.val.town/x/std/sqlite) or\n[`std/blob`](https://www.val.town/x/std/blob).\n\n**API routes.** The `Increment` button POSTs to `/api/increment`, which returns\nJSON. Mix server-rendered pages and JSON endpoints in the same Hono app:\n\n```ts\napp.post(\"/api/increment\", (c) => c.json({ count: ++count }));\n```\n\n**Static files.** `serveFile` reads the file by path and serves it with the\nright content-type. `.ts` files are transpiled to JS on request by Val Town — so\nyou can write `public/counter.ts` and reference it as\n`

Diff · v15