[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_450":451,"_452":453},"routes/_app.x.$handle.$name",{"_98":97,"_31":43,"_106":88,"_26":107,"_13":14,"_108":-5,"_11":109,"_110":111,"_112":113,"_114":115,"_116":117,"_20":-5,"_118":-5,"_119":-5,"_120":24,"_121":122,"_123":57,"_124":-5,"_125":-5,"_126":127},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_41":42,"_31":43,"_36":44,"_45":-5,"_46":24,"_47":48,"_49":48,"_50":51,"_52":-5,"_18":53,"_54":55,"_56":57,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":65,"_66":67,"_68":69,"_70":-5,"_71":72,"_73":72,"_74":75,"_76":75,"_77":75,"_78":72,"_79":72,"_80":81,"_82":83,"_84":85,"_86":24},"version",58,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":-7,"_23":24,"_25":24,"_26":27,"_28":27},"file",{"_29":30,"_31":27,"_32":33,"_34":35,"_36":37,"_38":14,"_39":40,"_23":24},"previous","deleted",false,"deletedHere","path","explorer_test.ts","latestPath","fileId","0b630698-b2ca-11f1-b1a2-1607ee4eb77e","name","fileType","script","content","import { explorerHtml } from './explorer_ui.ts';\nconst base='https://publisher-explorer-omri.val.run';\nconst key=Deno.env.get('AUDIT_EXPORT_KEY');\nif(!key)throw Error('Missing read-only key');\nconst results:any[]=[];\nfunction check(name:string,value:boolean,details:unknown=null){results.push({name,passed:!!value,...(details==null?{}:{details})});if(!value)throw Error('Test failed: '+name);}\nconst js=explorerHtml.match(/`;",[237],"a9517787-006d-4b4c-9b3c-7cbbca82e160",["D",1789671626917],[27],"b4e29649-58c5-44c5-97e4-785f7701c47b","explorer_prepare.ts","51423888-b2c9-11f1-9b98-1607ee4eb77e",55,["D",1789671314654],"import { sqlite } from 'https://esm.town/v/std/sqlite/main.ts';\n// A compact read model for the browser. Original research tables are unchanged.\nawait sqlite.execute(`CREATE TABLE IF NOT EXISTS explorer_sites (\n domain TEXT PRIMARY KEY, status TEXT NOT NULL, listed_networks TEXT, networks TEXT,\n site_type TEXT, reviewed INTEGER, confidence TEXT, title TEXT, niche TEXT,\n page_count INTEGER, sitemap_status TEXT, ads_checked_at TEXT, enriched_at TEXT,\n source_summary TEXT, source_disclosed INTEGER, search_text TEXT\n)`);\nawait sqlite.execute(`INSERT OR REPLACE INTO explorer_sites\nSELECT s.domain,COALESCE(c.status,'not_checked'),s.networks,\n COALESCE((SELECT group_concat(DISTINCT json_extract(j.value,'$.network')) FROM json_each(c.evidence_json) j),''),\n COALESCE(m.classification,json_extract(e.result_json,'$.classification.type'),'unknown'),\n CASE WHEN m.review_status IS NOT NULL THEN 1 ELSE 0 END,\n COALESCE(json_extract(e.result_json,'$.classification.confidence'),'low'),\n COALESCE(json_extract(e.result_json,'$.homepage.title'),''),COALESCE(m.niche,''),\n CASE WHEN e.sitemap_count=0 AND e.sitemap_status!='complete_discovered_tree' THEN NULL ELSE e.sitemap_count END,\n COALESCE(e.sitemap_status,'not_attempted'),c.checked_at,e.checked_at,\n COALESCE((SELECT group_concat(json_extract(j.value,'$.source'),'; ') FROM json_each(m.data_sources_json) j WHERE json_extract(j.value,'$.source') IS NOT NULL),''),\n CASE WHEN EXISTS(SELECT 1 FROM json_each(m.data_sources_json) j WHERE json_extract(j.value,'$.status')='publisher_disclosed' AND json_extract(j.value,'$.source') IS NOT NULL) THEN 1 ELSE 0 END,\n lower(s.domain||' '||COALESCE(m.niche,'')||' '||COALESCE(json_extract(e.result_json,'$.homepage.title'),'')||' '||COALESCE((SELECT group_concat(json_extract(j.value,'$.source'),' ') FROM json_each(m.data_sources_json) j),''))\nFROM (SELECT domain,group_concat(DISTINCT network) networks FROM sellers WHERE domain<>'' GROUP BY domain) s\nLEFT JOIN checks c ON c.domain=s.domain\nLEFT JOIN enrichment e ON e.domain=s.domain\nLEFT JOIN manual_reviews m ON m.domain=s.domain`);\nawait sqlite.batch([\n {sql:'CREATE INDEX IF NOT EXISTS explorer_status_type ON explorer_sites(status,site_type,domain)'},\n {sql:'CREATE INDEX IF NOT EXISTS explorer_status_pages ON explorer_sites(status,page_count,domain)'},\n {sql:'CREATE INDEX IF NOT EXISTS explorer_reviewed ON explorer_sites(reviewed,status,domain)'}\n]);\nconsole.log(JSON.stringify({rows:(await sqlite.execute('SELECT COUNT(*) n FROM explorer_sites')).rows,statuses:(await sqlite.execute('SELECT status,COUNT(*) n FROM explorer_sites GROUP BY status')).rows,reviewed:(await sqlite.execute('SELECT reviewed,COUNT(*) n FROM explorer_sites GROUP BY reviewed')).rows}));",[247],"69e1606f-3b30-4175-9675-7bd1fba3e94d","enrich_core.ts","01a0b075-429a-70af-87d6-fa93cd9ce0e4",43,["D",1789669472058],8,"import { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\nimport { gunzipSync } from \"node:zlib\";\nimport { XMLParser } from \"npm:fast-xml-parser@4.5.3\";\nimport { classifySite } from './classify.ts';\nexport async function runEnrich(shard=0,shards=1){\n const started=Date.now(),BUDGET=30000,WORKERS=100,LIMIT=3000,SITE_MS=18000;\n const rows=await sqlite.execute({sql:\"SELECT c.domain,c.attempts_json FROM checks c LEFT JOIN enrichment e ON c.domain=e.domain WHERE c.status='confirmed' AND e.domain IS NULL AND ((unicode(substr(c.domain,1,1))*31+unicode(substr(c.domain,2,1))+length(c.domain)) % ?) = ? ORDER BY CASE WHEN c.domain LIKE '%directory%' OR c.domain LIKE '%locator%' OR c.domain LIKE '%finder%' OR c.domain IN ('worldpopulationreview.com','campsitephotos.com') THEN 0 ELSE 1 END,c.domain LIMIT ?\",args:[shards,shard,LIMIT]});\n const domains=rows.rows.map(x=>String(x.domain));\n const baseByDomain=new Map();for(const r of rows.rows){let base='https://'+r.domain;try{const a=JSON.parse(String(r.attempts_json??'[]')).find((a:any)=>a.status===200&&!a.invalid_body&&a.parsed_lines!=null);if(a?.url)base=new URL(a.url).origin;}catch{}baseByDomain.set(String(r.domain),base);}\n const parser=new XMLParser({ignoreAttributes:true,removeNSPrefix:true,parseTagValue:false,processEntities:false,trimValues:true});\n const arr=(x:any)=>x==null?[]:Array.isArray(x)?x:[x];\n const unesc=(s:string)=>s.replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'\"').replace(/'/g,\"'\");\n const plain=(s:string)=>unesc(s.replace(/]*>[\\s\\S]*?<\\/script>/gi,' ').replace(/]*>[\\s\\S]*?<\\/style>/gi,' ').replace(/<[^>]+>/g,' ').replace(/\\s+/g,' ')).trim();\n async function get(url:string,end:number,max=7000000):Promise{try{const left=end-Date.now();if(left<300)return {url,error:'site_time_budget'};const r=await fetch(url,{signal:AbortSignal.timeout(Math.min(8000,left)),headers:{'User-Agent':'PublisherNetworkResearch/1.0 (sitemap and public-page research)'}});const rd=r.body?.getReader();const chunks:Uint8Array[]=[];let n=0;if(rd)while(true){const p=await rd.read();if(p.done)break;n+=p.value.length;if(n>max){await rd.cancel();return {url,final_url:r.url,status:r.status,error:'body_size_limit'};}chunks.push(p.value);}let b=new Uint8Array(n);let offset=0;for(const c of chunks){b.set(c,offset);offset+=c.length;}if(b[0]===31&&b[1]===139)b=new Uint8Array(gunzipSync(b,{maxOutputLength:20000000}));return {url,final_url:r.url,status:r.status,body:new TextDecoder().decode(b),bytes:b.length};}catch(e){return {url,error:String(e)};}}\n function links(html:string,base:string){const out:any[]=[];for(const m of html.matchAll(/]*href\\s*=\\s*[\"']([^\"']+)[\"'][^>]*>([\\s\\S]*?)<\\/a>/gi)){try{const u=new URL(unesc(m[1]),base);if(!/^https?:$/.test(u.protocol))continue;out.push({url:u.href,text:plain(m[2]).slice(0,100)});}catch{}}return out;}\n function metadata(body:string){const title=plain(body.match(/]*>([\\s\\S]*?)<\\/title>/i)?.[1]??'');const desc=body.match(/]*name\\s*=\\s*[\"']description[\"'][^>]*content\\s*=\\s*[\"']([^\"']*)/i)?.[1]??body.match(/]*content\\s*=\\s*[\"']([^\"']*)[\"'][^>]*name\\s*=\\s*[\"']description/i)?.[1]??'';const schema=[...body.matchAll(/\"@type\"\\s*:\\s*\"([^\"]+)\"/g)].map(x=>x[1]);return {title,description:unesc(desc),schema_types:[...new Set(schema)],wordpress:/wp-content|wp-includes|wp-json/i.test(body)};}\n function patterns(urls:string[]){const first:Record={};const paths=urls.map(x=>{try{return new URL(x).pathname;}catch{return '';}});for(const p of paths){const s=p.split('/').filter(Boolean);if(s[0])first[s[0]]=(first[s[0]]??0)+1;}const counts=new Map();for(let i=0;i/^\\d{4}$/.test(x)?'{year}':/^\\d+$/.test(x)?'{id}':j===0&&s.length>1&&(first[x]??0)>=3?x:x.endsWith('.html')?'{slug}.html':x.endsWith('.php')?'{slug}.php':'{slug}').join('/')+(p.endsWith('/')&&s.length?'/':'');const c=counts.get(k)??{count:0,examples:[]};c.count++;if(c.examples.length<3)c.examples.push(urls[i]);counts.set(k,c);}return [...counts.entries()].map(([pattern,v])=>({pattern,...v})).sort((a,b)=>b.count-a.count).slice(0,20);}\n async function enrich(domain:string){const end=Date.now()+SITE_MS,base=baseByDomain.get(domain)??'https://'+domain;const [home,robots]=await Promise.all([get(base+'/',end,1500000),get(base+'/robots.txt',end,500000)]);const origin=home.final_url?new URL(home.final_url).origin:base;const canonicalHost=new URL(origin).hostname.replace(/^www\\./,'');const meta=metadata(home.status===200?home.body??'':'');const homeText=plain(home.status===200?home.body??'':'');const declared=(robots.status===200&&robots.body)?[...robots.body.matchAll(/^\\s*sitemap\\s*:\\s*(\\S+)\\s*$/gim)].map((m:any)=>unesc(m[1])):[];\n const roots=[...new Set(declared)];const queue:string[]=[...roots],seen=new Set(),pageUrls=new Set(),files:any[]=[];let errors=0,valid=0,offsite=0,assets=0,capped=false;\n async function parseMap(url:string){if(seen.has(url))return;seen.add(url);const f=await get(url,end);const rec:any={url,final_url:f.final_url??'',http_status:f.status??null,bytes:f.bytes??0};files.push(rec);if(f.status!==200||f.error){rec.error=f.error??'HTTP '+f.status;errors++;return;}try{if(!/<(?:[\\w-]+:)?(?:sitemapindex|urlset)\\b/.test(f.body)){rec.error='not_xml_sitemap';errors++;return;}const doc=parser.parse(f.body);if(doc.sitemapindex){const children=arr(doc.sitemapindex.sitemap).map(x=>x?.loc).filter((x:any)=>typeof x==='string');rec.kind='index';rec.child_count=children.length;valid++;for(const x of children){try{const u=new URL(unesc(x),f.final_url||url).href;if(!/^https?:/.test(u))continue;if(!seen.has(u)&&!queue.includes(u))queue.push(u);}catch{errors++;}}}else if(doc.urlset){rec.kind='urlset';let n=0;valid++;for(const item of arr(doc.urlset.url)){if(typeof item?.loc!=='string')continue;let u:URL;try{u=new URL(unesc(item.loc));u.hash='';if(!/^https?:$/.test(u.protocol))continue;}catch{continue;}const h=u.hostname.replace(/^www\\./,'');if(h!==domain&&h!==canonicalHost&&!h.endsWith('.'+domain)){offsite++;continue;}if(/\\.(?:jpg|jpeg|png|gif|webp|svg|pdf|mp4|mp3|css|js|zip|kml|xml|json|csv|txt|xlsx|rss|gz)$/i.test(u.pathname)){assets++;continue;}if(pageUrls.size>=100000){capped=true;break;}pageUrls.add(u.href);n++;}rec.page_url_entries=n;}else{rec.error='unknown_xml_root';errors++;}}catch(e){rec.error=String(e);errors++;}}\n if(!queue.length){for(const name of ['/sitemap.xml','/sitemap_index.xml','/wp-sitemap.xml']){await parseMap(origin+name);if(valid){roots.push(origin+name);errors=0;break;}if(Date.now()>end-1500)break;}}\n while(queue.length&&Date.now()!seen.has(x));await Promise.all(batch.map(parseMap));}\n if(queue.length||capped)capped=true;\n const urls=[...pageUrls],cls=classifySite(domain,meta,homeText,urls,files),homeLinks=links(home.body??'',home.final_url??origin);\n const rank=(u:string)=>/\\/(?:methodology|data-sources?|sources)(?:\\/|$|[?#])/i.test(u)?3:/\\/(?:about(?:-us)?|aboutus)(?:[./]|$)/i.test(u)?2:1;\n const about=[...new Map(homeLinks.filter(x=>{try{return new URL(x.url).hostname.replace(/^www\\./,'')===canonicalHost&&/\\/(?:about(?:-us)?|aboutus|methodology|data-sources?|sources|faq)(?:[./?#]|$)/i.test(x.url);}catch{return false;}}).map(x=>[x.url,x])).values()].sort((a,b)=>rank(b.url)-rank(a.url)).slice(0,5);\n const sourcePages:any[]=[];if(['directory','database_reference'].includes(cls.type)&&about[0]&&Date.now()=10)break;}if(provenance.length>=10)break;}\n const status=valid?(errors||capped?'partial':'complete_discovered_tree'):'unavailable';const result={domain,checked_at:new Date().toISOString(),homepage:{url:home.final_url??origin,http_status:home.status??null,error:home.error??null,...meta,text_excerpt:homeText.slice(0,1800)},classification:cls,sitemap:{status,page_count:valid?urls.length:null,count_definition:'Unique non-asset page URLs in fetched sitemaps; not indexed pages or total site pages',roots,files,remaining_files:queue.length,offsite_urls_skipped:offsite,asset_urls_skipped:assets,limits:{max_files:80,max_unique_urls:100000,site_ms:SITE_MS},robots_status:robots.status??null},url_patterns:patterns(urls),sample_urls:urls.length>60?Array.from({length:60},(_,i)=>urls[Math.floor(i*urls.length/60)]):urls,about_links:about,source_evidence:provenance,data_source_status:provenance.length?'mentions_found_requires_review':home.status===200?'unknown_not_disclosed_in_checked_pages':'unknown_pages_unavailable',source_pages:sourcePages};\n return [domain,result.checked_at,'done',result.sitemap.page_count,status,JSON.stringify(result)];\n }\n const INSERT=\"INSERT OR REPLACE INTO enrichment SELECT json_extract(value,'$[0]'),json_extract(value,'$[1]'),json_extract(value,'$[2]'),json_extract(value,'$[3]'),json_extract(value,'$[4]'),json_extract(value,'$[5]') FROM json_each(?)\";\n let i=0,saved=0,failedWrites=0,buf:any[]=[],bufSize=0,chain=Promise.resolve();\n function enqueue(){if(!buf.length)return;const batch=buf;buf=[];bufSize=0;chain=chain.then(async()=>{for(let a=0;a<3;a++){try{await sqlite.execute({sql:INSERT,args:[JSON.stringify(batch)]});saved+=batch.length;return;}catch(e){if(a===2){failedWrites+=batch.length;console.log(JSON.stringify({write_error:String(e),domains:batch.map(r=>r[0])}));}else await new Promise(r=>setTimeout(r,400*(a+1)));}}});}\n await Promise.all(Array.from({length:WORKERS},async()=>{while(i110000)enqueue();buf.push(row);bufSize+=n;if(buf.length>=12)enqueue();}}));enqueue();await chain;\n return {attempted:i,enriched:saved,failedWrites,duration_ms:Date.now()-started};\n}",[254],"4b6d882c-9f29-4b36-9c98-1022fb8088dd","enrich.ts","11ddbb7e-b2bd-11f1-b912-1607ee4eb77e",15,["D",1789666680342],"import { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\nimport { gunzipSync } from \"node:zlib\";\nimport { XMLParser } from \"npm:fast-xml-parser@4.5.3\";\nconst started=Date.now(),BUDGET=50000,WORKERS=160,LIMIT=5000;\nconst rows=await sqlite.execute({sql:\"SELECT c.domain FROM checks c LEFT JOIN enrichment e ON c.domain=e.domain WHERE c.status='confirmed' AND e.domain IS NULL ORDER BY CASE WHEN c.domain LIKE '%directory%' OR c.domain LIKE '%locator%' OR c.domain LIKE '%finder%' OR c.domain IN ('worldpopulationreview.com','campsitephotos.com') THEN 0 ELSE 1 END,c.domain LIMIT ?\",args:[LIMIT]});\nconst domains=rows.rows.map(x=>String(x.domain));\nconst parser=new XMLParser({ignoreAttributes:true,removeNSPrefix:true,parseTagValue:false,processEntities:false,trimValues:true});\nconst arr=(x:any)=>x==null?[]:Array.isArray(x)?x:[x];\nconst unesc=(s:string)=>s.replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'\"').replace(/'/g,\"'\");\nconst plain=(s:string)=>unesc(s.replace(/]*>[\\s\\S]*?<\\/script>/gi,' ').replace(/]*>[\\s\\S]*?<\\/style>/gi,' ').replace(/<[^>]+>/g,' ').replace(/\\s+/g,' ')).trim();\nasync function get(url:string,end:number,max=7000000):Promise{try{const left=end-Date.now();if(left<300)return {url,error:'site_time_budget'};const r=await fetch(url,{signal:AbortSignal.timeout(Math.min(6500,left)),headers:{'User-Agent':'PublisherNetworkResearch/1.0 (sitemap and public-page research)'}});const rd=r.body?.getReader();const chunks:Uint8Array[]=[];let n=0;if(rd)while(true){const p=await rd.read();if(p.done)break;n+=p.value.length;if(n>max){await rd.cancel();return {url,final_url:r.url,status:r.status,error:'body_size_limit'};}chunks.push(p.value);}let b=new Uint8Array(n);let offset=0;for(const c of chunks){b.set(c,offset);offset+=c.length;}if(b[0]===31&&b[1]===139)b=new Uint8Array(gunzipSync(b,{maxOutputLength:20000000}));return {url,final_url:r.url,status:r.status,body:new TextDecoder().decode(b),bytes:b.length};}catch(e){return {url,error:String(e)};}}\nfunction links(html:string,base:string){const out:any[]=[];for(const m of html.matchAll(/]*href\\s*=\\s*[\"']([^\"']+)[\"'][^>]*>([\\s\\S]*?)<\\/a>/gi)){try{const u=new URL(unesc(m[1]),base);if(!/^https?:$/.test(u.protocol))continue;out.push({url:u.href,text:plain(m[2]).slice(0,100)});}catch{}}return out;}\nfunction metadata(body:string){const title=plain(body.match(/]*>([\\s\\S]*?)<\\/title>/i)?.[1]??'');const desc=body.match(/]*name\\s*=\\s*[\"']description[\"'][^>]*content\\s*=\\s*[\"']([^\"']*)/i)?.[1]??body.match(/]*content\\s*=\\s*[\"']([^\"']*)[\"'][^>]*name\\s*=\\s*[\"']description/i)?.[1]??'';const schema=[...body.matchAll(/\"@type\"\\s*:\\s*\"([^\"]+)\"/g)].map(x=>x[1]);return {title,description:unesc(desc),schema_types:[...new Set(schema)],wordpress:/wp-content|wp-includes|wp-json/i.test(body)};}\nfunction categorize(domain:string,meta:any,text:string,urls:string[],maps:any[]){const head=(meta.title+' '+meta.description+' '+text.slice(0,500)).toLowerCase();const d=/\\bdirector(?:y|ies)\\b|\\blocator\\b|find (?:a |the |local )?(?:business|clinic|campground|store|boat ramp|toilet|doctor|facility|facilities|dentist)/i.test(head)||/directory|locator/.test(domain);const places=meta.schema_types.some((x:string)=>/LocalBusiness|MedicalClinic|Place|CivicStructure|LodgingBusiness/.test(x));const tool=/calculator|converter|generator|checker|word finder|bpm finder|fare finder|ip location/.test(head);const reference=/population|census|dictionary|encyclopedia|database|surname|lyrics|postal code|zip code/.test(head);const forum=/forum|community forums|discussion board/.test(meta.title.toLowerCase())||urls.some(x=>/\\/(?:threads|forums)\\//.test(x));const posts=maps.some(x=>/post-sitemap|wp-sitemap-posts-post/.test(x.url));const blog=posts||meta.schema_types.some((x:string)=>/BlogPosting|Article|NewsArticle/.test(x))||/\\bblog\\b|recipes|latest articles|latest posts/.test(head);let type='unknown',confidence='low',reason='Insufficient distinctive evidence';if(d){type='directory';confidence=d?'medium':'low';reason='Directory/locator language or place/business schema; automated classification';}else if(tool){type='tool';confidence='medium';reason='Calculator/converter/generator/finder language';}else if(forum){type='forum';confidence='medium';reason='Forum language or thread URL structure';}else if(reference){type='database_reference';confidence='medium';reason='Structured reference/database language';}else if(blog){type='blog';confidence='medium';reason=posts?'Post sitemap discovered':'Article/blog/recipe language or schema';}return {type,confidence,reason,review_status:'automated_not_manually_reviewed',has_editorial_section:blog};}\nfunction patterns(urls:string[]){const first:Record={};const paths=urls.map(x=>{try{return new URL(x).pathname;}catch{return '';}});for(const p of paths){const s=p.split('/').filter(Boolean);if(s[0])first[s[0]]=(first[s[0]]??0)+1;}const counts=new Map();for(let i=0;i/^\\d{4}$/.test(x)?'{year}':/^\\d+$/.test(x)?'{id}':j===0&&s.length>1&&(first[x]??0)>=3?x:x.endsWith('.html')?'{slug}.html':x.endsWith('.php')?'{slug}.php':'{slug}').join('/')+(p.endsWith('/')&&s.length?'/':'');const c=counts.get(k)??{count:0,examples:[]};c.count++;if(c.examples.length<3)c.examples.push(urls[i]);counts.set(k,c);}return [...counts.entries()].map(([pattern,v])=>({pattern,...v})).sort((a,b)=>b.count-a.count).slice(0,15);}\nasync function enrich(domain:string){const end=Date.now()+18000;const [home,robots]=await Promise.all([get('https://'+domain+'/',end,1500000),get('https://'+domain+'/robots.txt',end,500000)]);const origin=home.final_url?new URL(home.final_url).origin:'https://'+domain;const meta=metadata(home.body??'');const homeText=plain(home.body??'');const declared=(robots.status===200&&robots.body)?[...robots.body.matchAll(/^\\s*sitemap\\s*:\\s*(\\S+)\\s*$/gim)].map((m:any)=>unesc(m[1])):[];\n const roots=[...new Set(declared)];const queue:string[]=[...roots],seen=new Set(),pageUrls=new Set(),files:any[]=[];let errors=0,valid=0,offsite=0,assets=0,capped=false;\n async function parseMap(url:string){if(seen.has(url))return;seen.add(url);const f=await get(url,end);const rec:any={url,final_url:f.final_url??'',http_status:f.status??null,bytes:f.bytes??0};files.push(rec);if(f.status!==200||f.error){rec.error=f.error??'HTTP '+f.status;errors++;return;}try{if(!/<(?:[\\w-]+:)?(?:sitemapindex|urlset)\\b/.test(f.body)){rec.error='not_xml_sitemap';errors++;return;}const doc=parser.parse(f.body);if(doc.sitemapindex){const children=arr(doc.sitemapindex.sitemap).map(x=>x?.loc).filter((x:any)=>typeof x==='string');rec.kind='index';rec.child_count=children.length;valid++;for(const x of children){try{const u=new URL(unesc(x),f.final_url||url).href;if(!seen.has(u)&&!queue.includes(u))queue.push(u);}catch{errors++;}}}else if(doc.urlset){rec.kind='urlset';let n=0;valid++;for(const item of arr(doc.urlset.url)){if(typeof item?.loc!=='string')continue;let u:URL;try{u=new URL(unesc(item.loc));u.hash='';}catch{continue;}if(u.hostname.replace(/^www\\./,'')!==domain&&!u.hostname.endsWith('.'+domain)){offsite++;continue;}if(/\\.(?:jpg|jpeg|png|gif|webp|svg|pdf|mp4|mp3|css|js|zip|kml|xml|json|csv|txt|xlsx|rss|gz)$/i.test(u.pathname)){assets++;continue;}pageUrls.add(u.href);n++;if(pageUrls.size>=100000){capped=true;break;}}rec.page_url_entries=n;}else{rec.error='unknown_xml_root';errors++;}}catch(e){rec.error=String(e);errors++;}}\n if(!queue.length){for(const name of ['/sitemap.xml','/sitemap_index.xml','/wp-sitemap.xml']){await parseMap(origin+name);if(valid){roots.push(origin+name);errors=0;break;}if(Date.now()>end-1500)break;}}\n while(queue.length&&Date.now()!seen.has(x));await Promise.all(batch.map(parseMap));}\n if(queue.length||seen.size>=80||capped)capped=true;\n const urls=[...pageUrls];const cls=categorize(domain,meta,homeText,urls,files);const homeLinks=links(home.body??'',home.final_url??origin);const about=homeLinks.filter(x=>{try{return new URL(x.url).hostname.replace(/^www\\./,'')===domain&&/about|methodology|data-source|sources|faq/i.test(x.url+' '+x.text);}catch{return false;}}).sort((a,b)=>Number(/methodology|sources/.test(b.url))-Number(/methodology|sources/.test(a.url))).slice(0,5);\n const sourcePages:any[]=[];if(cls.type==='directory'||cls.type==='database_reference'){if(about[0]&&Date.now()=12)break;}}\n const status=valid?(errors||capped?'partial':'complete_discovered_tree'):'unavailable';const result={domain,checked_at:new Date().toISOString(),homepage:{url:home.final_url??origin,http_status:home.status??null,error:home.error??null,...meta,text_excerpt:homeText.slice(0,1800)},classification:cls,sitemap:{status,page_count:valid?urls.length:null,count_definition:'Unique non-asset page URLs in fetched sitemaps; not indexed pages or total site pages',roots,files,remaining_files:queue.length,offsite_urls_skipped:offsite,asset_urls_skipped:assets,limits:{max_files:80,max_unique_urls:100000,site_ms:18000},robots_status:robots.status??null},url_patterns:patterns(urls),sample_urls:urls.slice(0,12),about_links:about,source_evidence:provenance,data_source_status:provenance.length?'mentions_found_requires_review':'unknown_not_disclosed_in_checked_pages',source_pages:sourcePages};\n return [domain,result.checked_at,'done',result.sitemap.page_count,status,JSON.stringify(result)];}\nlet i=0;const out:any[]=[];await Promise.all(Array.from({length:WORKERS},async()=>{while(i180000)await save();chunk.push(r);size+=n;}await save();console.log(JSON.stringify({enriched:out.length,duration_ms:Date.now()-started,totals:(await sqlite.execute(\"SELECT json_extract(result_json,'$.classification.type') type,sitemap_status,COUNT(*) n FROM enrichment GROUP BY 1,2\")).rows,examples:out.slice(0,4).map(r=>{const d=JSON.parse(r[5]);return {domain:r[0],count:r[3],status:r[4],classification:d.classification};})}));",[262],"e22921c3-81aa-46a0-b970-aa99c953c531","download.ts","7645eebc-b2c0-11f1-a2dc-1607ee4eb77e",50,["D",1789670391403],"import { DATASETS, csvChunks } from './export_core.ts';\nimport { gzipSync } from 'node:zlib';\nimport { Buffer } from 'node:buffer';\nexport default async function(req:Request){\n const u=new URL(req.url),expected=Deno.env.get('AUDIT_EXPORT_KEY');\n const supplied=req.headers.get('authorization')?.replace(/^Bearer /,'')??u.searchParams.get('key');\n if(!expected||supplied!==expected)return new Response('Unauthorized',{status:401});\n if(req.method!=='GET')return new Response('GET required',{status:405});\n const kind=u.searchParams.get('kind')??'confirmed';\n if(!DATASETS.includes(kind))return new Response('Invalid dataset',{status:400});\n const compressed=u.searchParams.get('compressed')!=='0';\n const parts:Uint8Array[]=[];for await(const part of csvChunks(kind))parts.push(part);\n const csv=Buffer.concat(parts),body=compressed?gzipSync(csv):csv;\n return new Response(body,{headers:{'content-type':compressed?'application/gzip':'text/csv; charset=utf-8','content-length':String(body.length),'content-disposition':'attachment; filename=\"publisher-audit-'+kind+'-2026-09-17.csv'+(compressed?'.gz':'')+'\"','cache-control':'private, no-store','referrer-policy':'no-referrer','x-content-type-options':'nosniff','access-control-allow-origin':'*'}});\n}",[269],"62756cac-b216-4a2e-a06d-e36b33990c22","directory_research.ts","43ca09aa-b2c5-11f1-8c69-1607ee4eb77e",44,["D",1789669574067],"import { sqlite } from 'https://esm.town/v/std/sqlite/main.ts';\nawait sqlite.execute('CREATE TABLE IF NOT EXISTS directory_research(domain TEXT PRIMARY KEY,checked_at TEXT,result_json TEXT)');\nconst start=Date.now(),BUDGET=26000,WORKERS=16;\nconst rows=(await sqlite.execute(\"SELECT e.domain,e.result_json FROM enrichment e LEFT JOIN directory_research d ON e.domain=d.domain WHERE json_extract(e.result_json,'$.classification.type') IN ('directory','database_reference') AND d.domain IS NULL ORDER BY e.sitemap_count DESC LIMIT 400\")).rows;\nconst plain=(s:string)=>s.replace(/]*>[\\s\\S]*?<\\/script>/gi,' ').replace(/]*>[\\s\\S]*?<\\/style>/gi,' ').replace(/<[^>]+>/g,' ').replace(/&/g,'&').replace(/ /g,' ').replace(/\\s+/g,' ').trim();\nasync function page(url:string){try{const r=await fetch(url,{signal:AbortSignal.timeout(11000),headers:{'User-Agent':'PublisherNetworkResearch/1.0 (directory source research)'}});if(!r.ok){await r.body?.cancel();return {url,final_url:r.url,status:r.status};}const reader=r.body?.getReader(),chunks:Uint8Array[]=[];let n=0;if(reader)while(true){const p=await reader.read();if(p.done)break;n+=p.value.length;if(n>1800000){await reader.cancel();return {url,final_url:r.url,status:r.status,error:'body_size_limit'};}chunks.push(p.value);}const b=new Uint8Array(n);let k=0;for(const c of chunks){b.set(c,k);k+=c.length;}const html=new TextDecoder().decode(b),text=plain(html),title=plain(html.match(/]*>([\\s\\S]*?)<\\/title>/i)?.[1]??'');\n const headings=[...html.matchAll(/]*>([\\s\\S]*?)<\\/h[1-3]>/gi)].map(x=>plain(x[1])).slice(0,30);\n const links:any[]=[];for(const m of html.matchAll(/]*href\\s*=\\s*[\"']([^\"']+)[\"'][^>]*>([\\s\\S]*?)<\\/a>/gi)){try{const u=new URL(m[1].replace(/&/g,'&'),r.url);if(!/^https?:$/.test(u.protocol))continue;const label=plain(m[2]).slice(0,100);if(u.hostname!==new URL(r.url).hostname||/about|methodology|sources|submit|add.?listing|claim|contribut|correct|contact/i.test(u.pathname+' '+label))links.push({url:u.href,label});}catch{}}\n const snippets:any[]=[];const re=/sourced from|provided by|obtained from|compiled from|collected from|data (?:is|are|comes)|our data|we (?:use|collect|gather|compile)|OpenStreetMap|Census Bureau|U\\.S\\. Census|USGS|Recreation\\.gov|Wikidata|MusicBrainz|Google Places|Google Maps|HRSA|Centers for Medicare|\\bCMS\\b|NHS|submitted by|community.contribut|publicly.available|official.sources|licensed.data|scrap(?:ing|ed)/gi;for(const m of text.matchAll(re)){const i=m.index??0;snippets.push(text.slice(Math.max(0,i-100),i+260));if(snippets.length>=14)break;}\n const schemaKeys=[...new Set([...html.matchAll(/\"(address|telephone|openingHours|openingHoursSpecification|geo|latitude|longitude|aggregateRating|review|priceRange|sameAs|url|name|description|medicalSpecialty|servesCuisine|amenityFeature|dateModified|datePublished|identifier)\"\\s*:/g)].map(m=>m[1]))];\n const signals={owner_submission:/submit (?:a |your )?(?:business|listing|location|clinic|church)|add (?:a |your )?(?:business|listing|location|clinic|church)|claim (?:this |your )?(?:listing|business|clinic)|suggest an update/i.test(text),reviews:/reviews|ratings|testimonials/i.test(text),google_maps:/maps\\.google|maps\\.googleapis|maps\\.app\\.goo|google\\.com\\/maps/.test(html),openstreetmap:/openstreetmap|osm\\.org/.test(html)};\n return {url,final_url:r.url,status:r.status,title,headings,summary_text:text.slice(0,2400),source_snippets:snippets,relevant_links:[...new Map(links.map(x=>[x.url,x])).values()].slice(0,50),schema_keys:schemaKeys,signals};\n}catch(e){return {url,error:String(e)};}}\nlet i=0,saved=0;\nawait Promise.all(Array.from({length:WORKERS},async()=>{while(ix.url),...(e.url_patterns??[]).filter((x:any)=>x.examples?.length).slice(0,2).map((x:any)=>x.examples[0])];const urls=[...new Set(picks)].slice(0,5);const pages=await Promise.all(urls.map(page));const result={domain:row.domain,checked_at:new Date().toISOString(),status:'automated_page_research_requires_review',pages,observed_schema_fields:[...new Set(pages.flatMap((p:any)=>p.schema_keys??[]))],source_status:'unknown_until_disclosure_is_reviewed',note:'Map and platform signals do not prove the listing database source.'};await sqlite.execute({sql:'INSERT OR REPLACE INTO directory_research VALUES (?,?,?)',args:[String(row.domain),result.checked_at,JSON.stringify(result)]});saved++;}}));\nconsole.log(JSON.stringify({researched:saved,elapsed_ms:Date.now()-start,remaining:(await sqlite.execute(\"SELECT COUNT(*) n FROM enrichment e LEFT JOIN directory_research d ON e.domain=d.domain WHERE json_extract(e.result_json,'$.classification.type') IN ('directory','database_reference') AND d.domain IS NULL\")).rows}));",[276],"558d7440-dd48-40ee-a096-168391b2a1da","classify.ts","aac5576c-b2c1-11f1-bee6-1607ee4eb77e",28,["D",1789668028848],"export function classifySite(domain:string,meta:any,text:string,urls:string[],maps:any[]){\n const title=String(meta.title??'').toLowerCase(),head=(title+' '+String(meta.description??'')).toLowerCase();\n const body=String(text??'').slice(0,1800).toLowerCase();\n const schemas=Array.isArray(meta.schema_types)?meta.schema_types:[];\n const sample=urls.length>300?Array.from({length:300},(_,i)=>urls[Math.floor(i*urls.length/300)]):urls;\n const paths=sample.map(s=>{try{return new URL(s).pathname.toLowerCase();}catch{return '';}});\n const ratio=(r:RegExp)=>paths.length?paths.filter(p=>r.test(p)).length/paths.length:0;\n const articlePaths=ratio(/^\\/(?:blog|blogs|article|articles|posts|news|recipes|recetas|rezepte|guides)\\//);\n const entityPaths=ratio(/^\\/(?:hospital|hospitals|surgery-center|ramps|campgrounds|businesses|clinics|prisons|dentists|nursing-homes|shelters)\\//);\n const geoPaths=ratio(/^\\/(?:states|locations|places|stores|cities)\\/[^/]+\\/[^/]+/);\n const forum=/\\bforums?\\b|\\bforo\\b|discussion board|community discussions/.test(head)||ratio(/^\\/(?:threads|forums|discussions)\\//)>.3;\n const tool=/calculator|converter|generator|word finder|word solver|bpm finder|fare finder|ip location|tracking tool|online tracker/.test(head)||schemas.includes('SoftwareApplication')&&/calculate|convert|generate|track/.test(body);\n const reference=/encyclopedia|encyclopaedia|dictionary|gazetteer|\\bdatabase\\b|\\bwiki\\b|world population|drop sources|postal codes|zip codes/.test(head);\n const directDirectory=/\\bdirector(?:y|ies)\\b|\\blocator\\b|business listings|\\bannuaire\\b|branchenbuch|find.{0,40}(?:near you|near me|business|clinic|campground|boat ramp|dentist|shelter)/.test(head);\n const nameDirectory=/directory|locator|annuaire/.test(domain);\n const postMap=maps.some(m=>/(?:^|\\/)post-sitemap|\\/wp-sitemap-posts-(?:post|recipe)|\\/sitemap-posts(?:[-_.?]|$)|\\/sitemap[._-]post(?:[-_.?]|$)/i.test(String(m.url??'')));\n const editorial=postMap||articlePaths>.35||schemas.some((s:string)=>['Blog','BlogPosting','Article','NewsArticle','Recipe'].includes(s))||/\\bblog\\b|\\brecipes?\\b|recetas|ricette|receitas|recettes|rezepte|przepisy|рецепт|cooking|lifestyle|travel tips|diy projects|gardening tips|latest articles|latest posts/.test(head+' '+body.slice(0,300));\n let type='unknown',confidence='low',reason='Insufficient accessible evidence to assign a primary site type';\n if(forum){type='forum';confidence='medium';reason='Forum title/description or repeated discussion-thread URLs';}\n else if(tool){type='tool';confidence='medium';reason='Title/description or application schema describes an interactive utility';}\n else if(reference&&!directDirectory){type='database_reference';confidence='medium';reason='Reference/database/encyclopedia language in title or description';}\n else if(directDirectory||entityPaths>.4||geoPaths>.5){type='directory';confidence='medium';reason=directDirectory?'Directory or entity-search language in title/description':'Repeated entity or geographic listing URL structure';}\n else if(nameDirectory&&!editorial){type='directory';confidence='low';reason='Domain-name hint only; manual review required';}\n else if(editorial){type='blog';confidence='medium';reason=postMap?'Editorial post sitemap':articlePaths>.35?'Predominantly article/blog/recipe URL structure':'Editorial/recipe language or article schema';}\n return {type,confidence,reason,review_status:'automated_v2_not_manually_reviewed',has_editorial_section:editorial,version:2,evidence:{article_url_ratio:Number(articlePaths.toFixed(3)),entity_url_ratio:Number(entityPaths.toFixed(3)),geographic_url_ratio:Number(geoPaths.toFixed(3)),post_sitemap:postMap}};\n}",[283],"75e758c0-6550-4a8e-a4bf-d244768ed56e","batch_scan.ts","19a948f8-b2bf-11f1-b5e0-1607ee4eb77e",29,["D",1789668108011],2,"import { sqlite } from 'https://esm.town/v/std/sqlite/main.ts';\nconst endpoint='https://omriluz--0eb0b968b2bf11f196131607ee4eb77e.web.val.run';\nconst start=Date.now(),shards=8;\nconst pending=(await sqlite.execute({sql:\"SELECT DISTINCT ((unicode(substr(s.domain,1,1))*31+unicode(substr(s.domain,2,1))+length(s.domain)) % ?) shard FROM sellers s LEFT JOIN checks c ON s.domain=c.domain WHERE s.domain<>'' AND c.domain IS NULL ORDER BY shard\",args:[shards]})).rows.map(r=>Number(r.shard));\nconst results=await Promise.all(pending.map(async shard=>{try{const r=await fetch(endpoint,{method:'POST',headers:{'content-type':'application/json',authorization:'Bearer '+Deno.env.get('AUDIT_JOB_KEY')},body:JSON.stringify({kind:'scan',shard,shards}),signal:AbortSignal.timeout(75000)});const text=await r.text();try{const value=JSON.parse(text);return {shard,status:r.status,...(value&&typeof value==='object'?value:{error:value})};}catch{return {shard,status:r.status,body:text.slice(0,600)};}}catch(e){return {shard,error:String(e)};}}));\nconsole.log(JSON.stringify({duration_ms:Date.now()-start,results,totals:(await sqlite.execute('SELECT status,COUNT(*) n FROM checks GROUP BY status')).rows,remaining:(await sqlite.execute(\"SELECT COUNT(DISTINCT s.domain) n FROM sellers s LEFT JOIN checks c ON s.domain=c.domain WHERE s.domain<>'' AND c.domain IS NULL\")).rows}));",[290],"487510d4-a261-4fb6-9156-e600336de14a","batch_retry.ts","c332e11a-b2c2-11f1-8885-1607ee4eb77e",37,["D",1789668499337],"import { sqlite } from 'https://esm.town/v/std/sqlite/main.ts';\nconst endpoint='https://omriluz--0eb0b968b2bf11f196131607ee4eb77e.web.val.run';\nconst start=Date.now(),shards=8;\nconst pending=(await sqlite.execute({sql:\"SELECT DISTINCT ((unicode(substr(c.domain,1,1))*31+unicode(substr(c.domain,2,1))+length(c.domain)) % ?) shard FROM checks c WHERE c.status='unverified' AND c.checked_at < (SELECT value FROM audit_settings WHERE key='retry_cutoff') ORDER BY shard\",args:[shards]})).rows.map(r=>Number(r.shard));\nconst results=await Promise.all(pending.map(async shard=>{try{const r=await fetch(endpoint,{method:'POST',headers:{'content-type':'application/json',authorization:'Bearer '+Deno.env.get('AUDIT_JOB_KEY')},body:JSON.stringify({kind:'retry',shard,shards}),signal:AbortSignal.timeout(75000)});const text=await r.text();try{const value=JSON.parse(text);return {shard,status:r.status,...(value&&typeof value==='object'?value:{error:value})};}catch{return {shard,status:r.status,body:text.slice(0,400)};}}catch(e){return {shard,error:String(e)};}}));\nconsole.log(JSON.stringify({duration_ms:Date.now()-start,results,totals:(await sqlite.execute('SELECT status,COUNT(*) n FROM checks GROUP BY status')).rows,remaining_retries:(await sqlite.execute(\"SELECT COUNT(*) n FROM checks c WHERE c.status='unverified' AND c.checked_at < (SELECT value FROM audit_settings WHERE key='retry_cutoff')\")).rows}));",[298],"614711a8-16a6-49a3-b849-5b107926bb2d","batch_enrich.ts","6381902c-b2c2-11f1-809f-1607ee4eb77e",31,["D",1789668338773],"import { sqlite } from 'https://esm.town/v/std/sqlite/main.ts';\nconst endpoint='https://omriluz--0eb0b968b2bf11f196131607ee4eb77e.web.val.run';\nconst start=Date.now(),shards=8;\nconst pending=(await sqlite.execute({sql:\"SELECT DISTINCT ((unicode(substr(c.domain,1,1))*31+unicode(substr(c.domain,2,1))+length(c.domain)) % ?) shard FROM checks c LEFT JOIN enrichment e ON c.domain=e.domain WHERE c.status='confirmed' AND e.domain IS NULL ORDER BY shard\",args:[shards]})).rows.map(r=>Number(r.shard));\nconst results=await Promise.all(pending.map(async shard=>{try{const r=await fetch(endpoint,{method:'POST',headers:{'content-type':'application/json',authorization:'Bearer '+Deno.env.get('AUDIT_JOB_KEY')},body:JSON.stringify({kind:'enrich',shard,shards}),signal:AbortSignal.timeout(75000)});const text=await r.text();try{const value=JSON.parse(text);return {shard,status:r.status,...(value&&typeof value==='object'?value:{error:value})};}catch{return {shard,status:r.status,body:text.slice(0,600)};}}catch(e){return {shard,error:String(e)};}}));\nconsole.log(JSON.stringify({duration_ms:Date.now()-start,results,totals:(await sqlite.execute('SELECT sitemap_status,COUNT(*) n FROM enrichment GROUP BY sitemap_status')).rows,remaining:(await sqlite.execute(\"SELECT COUNT(*) n FROM checks c LEFT JOIN enrichment e ON c.domain=e.domain WHERE c.status='confirmed' AND e.domain IS NULL\")).rows}));",[305],"af49aca0-800e-4cdb-a656-605cafe7a1cb","README.md","01a0b061-5e61-74b2-9c7e-77e87e2983ad",["D",1789665566213],"# hono-jsx-starter\n\nA minimal [Hono](https://hono.dev) + Hono JSX starter for Val Town.\nServer-rendered HTML, no build step, no React.\n\n## Picking a starter\n\n| Starter | When to pick it |\n| --- | --- |\n| [`basic-html-starter`](https://www.val.town/x/templates/basic-html-starter) | Static HTML pages. Sprinkle of vanilla JS. No server-side rendering. |\n| **[`hono-jsx-starter`](https://www.val.town/x/templates/hono-jsx-starter)** *(you're here)* | Dynamic server-rendered HTML with JSX. API routes. No React, no SPA. |\n| [`react-hono-starter`](https://www.val.town/x/templates/react-hono-starter) | Client-side React SPA with a Hono API backend. |\n\n## What's here\n\n```\nindex.tsx Hono app, Layout, routes\npublic/\n favicon.svg Static asset\n counter.ts Tiny client-side script\n```\n\nThe whole app fits in one file. As you grow it, the natural next moves are:\n\n- Pull `Layout` into its own `.tsx` file once you have 3+ pages.\n- Split routes with `app.route(\"/api\", api)` if `/api/*` grows.\n- Add more files under `public/` — they're served by a single\n `app.get(\"/public/*\", ...)` route.\n\n## Patterns worth knowing\n\n**JSX pragma.** Every `.tsx` file that uses JSX needs this at the top:\n\n```tsx\n/** @jsxImportSource npm:hono@4/jsx */\n```\n\n**Server state.** The counter on `/` is `let count = 0` at module scope. It\nlives in memory on the Val Town server and survives across requests — but resets\nwhenever the val restarts (deploys, idle timeouts, etc.). For persistent state,\nswap it for [`std/sqlite`](https://www.val.town/x/std/sqlite) or\n[`std/blob`](https://www.val.town/x/std/blob).\n\n**API routes.** The `Increment` button POSTs to `/api/increment`, which returns\nJSON. Mix server-rendered pages and JSON endpoints in the same Hono app:\n\n```ts\napp.post(\"/api/increment\", (c) => c.json({ count: ++count }));\n```\n\n**Static files.** `serveFile` reads the file by path and serves it with the\nright content-type. `.ts` files are transpiled to JS on request by Val Town — so\nyou can write `public/counter.ts` and reference it as\n`

Diff · v58