[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8},"actionData","errors","root",{"_106":107,"_108":109},"routes/embed.x.$handle.$name.$",{"_9":10,"_11":12,"_13":14,"_15":16,"_17":18,"_19":20,"_21":-5},"handle","omriluz","name","publisher-network-audit","path","enrich_core.ts","project",{"_22":45,"_11":12,"_33":46,"_47":-5,"_48":42,"_49":50,"_51":50,"_52":53,"_54":-5,"_55":56,"_57":58,"_59":32,"_60":-5,"_61":-5,"_62":63,"_64":65,"_66":67,"_68":69,"_70":71,"_72":-5,"_73":74,"_75":74,"_76":77,"_78":77,"_79":77,"_80":74,"_81":74,"_82":83,"_84":85,"_86":87,"_88":42},"file",{"_22":23,"_11":14,"_24":25,"_26":-5,"_27":28,"_29":30,"_31":32,"_33":34,"_35":36,"_37":38,"_39":40,"_41":42,"_43":-5,"_13":44},"permalink","https://www.val.town/x/omriluz/publisher-network-audit/code/enrich_core.ts","extension","id","69e1606f-3b30-4175-9675-7bd1fba3e94d","fileId","01a0b075-429a-70af-87d6-fa93cd9ce0e4","parentId","branchId","cd292256-c961-4ceb-b973-27c236575749","branchVersion",43,"latest",true,"createdAt",["D",1789669472058],"version",8,"fileType","script","content","import { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\nimport { gunzipSync } from \"node:zlib\";\nimport { XMLParser } from \"npm:fast-xml-parser@4.5.3\";\nimport { classifySite } from './classify.ts';\nexport async function runEnrich(shard=0,shards=1){\n const started=Date.now(),BUDGET=30000,WORKERS=100,LIMIT=3000,SITE_MS=18000;\n const rows=await sqlite.execute({sql:\"SELECT c.domain,c.attempts_json FROM checks c LEFT JOIN enrichment e ON c.domain=e.domain WHERE c.status='confirmed' AND e.domain IS NULL AND ((unicode(substr(c.domain,1,1))*31+unicode(substr(c.domain,2,1))+length(c.domain)) % ?) = ? ORDER BY CASE WHEN c.domain LIKE '%directory%' OR c.domain LIKE '%locator%' OR c.domain LIKE '%finder%' OR c.domain IN ('worldpopulationreview.com','campsitephotos.com') THEN 0 ELSE 1 END,c.domain LIMIT ?\",args:[shards,shard,LIMIT]});\n const domains=rows.rows.map(x=>String(x.domain));\n const baseByDomain=new Map();for(const r of rows.rows){let base='https://'+r.domain;try{const a=JSON.parse(String(r.attempts_json??'[]')).find((a:any)=>a.status===200&&!a.invalid_body&&a.parsed_lines!=null);if(a?.url)base=new URL(a.url).origin;}catch{}baseByDomain.set(String(r.domain),base);}\n const parser=new XMLParser({ignoreAttributes:true,removeNSPrefix:true,parseTagValue:false,processEntities:false,trimValues:true});\n const arr=(x:any)=>x==null?[]:Array.isArray(x)?x:[x];\n const unesc=(s:string)=>s.replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'\"').replace(/'/g,\"'\");\n const plain=(s:string)=>unesc(s.replace(/]*>[\\s\\S]*?<\\/script>/gi,' ').replace(/]*>[\\s\\S]*?<\\/style>/gi,' ').replace(/<[^>]+>/g,' ').replace(/\\s+/g,' ')).trim();\n async function get(url:string,end:number,max=7000000):Promise{try{const left=end-Date.now();if(left<300)return {url,error:'site_time_budget'};const r=await fetch(url,{signal:AbortSignal.timeout(Math.min(8000,left)),headers:{'User-Agent':'PublisherNetworkResearch/1.0 (sitemap and public-page research)'}});const rd=r.body?.getReader();const chunks:Uint8Array[]=[];let n=0;if(rd)while(true){const p=await rd.read();if(p.done)break;n+=p.value.length;if(n>max){await rd.cancel();return {url,final_url:r.url,status:r.status,error:'body_size_limit'};}chunks.push(p.value);}let b=new Uint8Array(n);let offset=0;for(const c of chunks){b.set(c,offset);offset+=c.length;}if(b[0]===31&&b[1]===139)b=new Uint8Array(gunzipSync(b,{maxOutputLength:20000000}));return {url,final_url:r.url,status:r.status,body:new TextDecoder().decode(b),bytes:b.length};}catch(e){return {url,error:String(e)};}}\n function links(html:string,base:string){const out:any[]=[];for(const m of html.matchAll(/]*href\\s*=\\s*[\"']([^\"']+)[\"'][^>]*>([\\s\\S]*?)<\\/a>/gi)){try{const u=new URL(unesc(m[1]),base);if(!/^https?:$/.test(u.protocol))continue;out.push({url:u.href,text:plain(m[2]).slice(0,100)});}catch{}}return out;}\n function metadata(body:string){const title=plain(body.match(/]*>([\\s\\S]*?)<\\/title>/i)?.[1]??'');const desc=body.match(/]*name\\s*=\\s*[\"']description[\"'][^>]*content\\s*=\\s*[\"']([^\"']*)/i)?.[1]??body.match(/]*content\\s*=\\s*[\"']([^\"']*)[\"'][^>]*name\\s*=\\s*[\"']description/i)?.[1]??'';const schema=[...body.matchAll(/\"@type\"\\s*:\\s*\"([^\"]+)\"/g)].map(x=>x[1]);return {title,description:unesc(desc),schema_types:[...new Set(schema)],wordpress:/wp-content|wp-includes|wp-json/i.test(body)};}\n function patterns(urls:string[]){const first:Record={};const paths=urls.map(x=>{try{return new URL(x).pathname;}catch{return '';}});for(const p of paths){const s=p.split('/').filter(Boolean);if(s[0])first[s[0]]=(first[s[0]]??0)+1;}const counts=new Map();for(let i=0;i/^\\d{4}$/.test(x)?'{year}':/^\\d+$/.test(x)?'{id}':j===0&&s.length>1&&(first[x]??0)>=3?x:x.endsWith('.html')?'{slug}.html':x.endsWith('.php')?'{slug}.php':'{slug}').join('/')+(p.endsWith('/')&&s.length?'/':'');const c=counts.get(k)??{count:0,examples:[]};c.count++;if(c.examples.length<3)c.examples.push(urls[i]);counts.set(k,c);}return [...counts.entries()].map(([pattern,v])=>({pattern,...v})).sort((a,b)=>b.count-a.count).slice(0,20);}\n async function enrich(domain:string){const end=Date.now()+SITE_MS,base=baseByDomain.get(domain)??'https://'+domain;const [home,robots]=await Promise.all([get(base+'/',end,1500000),get(base+'/robots.txt',end,500000)]);const origin=home.final_url?new URL(home.final_url).origin:base;const canonicalHost=new URL(origin).hostname.replace(/^www\\./,'');const meta=metadata(home.status===200?home.body??'':'');const homeText=plain(home.status===200?home.body??'':'');const declared=(robots.status===200&&robots.body)?[...robots.body.matchAll(/^\\s*sitemap\\s*:\\s*(\\S+)\\s*$/gim)].map((m:any)=>unesc(m[1])):[];\n const roots=[...new Set(declared)];const queue:string[]=[...roots],seen=new Set(),pageUrls=new Set(),files:any[]=[];let errors=0,valid=0,offsite=0,assets=0,capped=false;\n async function parseMap(url:string){if(seen.has(url))return;seen.add(url);const f=await get(url,end);const rec:any={url,final_url:f.final_url??'',http_status:f.status??null,bytes:f.bytes??0};files.push(rec);if(f.status!==200||f.error){rec.error=f.error??'HTTP '+f.status;errors++;return;}try{if(!/<(?:[\\w-]+:)?(?:sitemapindex|urlset)\\b/.test(f.body)){rec.error='not_xml_sitemap';errors++;return;}const doc=parser.parse(f.body);if(doc.sitemapindex){const children=arr(doc.sitemapindex.sitemap).map(x=>x?.loc).filter((x:any)=>typeof x==='string');rec.kind='index';rec.child_count=children.length;valid++;for(const x of children){try{const u=new URL(unesc(x),f.final_url||url).href;if(!/^https?:/.test(u))continue;if(!seen.has(u)&&!queue.includes(u))queue.push(u);}catch{errors++;}}}else if(doc.urlset){rec.kind='urlset';let n=0;valid++;for(const item of arr(doc.urlset.url)){if(typeof item?.loc!=='string')continue;let u:URL;try{u=new URL(unesc(item.loc));u.hash='';if(!/^https?:$/.test(u.protocol))continue;}catch{continue;}const h=u.hostname.replace(/^www\\./,'');if(h!==domain&&h!==canonicalHost&&!h.endsWith('.'+domain)){offsite++;continue;}if(/\\.(?:jpg|jpeg|png|gif|webp|svg|pdf|mp4|mp3|css|js|zip|kml|xml|json|csv|txt|xlsx|rss|gz)$/i.test(u.pathname)){assets++;continue;}if(pageUrls.size>=100000){capped=true;break;}pageUrls.add(u.href);n++;}rec.page_url_entries=n;}else{rec.error='unknown_xml_root';errors++;}}catch(e){rec.error=String(e);errors++;}}\n if(!queue.length){for(const name of ['/sitemap.xml','/sitemap_index.xml','/wp-sitemap.xml']){await parseMap(origin+name);if(valid){roots.push(origin+name);errors=0;break;}if(Date.now()>end-1500)break;}}\n while(queue.length&&Date.now()!seen.has(x));await Promise.all(batch.map(parseMap));}\n if(queue.length||capped)capped=true;\n const urls=[...pageUrls],cls=classifySite(domain,meta,homeText,urls,files),homeLinks=links(home.body??'',home.final_url??origin);\n const rank=(u:string)=>/\\/(?:methodology|data-sources?|sources)(?:\\/|$|[?#])/i.test(u)?3:/\\/(?:about(?:-us)?|aboutus)(?:[./]|$)/i.test(u)?2:1;\n const about=[...new Map(homeLinks.filter(x=>{try{return new URL(x.url).hostname.replace(/^www\\./,'')===canonicalHost&&/\\/(?:about(?:-us)?|aboutus|methodology|data-sources?|sources|faq)(?:[./?#]|$)/i.test(x.url);}catch{return false;}}).map(x=>[x.url,x])).values()].sort((a,b)=>rank(b.url)-rank(a.url)).slice(0,5);\n const sourcePages:any[]=[];if(['directory','database_reference'].includes(cls.type)&&about[0]&&Date.now()=10)break;}if(provenance.length>=10)break;}\n const status=valid?(errors||capped?'partial':'complete_discovered_tree'):'unavailable';const result={domain,checked_at:new Date().toISOString(),homepage:{url:home.final_url??origin,http_status:home.status??null,error:home.error??null,...meta,text_excerpt:homeText.slice(0,1800)},classification:cls,sitemap:{status,page_count:valid?urls.length:null,count_definition:'Unique non-asset page URLs in fetched sitemaps; not indexed pages or total site pages',roots,files,remaining_files:queue.length,offsite_urls_skipped:offsite,asset_urls_skipped:assets,limits:{max_files:80,max_unique_urls:100000,site_ms:SITE_MS},robots_status:robots.status??null},url_patterns:patterns(urls),sample_urls:urls.length>60?Array.from({length:60},(_,i)=>urls[Math.floor(i*urls.length/60)]):urls,about_links:about,source_evidence:provenance,data_source_status:provenance.length?'mentions_found_requires_review':home.status===200?'unknown_not_disclosed_in_checked_pages':'unknown_pages_unavailable',source_pages:sourcePages};\n return [domain,result.checked_at,'done',result.sitemap.page_count,status,JSON.stringify(result)];\n }\n const INSERT=\"INSERT OR REPLACE INTO enrichment SELECT json_extract(value,'$[0]'),json_extract(value,'$[1]'),json_extract(value,'$[2]'),json_extract(value,'$[3]'),json_extract(value,'$[4]'),json_extract(value,'$[5]') FROM json_each(?)\";\n let i=0,saved=0,failedWrites=0,buf:any[]=[],bufSize=0,chain=Promise.resolve();\n function enqueue(){if(!buf.length)return;const batch=buf;buf=[];bufSize=0;chain=chain.then(async()=>{for(let a=0;a<3;a++){try{await sqlite.execute({sql:INSERT,args:[JSON.stringify(batch)]});saved+=batch.length;return;}catch(e){if(a===2){failedWrites+=batch.length;console.log(JSON.stringify({write_error:String(e),domains:batch.map(r=>r[0])}));}else await new Promise(r=>setTimeout(r,400*(a+1)));}}});}\n await Promise.all(Array.from({length:WORKERS},async()=>{while(i110000)enqueue();buf.push(row);bufSize+=n;if(buf.length>=12)enqueue();}}));enqueue();await chain;\n return {attempted:i,enriched:saved,failedWrites,duration_ms:Date.now()-started};\n}","deleted",false,"label",[14],"7e58a892-b130-4895-9cbc-de82614ff56d",["D",1789665566213],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Browse publishers, directory teardowns and ads.txt evidence.","imageUrl","userId","f5be24e4-c798-4816-91e5-037b60b986f5","pinnedFileId","01a0b061-5e61-74b2-9c7e-7b033afc92fc","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","7f67379d-2d38-49bd-81ac-2c7f92a739e2","tursoName","7e58a892b13048959cbcde82614ff56d","tursoHostname","7e58a892b13048959cbcde82614ff56d-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789665692456],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_9":10,"_105":101},"org",{"_9":10,"_99":-5,"_100":101,"_55":56,"_102":103,"_104":42},"branch",{"_35":89,"_11":90,"_22":28,"_91":45,"_92":-5,"_93":-5,"_94":-5,"_95":-5,"_96":-5,"_97":12,"_98":10},"isOwned",59,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18ycWxkOWxiOE00SkVtZENmRWM4YWxLWTRVUFMifQ","tier","free","canRestrictVals","profileImageUrl","ctx",{"_82":-5,"_139":140,"_141":142,"_143":144,"_145":42,"_146":42,"_147":42,"_148":32,"_149":-5,"_150":-7,"_151":-7,"_152":63,"_153":42,"_154":42,"_155":-5,"_156":42,"_157":42,"_158":42,"_159":-5},"ENV",{"_110":111,"_112":113,"_114":90,"_115":116,"_117":118,"_119":120,"_121":122,"_123":124,"_125":126,"_127":128,"_129":130,"_131":132,"_133":134,"_135":136,"_137":138},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","71c1b20804b3232a7e513be260cfffc1932a9e97","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town","orgs",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]