[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8},"actionData","errors","root",{"_105":106,"_107":108},"routes/embed.x.$handle.$name.$",{"_9":10,"_11":12,"_13":14,"_15":16,"_17":18,"_19":20,"_21":-5},"handle","omriluz","name","publisher-network-audit","path","directory_research.ts","project",{"_22":45,"_11":12,"_33":46,"_47":-5,"_48":42,"_49":50,"_51":50,"_52":53,"_54":-5,"_55":56,"_57":58,"_59":32,"_60":-5,"_61":-5,"_62":36,"_63":64,"_65":66,"_67":68,"_69":70,"_71":-5,"_72":73,"_74":73,"_75":76,"_77":76,"_78":76,"_79":73,"_80":73,"_81":82,"_83":84,"_85":86,"_87":42},"file",{"_22":23,"_11":14,"_24":25,"_26":-5,"_27":28,"_29":30,"_31":32,"_33":34,"_35":36,"_37":38,"_39":40,"_41":42,"_43":-5,"_13":44},"permalink","https://www.val.town/x/omriluz/publisher-network-audit/code/directory_research.ts","extension","id","62756cac-b216-4a2e-a06d-e36b33990c22","fileId","43ca09aa-b2c5-11f1-8c69-1607ee4eb77e","parentId","branchId","cd292256-c961-4ceb-b973-27c236575749","branchVersion",44,"latest",true,"createdAt",["D",1789669574067],"version",0,"fileType","script","content","import { sqlite } from 'https://esm.town/v/std/sqlite/main.ts';\nawait sqlite.execute('CREATE TABLE IF NOT EXISTS directory_research(domain TEXT PRIMARY KEY,checked_at TEXT,result_json TEXT)');\nconst start=Date.now(),BUDGET=26000,WORKERS=16;\nconst rows=(await sqlite.execute(\"SELECT e.domain,e.result_json FROM enrichment e LEFT JOIN directory_research d ON e.domain=d.domain WHERE json_extract(e.result_json,'$.classification.type') IN ('directory','database_reference') AND d.domain IS NULL ORDER BY e.sitemap_count DESC LIMIT 400\")).rows;\nconst plain=(s:string)=>s.replace(/]*>[\\s\\S]*?<\\/script>/gi,' ').replace(/]*>[\\s\\S]*?<\\/style>/gi,' ').replace(/<[^>]+>/g,' ').replace(/&/g,'&').replace(/ /g,' ').replace(/\\s+/g,' ').trim();\nasync function page(url:string){try{const r=await fetch(url,{signal:AbortSignal.timeout(11000),headers:{'User-Agent':'PublisherNetworkResearch/1.0 (directory source research)'}});if(!r.ok){await r.body?.cancel();return {url,final_url:r.url,status:r.status};}const reader=r.body?.getReader(),chunks:Uint8Array[]=[];let n=0;if(reader)while(true){const p=await reader.read();if(p.done)break;n+=p.value.length;if(n>1800000){await reader.cancel();return {url,final_url:r.url,status:r.status,error:'body_size_limit'};}chunks.push(p.value);}const b=new Uint8Array(n);let k=0;for(const c of chunks){b.set(c,k);k+=c.length;}const html=new TextDecoder().decode(b),text=plain(html),title=plain(html.match(/]*>([\\s\\S]*?)<\\/title>/i)?.[1]??'');\n const headings=[...html.matchAll(/]*>([\\s\\S]*?)<\\/h[1-3]>/gi)].map(x=>plain(x[1])).slice(0,30);\n const links:any[]=[];for(const m of html.matchAll(/]*href\\s*=\\s*[\"']([^\"']+)[\"'][^>]*>([\\s\\S]*?)<\\/a>/gi)){try{const u=new URL(m[1].replace(/&/g,'&'),r.url);if(!/^https?:$/.test(u.protocol))continue;const label=plain(m[2]).slice(0,100);if(u.hostname!==new URL(r.url).hostname||/about|methodology|sources|submit|add.?listing|claim|contribut|correct|contact/i.test(u.pathname+' '+label))links.push({url:u.href,label});}catch{}}\n const snippets:any[]=[];const re=/sourced from|provided by|obtained from|compiled from|collected from|data (?:is|are|comes)|our data|we (?:use|collect|gather|compile)|OpenStreetMap|Census Bureau|U\\.S\\. Census|USGS|Recreation\\.gov|Wikidata|MusicBrainz|Google Places|Google Maps|HRSA|Centers for Medicare|\\bCMS\\b|NHS|submitted by|community.contribut|publicly.available|official.sources|licensed.data|scrap(?:ing|ed)/gi;for(const m of text.matchAll(re)){const i=m.index??0;snippets.push(text.slice(Math.max(0,i-100),i+260));if(snippets.length>=14)break;}\n const schemaKeys=[...new Set([...html.matchAll(/\"(address|telephone|openingHours|openingHoursSpecification|geo|latitude|longitude|aggregateRating|review|priceRange|sameAs|url|name|description|medicalSpecialty|servesCuisine|amenityFeature|dateModified|datePublished|identifier)\"\\s*:/g)].map(m=>m[1]))];\n const signals={owner_submission:/submit (?:a |your )?(?:business|listing|location|clinic|church)|add (?:a |your )?(?:business|listing|location|clinic|church)|claim (?:this |your )?(?:listing|business|clinic)|suggest an update/i.test(text),reviews:/reviews|ratings|testimonials/i.test(text),google_maps:/maps\\.google|maps\\.googleapis|maps\\.app\\.goo|google\\.com\\/maps/.test(html),openstreetmap:/openstreetmap|osm\\.org/.test(html)};\n return {url,final_url:r.url,status:r.status,title,headings,summary_text:text.slice(0,2400),source_snippets:snippets,relevant_links:[...new Map(links.map(x=>[x.url,x])).values()].slice(0,50),schema_keys:schemaKeys,signals};\n}catch(e){return {url,error:String(e)};}}\nlet i=0,saved=0;\nawait Promise.all(Array.from({length:WORKERS},async()=>{while(ix.url),...(e.url_patterns??[]).filter((x:any)=>x.examples?.length).slice(0,2).map((x:any)=>x.examples[0])];const urls=[...new Set(picks)].slice(0,5);const pages=await Promise.all(urls.map(page));const result={domain:row.domain,checked_at:new Date().toISOString(),status:'automated_page_research_requires_review',pages,observed_schema_fields:[...new Set(pages.flatMap((p:any)=>p.schema_keys??[]))],source_status:'unknown_until_disclosure_is_reviewed',note:'Map and platform signals do not prove the listing database source.'};await sqlite.execute({sql:'INSERT OR REPLACE INTO directory_research VALUES (?,?,?)',args:[String(row.domain),result.checked_at,JSON.stringify(result)]});saved++;}}));\nconsole.log(JSON.stringify({researched:saved,elapsed_ms:Date.now()-start,remaining:(await sqlite.execute(\"SELECT COUNT(*) n FROM enrichment e LEFT JOIN directory_research d ON e.domain=d.domain WHERE json_extract(e.result_json,'$.classification.type') IN ('directory','database_reference') AND d.domain IS NULL\")).rows}));","deleted",false,"label",[14],"7e58a892-b130-4895-9cbc-de82614ff56d",["D",1789665566213],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Browse publishers, directory teardowns and ads.txt evidence.","imageUrl","userId","f5be24e4-c798-4816-91e5-037b60b986f5","pinnedFileId","01a0b061-5e61-74b2-9c7e-7b033afc92fc","pinPreview","pinMigrated","screenshotVersion","likeCount","orgId","7f67379d-2d38-49bd-81ac-2c7f92a739e2","tursoName","7e58a892b13048959cbcde82614ff56d","tursoHostname","7e58a892b13048959cbcde82614ff56d-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789665692456],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_9":10,"_104":100},"org",{"_9":10,"_98":-5,"_99":100,"_55":56,"_101":102,"_103":42},"branch",{"_35":88,"_11":89,"_22":28,"_90":45,"_91":-5,"_92":-5,"_93":-5,"_94":-5,"_95":-5,"_96":12,"_97":10},"isOwned",59,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18ycWxkOWxiOE00SkVtZENmRWM4YWxLWTRVUFMifQ","tier","free","canRestrictVals","profileImageUrl","ctx",{"_81":-5,"_138":139,"_140":141,"_142":143,"_144":42,"_145":42,"_146":42,"_147":32,"_148":-5,"_149":-7,"_150":-7,"_151":36,"_152":42,"_153":42,"_154":-5,"_155":42,"_156":42,"_157":42,"_158":-5},"ENV",{"_109":110,"_111":112,"_113":89,"_114":115,"_116":117,"_118":119,"_120":121,"_122":123,"_124":125,"_126":127,"_128":129,"_130":131,"_132":133,"_134":135,"_136":137},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","71c1b20804b3232a7e513be260cfffc1932a9e97","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town","orgs",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]