[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_363":364,"_365":366},"routes/_app.x.$handle.$name",{"_11":12,"_13":14,"_15":16,"_17":18,"_41":-7,"_154":-5,"_19":155,"_156":157,"_158":159,"_160":161,"_162":163,"_21":164,"_165":-5,"_166":-5,"_167":24,"_168":169,"_170":113,"_171":-5,"_172":-5,"_173":174},"routes/_app.x.$handle.$name.branch.$branch.history_.$",{"_11":12,"_13":14,"_15":16,"_17":18,"_19":20,"_21":22,"_23":24,"_25":26,"_27":28,"_29":24,"_30":-5},"handle","logan1","name","housing","branchName","main","path","cron/sources.ts","project",{"_35":100,"_13":14,"_39":101,"_102":-5,"_103":24,"_104":105,"_106":105,"_107":108,"_109":-5,"_30":72,"_110":111,"_112":113,"_114":-5,"_115":-5,"_116":26,"_43":44,"_117":118,"_119":120,"_121":122,"_123":-5,"_124":125,"_126":125,"_127":128,"_129":128,"_130":128,"_131":125,"_132":125,"_56":133,"_134":135,"_136":137,"_138":24},"file",{"_66":67,"_13":68,"_69":70,"_71":24,"_99":-5},"isDirectory",false,"page",0,"history",[31,32,33,34],"hasNext","userId",{"_35":91,"_37":38,"_39":92,"_41":93,"_43":44,"_45":94,"_47":-5,"_48":-5,"_49":-5,"_50":-5,"_51":-5,"_52":53,"_54":55,"_56":95,"_21":96,"_59":-5,"_60":-5,"_61":-5,"_62":-5,"_63":97},{"_35":83,"_37":38,"_39":84,"_41":85,"_43":44,"_45":86,"_47":-5,"_48":-5,"_49":-5,"_50":-5,"_51":-5,"_52":53,"_54":55,"_56":87,"_21":88,"_59":-5,"_60":-5,"_61":-5,"_62":-5,"_63":89},{"_35":75,"_37":38,"_39":76,"_41":77,"_43":44,"_45":78,"_47":-5,"_48":-5,"_49":-5,"_50":-5,"_51":-5,"_52":53,"_54":55,"_56":79,"_21":80,"_59":-5,"_60":-5,"_61":-5,"_62":-5,"_63":81},{"_35":36,"_37":38,"_39":40,"_41":42,"_43":44,"_45":46,"_47":-5,"_48":-5,"_49":-5,"_50":-5,"_51":-5,"_52":53,"_54":55,"_56":57,"_21":58,"_59":-5,"_60":-5,"_61":-5,"_62":-5,"_63":64},"id","7512c4d1-2bf8-4129-b2d3-47ea5526f7fb","branchId","a4c5838e-3adb-4640-ab71-e9f3d2b72f0a","createdAt",["D",1789298913362],"version",26,"orgId","01a09786-a61b-74b5-8910-af3116d3de5c","parentId","26d23258-0842-45c2-b2f6-5f4f97c626f4","incomingCommitId","aiChatMessageId","aiChatThreadId","aiChatThreadUserId","aiChatThreadTitle","source","mcp","clientLabel","Claude Code (plugin:valtown:valtown)","user",{"_35":72,"_43":44,"_11":12,"_73":74},{"_66":67,"_13":68,"_69":70,"_71":24},"pull","merge","revert","multiple","commits",[65],{"_35":36,"_37":38,"_39":40,"_41":42,"_43":44,"_45":46,"_47":-5,"_48":-5,"_49":-5,"_50":-5,"_51":-5,"_52":53,"_54":55,"_56":57,"_21":58,"_59":-5,"_60":-5,"_61":-5},"fileId","40cdd694-af66-11f1-ad84-1607ee4eb77e","sources.ts","fileType","interval","deleted","87445237-7d14-45c7-b3a2-72e031f10ec1","avatar","https://lh3.googleusercontent.com/a/ACg8ocIFxddfJftSiTFZN49YShT3nY7q8GOQ7dXYM2vJ6XkajnQMGA=s96-c","aa4a47ba-cc91-4760-a05a-f21768939667",["D",1789299285354],31,"0edb9702-00ad-4fba-a01e-3649e07235b0",{"_35":72,"_43":44,"_11":12,"_73":74},{"_66":67,"_13":68,"_69":70,"_71":24},[82],{"_35":75,"_37":38,"_39":76,"_41":77,"_43":44,"_45":78,"_47":-5,"_48":-5,"_49":-5,"_50":-5,"_51":-5,"_52":53,"_54":55,"_56":79,"_21":80,"_59":-5,"_60":-5,"_61":-5},"e19d8f6d-a53e-4239-ba02-3b647c45d792",["D",1789302854749],38,"84e5ba3b-225b-48dd-bbc0-6e651dc3e955",{"_35":72,"_43":44,"_11":12,"_73":74},{"_66":67,"_13":68,"_69":70,"_71":24},[90],{"_35":83,"_37":38,"_39":84,"_41":85,"_43":44,"_45":86,"_47":-5,"_48":-5,"_49":-5,"_50":-5,"_51":-5,"_52":53,"_54":55,"_56":87,"_21":88,"_59":-5,"_60":-5,"_61":-5},"8115ca75-605f-464c-9d57-cdffdfb51210",["D",1789307296223],62,"36f8f97b-2426-4b23-ba79-2198e06fcabe",{"_35":72,"_43":44,"_11":12,"_73":74},{"_66":67,"_13":68,"_69":70,"_71":24},[98],{"_35":91,"_37":38,"_39":92,"_41":93,"_43":44,"_45":94,"_47":-5,"_48":-5,"_49":-5,"_50":-5,"_51":-5,"_52":53,"_54":55,"_56":95,"_21":96,"_59":-5,"_60":-5,"_61":-5},"deletedAtVersion","b7ccec7e-ebe9-4364-ba98-e367c8d8849b",["D",1789248987477],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","DC housing search pipeline","imageUrl","pinnedFileId","0544e21c-aef2-11f1-a6f0-1607ee4eb77e","pinPreview",true,"pinMigrated","screenshotVersion","likeCount","tursoName","b7ccec7eebe94364ba98e367c8d8849b","tursoHostname","b7ccec7eebe94364ba98e367c8d8849b-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789519256886],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm",{"_11":12,"_153":74},"org",{"_11":12,"_148":149,"_73":74,"_30":72,"_150":151,"_152":24},"branch",{"_41":139,"_13":16,"_35":38,"_140":100,"_141":-5,"_142":-5,"_143":-5,"_144":-5,"_145":-5,"_146":14,"_147":12},"isOwned",71,"projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","displayName","Logan","tier","free","canRestrictVals","profileImageUrl","adoptOrgId",{"_35":100,"_13":14,"_39":358,"_102":-5,"_103":24,"_104":105,"_106":105,"_107":108,"_109":-5,"_30":72,"_110":111,"_112":113,"_114":-5,"_115":-5,"_116":26,"_43":44,"_117":118,"_119":120,"_121":359,"_123":-5,"_124":125,"_126":125,"_127":128,"_129":128,"_130":128,"_131":125,"_132":125,"_56":360,"_134":361,"_136":362,"_138":24},"tree",[198,203,210,212],"files",[198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216],"counts",{"_186":26,"_187":188,"_189":26,"_190":191,"_192":193},"contents",[],{"_35":175,"_13":68,"_66":67,"_45":176,"_37":38,"_177":93,"_178":113,"_39":179,"_41":180,"_69":70,"_181":182,"_71":24,"_183":-5,"_17":184},"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},"6530a678-015a-407c-8eb5-aefba5bf288b","041eaecc-aef2-11f1-9a8e-1607ee4eb77e","branchVersion","latest",["D",1789307296223],3,"content","// Generic multi-source poller (cron, every 30 min). Each run processes the registry entries whose\r\n// `every` interval has elapsed (most overdue first) inside a 45 s budget; the rest wait for the next\r\n// run. Every hit goes through upsertListing (dedupe, score, push) with source = registry key.\r\n// State per source lives in `source_state`; results in poll_log as job='sources:'.\r\n// Admin: ?job=sources ?job=sources&only= ?job=sources&force=1\r\nimport { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\r\nimport {\r\n ensureSchema, upsertListing, logPoll, inferArea, inferPark, inferType, inferLL, inferDog, inferBeds, clampBeds,\r\n UA, sleep, type Area, type Listing,\r\n} from \"../lib/core.ts\";\r\nimport { REGISTRY, PARSERS, nestfullyDetail, FULL_INVENTORY_PARSERS, type Source, type Hit } from \"../lib/sources.ts\";\r\nimport { applyMissed } from \"../lib/enrich.ts\";\r\n\r\nconst TIME_BUDGET_MS = 45_000;\r\nconst PRICE_LO = 1800, PRICE_HI = 2800;\r\nconst AREA_BY_ZIP: Record = {\r\n \"20011\": \"Petworth\", \"20010\": \"Park View / 16th St Hts\", \"20012\": \"Brightwood\", \"22206\": \"Fairlington / Shirlington\",\r\n \"22204\": \"Columbia Pike\", \"22301\": \"Del Ray\", \"22305\": \"Del Ray\", \"20781\": \"Hyattsville\", \"20782\": \"Hyattsville\",\r\n};\r\n\r\nasync function fetchText(url: string, src: Source): Promise<{ status: number; text: string }> {\r\n const headers: Record = {\r\n \"user-agent\": UA,\r\n \"accept\": src.accept === \"json\" ? \"application/json,text/plain,*/*\" : src.accept === \"xml\" ? \"application/rss+xml,application/atom+xml,application/xml,text/xml,*/*\" : \"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\",\r\n \"accept-language\": \"en-US,en;q=0.9\",\r\n ...(src.headers || {}),\r\n };\r\n let res = await fetch(url, { headers, redirect: \"follow\" });\r\n if (res.status === 403 || res.status === 429) { await sleep(3000); res = await fetch(url, { headers, redirect: \"follow\" }); } // one polite retry\r\n return { status: res.status, text: res.ok ? await res.text() : \"\" };\r\n}\r\n\r\nexport async function runSource(src: Source, started: number, opts: { detailBudget: { n: number } }): Promise<{ fetched: number; inserted: number; updated: number; pushed: number; ok: boolean; detail: string }> {\r\n const stats = { fetched: 0, inserted: 0, updated: 0, pushed: 0 };\r\n const notes: string[] = [];\r\n const kind = src.kind || \"listing\";\r\n const parser = PARSERS[src.parser];\r\n const hits: Hit[] = [];\r\n let failed = 0, pages = 0, complete = true; // complete = every page fetched, nothing truncated (needed for the missed check)\r\n const ctxZip = src.zip;\r\n const ctxArea = ctxZip ? AREA_BY_ZIP[ctxZip] : undefined;\r\n\r\n for (const base of src.urls) {\r\n let url = base;\r\n for (let page = 1; page <= (src.maxPages || 1); page++) {\r\n if (Date.now() - started > TIME_BUDGET_MS) { notes.push(\"budget hit before all pages\"); complete = false; break; }\r\n if (page > 1) { if (!src.pageUrl) { complete = false; break; } url = src.pageUrl(base, page); }\r\n try {\r\n const { status, text } = await fetchText(url, src);\r\n pages++;\r\n if (status === 404 && page > 1) break; // past the last page\r\n if (status !== 200 || !text) { failed++; notes.push(`http ${status} ${url.slice(0, 80)}`); break; }\r\n const zipFromUrl = (url.match(/\\b(2001[012]|2220[46]|2230[15]|2078[12])\\b/) || [])[1];\r\n const r = parser(text, { url, zip: ctxZip || zipFromUrl, area: ctxArea || (zipFromUrl ? AREA_BY_ZIP[zipFromUrl] : undefined) });\r\n hits.push(...r.hits);\r\n if (!r.more) break;\r\n if (page === (src.maxPages || 1)) complete = false; // more pages exist than we fetch - inventory not complete\r\n } catch (e) { failed++; complete = false; notes.push(`error ${String(e).slice(0, 80)} ${url.slice(0, 60)}`); break; }\r\n await sleep(src.spacingMs ?? 1500);\r\n }\r\n }\r\n stats.fetched = hits.length;\r\n\r\n // dedupe within the run, then upsert\r\n const seen = new Set();\r\n let skipped = 0, details = 0;\r\n for (const h of hits) {\r\n if (seen.has(h.extId)) continue; seen.add(h.extId);\r\n if (Date.now() - started > TIME_BUDGET_MS) { notes.push(\"budget hit during upserts; remainder next run\"); break; }\r\n const zip = h.zip || (h.text.match(/\\b(2001[012]|2220[46]|2230[15]|2078[12])\\b/) || [])[1] || null;\r\n const area = inferArea([zip || \"\", h.addr || \"\", h.text].join(\" \"), h.lat, h.lon, ctxArea);\r\n if (!area) { skipped++; continue; } // outside our areas (AppFolio full-inventory pages etc.)\r\n if (zip && !AREA_BY_ZIP[zip]) { skipped++; continue; }\r\n\r\n let bedsRaw = h.beds ?? inferBeds(h.text);\r\n if (kind === \"listing\") {\r\n if (bedsRaw === 0) { skipped++; continue; } // studio\r\n if (h.price != null && (h.price < PRICE_LO || h.price > PRICE_HI)) { skipped++; continue; }\r\n }\r\n let beds = kind === \"listing\" ? clampBeds(bedsRaw) : 1;\r\n if (beds === 1 && /\\bden\\b|\\boffice\\b|bonus room/i.test(h.text)) beds = 1.5;\r\n\r\n let park = h.park ?? inferPark(h.text);\r\n let type = h.type ?? inferType(h.text);\r\n let dog: boolean | null = h.dog !== undefined ? h.dog : inferDog(h.text);\r\n let ll = h.ll ?? inferLL(h.text, src.ll);\r\n let note = h.note || \"\";\r\n let addr = h.addr;\r\n\r\n // Nestfully stage 2: structured Parking / Pets / Availability from the detail page, new listings only\r\n if (h.detailUrl && kind === \"listing\" && opts.detailBudget.n < (src.detailCap ?? 0)) {\r\n const known = await sqlite.execute({ sql: `select 1 from listings where source=? and external_id=? limit 1`, args: [src.key, h.extId] });\r\n if (!known.rows.length) {\r\n opts.detailBudget.n++; details++;\r\n try {\r\n await sleep(src.spacingMs ?? 3000);\r\n const { status, text } = await fetchText(h.detailUrl, src);\r\n if (status === 200) {\r\n const d = nestfullyDetail(text);\r\n if (d.park) park = d.park;\r\n if (d.pets != null) dog = d.pets;\r\n if (d.beds != null && bedsRaw == null) beds = clampBeds(d.beds);\r\n if (d.office) ll = /property management|management|pm\\b|rentals? llc/i.test(d.office) ? \"pm\" : \"individual\"; // agent-listed unit = owner behind a brokerage\r\n note = [note, d.mls ? `MLS ${d.mls}` : \"\", d.parking ? `Parking: ${d.parking}` : \"\", d.pets != null ? `Pets: ${d.pets ? \"Yes\" : \"No\"}` : \"\", d.avail ? `avail ${d.avail}` : \"\", d.development ? d.development : \"\", d.office ? `office ${d.office}` : \"\"].filter(Boolean).join(\" | \");\r\n } else notes.push(`detail ${h.extId} http ${status}`);\r\n } catch (e) { notes.push(`detail ${h.extId} error ${String(e).slice(0, 60)}`); }\r\n }\r\n }\r\n\r\n try {\r\n const r = await upsertListing({\r\n addr, area, price: h.price, beds, source: src.key, externalId: h.extId, url: h.url,\r\n park, type, ll, dog, notes: note.slice(0, 400), lat: h.lat ?? null, lon: h.lon ?? null, raw: h.text.slice(0, 4000), postedAt: h.posted ?? null,\r\n }, { push: kind === \"listing\" && src.push !== false, pushMin: src.pushMin });\r\n if (r.action === \"inserted\") stats.inserted++; else stats.updated++;\r\n if (r.pushed) stats.pushed++;\r\n } catch (e) { notes.push(`upsert ${h.extId} ${String(e).slice(0, 60)}`); }\r\n }\r\n const ok = failed === 0 && pages > 0;\r\n // Part 4: full-inventory sources - after a successful, complete fetch, rows of this source not in the result are missing\r\n if (ok && complete && FULL_INVENTORY_PARSERS.has(src.parser) && hits.length && !notes.some((n) => n.startsWith(\"budget hit\"))) {\r\n try {\r\n const m = await applyMissed(src.key, new Set(hits.map((h) => h.extId)));\r\n if (m.inc || m.dead) notes.push(`missed +${m.inc}, dead ${m.dead}`);\r\n } catch (e) { notes.push(`missed check failed ${String(e).slice(0, 60)}`); }\r\n }\r\n const detail = `${pages} pages, ${hits.length} hits, ${skipped} skipped, ${details} detail fetches, ${failed} failed. ${notes.join(\"; \")}`.slice(0, 1900);\r\n return { ...stats, ok, detail };\r\n}\r\n\r\nexport async function ensureSourceState() {\r\n await sqlite.execute(`create table if not exists source_state (\r\n source text primary key, last_run text, last_ok text, fail_count integer not null default 0,\r\n runs integer not null default 0, inserted_total integer not null default 0, detail text)`);\r\n}\r\n\r\nexport default async function run(opts: { only?: string; force?: boolean } = {}) {\r\n const started = Date.now();\r\n await ensureSchema(); await ensureSourceState();\r\n const state = await sqlite.execute(`select source, last_run, fail_count from source_state`);\r\n const last = new Map();\r\n for (const r of state.rows as any[]) last.set(r.source, { last_run: r.last_run, fail_count: Number(r.fail_count) });\r\n\r\n // pick: enabled, due (or forced/only), most overdue first\r\n const now = Date.now();\r\n const due = REGISTRY.filter((s) => s.enabled !== false && (!opts.only || s.key === opts.only)).map((s) => {\r\n const lr = last.get(s.key)?.last_run; const age = lr ? (now - Date.parse(lr)) / 60000 : Infinity;\r\n return { s, overdue: age - s.every };\r\n }).filter((x) => opts.force || opts.only || x.overdue >= 0).sort((a, b) => b.overdue - a.overdue);\r\n\r\n const results: Record = {};\r\n const detailBudget = { n: 0 };\r\n const groupsRan = new Set();\r\n for (const { s } of due) {\r\n if (Date.now() - started > TIME_BUDGET_MS - 5000) { results._skipped = `${due.length - Object.keys(results).length} sources deferred to next run (time budget)`; break; }\r\n if (s.group && !opts.only) { if (groupsRan.has(s.group)) continue; groupsRan.add(s.group); }\r\n const r = await runSource(s, started, { detailBudget });\r\n results[s.key] = r;\r\n await logPoll(`sources:${s.key}`, r.ok, r, r.detail);\r\n const fc = r.ok ? 0 : (last.get(s.key)?.fail_count || 0) + 1;\r\n await sqlite.execute({\r\n sql: `insert into source_state (source, last_run, last_ok, fail_count, runs, inserted_total, detail) values (?,?,?,?,1,?,?)\r\n on conflict(source) do update set last_run=excluded.last_run, last_ok=coalesce(excluded.last_ok, source_state.last_ok), fail_count=?, runs=runs+1, inserted_total=inserted_total+?, detail=excluded.detail`,\r\n args: [s.key, new Date().toISOString(), r.ok ? new Date().toISOString() : null, fc, r.inserted, r.detail.slice(0, 500), fc, r.inserted],\r\n });\r\n }\r\n const summary = { ran: Object.keys(results).filter((k) => !k.startsWith(\"_\")).length, due: due.length, seconds: Math.round((Date.now() - started) / 1000), results };\r\n console.log(\"sources\", JSON.stringify(summary).slice(0, 2000));\r\n return summary;\r\n}\r\n","label",[185,68],"cron","pulls","branches",1,"forks","envVars",{"_194":196,"_197":26},"connections",{"_194":26,"_195":26},"count","unfilledCount",2,"emptyCount",{"_35":354,"_13":329,"_66":322,"_45":-5,"_37":38,"_177":196,"_178":113,"_39":355,"_41":26,"_69":256,"_181":-5,"_71":24,"_183":-5,"_225":356,"_17":357},{"_35":347,"_13":348,"_66":111,"_45":322,"_37":38,"_177":349,"_178":113,"_39":350,"_41":196,"_69":325,"_181":351,"_71":24,"_183":-5,"_225":352,"_17":353},{"_35":338,"_13":339,"_66":340,"_45":322,"_37":38,"_177":341,"_178":113,"_39":342,"_41":343,"_69":325,"_181":344,"_71":24,"_183":-5,"_225":345,"_17":346},{"_35":330,"_13":331,"_66":332,"_45":322,"_37":38,"_177":333,"_178":113,"_39":334,"_41":196,"_69":325,"_181":335,"_71":24,"_183":-5,"_225":336,"_17":337},{"_35":319,"_13":320,"_66":321,"_45":322,"_37":38,"_177":323,"_178":113,"_39":324,"_41":180,"_69":325,"_181":326,"_71":24,"_183":-5,"_225":327,"_17":328},{"_35":315,"_13":185,"_66":176,"_45":-5,"_37":38,"_177":180,"_178":113,"_39":316,"_41":26,"_69":256,"_181":-5,"_71":24,"_183":-5,"_225":317,"_17":318},{"_35":307,"_13":308,"_66":309,"_45":176,"_37":38,"_177":310,"_178":113,"_39":311,"_41":26,"_69":223,"_181":312,"_71":24,"_183":-5,"_225":313,"_17":314},{"_35":299,"_13":300,"_66":301,"_45":176,"_37":38,"_177":302,"_178":113,"_39":303,"_41":180,"_69":70,"_181":304,"_71":24,"_183":-5,"_225":305,"_17":306},{"_35":291,"_13":292,"_66":293,"_45":176,"_37":38,"_177":294,"_178":113,"_39":295,"_41":233,"_69":70,"_181":296,"_71":24,"_183":-5,"_225":297,"_17":298},{"_35":284,"_13":238,"_66":285,"_45":176,"_37":38,"_177":286,"_178":113,"_39":287,"_41":188,"_69":70,"_181":288,"_71":24,"_183":-5,"_225":289,"_17":290},{"_35":276,"_13":277,"_66":278,"_45":176,"_37":38,"_177":279,"_178":113,"_39":280,"_41":196,"_69":70,"_181":281,"_71":24,"_183":-5,"_225":282,"_17":283},{"_35":175,"_13":68,"_66":67,"_45":176,"_37":38,"_177":93,"_178":113,"_39":273,"_41":180,"_69":70,"_181":182,"_71":24,"_183":-5,"_225":274,"_17":275},{"_35":269,"_13":265,"_66":262,"_45":-5,"_37":38,"_177":263,"_178":113,"_39":270,"_41":26,"_69":256,"_181":-5,"_71":24,"_183":-5,"_225":271,"_17":272},{"_35":259,"_13":260,"_66":261,"_45":262,"_37":38,"_177":263,"_178":113,"_39":264,"_41":26,"_69":265,"_181":266,"_71":24,"_183":-5,"_225":267,"_17":268},{"_35":254,"_13":228,"_66":220,"_45":-5,"_37":38,"_177":188,"_178":113,"_39":255,"_41":26,"_69":256,"_181":-5,"_71":24,"_183":-5,"_225":257,"_17":258},{"_35":245,"_13":246,"_66":247,"_45":220,"_37":38,"_177":248,"_178":113,"_39":249,"_41":250,"_69":21,"_181":251,"_71":24,"_183":-5,"_225":252,"_17":253},{"_35":237,"_13":238,"_66":239,"_45":220,"_37":38,"_177":240,"_178":113,"_39":241,"_41":196,"_69":223,"_181":242,"_71":24,"_183":-5,"_225":243,"_17":244},{"_35":229,"_13":68,"_66":230,"_45":220,"_37":38,"_177":231,"_178":113,"_39":232,"_41":233,"_69":223,"_181":234,"_71":24,"_183":-5,"_225":235,"_17":236},{"_35":217,"_13":218,"_66":219,"_45":220,"_37":38,"_177":221,"_178":113,"_39":222,"_41":188,"_69":223,"_181":224,"_71":24,"_183":-5,"_225":226,"_17":227},"a733d1b9-01d9-4909-bca8-894bbf6883d2","vibe.ts","e570c70e-af77-11f1-95c6-1607ee4eb77e","03892172-aef2-11f1-9a6d-1607ee4eb77e",63,["D",1789308398032],"script","// Neighborhood vibe layer (Part 5). Geocoding (Census batch + DC address points fallback), OSM POIs\r\n// (loaded by phase3/pois.ts -> POST {pois:[...]}), per-listing vibe metrics and a 0-5 vibe_score.\r\n// The vibe_score is a SEPARATE number - it never feeds score() (fit) and never gates a push.\r\nimport { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\r\nimport { streetKey, today, sleep, UA } from \"./core.ts\";\r\n\r\n// Nightlife strips (coordinates checked on Google Maps satellite/road view 2026-09-13 - each sits on the strip itself)\r\nexport const STRIPS: { name: string; lat: number; lon: number; area: string }[] = [\r\n { name: \"Upshur St NW @ 9th\", lat: 38.9424, lon: -77.0238, area: \"Petworth\" },\r\n { name: \"Georgia Ave @ Petworth Metro\", lat: 38.9367, lon: -77.0242, area: \"Petworth\" },\r\n { name: \"Kennedy St NW @ 5th\", lat: 38.9564, lon: -77.0195, area: \"Brightwood\" },\r\n { name: \"11th St NW Columbia Heights\", lat: 38.9290, lon: -77.0275, area: \"Park View / 16th St Hts\" },\r\n { name: \"Mt Vernon Ave Del Ray\", lat: 38.8285, lon: -77.0585, area: \"Del Ray\" },\r\n { name: \"Shirlington Village\", lat: 38.8405, lon: -77.0890, area: \"Fairlington / Shirlington\" },\r\n { name: \"Columbia Pike @ Walter Reed Dr\", lat: 38.8622, lon: -77.0906, area: \"Columbia Pike\" },\r\n { name: \"Route 1 Hyattsville Arts District\", lat: 38.9556, lon: -76.9410, area: \"Hyattsville\" },\r\n];\r\nexport const SHIRLINGTON = { lat: 38.8404, lon: -77.0872 }; // friend anchor\r\n// venues still tagged in OSM but closed (checked 2026-09-13): dropped from counts and names\r\nconst CLOSED = new Set([\"Slash Run\", \"Cinder Neighborhood Barbeque\", \"Cinder BBQ\"]);\r\n\r\n// Two-sentence blurbs. Venues verified 2026-09-13 (WebSearch / OSM): Timber Pizza 809 Upshur, Looking Glass Lounge 3634 Georgia,\r\n// Reliable Tavern 3655 Georgia, Brightwood Pizza & Bottle by ANXO 711 Kennedy, Wonderland Ballroom 11th & Kenyon, Cheesetique +\r\n// Pork Barrel BBQ on Mt Vernon Ave, Busboys and Poets + Samuel Beckett's in Shirlington, Lost Dog Cafe on the Pike, Franklins +\r\n// Busboys and Poets on Route 1. Slash Run (closed 2024) and Cinder (closed since the Jan 2026 fire) are deliberately not named.\r\nexport const AREA_BLURBS: Record = {\r\n \"Petworth\": \"Two short strips - Upshur St at 9th (Timber Pizza and a handful of taverns) and Georgia Ave around the Metro (Looking Glass Lounge, Reliable Tavern) - with quiet rowhouse blocks in every direction. The housing stock is exactly the search target (rowhouses with alley garages and English basements), so the trade-off is only that the nightlife is small-scale and closes early on weeknights.\",\r\n \"Park View / 16th St Hts\": \"Park View leans on Georgia Ave and, a 10-minute walk south, the 11th St corridor in Columbia Heights (Wonderland Ballroom, Red Rocks) - the densest bar strip in the search. Rowhouses and basement units are plentiful on the Park View side while 16th St Heights is larger detached and semi-detached houses with driveways; the trade-off is that the best-vibe blocks near 11th St price above budget and garages get rarer as you go south.\",\r\n \"Brightwood\": \"Kennedy St NW is the strip - Brightwood Pizza & Bottle by ANXO is the anchor and the rest is carry-outs and a slow trickle of new spots. Brightwood and Brightwood Park are the deepest pool of alley-garage rowhouses at the lowest DC prices in the search, at the cost of the thinnest nightlife and a longer ride to Shirlington.\",\r\n \"Fairlington / Shirlington\": \"Shirlington Village (Busboys and Poets, Samuel Beckett's, Palette 22) is a real restaurant row and it is the friend anchor - vibe here is walking distance to dinner, not late-night bars. Fairlington is 1940s brick condo townhouses: dog-friendly, green, garage-free (some carports/assigned spaces), so a Fairlington match trades the garage bonus for the strip and the friends.\",\r\n \"Columbia Pike\": \"The Pike around Walter Reed Dr (Lost Dog Cafe, Rebellion on the Pike, Penrose Square) is a bus-corridor main street: plenty of restaurants, a few bars, no Metro. Housing is garden apartments and some townhouses with surface parking; garages are rare, but prices and dog policies are friendlier than DC and Shirlington is a short drive.\",\r\n \"Del Ray\": \"Mt Vernon Ave is a classic small-town main street (Cheesetique, Pork Barrel BBQ, Taqueria Poblano) - family-hour busy, quiet by 10. Detached and semi-detached houses on small lots, many with detached garages off driveways, and Potomac Yard Metro at the south end; the trade-off is that whole-house rentals in budget are scarce and basement units rarer than in DC.\",\r\n \"Hyattsville\": \"The Route 1 Arts District (Franklins, Busboys and Poets, Pizzeria Paradiso, Vigilante Coffee) is a compact strip with two Metro stations within a mile. Housing is 1920s-40s bungalows and Cape Cods with driveways and detached garages at the lowest prices in the search; the trade-off is a longer commute to everything in Virginia and a strip that is one street, not a neighborhood of them.\",\r\n};\r\n\r\nconst R_MI = 3958.8;\r\nexport function haversineMi(lat1: number, lon1: number, lat2: number, lon2: number): number {\r\n const toR = (d: number) => d * Math.PI / 180;\r\n const dLat = toR(lat2 - lat1), dLon = toR(lon2 - lon1);\r\n const a = Math.sin(dLat / 2) ** 2 + Math.cos(toR(lat1)) * Math.cos(toR(lat2)) * Math.sin(dLon / 2) ** 2;\r\n return 2 * R_MI * Math.asin(Math.sqrt(a));\r\n}\r\nexport type Poi = { id: string; kind: string; name: string; lat: number; lon: number; area: string };\r\nexport type Vibe = {\r\n strip: string; strip_mi: number; strip_min: number; bars: number; eats: number; bar_names: string;\r\n metro: string | null; metro_mi: number | null; dog_park: string | null; dog_mi: number | null; grocery_mi: number | null;\r\n shirlington_min: number; vibe_score: number; line: string;\r\n};\r\nconst r1 = (x: number) => Math.round(x * 10) / 10;\r\n// pure: metrics + 0-5 score from a coordinate and the POI set\r\nexport function computeVibe(lat: number, lon: number, pois: Poi[]): Vibe {\r\n const d = (p: { lat: number; lon: number }) => haversineMi(lat, lon, p.lat, p.lon);\r\n const strip = STRIPS.map((s) => ({ s, mi: d(s) })).sort((a, b) => a.mi - b.mi)[0];\r\n const live = pois.filter((p) => !CLOSED.has(p.name));\r\n const bars = live.filter((p) => p.kind === \"bar\").map((p) => ({ p, mi: d(p) })).filter((x) => x.mi <= 0.5).sort((a, b) => a.mi - b.mi);\r\n const eats = live.filter((p) => p.kind === \"eat\" && d(p) <= 0.5).length;\r\n const nearest = (kind: string, maxMi: number) => { const c = live.filter((p) => p.kind === kind).map((p) => ({ p, mi: d(p) })).sort((a, b) => a.mi - b.mi)[0]; return c && c.mi <= maxMi ? c : null; };\r\n const metro = nearest(\"metro\", 3), dog = nearest(\"dog\", 1.5), groc = nearest(\"grocery\", 2);\r\n const shirl = Math.round(haversineMi(lat, lon, SHIRLINGTON.lat, SHIRLINGTON.lon) * 2.8);\r\n let score = 0;\r\n if (strip.mi <= 0.5) score += 2; else if (strip.mi <= 0.8) score += 1;\r\n if (bars.length >= 5) score += 1;\r\n if (metro && metro.mi <= 0.6) score += 1;\r\n if (dog && dog.mi <= 0.5) score += 1;\r\n const stripMin = Math.round(strip.mi * 20);\r\n const line = `Vibe ${score}/5 - ${strip.s.name} ${r1(strip.mi)} mi (${stripMin} min) - ${bars.length} bar${bars.length === 1 ? \"\" : \"s\"} - ${metro ? `${metro.p.name} ${r1(metro.mi)} mi` : \"no Metro <3 mi\"} - ~${shirl} min to Shirlington`;\r\n return {\r\n strip: strip.s.name, strip_mi: r1(strip.mi), strip_min: stripMin, bars: bars.length, eats,\r\n bar_names: bars.slice(0, 3).map((x) => x.p.name).filter(Boolean).join(\", \"),\r\n metro: metro ? metro.p.name : null, metro_mi: metro ? r1(metro.mi) : null,\r\n dog_park: dog ? dog.p.name : null, dog_mi: dog ? r1(dog.mi) : null, grocery_mi: groc ? r1(groc.mi) : null,\r\n shirlington_min: shirl, vibe_score: score, line,\r\n };\r\n}\r\n\r\n// ---------- schema ----------\r\nexport async function ensureVibeSchema() {\r\n await sqlite.batch([\r\n `create table if not exists pois (id text primary key, kind text not null, name text not null default '', lat real not null, lon real not null, area text)`,\r\n `create table if not exists vibe (rowid_pk integer primary key, strip text, strip_mi real, strip_min integer, bars integer, eats integer, bar_names text,\r\n metro text, metro_mi real, dog_park text, dog_mi real, grocery_mi real, shirlington_min integer, vibe_score integer, line text, computed text)`,\r\n ]);\r\n}\r\nlet poiCache: { at: number; pois: Poi[] } | null = null;\r\nexport async function loadPois(): Promise {\r\n if (poiCache && Date.now() - poiCache.at < 10 * 60_000) return poiCache.pois;\r\n const r = await sqlite.execute(`select id, kind, name, lat, lon, area from pois`);\r\n poiCache = { at: Date.now(), pois: (r.rows as any[]).map((x) => ({ id: String(x.id), kind: String(x.kind), name: String(x.name || \"\"), lat: Number(x.lat), lon: Number(x.lon), area: String(x.area || \"\") })) };\r\n return poiCache.pois;\r\n}\r\nexport async function upsertVibe(rowid: number, v: Vibe) {\r\n await sqlite.execute({\r\n sql: `insert or replace into vibe (rowid_pk, strip, strip_mi, strip_min, bars, eats, bar_names, metro, metro_mi, dog_park, dog_mi, grocery_mi, shirlington_min, vibe_score, line, computed) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,\r\n args: [rowid, v.strip, v.strip_mi, v.strip_min, v.bars, v.eats, v.bar_names, v.metro, v.metro_mi, v.dog_park, v.dog_mi, v.grocery_mi, v.shirlington_min, v.vibe_score, v.line, today()],\r\n });\r\n await sqlite.execute({ sql: `update listings set vibe_score=? where rowid_pk=?`, args: [v.vibe_score, rowid] });\r\n}\r\n// vibe for a freshly upserted row that already has coordinates (Craigslist, add endpoint); no-op without POIs\r\nexport async function vibeForRow(rowid: number, lat: number | null | undefined, lon: number | null | undefined): Promise {\r\n if (lat == null || lon == null) return null;\r\n const pois = await loadPois(); if (!pois.length) return null;\r\n const v = computeVibe(Number(lat), Number(lon), pois);\r\n await upsertVibe(rowid, v);\r\n return v;\r\n}\r\n\r\n// ---------- geocoding ----------\r\nconst SYNTHETIC_RE = /^[A-Z0-9-]+ [A-Za-z0-9]+$/; // \"NESTFULLY-20782 428581813\", \"CRAIGSLIST kdRP...\" - ids with no address\r\nconst CITY: Record = {\r\n \"Petworth\": [\"Washington\", \"DC\"], \"Park View / 16th St Hts\": [\"Washington\", \"DC\"], \"Brightwood\": [\"Washington\", \"DC\"],\r\n \"Fairlington / Shirlington\": [\"Arlington\", \"VA\"], \"Columbia Pike\": [\"Arlington\", \"VA\"], \"Del Ray\": [\"Alexandria\", \"VA\"], \"Hyattsville\": [\"Hyattsville\", \"MD\"],\r\n};\r\nexport function geocodable(id: string, addrNorm: string): boolean {\r\n return !!addrNorm && !SYNTHETIC_RE.test(id) && /^\\d+[A-Z]?\\s+\\S/.test(addrNorm) && !/\\bNEAR\\b/.test(addrNorm);\r\n}\r\nfunction csvCell(s: string) { return `\"${String(s).replace(/\"/g, \"'\")}\"`; }\r\nexport async function censusBatch(rows: { rowid: number; street: string; city: string; state: string; zip: string }[]): Promise> {\r\n const out = new Map();\r\n if (!rows.length) return out;\r\n const csv = rows.map((r) => [r.rowid, csvCell(r.street), csvCell(r.city), r.state, r.zip].join(\",\")).join(\"\\n\");\r\n const fd = new FormData();\r\n fd.append(\"addressFile\", new Blob([csv], { type: \"text/csv\" }), \"addresses.csv\");\r\n fd.append(\"benchmark\", \"Public_AR_Current\");\r\n const res = await fetch(\"https://geocoding.geo.census.gov/geocoder/locations/addressbatch\", { method: \"POST\", body: fd, headers: { \"user-agent\": UA } });\r\n if (!res.ok) throw new Error(`census http ${res.status}`);\r\n const text = await res.text();\r\n for (const line of text.split(\"\\n\")) {\r\n // \"id\",\"input\",\"Match\",\"Exact\",\"matched address\",\"lon,lat\",\"tigerline\",\"side\"\r\n const m = line.match(/^\"(\\d+)\",\".*?\",\"Match\",\"[^\"]*\",\"[^\"]*\",\"(-?[\\d.]+),(-?[\\d.]+)\"/);\r\n if (m) out.set(Number(m[1]), { lon: Number(m[2]), lat: Number(m[3]) });\r\n }\r\n return out;\r\n}\r\nconst EXPAND: Record = { ST: \"STREET\", AVE: \"AVENUE\", RD: \"ROAD\", PL: \"PLACE\", DR: \"DRIVE\", CT: \"COURT\", LN: \"LANE\", BLVD: \"BOULEVARD\", TER: \"TERRACE\", CIR: \"CIRCLE\" };\r\nexport async function dcAddressPoint(addrNorm: string): Promise<{ lat: number; lon: number } | null> {\r\n const k = streetKey(addrNorm);\r\n const m = k.match(/^(\\d+)[A-Z]?\\s+(.+?)\\s+(ST|AVE|RD|PL|DR|CT|LN|BLVD|TER|CIR)\\s+(NW|NE|SW|SE)$/);\r\n if (!m) return null;\r\n const where = `ADDRESS_NUMBER=${Number(m[1])} AND STREET_NAME='${m[2].replace(/'/g, \"''\")}' AND STREET_TYPE='${EXPAND[m[3]] || m[3]}' AND QUADRANT='${m[4]}'`;\r\n const u = `https://maps2.dcgis.dc.gov/dcgis/rest/services/DCGIS_DATA/Location_WebMercator/MapServer/0/query?where=${encodeURIComponent(where)}&outFields=ADDRESS&returnGeometry=true&outSR=4326&f=json`;\r\n const res = await fetch(u, { headers: { \"user-agent\": UA } });\r\n if (!res.ok) return null;\r\n const j = await res.json(); const g = j.features?.[0]?.geometry;\r\n return g ? { lat: Number(g.y), lon: Number(g.x) } : null;\r\n}\r\nexport async function geocodeStep(started: number, budget: number, cap = 200): Promise {\r\n await ensureVibeSchema();\r\n const rows = await sqlite.execute(`select rowid_pk, id, address_norm, addr, area from listings where lat is null and geocode_src is null\r\n and source not like 'signal-%' and source <> 'cl-parking' and status <> 'dead' order by score desc limit ${cap}`);\r\n const cand: any[] = [];\r\n let skipped = 0;\r\n for (const r of rows.rows as any[]) {\r\n if (!geocodable(String(r.id), String(r.address_norm))) { skipped++; await sqlite.execute({ sql: `update listings set geocode_src='skip' where rowid_pk=?`, args: [r.rowid_pk] }); continue; }\r\n const [city, state] = CITY[String(r.area)] || [\"Washington\", \"DC\"];\r\n const zip = (String(r.addr).match(/\\b(2\\d{4})\\b/) || [])[1] || \"\";\r\n const street = String(r.addr).split(\",\")[0].replace(/\\b(unit|apt|ste|#)\\s*\\S+$/i, \"\").replace(/\\s+[A-Z]$/i, \"\").trim();\r\n cand.push({ rowid: Number(r.rowid_pk), street, city, state, zip, addrNorm: String(r.address_norm) });\r\n }\r\n let census = 0, dc = 0, miss = 0;\r\n if (cand.length) {\r\n const hits = await censusBatch(cand);\r\n for (const c of cand) {\r\n if (Date.now() - started > budget) break;\r\n let pt = hits.get(c.rowid); let src = \"census\";\r\n if (!pt && c.state === \"DC\") { try { pt = (await dcAddressPoint(c.addrNorm)) || undefined; src = \"dc-mar\"; } catch { /* ignore */ } await sleep(150); }\r\n if (pt) { await sqlite.execute({ sql: `update listings set lat=?, lon=?, geocode_src=? where rowid_pk=?`, args: [pt.lat, pt.lon, src, c.rowid] }); if (src === \"census\") census++; else dc++; }\r\n else { await sqlite.execute({ sql: `update listings set geocode_src='miss' where rowid_pk=?`, args: [c.rowid] }); miss++; }\r\n }\r\n }\r\n return { candidates: cand.length, census, dc_mar: dc, miss, skipped };\r\n}\r\n\r\n// ---------- vibe step ----------\r\nexport async function vibeStep(started: number, budget: number, cap = 300): Promise {\r\n await ensureVibeSchema();\r\n const pois = await loadPois();\r\n if (!pois.length) return { skipped: \"no pois loaded (POST {pois:[...]} from phase3/pois-load-*.json)\" };\r\n const rows = await sqlite.execute(`select l.rowid_pk, l.lat, l.lon from listings l left join vibe v on v.rowid_pk=l.rowid_pk\r\n where l.lat is not null and l.lon is not null and v.rowid_pk is null and l.source not like 'signal-%' and l.source <> 'cl-parking' order by l.score desc limit ${cap}`);\r\n let n = 0;\r\n for (const r of rows.rows as any[]) {\r\n if (Date.now() - started > budget) break;\r\n await upsertVibe(Number(r.rowid_pk), computeVibe(Number(r.lat), Number(r.lon), pois));\r\n n++;\r\n }\r\n return { computed: n, pois: pois.length };\r\n}\r\n","children",[],[228,218],"lib","0bccf9a5-ef34-4edd-a6be-f86fa0f157d3","18a74ccc-af66-11f1-9858-1607ee4eb77e",61,["D",1789307288593],4,"// Multi-source registry + one parser per platform. Pure functions (text -> hits) so they are\r\n// unit-testable against scripts/fixtures/*. The cron (cron/sources.ts) does the fetching,\r\n// rotation, dedupe and upsert. Adding a source = one REGISTRY entry (+ a parser if new platform).\r\n// Per-source gotchas are documented in valtown/SOURCES.md.\r\nimport {\r\n decodeEntities, stripTags, inferArea, inferBeds, inferPrice, parsePrice, type Area, type Listing,\r\n} from \"./core.ts\";\r\nimport { parseSearch as clParseSearch } from \"../cron/craigslist.ts\";\r\n\r\nexport type Kind = \"listing\" | \"parking\" | \"signal\";\r\nexport interface Hit {\r\n extId: string; // stable per-source id (dedupe key)\r\n url: string;\r\n addr: string | null; // street address if known\r\n price: number | null;\r\n beds: number | null; // null = unknown (caller clamps), 0 = studio\r\n text: string; // everything we know, for the infer* helpers\r\n zip?: string | null;\r\n lat?: number | null; lon?: number | null;\r\n park?: Listing[\"park\"]; type?: Listing[\"type\"]; ll?: Listing[\"ll\"]; dog?: boolean | null; // structured overrides\r\n note?: string;\r\n posted?: string | number | null; // source's first-posted date when it exposes one (listing date, pubDate, issue date)\r\n detailUrl?: string; // fetch this for structured fields (Nestfully)\r\n}\r\nexport interface ParseResult { hits: Hit[]; more?: boolean; }\r\nexport type Parser = (text: string, ctx: { url: string; zip?: string; area?: Area }) => ParseResult;\r\n\r\nexport interface Source {\r\n key: string; // becomes listings.source and poll_log job 'sources:'\r\n parser: keyof typeof PARSERS;\r\n urls: string[]; // fetched in order; parser may request more pages via ParseResult.more + pageUrl()\r\n every: number; // minutes between runs\r\n ll: Listing[\"ll\"]; // default landlord type for this source\r\n kind?: Kind; // default 'listing'\r\n push?: boolean; // default true for listings; parking/signals never push\r\n pushMin?: number; // override PUSH_THRESHOLD for this source (RSS = 6)\r\n enabled?: boolean; // false = skipped without redeploying parsers\r\n zip?: string; // zip context for parsers that don't carry one (CL sub/prk)\r\n headers?: Record;\r\n accept?: \"html\" | \"json\" | \"xml\";\r\n pageUrl?: (base: string, n: number) => string; // n >= 2\r\n maxPages?: number;\r\n detailCap?: number; // detail fetches per run (Nestfully)\r\n spacingMs?: number; // delay between fetches to the same source\r\n group?: string; // at most one source per group runs per invocation (Cloudflare bursts: Nestfully)\r\n}\r\n\r\nexport const TARGET_ZIPS = [\"20010\", \"20011\", \"20012\", \"22206\", \"22204\", \"22301\", \"22305\", \"20781\", \"20782\"];\r\nexport const ZIP_RE = /\\b(2001[012]|2220[46]|2230[15]|2078[12])\\b/;\r\nconst NAV_HEADERS = {\r\n \"accept\": \"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\",\r\n \"accept-language\": \"en-US,en;q=0.9\",\r\n \"upgrade-insecure-requests\": \"1\",\r\n \"sec-fetch-mode\": \"navigate\", \"sec-fetch-dest\": \"document\", \"sec-fetch-site\": \"none\",\r\n};\r\n\r\n// ---------- helpers ----------\r\nconst clean = (s: string) => decodeEntities(s).replace(/\\s+/g, \" \").trim();\r\nfunction pipeText(html: string): string {\r\n // tags -> \" | \" separators so label/value pairs survive\r\n const t = html.replace(//g, \"\").replace(/|/gi, \"\").replace(/<[^>]+>/g, \" | \");\r\n return decodeEntities(t).replace(/\\s+/g, \" \").replace(/(\\s*\\|\\s*)+/g, \" | \").trim();\r\n}\r\nfunction labelValue(pt: string, label: string): string | null {\r\n const m = pt.match(new RegExp(label.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\") + \"\\\\s*:?\\\\s*\\\\|\\\\s*([^|]+)\"));\r\n return m ? m[1].trim() : null;\r\n}\r\nexport function inferZip(s: string): string | null { return (s.match(ZIP_RE) || [])[1] || null; }\r\nfunction bedsFrom(s: string): number | null { return inferBeds(s); }\r\n\r\n// ---------- 1a. Nestfully (Bright MLS consumer portal) ----------\r\nexport const nestfully: Parser = (html, ctx) => {\r\n const hits: Hit[] = [];\r\n const descByUrl = new Map();\r\n const postedByUrl = new Map();\r\n for (const m of html.matchAll(/\n\n`;\n\nexport default async function (req: Request): Promise {\n if (req.method !== \"GET\") return json({ error: \"GET only\" }, 405);\n if (!checkToken(req)) return new Response(\"unauthorized: open the tracker link that includes ?token=\", { status: 401 });\n return new Response(HTML, { headers: { \"content-type\": \"text/html; charset=utf-8\", \"cache-control\": \"no-store\", \"x-robots-tag\": \"noindex\" } });\n}\n",[],[329,320],"api","9d7d683c-4161-4851-a574-edb97359a76c","feed.ts","05d34138-aef2-11f1-8674-1607ee4eb77e",43,["D",1789302877522],"// housing-feed: JSON feed of active listings (GET). The Listing Tracker artifact\r\n// pulls this on open. Token via X-Housing-Token header or ?token= query.\r\n// Returns a bare JSON array of { id, addr, area, price, beds, park, type, ll, dog,\r\n// url, src, note, fs, lv, status, missed }.\r\n//\r\n// ?days=N window on last_seen (default 45)\r\n// ?kinds=all include signal-* and cl-parking rows (excluded by default)\r\n// ?v=2 add age_days + posted_at to each row (default contract unchanged for the tracker)\r\n// ?stats=1 returns a stats object instead of the array (for validation/debug)\r\nimport { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\r\nimport { ensureSchema, activeListings, checkToken, json, CORS } from \"../lib/core.ts\";\r\n\r\nexport default async function (req: Request): Promise {\r\n if (req.method === \"OPTIONS\") return new Response(null, { status: 204, headers: CORS });\r\n if (req.method !== \"GET\") return json({ error: \"GET only\" }, 405);\r\n if (!checkToken(req)) return json({ error: \"unauthorized\" }, 401);\r\n await ensureSchema();\r\n const url = new URL(req.url);\r\n\r\n if (url.searchParams.get(\"stats\")) {\r\n const bySrc = await sqlite.execute(`select source, count(*) n, max(last_seen) last from listings group by source`);\r\n const byArea = await sqlite.execute(`select area, count(*) n, round(avg(price)) avg_price from listings group by area`);\r\n const recent = await sqlite.execute(`select ts, job, ok, fetched, inserted, updated, pushed, detail from poll_log order by id desc limit 15`);\r\n const total = await sqlite.execute(`select count(*) n, sum(score>=8) hot, sum(pushed) pushed from listings`);\r\n return json({ total: total.rows[0], bySource: bySrc.rows, byArea: byArea.rows, recentPolls: recent.rows });\r\n }\r\n\r\n const days = Math.min(365, Math.max(1, Number(url.searchParams.get(\"days\") || 45)));\r\n const list = await activeListings(days, url.searchParams.get(\"kinds\") === \"all\" ? \"all\" : \"listings\", Number(url.searchParams.get(\"v\") || 1)); // ?kinds=all includes signal-*/cl-parking rows; ?v=2 adds age_days/posted_at\r\n return json(list, 200, { \"cache-control\": \"no-store\" });\r\n}\r\n",[],[329,331],"e9cee830-9665-4d39-825b-5703d3abf333","admin.ts","066872bc-aef2-11f1-8bc7-1607ee4eb77e",69,["D",1789308601983],13,"// housing-admin: token-checked maintenance endpoint for validation and manual runs.\r\n// GET ?job=craigslist | popville | digest | sources run a job now (digest sends immediately)\r\n// ?job=sources&only= ?job=sources&force=1 one source / ignore the `every` interval\r\n// GET ?sql=select ... read-only query (SELECT only)\r\n// POST { id, status } set a listing's status (new|contacted|touring|applied|dead)\r\n// POST { targets: [...] } load Phase 3 landlord targets ({addr, area, confidence A|B|C, verified_at?, ...}; bonus A +3, B +2, C 0)\r\n// POST { pois: [...] } load vibe-layer POIs (phase3/pois-load-*.json)\r\n// POST { verify_target: { address_norm, result: garage|none|probable } } satellite verdict -> A / demote to C / keep B, stamps verified_at\r\n// GET ?job=calibration[&email=1] weekly calibration report as Markdown (email=1 sends it via std/email)\r\n// GET ?job=enrich[&force=1][&step=queue|bbl|license|price|geocode|vibe] enrichment cron steps now (force = BBL refresh regardless of day)\r\n// GET ?job=backfill_posted[&cap=40] re-fetch Craigslist / Nestfully pages to fill posted_at for score>=6 rows\r\n// GET ?testpush=1 send a test Discord push\r\nimport { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\r\nimport { ensureSchema, checkToken, json, CORS, STATUS, pushDiscord, actionLinks, PUSH_THRESHOLD, normalizeAddress, streetKey, toIsoDate, UA, sleep } from \"../lib/core.ts\";\r\nimport { parseDetail as clParseDetail } from \"../cron/craigslist.ts\";\r\nimport { PARSERS, REGISTRY } from \"../lib/sources.ts\";\r\nimport craigslist from \"../cron/craigslist.ts\";\r\nimport popville from \"../cron/popville.ts\";\r\nimport { sendDigest } from \"../cron/digest.ts\";\r\nimport sources from \"../cron/sources.ts\";\r\nimport enrich from \"../cron/enrich.ts\";\r\nimport { calibrationReport, sendCalibration } from \"../cron/calibration.ts\";\r\n\r\nexport default async function (req: Request): Promise {\r\n if (req.method === \"OPTIONS\") return new Response(null, { status: 204, headers: CORS });\r\n if (!checkToken(req)) return json({ error: \"unauthorized\" }, 401);\r\n await ensureSchema();\r\n const url = new URL(req.url);\r\n\r\n if (req.method === \"POST\") {\r\n const b = await req.json().catch(() => ({}));\r\n // POST { delete: { id? | source? | url_like? | before? } } - remove rows matching ALL given filters (at least one)\r\n if (b.delete && typeof b.delete === \"object\") {\r\n const w: string[] = []; const a: any[] = [];\r\n if (b.delete.id != null) { w.push(\"id=?\"); a.push(b.delete.id); }\r\n if (b.delete.rowid != null) { w.push(\"rowid_pk=?\"); a.push(Number(b.delete.rowid)); }\r\n if (b.delete.source) { w.push(\"source=?\"); a.push(b.delete.source); }\r\n if (b.delete.url_like) { w.push(\"url like ?\"); a.push(b.delete.url_like); }\r\n if (b.delete.before) { w.push(\"first_seen t.addr && t.area).map((t: any) => ({\r\n sql: `insert or replace into targets (address_norm, addr, area, owner, mailing, garage_evidence, units, beds, bbl, rank, added, confidence, verified_at) values (?,?,?,?,?,?,?,?,?,?,date('now'),?,?)`,\r\n args: [streetKey(normalizeAddress(String(t.addr).split(\",\")[0])), String(t.addr).split(\",\")[0].trim(), t.area, t.owner || \"\", t.mailing || \"\", t.garage_evidence || \"\", t.units ?? null, t.beds ?? null, t.bbl ? 1 : 0, t.rank ?? null,\r\n /^[ABC]$/.test(String(t.confidence || \"\")) ? t.confidence : \"C\", t.verified_at || null],\r\n }));\r\n if (stmts.length) await sqlite.batch(stmts);\r\n const n = await sqlite.execute(`select count(*) n, sum(confidence='A') a, sum(confidence='B') b, sum(confidence='C') c from targets`);\r\n return json({ ok: true, upserted: stmts.length, totals: n.rows[0] });\r\n }\r\n // POST { pois: [{id, kind, name, lat, lon, area}] } - vibe layer POIs from phase3/pois.ts (<= 400 per call)\r\n if (Array.isArray(b.pois)) {\r\n const rows = b.pois.slice(0, 400).filter((p: any) => p.id && p.kind && p.lat != null && p.lon != null);\r\n if (rows.length) await sqlite.batch(rows.map((p: any) => ({ sql: `insert or replace into pois (id, kind, name, lat, lon, area) values (?,?,?,?,?,?)`, args: [String(p.id), String(p.kind), String(p.name || \"\"), Number(p.lat), Number(p.lon), p.area || null] })));\r\n const n = await sqlite.execute(`select kind, count(*) n from pois group by kind`);\r\n return json({ ok: true, upserted: rows.length, totals: n.rows });\r\n }\r\n // POST { verify_target: { address_norm, result } } - the Chrome sweep's satellite verdict for a \"VERIFY GARAGE\" push\r\n if (b.verify_target && typeof b.verify_target === \"object\") {\r\n const key = streetKey(normalizeAddress(String(b.verify_target.address_norm || b.verify_target.addr || \"\").split(\",\")[0]));\r\n const result = String(b.verify_target.result || \"\");\r\n if (!key || ![\"garage\", \"none\", \"probable\"].includes(result)) return json({ error: \"need address_norm and result in garage|none|probable\" }, 400);\r\n const conf = result === \"garage\" ? \"A\" : result === \"none\" ? \"C\" : \"B\";\r\n const r = await sqlite.execute({ sql: `update targets set confidence=?, verified_at=date('now') where address_norm=?`, args: [conf, key] });\r\n return json({ ok: true, address_norm: key, confidence: conf, rowsAffected: r.rowsAffected });\r\n }\r\n if (!b.id || !STATUS.includes(b.status)) return json({ error: `need id and status in ${STATUS.join(\"|\")}` }, 400);\r\n const r = await sqlite.execute({ sql: `update listings set status=? where id=?`, args: [b.status, b.id] });\r\n return json({ ok: true, rowsAffected: r.rowsAffected });\r\n }\r\n\r\n const job = url.searchParams.get(\"job\");\r\n if (job === \"craigslist\") return json(await craigslist());\r\n if (job === \"popville\") return json(await popville());\r\n if (job === \"digest\") return json(await sendDigest(true));\r\n if (job === \"sources\") return json(await sources({ only: url.searchParams.get(\"only\") || undefined, force: !!url.searchParams.get(\"force\") }));\r\n if (job === \"calibration\") {\r\n if (url.searchParams.get(\"email\")) return json(await sendCalibration());\r\n const r = await calibrationReport();\r\n return new Response(r.markdown, { headers: { \"content-type\": \"text/markdown; charset=utf-8\", ...CORS } });\r\n }\r\n if (job === \"enrich\") return json(await enrich({ force: !!url.searchParams.get(\"force\"), step: url.searchParams.get(\"step\") || undefined }));\r\n if (job === \"backfill_posted\") return json(await backfillPosted(Number(url.searchParams.get(\"cap\") || 40)));\r\n\r\n // push every active score>=8 row that never got a Discord alert (webhook was down / not yet configured)\r\n if (url.searchParams.get(\"pushpending\")) {\r\n const r = await sqlite.execute(`select * from listings where score >= ${PUSH_THRESHOLD} and pushed = 0 and status <> 'dead' and last_seen >= date('now','-14 day') order by score desc`);\r\n const out: any[] = [];\r\n for (const row of r.rows as any[]) {\r\n const ok = await pushDiscord({ id: row.id, addr: row.addr, area: row.area, price: row.price, beds: Number(row.beds), park: row.parking_type, type: row.htype, ll: row.landlord_type, dog: row.pet_ok == null ? null : !!row.pet_ok, url: row.url, source: row.source, score: row.score, notes: row.notes });\r\n if (ok) await sqlite.execute({ sql: `update listings set pushed=1 where rowid_pk=?`, args: [row.rowid_pk] });\r\n out.push({ id: row.id, score: row.score, pushed: ok });\r\n }\r\n return json({ pending: out.length, results: out });\r\n }\r\n\r\n if (url.searchParams.get(\"testpush\")) {\r\n const l = { id: \"719 QUEBEC PL NW\", rowid: null, addr: \"Test push - 719 Quebec Pl NW\", area: \"Park View / 16th St Hts\", price: 2400, beds: 2, park: \"garage\", type: \"basement\", ll: \"individual\", dog: true, url: \"https://example.com\", source: \"test\", score: 12, notes: \"Pipeline test; ignore.\", tier: \"B\", ageDays: 31, license: \"yes\", licenseDetail: \"500620000246 exp 2028-08-31\", anomaly: false, trail: [2500, 2450, 2400], motivated: true, lat: 38.9365, lon: -77.0223, vibe: \"Vibe 4/5 - Georgia Ave @ Petworth Metro 0.3 mi (6 min) - 6 bars - Petworth Metro 0.4 mi - ~14 min to Shirlington\" };\r\n const ok = await pushDiscord(l);\r\n return json({ pushed: ok, links: actionLinks(l) });\r\n }\r\n\r\n const sql = url.searchParams.get(\"sql\");\r\n if (sql) {\r\n if (!/^\\s*select\\b/i.test(sql)) return json({ error: \"SELECT only\" }, 400);\r\n const r = await sqlite.execute(sql);\r\n return json({ columns: r.columns, rows: r.rows });\r\n }\r\n return json({ usage: \"?job=craigslist|popville|digest|sources[&only=key][&force=1] ?sql=select... ?testpush=1 POST {id,status}\" });\r\n}\r\n\r\n// Fix 2 backfill: posted_at for Craigslist rows (detail page ) and Nestfully rows (list page datePosted),\r\n// score >= 6 only, capped, polite. Everything else stays null (first_seen is the age fallback).\r\nasync function backfillPosted(cap: number) {\r\n const headers = { \"user-agent\": UA, \"accept\": \"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\", \"accept-language\": \"en-US,en;q=0.9\", \"upgrade-insecure-requests\": \"1\", \"sec-fetch-mode\": \"navigate\", \"sec-fetch-dest\": \"document\", \"sec-fetch-site\": \"none\" };\r\n const out: any[] = []; let fetches = 0;\r\n // step 0 (no fetch): Nestfully / add-endpoint notes already carry \"posted YYYY-MM-DD\" from the list page\r\n const fromNotes = await sqlite.execute(`select rowid_pk, notes from listings where posted_at is null and notes like '%posted 20__-__-__%'`);\r\n let notesFilled = 0;\r\n for (const row of fromNotes.rows as any[]) {\r\n const m = String(row.notes).match(/posted (\\d{4}-\\d{2}-\\d{2})/); const d = m ? toIsoDate(m[1]) : null;\r\n if (d) { await sqlite.execute({ sql: `update listings set posted_at=? where rowid_pk=? and posted_at is null`, args: [d, row.rowid_pk] }); notesFilled++; }\r\n }\r\n out.push({ step: \"posted_at from notes\", filled: notesFilled });\r\n const cl = await sqlite.execute(`select rowid_pk, url, addr from listings where source='craigslist' and posted_at is null and score >= 6 order by score desc`);\r\n for (const row of cl.rows as any[]) {\r\n if (fetches >= cap) break;\r\n fetches++;\r\n try {\r\n const res = await fetch(String(row.url), { headers }); const text = res.ok ? await res.text() : \"\";\r\n const posted = text ? toIsoDate(clParseDetail(text).posted) : null;\r\n if (posted) await sqlite.execute({ sql: `update listings set posted_at=? where rowid_pk=?`, args: [posted, row.rowid_pk] });\r\n out.push({ addr: row.addr, http: res.status, posted });\r\n } catch (e) { out.push({ addr: row.addr, error: String(e).slice(0, 80) }); }\r\n await sleep(1500);\r\n }\r\n const nf = await sqlite.execute(`select source, count(*) n from listings where source like 'nestfully-%' and posted_at is null and score >= 6 group by source`);\r\n for (const g of nf.rows as any[]) {\r\n if (fetches >= cap) break;\r\n const zip = String(g.source).replace(\"nestfully-\", \"\");\r\n const reg = REGISTRY.find((s) => s.key === g.source); if (!reg) continue;\r\n const byId = new Map();\r\n await sleep(6000); // Cloudflare 403s Nestfully bursts - one zip at a time, spaced\r\n for (let page = 1; page <= 3 && fetches < cap; page++) {\r\n fetches++;\r\n const u = page > 1 && reg.pageUrl ? reg.pageUrl(reg.urls[0], page) : reg.urls[0];\r\n try {\r\n let res = await fetch(u, { headers: { \"user-agent\": UA, ...(reg.headers || headers) } });\r\n if (res.status === 403 || res.status === 429) { await sleep(8000); res = await fetch(u, { headers: { \"user-agent\": UA, ...(reg.headers || headers) } }); }\r\n if (!res.ok) { out.push({ source: g.source, page, http: res.status }); break; }\r\n const r = PARSERS.nestfully(await res.text(), { url: u, zip });\r\n for (const h of r.hits) if (h.posted) byId.set(h.extId, String(h.posted).slice(0, 10));\r\n if (!r.more) break;\r\n } catch (e) { out.push({ source: g.source, page, error: String(e).slice(0, 80) }); break; }\r\n await sleep(5000);\r\n }\r\n let updated = 0;\r\n for (const [extId, posted] of byId) {\r\n const r = await sqlite.execute({ sql: `update listings set posted_at=? where source=? and external_id=? and posted_at is null`, args: [posted, g.source, extId] });\r\n updated += Number(r.rowsAffected || 0);\r\n }\r\n out.push({ source: g.source, listPagesParsed: byId.size, updated });\r\n }\r\n // re-derive STALE prefixes for rows that now have a posted_at\r\n const stale = await sqlite.execute(`select rowid_pk, notes, posted_at, first_seen from listings where posted_at is not null and score >= 6`);\r\n let restamped = 0;\r\n for (const row of stale.rows as any[]) {\r\n const age = Math.round((Date.now() - Date.parse(String(row.posted_at || row.first_seen))) / 86400000);\r\n const base = String(row.notes || \"\").replace(/^STALE \\d+d \\| /, \"\");\r\n const notes = age > 45 ? `STALE ${age}d | ${base}` : base;\r\n if (notes !== row.notes) { await sqlite.execute({ sql: `update listings set notes=? where rowid_pk=?`, args: [notes, row.rowid_pk] }); restamped++; }\r\n }\r\n return { fetches, restamped, out };\r\n}\r\n",[],[329,339],"1fba8fc3-f36b-4b1c-90e3-1d45a5867818","add.ts",42,["D",1789302872872],"// housing-add: manual ingestion endpoint (POST). Claude in Chrome and Logan post\r\n// FB / sign / portal finds here. Shared-secret header X-Housing-Token.\r\n//\r\n// Body (JSON, single object or array):\r\n// { addr, area?, price?, beds?, park?, type?, ll?, dog?, url, src, note?, text?, posted? (yyyy-mm-dd first-posted date) }\r\n// `text` is optional free text (the listing body) used to infer any field left blank.\r\nimport {\r\n AREAS, PARK, HTYPE, LL, ensureSchema, upsertListing, inferArea, inferPark, inferType, inferLL, inferDog,\r\n inferBeds, clampBeds, parsePrice, inferPrice, checkToken, json, CORS, type Area,\r\n} from \"../lib/core.ts\";\r\n\r\nexport default async function (req: Request): Promise {\r\n if (req.method === \"OPTIONS\") return new Response(null, { status: 204, headers: CORS });\r\n if (req.method !== \"POST\") return json({ error: \"POST only\" }, 405);\r\n if (!checkToken(req)) return json({ error: \"unauthorized\" }, 401);\r\n\r\n let body: any;\r\n try { body = await req.json(); } catch { return json({ error: \"invalid JSON\" }, 400); }\r\n const items: any[] = Array.isArray(body) ? body : [body];\r\n if (items.length === 0 || items.length > 50) return json({ error: \"1-50 listings per request\" }, 400);\r\n\r\n await ensureSchema();\r\n const results: any[] = [];\r\n for (const it of items) {\r\n const text = [it.addr, it.title, it.note, it.text].filter(Boolean).join(\" \\n \");\r\n const url = String(it.url || \"\").trim();\r\n const src = String(it.src || it.source || \"manual\").toLowerCase().trim();\r\n if (!url && !it.addr) { results.push({ error: \"need url or addr\", input: it }); continue; }\r\n\r\n let area: Area | null = AREAS.includes(it.area) ? it.area : inferArea(text);\r\n if (!area) { results.push({ error: `area required; one of ${AREAS.join(\" | \")}`, input: it }); continue; }\r\n\r\n let beds = it.beds != null && it.beds !== \"\" ? Number(it.beds) : inferBeds(text);\r\n if (beds === 0) { results.push({ skipped: \"studio\", input: it }); continue; }\r\n beds = clampBeds(beds);\r\n\r\n const park = PARK.includes(it.park) ? it.park : inferPark(text);\r\n const type = HTYPE.includes(it.type) ? it.type : inferType(text);\r\n // FB/CL/sign posts default to individual-landlord signal; portals default to pm\r\n const llDefault = [\"fb\", \"fbm\", \"facebook\", \"craigslist\", \"sign\", \"popville\", \"manual\"].includes(src) ? \"individual\" : \"pm\";\r\n const ll = LL.includes(it.ll) ? it.ll : inferLL(text, llDefault);\r\n const dog = typeof it.dog === \"boolean\" ? it.dog : inferDog(text);\r\n const price = it.price != null && it.price !== \"\" ? parsePrice(it.price) : inferPrice(text);\r\n\r\n const r = await upsertListing({\r\n addr: it.addr ? String(it.addr).trim() : null,\r\n area, price, beds, source: src, externalId: it.externalId || it.external_id || null, url: url || \"\",\r\n park, type, ll, dog, notes: String(it.note || \"\").trim(), raw: it.text ? String(it.text) : undefined, unit: it.unit || null,\r\n postedAt: it.posted || it.posted_at || null,\r\n }, { push: true });\r\n results.push(r);\r\n }\r\n return json({ ok: true, results });\r\n}\r\n",[],[329,348],"5c11403c-d645-4b4e-b1c8-b1cc89a025ad",["D",1789248989432],[199,200,201,202],[329],["D",1789248987477],["D",1789519256886],{"_11":12,"_153":74},{"_11":12,"_148":149,"_73":74,"_30":72,"_150":151,"_152":24},{"_41":139,"_13":16,"_35":38,"_140":100,"_141":-5,"_142":-5,"_143":-5,"_144":-5,"_145":-5,"_146":14,"_147":12},"ctx",{"_56":-5,"_172":396,"_397":398,"_399":400,"_401":24,"_402":24,"_403":24,"_404":113,"_405":-5,"_406":-7,"_407":-7,"_408":26,"_409":24,"_410":24,"_411":-5,"_412":24,"_413":24,"_414":24,"_415":-5},"ENV",{"_367":368,"_369":370,"_371":16,"_372":373,"_374":375,"_376":377,"_378":379,"_380":381,"_382":383,"_384":385,"_386":387,"_388":389,"_390":391,"_392":393,"_394":395},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","fdb67791413f4386b4dbf18f354029a617f6160f","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

History · sources.ts