[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8},"actionData","errors","root",{"_110":111,"_112":113},"routes/embed.x.$handle.$name.$",{"_9":10,"_11":12,"_13":14,"_15":16,"_17":18,"_19":20,"_21":-5},"handle","logan1","name","housing","path","cron/sources.ts","project",{"_22":48,"_11":12,"_35":49,"_50":-5,"_51":44,"_52":53,"_54":53,"_55":56,"_57":-5,"_58":59,"_60":61,"_62":34,"_63":-5,"_64":-5,"_65":66,"_67":68,"_69":70,"_71":72,"_73":74,"_75":-5,"_76":77,"_78":77,"_79":80,"_81":80,"_82":80,"_83":77,"_84":77,"_85":86,"_87":88,"_89":90,"_91":44},"file",{"_22":23,"_11":24,"_25":26,"_27":28,"_29":30,"_31":32,"_33":34,"_35":36,"_37":38,"_39":40,"_41":42,"_43":44,"_45":-5,"_13":46},"permalink","https://www.val.town/x/logan1/housing/code/cron/sources.ts","extension","id","6530a678-015a-407c-8eb5-aefba5bf288b","sources.ts","fileId","40cdd694-af66-11f1-ad84-1607ee4eb77e","parentId","041eaecc-aef2-11f1-9a8e-1607ee4eb77e","branchId","a4c5838e-3adb-4640-ab71-e9f3d2b72f0a","branchVersion",62,"latest",true,"createdAt",["D",1789307296223],"version",3,"fileType","interval","content","// Generic multi-source poller (cron, every 30 min). Each run processes the registry entries whose\r\n// `every` interval has elapsed (most overdue first) inside a 45 s budget; the rest wait for the next\r\n// run. Every hit goes through upsertListing (dedupe, score, push) with source = registry key.\r\n// State per source lives in `source_state`; results in poll_log as job='sources:'.\r\n// Admin: ?job=sources ?job=sources&only= ?job=sources&force=1\r\nimport { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\r\nimport {\r\n ensureSchema, upsertListing, logPoll, inferArea, inferPark, inferType, inferLL, inferDog, inferBeds, clampBeds,\r\n UA, sleep, type Area, type Listing,\r\n} from \"../lib/core.ts\";\r\nimport { REGISTRY, PARSERS, nestfullyDetail, FULL_INVENTORY_PARSERS, type Source, type Hit } from \"../lib/sources.ts\";\r\nimport { applyMissed } from \"../lib/enrich.ts\";\r\n\r\nconst TIME_BUDGET_MS = 45_000;\r\nconst PRICE_LO = 1800, PRICE_HI = 2800;\r\nconst AREA_BY_ZIP: Record = {\r\n \"20011\": \"Petworth\", \"20010\": \"Park View / 16th St Hts\", \"20012\": \"Brightwood\", \"22206\": \"Fairlington / Shirlington\",\r\n \"22204\": \"Columbia Pike\", \"22301\": \"Del Ray\", \"22305\": \"Del Ray\", \"20781\": \"Hyattsville\", \"20782\": \"Hyattsville\",\r\n};\r\n\r\nasync function fetchText(url: string, src: Source): Promise<{ status: number; text: string }> {\r\n const headers: Record = {\r\n \"user-agent\": UA,\r\n \"accept\": src.accept === \"json\" ? \"application/json,text/plain,*/*\" : src.accept === \"xml\" ? \"application/rss+xml,application/atom+xml,application/xml,text/xml,*/*\" : \"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\",\r\n \"accept-language\": \"en-US,en;q=0.9\",\r\n ...(src.headers || {}),\r\n };\r\n let res = await fetch(url, { headers, redirect: \"follow\" });\r\n if (res.status === 403 || res.status === 429) { await sleep(3000); res = await fetch(url, { headers, redirect: \"follow\" }); } // one polite retry\r\n return { status: res.status, text: res.ok ? await res.text() : \"\" };\r\n}\r\n\r\nexport async function runSource(src: Source, started: number, opts: { detailBudget: { n: number } }): Promise<{ fetched: number; inserted: number; updated: number; pushed: number; ok: boolean; detail: string }> {\r\n const stats = { fetched: 0, inserted: 0, updated: 0, pushed: 0 };\r\n const notes: string[] = [];\r\n const kind = src.kind || \"listing\";\r\n const parser = PARSERS[src.parser];\r\n const hits: Hit[] = [];\r\n let failed = 0, pages = 0, complete = true; // complete = every page fetched, nothing truncated (needed for the missed check)\r\n const ctxZip = src.zip;\r\n const ctxArea = ctxZip ? AREA_BY_ZIP[ctxZip] : undefined;\r\n\r\n for (const base of src.urls) {\r\n let url = base;\r\n for (let page = 1; page <= (src.maxPages || 1); page++) {\r\n if (Date.now() - started > TIME_BUDGET_MS) { notes.push(\"budget hit before all pages\"); complete = false; break; }\r\n if (page > 1) { if (!src.pageUrl) { complete = false; break; } url = src.pageUrl(base, page); }\r\n try {\r\n const { status, text } = await fetchText(url, src);\r\n pages++;\r\n if (status === 404 && page > 1) break; // past the last page\r\n if (status !== 200 || !text) { failed++; notes.push(`http ${status} ${url.slice(0, 80)}`); break; }\r\n const zipFromUrl = (url.match(/\\b(2001[012]|2220[46]|2230[15]|2078[12])\\b/) || [])[1];\r\n const r = parser(text, { url, zip: ctxZip || zipFromUrl, area: ctxArea || (zipFromUrl ? AREA_BY_ZIP[zipFromUrl] : undefined) });\r\n hits.push(...r.hits);\r\n if (!r.more) break;\r\n if (page === (src.maxPages || 1)) complete = false; // more pages exist than we fetch - inventory not complete\r\n } catch (e) { failed++; complete = false; notes.push(`error ${String(e).slice(0, 80)} ${url.slice(0, 60)}`); break; }\r\n await sleep(src.spacingMs ?? 1500);\r\n }\r\n }\r\n stats.fetched = hits.length;\r\n\r\n // dedupe within the run, then upsert\r\n const seen = new Set();\r\n let skipped = 0, details = 0;\r\n for (const h of hits) {\r\n if (seen.has(h.extId)) continue; seen.add(h.extId);\r\n if (Date.now() - started > TIME_BUDGET_MS) { notes.push(\"budget hit during upserts; remainder next run\"); break; }\r\n const zip = h.zip || (h.text.match(/\\b(2001[012]|2220[46]|2230[15]|2078[12])\\b/) || [])[1] || null;\r\n const area = inferArea([zip || \"\", h.addr || \"\", h.text].join(\" \"), h.lat, h.lon, ctxArea);\r\n if (!area) { skipped++; continue; } // outside our areas (AppFolio full-inventory pages etc.)\r\n if (zip && !AREA_BY_ZIP[zip]) { skipped++; continue; }\r\n\r\n let bedsRaw = h.beds ?? inferBeds(h.text);\r\n if (kind === \"listing\") {\r\n if (bedsRaw === 0) { skipped++; continue; } // studio\r\n if (h.price != null && (h.price < PRICE_LO || h.price > PRICE_HI)) { skipped++; continue; }\r\n }\r\n let beds = kind === \"listing\" ? clampBeds(bedsRaw) : 1;\r\n if (beds === 1 && /\\bden\\b|\\boffice\\b|bonus room/i.test(h.text)) beds = 1.5;\r\n\r\n let park = h.park ?? inferPark(h.text);\r\n let type = h.type ?? inferType(h.text);\r\n let dog: boolean | null = h.dog !== undefined ? h.dog : inferDog(h.text);\r\n let ll = h.ll ?? inferLL(h.text, src.ll);\r\n let note = h.note || \"\";\r\n let addr = h.addr;\r\n\r\n // Nestfully stage 2: structured Parking / Pets / Availability from the detail page, new listings only\r\n if (h.detailUrl && kind === \"listing\" && opts.detailBudget.n < (src.detailCap ?? 0)) {\r\n const known = await sqlite.execute({ sql: `select 1 from listings where source=? and external_id=? limit 1`, args: [src.key, h.extId] });\r\n if (!known.rows.length) {\r\n opts.detailBudget.n++; details++;\r\n try {\r\n await sleep(src.spacingMs ?? 3000);\r\n const { status, text } = await fetchText(h.detailUrl, src);\r\n if (status === 200) {\r\n const d = nestfullyDetail(text);\r\n if (d.park) park = d.park;\r\n if (d.pets != null) dog = d.pets;\r\n if (d.beds != null && bedsRaw == null) beds = clampBeds(d.beds);\r\n if (d.office) ll = /property management|management|pm\\b|rentals? llc/i.test(d.office) ? \"pm\" : \"individual\"; // agent-listed unit = owner behind a brokerage\r\n note = [note, d.mls ? `MLS ${d.mls}` : \"\", d.parking ? `Parking: ${d.parking}` : \"\", d.pets != null ? `Pets: ${d.pets ? \"Yes\" : \"No\"}` : \"\", d.avail ? `avail ${d.avail}` : \"\", d.development ? d.development : \"\", d.office ? `office ${d.office}` : \"\"].filter(Boolean).join(\" | \");\r\n } else notes.push(`detail ${h.extId} http ${status}`);\r\n } catch (e) { notes.push(`detail ${h.extId} error ${String(e).slice(0, 60)}`); }\r\n }\r\n }\r\n\r\n try {\r\n const r = await upsertListing({\r\n addr, area, price: h.price, beds, source: src.key, externalId: h.extId, url: h.url,\r\n park, type, ll, dog, notes: note.slice(0, 400), lat: h.lat ?? null, lon: h.lon ?? null, raw: h.text.slice(0, 4000), postedAt: h.posted ?? null,\r\n }, { push: kind === \"listing\" && src.push !== false, pushMin: src.pushMin });\r\n if (r.action === \"inserted\") stats.inserted++; else stats.updated++;\r\n if (r.pushed) stats.pushed++;\r\n } catch (e) { notes.push(`upsert ${h.extId} ${String(e).slice(0, 60)}`); }\r\n }\r\n const ok = failed === 0 && pages > 0;\r\n // Part 4: full-inventory sources - after a successful, complete fetch, rows of this source not in the result are missing\r\n if (ok && complete && FULL_INVENTORY_PARSERS.has(src.parser) && hits.length && !notes.some((n) => n.startsWith(\"budget hit\"))) {\r\n try {\r\n const m = await applyMissed(src.key, new Set(hits.map((h) => h.extId)));\r\n if (m.inc || m.dead) notes.push(`missed +${m.inc}, dead ${m.dead}`);\r\n } catch (e) { notes.push(`missed check failed ${String(e).slice(0, 60)}`); }\r\n }\r\n const detail = `${pages} pages, ${hits.length} hits, ${skipped} skipped, ${details} detail fetches, ${failed} failed. ${notes.join(\"; \")}`.slice(0, 1900);\r\n return { ...stats, ok, detail };\r\n}\r\n\r\nexport async function ensureSourceState() {\r\n await sqlite.execute(`create table if not exists source_state (\r\n source text primary key, last_run text, last_ok text, fail_count integer not null default 0,\r\n runs integer not null default 0, inserted_total integer not null default 0, detail text)`);\r\n}\r\n\r\nexport default async function run(opts: { only?: string; force?: boolean } = {}) {\r\n const started = Date.now();\r\n await ensureSchema(); await ensureSourceState();\r\n const state = await sqlite.execute(`select source, last_run, fail_count from source_state`);\r\n const last = new Map();\r\n for (const r of state.rows as any[]) last.set(r.source, { last_run: r.last_run, fail_count: Number(r.fail_count) });\r\n\r\n // pick: enabled, due (or forced/only), most overdue first\r\n const now = Date.now();\r\n const due = REGISTRY.filter((s) => s.enabled !== false && (!opts.only || s.key === opts.only)).map((s) => {\r\n const lr = last.get(s.key)?.last_run; const age = lr ? (now - Date.parse(lr)) / 60000 : Infinity;\r\n return { s, overdue: age - s.every };\r\n }).filter((x) => opts.force || opts.only || x.overdue >= 0).sort((a, b) => b.overdue - a.overdue);\r\n\r\n const results: Record = {};\r\n const detailBudget = { n: 0 };\r\n const groupsRan = new Set();\r\n for (const { s } of due) {\r\n if (Date.now() - started > TIME_BUDGET_MS - 5000) { results._skipped = `${due.length - Object.keys(results).length} sources deferred to next run (time budget)`; break; }\r\n if (s.group && !opts.only) { if (groupsRan.has(s.group)) continue; groupsRan.add(s.group); }\r\n const r = await runSource(s, started, { detailBudget });\r\n results[s.key] = r;\r\n await logPoll(`sources:${s.key}`, r.ok, r, r.detail);\r\n const fc = r.ok ? 0 : (last.get(s.key)?.fail_count || 0) + 1;\r\n await sqlite.execute({\r\n sql: `insert into source_state (source, last_run, last_ok, fail_count, runs, inserted_total, detail) values (?,?,?,?,1,?,?)\r\n on conflict(source) do update set last_run=excluded.last_run, last_ok=coalesce(excluded.last_ok, source_state.last_ok), fail_count=?, runs=runs+1, inserted_total=inserted_total+?, detail=excluded.detail`,\r\n args: [s.key, new Date().toISOString(), r.ok ? new Date().toISOString() : null, fc, r.inserted, r.detail.slice(0, 500), fc, r.inserted],\r\n });\r\n }\r\n const summary = { ran: Object.keys(results).filter((k) => !k.startsWith(\"_\")).length, due: due.length, seconds: Math.round((Date.now() - started) / 1000), results };\r\n console.log(\"sources\", JSON.stringify(summary).slice(0, 2000));\r\n return summary;\r\n}\r\n","deleted",false,"label",[47,24],"cron","b7ccec7e-ebe9-4364-ba98-e367c8d8849b",["D",1789248987477],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","DC housing search pipeline","imageUrl","userId","87445237-7d14-45c7-b3a2-72e031f10ec1","pinnedFileId","0544e21c-aef2-11f1-a6f0-1607ee4eb77e","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","01a09786-a61b-74b5-8910-af3116d3de5c","tursoName","b7ccec7eebe94364ba98e367c8d8849b","tursoHostname","b7ccec7eebe94364ba98e367c8d8849b-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789519256886],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_9":10,"_109":105},"org",{"_9":10,"_102":103,"_104":105,"_58":59,"_106":107,"_108":44},"branch",{"_37":92,"_11":93,"_22":30,"_94":48,"_95":-5,"_96":-5,"_97":-5,"_98":-5,"_99":-5,"_100":12,"_101":10},"isOwned",71,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","displayName","Logan","avatar","https://lh3.googleusercontent.com/a/ACg8ocIFxddfJftSiTFZN49YShT3nY7q8GOQ7dXYM2vJ6XkajnQMGA=s96-c","tier","free","canRestrictVals","profileImageUrl","ctx",{"_85":-5,"_143":144,"_145":146,"_147":148,"_149":44,"_150":44,"_151":44,"_152":34,"_153":-5,"_154":-7,"_155":-7,"_156":66,"_157":44,"_158":44,"_159":-5,"_160":44,"_161":44,"_162":44,"_163":-5},"ENV",{"_114":115,"_116":117,"_118":93,"_119":120,"_121":122,"_123":124,"_125":126,"_127":128,"_129":130,"_131":132,"_133":134,"_135":136,"_137":138,"_139":140,"_141":142},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town","orgs",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]