[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8},"actionData","errors","root",{"_110":111,"_112":113},"routes/embed.x.$handle.$name.$",{"_9":10,"_11":12,"_13":14,"_15":16,"_17":18,"_19":20,"_21":-5},"handle","logan1","name","corvette","path","cron/sources.ts","project",{"_22":48,"_11":12,"_35":49,"_50":-5,"_51":44,"_52":53,"_54":53,"_55":56,"_57":-5,"_58":59,"_60":61,"_62":34,"_63":-5,"_64":-5,"_65":66,"_67":68,"_69":70,"_71":72,"_73":74,"_75":-5,"_76":77,"_78":77,"_79":80,"_81":80,"_82":80,"_83":77,"_84":77,"_85":86,"_87":88,"_89":90,"_91":44},"file",{"_22":23,"_11":24,"_25":26,"_27":28,"_29":30,"_31":32,"_33":34,"_35":36,"_37":38,"_39":40,"_41":42,"_43":44,"_45":-5,"_13":46},"permalink","https://www.val.town/x/logan1/corvette/code/cron/sources.ts","extension","id","06c58de1-08dd-4451-84c9-922e0ddc1c95","sources.ts","fileId","3fe17824-af86-11f1-ab68-1607ee4eb77e","parentId","1b0a73f2-af86-11f1-b58b-1607ee4eb77e","branchId","86e98f95-71bb-401c-ac30-44ee51ed36fc","branchVersion",51,"latest",true,"createdAt",["D",1789344569403],"version",4,"fileType","interval","content","// Generic multi-source poller (cron, every 30 min). Each run processes the registry entries whose `every`\r\n// interval has elapsed (most overdue first) inside a 45 s budget; the rest wait for the next run. Listing hits go\r\n// through upsertListing (dedupe, score, push) with source = registry key; comp hits go to the `comps` table.\r\n// State per source lives in `source_state`; results in poll_log as job='sources:'.\r\n// Admin: ?job=sources ?job=sources&only= ?job=sources&force=1\r\nimport { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\r\nimport { ensureSchema, upsertListing, logPoll, inferSeller, inferTrim, genFor, today, UA, NAV_HEADERS, sleep, PRICE_MIN, PRICE_MAX, YEAR_MIN, YEAR_MAX, MAX_INGEST_MI, distFrom16105, metroCoords } from \"../lib/core.ts\";\r\nimport { REGISTRY, PARSERS, FULL_INVENTORY_PARSERS, enrichHit, locFromAreaCode, type Source, type Hit } from \"../lib/sources.ts\";\r\nimport { applyMissed } from \"../lib/enrich.ts\";\r\n\r\nconst TIME_BUDGET_MS = 45_000;\r\n\r\nasync function fetchText(url: string, src: Source): Promise<{ status: number; text: string }> {\r\n const headers: Record = {\r\n \"user-agent\": UA, ...NAV_HEADERS,\r\n \"accept\": src.accept === \"json\" ? \"application/json,text/plain,*/*\" : src.accept === \"xml\" ? \"application/rss+xml,application/atom+xml,application/xml,text/xml,*/*\" : NAV_HEADERS.accept,\r\n ...(src.headers || {}),\r\n };\r\n const init: RequestInit = { headers, redirect: \"follow\", method: src.method || \"GET\", body: src.method === \"POST\" && src.body ? src.body(url) : undefined };\r\n let res = await fetch(url.replace(/#.*$/, \"\"), { ...init, signal: AbortSignal.timeout(15000) }); // a slow host (Farm and Dairy ~8-20 s) must not eat the 45 s budget or the 1-minute val limit\r\n if (res.status === 403 || res.status === 429) { await sleep(3000); res = await fetch(url.replace(/#.*$/, \"\"), { ...init, signal: AbortSignal.timeout(15000) }); }\r\n return { status: res.status, text: res.ok ? await res.text() : \"\" };\r\n}\r\n\r\nexport async function upsertComp(src: string, h: Hit): Promise<\"inserted\" | \"updated\"> {\r\n const ex = await sqlite.execute({ sql: `select 1 from comps where source=? and external_id=?`, args: [src, h.extId] });\r\n await sqlite.execute({\r\n sql: `insert into comps (source, external_id, year, gen, trans, body, miles, sold_price, sold_date, url, title, vin, added) values (?,?,?,?,?,?,?,?,?,?,?,?,?)\r\n on conflict(source, external_id) do update set sold_price=coalesce(excluded.sold_price, comps.sold_price), sold_date=coalesce(excluded.sold_date, comps.sold_date), miles=coalesce(excluded.miles, comps.miles)`,\r\n args: [src, h.extId, h.year ?? null, genFor(h.year ?? null), h.trans || \"unknown\", h.body || \"unknown\", h.miles ?? null, h.soldPrice ?? null, h.soldDate ?? null, h.url, h.title.slice(0, 200), h.vin ?? null, today()],\r\n });\r\n return ex.rows.length ? \"updated\" : \"inserted\";\r\n}\r\n\r\nexport async function runSource(src: Source, started: number): Promise<{ fetched: number; inserted: number; updated: number; pushed: number; ok: boolean; detail: string }> {\r\n const stats = { fetched: 0, inserted: 0, updated: 0, pushed: 0 };\r\n const notes: string[] = [];\r\n const kind = src.kind || \"listing\";\r\n const parser = PARSERS[src.parser];\r\n const hits: Hit[] = [];\r\n let failed = 0, pages = 0, complete = true;\r\n for (const base of src.urls) {\r\n let url = base;\r\n for (let page = 1; page <= (src.maxPages || 1); page++) {\r\n if (Date.now() - started > TIME_BUDGET_MS) { notes.push(\"budget hit before all pages\"); complete = false; break; }\r\n if (page > 1) { if (!src.pageUrl) { complete = false; break; } url = src.pageUrl(base, page); }\r\n try {\r\n const { status, text } = await fetchText(url, src);\r\n pages++;\r\n if (status === 404 && page > 1) break;\r\n if (status !== 200 || !text) { failed++; notes.push(`http ${status} ${url.slice(0, 80)}`); break; }\r\n const r = parser(text, { url });\r\n hits.push(...r.hits);\r\n if (!r.more) break;\r\n if (page === (src.maxPages || 1)) complete = false;\r\n } catch (e) { failed++; complete = false; notes.push(`error ${String(e).slice(0, 80)} ${url.slice(0, 60)}`); break; }\r\n await sleep(src.spacingMs ?? 1500);\r\n }\r\n }\r\n stats.fetched = hits.length;\r\n const seen = new Set();\r\n let skipped = 0;\r\n for (const raw of hits) {\r\n if (seen.has(raw.extId)) continue; seen.add(raw.extId);\r\n if (Date.now() - started > TIME_BUDGET_MS) { notes.push(\"budget hit during upserts; remainder next run\"); break; }\r\n try {\r\n if (kind === \"comp\") { const a = await upsertComp(src.key, raw); if (a === \"inserted\") stats.inserted++; else stats.updated++; continue; }\r\n const h = enrichHit(raw);\r\n if (h.year == null || h.year < YEAR_MIN || h.year > YEAR_MAX) { skipped++; continue; }\r\n if (h.price != null && (h.price < PRICE_MIN || h.price > PRICE_MAX) && h.seller !== \"auction\") { skipped++; continue; } // auction lots carry a running bid, not an asking price\r\n const text = `${h.title} \\n ${h.text}`;\r\n const seller = h.seller ?? inferSeller(text, src.seller);\r\n const loc = h.loc || locFromAreaCode(text) || \"\"; // forum/NCRS rows: the seller's phone area code is the only location hint\r\n const approx = h.lat != null && h.lon != null ? distFrom16105(h.lat, h.lon) : (metroCoords(loc) ? distFrom16105(metroCoords(loc)!.lat, metroCoords(loc)!.lon) : h.dist ?? null);\r\n if (approx != null && approx > MAX_INGEST_MI && !(h.seller === \"auction\" && h.auctionDate)) { skipped++; continue; } // expansion-plan fix 6: nothing beyond ~600 drive mi\r\n const r = await upsertListing({\r\n vin: h.vin ?? null, year: h.year, trim: inferTrim(h.title), trans: h.trans || \"unknown\", body: h.body || \"unknown\", miles: h.miles ?? null, price: h.price ?? null, title: h.titleStatus || \"unknown\", seller,\r\n loc, lat: h.lat ?? null, lon: h.lon ?? null, dist: h.dist ?? null, source: src.key, externalId: h.extId, url: h.url, notes: (h.note || \"\").slice(0, 300), raw: text.slice(0, 6000), text, postedAt: h.posted ?? null, auctionDate: h.auctionDate ?? null,\r\n }, { push: src.push !== false, pushMin: src.pushMin });\r\n if (r.action === \"inserted\") stats.inserted++; else stats.updated++;\r\n if (r.pushed) stats.pushed++;\r\n } catch (e) { notes.push(`upsert ${raw.extId} ${String(e).slice(0, 60)}`); }\r\n }\r\n const ok = failed === 0 && pages > 0;\r\n if (ok && complete && kind === \"listing\" && (src.full || FULL_INVENTORY_PARSERS.has(src.parser)) && hits.length && !notes.some((n) => n.startsWith(\"budget hit\"))) {\r\n try { const m = await applyMissed(src.key, new Set(hits.map((h) => h.extId))); if (m.inc || m.dead) notes.push(`missed +${m.inc}, dead ${m.dead}`); } catch (e) { notes.push(`missed check failed ${String(e).slice(0, 60)}`); }\r\n }\r\n const detail = `${pages} pages, ${hits.length} hits, ${skipped} skipped, ${failed} failed. ${notes.join(\"; \")}`.slice(0, 1900);\r\n return { ...stats, ok, detail };\r\n}\r\n\r\nexport default async function run(opts: { only?: string; force?: boolean } = {}) {\r\n const started = Date.now();\r\n await ensureSchema();\r\n const state = await sqlite.execute(`select source, last_run, fail_count from source_state`);\r\n const last = new Map();\r\n for (const r of state.rows as any[]) last.set(r.source, { last_run: r.last_run, fail_count: Number(r.fail_count) });\r\n const now = Date.now();\r\n const due = REGISTRY.filter((s) => s.enabled !== false && (!opts.only || s.key === opts.only)).map((s) => {\r\n const lr = last.get(s.key)?.last_run; const age = lr ? (now - Date.parse(lr)) / 60000 : Infinity;\r\n return { s, overdue: age - s.every };\r\n }).filter((x) => opts.force || opts.only || x.overdue >= 0).sort((a, b) => b.overdue - a.overdue);\r\n const results: Record = {};\r\n const groupsRan = new Set();\r\n for (const { s } of due) {\r\n if (Date.now() - started > TIME_BUDGET_MS - 5000) { results._skipped = `${due.length - Object.keys(results).length} sources deferred to next run (time budget)`; break; }\r\n if (s.group && !opts.only) { if (groupsRan.has(s.group)) continue; groupsRan.add(s.group); }\r\n const r = await runSource(s, started);\r\n results[s.key] = r;\r\n await logPoll(`sources:${s.key}`, r.ok, r, r.detail);\r\n const fc = r.ok ? 0 : (last.get(s.key)?.fail_count || 0) + 1;\r\n await sqlite.execute({\r\n sql: `insert into source_state (source, last_run, last_ok, fail_count, runs, inserted_total, detail) values (?,?,?,?,1,?,?)\r\n on conflict(source) do update set last_run=excluded.last_run, last_ok=coalesce(excluded.last_ok, source_state.last_ok), fail_count=?, runs=runs+1, inserted_total=inserted_total+?, detail=excluded.detail`,\r\n args: [s.key, new Date().toISOString(), r.ok ? new Date().toISOString() : null, fc, r.inserted, r.detail.slice(0, 500), fc, r.inserted],\r\n });\r\n }\r\n const summary = { ran: Object.keys(results).filter((k) => !k.startsWith(\"_\")).length, due: due.length, seconds: Math.round((Date.now() - started) / 1000), results };\r\n console.log(\"sources\", JSON.stringify(summary).slice(0, 2000));\r\n return summary;\r\n}\r\n","deleted",false,"label",[47,24],"cron","00cc23ab-c212-43f5-a7c5-1ba797ecaa18",["D",1789312547277],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","C4/C5 Corvette search pipeline (CL, forums, auctions, comps)","imageUrl","userId","87445237-7d14-45c7-b3a2-72e031f10ec1","pinnedFileId","2b16e0f0-af8b-11f1-8152-1607ee4eb77e","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","01a09786-a61b-74b5-8910-af3116d3de5c","tursoName","00cc23abc21243f5a7c51ba797ecaa18","tursoHostname","00cc23abc21243f5a7c51ba797ecaa18-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789486866837],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_9":10,"_109":105},"org",{"_9":10,"_102":103,"_104":105,"_58":59,"_106":107,"_108":44},"branch",{"_37":92,"_11":93,"_22":30,"_94":48,"_95":-5,"_96":-5,"_97":-5,"_98":-5,"_99":-5,"_100":12,"_101":10},"isOwned",79,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","displayName","Logan","avatar","https://lh3.googleusercontent.com/a/ACg8ocIFxddfJftSiTFZN49YShT3nY7q8GOQ7dXYM2vJ6XkajnQMGA=s96-c","tier","free","canRestrictVals","profileImageUrl","ctx",{"_85":-5,"_143":144,"_145":146,"_147":148,"_149":44,"_150":44,"_151":44,"_152":34,"_153":-5,"_154":-7,"_155":-7,"_156":66,"_157":44,"_158":44,"_159":-5,"_160":44,"_161":44,"_162":44,"_163":-5},"ENV",{"_114":115,"_116":117,"_118":93,"_119":120,"_121":122,"_123":124,"_125":126,"_127":128,"_129":130,"_131":132,"_133":134,"_135":136,"_137":138,"_139":140,"_141":142},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town","orgs",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]