[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_231":232,"_233":234},"routes/_app.x.$handle.$name",{"_98":97,"_32":45,"_106":88,"_27":37,"_13":14,"_107":-5,"_11":108,"_109":110,"_111":112,"_113":114,"_115":116,"_20":-5,"_117":-5,"_118":-5,"_119":49,"_120":121,"_122":49,"_123":-5,"_124":-5,"_125":126},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_43":44,"_32":45,"_38":46,"_47":-5,"_48":49,"_50":51,"_52":51,"_53":37,"_54":-5,"_18":55,"_56":-5,"_57":49,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":65,"_66":67,"_68":69,"_70":-5,"_71":72,"_73":72,"_74":75,"_76":75,"_77":75,"_78":72,"_79":72,"_80":81,"_82":83,"_84":85,"_86":49},"version",223,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":33,"_34":35,"_36":37,"_38":39,"_40":14,"_41":42,"_24":25},"previous",{"_30":31,"_32":33,"_34":35,"_36":37},"deleted",true,"deletedHere","path","scripts/test.js","latestPath","fileId","56127318-af00-11f1-b41d-1607ee4eb77e","name","test.js","fileType","script","content","","createdAt",["D",1789256710792],"branchVersion","branchId","1dc3dd44-cc65-414f-8e4a-735d1ac2a5b5","id","feed5b42-8d51-40be-9982-13fc813b4b7c","warData",["D",1788316219773],"deletedAt","legacyEnvVars",false,"privacy","public","httpPrivacy","description","imageUrl","0762e08b-0f1b-495a-871c-4fba2da9304a","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","019f5a38-1aa2-74ee-aaf5-8eac040acbf4","tursoName","feed5b428d5140be998213fc813b4b7c","tursoHostname","feed5b428d5140be998213fc813b4b7c-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789458074143],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_98":97,"_105":101},"org",{"_98":97,"_99":97,"_100":101,"_18":55,"_102":103,"_104":49},"branch",{"_13":87,"_32":88,"_43":42,"_89":44,"_90":-5,"_91":-5,"_92":-5,"_93":-5,"_94":-5,"_95":45,"_96":97},"isOwned",237,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","mustedpual","handle","displayName","avatar","https://avatars.githubusercontent.com/u/157880878?v=4","tier","free","canRestrictVals","profileImageUrl","branchName","adoptOrgId",{"_43":44,"_32":45,"_38":226,"_47":-5,"_48":49,"_50":51,"_52":51,"_53":37,"_54":-5,"_18":55,"_56":-5,"_57":49,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":65,"_66":67,"_68":227,"_70":-5,"_71":72,"_73":72,"_74":75,"_76":75,"_77":75,"_78":72,"_79":72,"_80":228,"_82":229,"_84":230,"_86":49},"tree",[187,190,191,192,193],"files",[187,188,189,190,191,192,193],"counts",{"_176":61,"_177":173,"_178":61,"_179":180,"_181":182},"contents",[127,128,129,130,131],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_43":168,"_32":169,"_30":170,"_135":-5,"_41":42,"_40":171,"_137":25,"_38":172,"_13":173,"_34":174,"_36":-5,"_24":49,"_142":-5,"_27":175},{"_43":160,"_32":161,"_30":162,"_135":-5,"_41":42,"_40":163,"_137":25,"_38":164,"_13":165,"_34":140,"_36":166,"_24":49,"_142":-5,"_27":167},{"_43":152,"_32":153,"_30":154,"_135":-5,"_41":42,"_40":155,"_137":25,"_38":156,"_13":157,"_34":140,"_36":158,"_24":49,"_142":-5,"_27":159},{"_43":144,"_32":145,"_30":146,"_135":-5,"_41":42,"_40":147,"_137":25,"_38":148,"_13":149,"_34":140,"_36":150,"_24":49,"_142":-5,"_27":151},{"_43":132,"_32":133,"_30":134,"_135":-5,"_41":42,"_40":136,"_137":49,"_38":138,"_13":139,"_34":140,"_36":141,"_24":49,"_142":-5,"_27":143},"80b6056b-9843-4880-9a09-1eb0fdf24994","warEvent.js","d75184f6-a8f1-11f1-abed-1607ee4eb77e","parentId",220,"latest",["D",1789201435968],80,"interval","import { DOMParser } from \"npm:linkedom\";\nimport puppeteer from \"npm:puppeteer-core\";\nimport { translateToId } from \"./scripts/translate.js\";\n\n// ---------------------------------------------------------------------------\n// Configuration & Credentials\n// ---------------------------------------------------------------------------\nconst TURSO_URL = Deno.env.get(\"TURSO_URL\");\nconst TURSO_AUTH_TOKEN = Deno.env.get(\"TURSO_AUTH_TOKEN\");\nconst BROWSERLESS_TOKEN = Deno.env.get(\"BROWSERLESS_TOKEN\");\n\n// ---------------------------------------------------------------------------\n// Architecture (unchanged shape, cheaper Phase C)\n//\n// PHASE A — POST /unblock once on liveuamap.com.\n// • Solves Cloudflare in a real browser session (costs proxy+captcha units).\n// • Returns main-page HTML (→ extract sublinks) AND a live\n// browserWSEndpoint (→ keep the cleared session alive).\n//\n// PHASE B — Turso lookup: which sublinks are not yet stored.\n//\n// PHASE C — Connect Puppeteer to the browserWSEndpoint from Phase A.\n// Sublink #1: real page.goto() — lands the page on the right origin.\n// Sublinks #2..N: in-page `fetch()` via page.evaluate() — same cookies,\n// same IP, same TLS fingerprint as the cleared session, but no asset\n// loading / render lifecycle. This is the \"handover\" step: cheap on\n// Browserless proxy + time units because nothing is being rendered.\n// ---------------------------------------------------------------------------\n\nconst TARGET_URL = \"https://liveuamap.com\";\nconst PROXY_NETWORK = \"residential\"; // datacenter IPs get pre-flagged\nconst PROXY_COUNTRY = \"de\";\n\nconst RUN_BUDGET_MS = 55_000; // total cron budget\nconst HEADROOM_MS = 10_000; // stop scraping when this remains\nconst UNBLOCK_TIMEOUT_MS = 45_000; // FIX: was 120_000 — see note below\nconst UNBLOCK_TTL_MS = 65_000; // FIX: was 120_000. Val Town free tier hard-kills\n// the val at 60s no matter what — if a run gets killed mid-flight, its\n// Browserless session should die shortly after too, not squat on one of\n// your 2 concurrency slots for another 2 minutes.\nconst PUPPETEER_NAV_TIMEOUT = 15_000; // per-sublink navigation timeout\nconst PUPPETEER_WAIT_TIMEOUT = 10_000; // waitForSelector timeout\nconst PUPPETEER_FETCH_TIMEOUT = 8_000; // in-page fetch timeout (handover step)\n\nconst START = Date.now();\nconst elapsed = () => Date.now() - START;\nconst timeLeft = () => RUN_BUDGET_MS - elapsed();\n\n// ---------------------------------------------------------------------------\n// Turso helpers (unchanged)\n// ---------------------------------------------------------------------------\nfunction typedArg(value) {\n if (value === null || value === undefined) {\n return { type: \"null\", value: null };\n }\n if (typeof value === \"number\") {\n // FIX: Turso's pipeline API rejects integers sent as JSON numbers —\n // it wants them as strings (to preserve 64-bit precision). Floats are\n // fine as-is; this only bit us once the run lock started sending an\n // integer timestamp.\n return Number.isInteger(value)\n ? { type: \"integer\", value: String(value) }\n : { type: \"float\", value };\n }\n return { type: \"text\", value: String(value) };\n}\n\nasync function tursoPipeline(stmts) {\n const base = TURSO_URL.replace(/\\/+$/, \"\");\n const requests = [\n ...stmts.map(({ sql, args }) => ({\n type: \"execute\",\n stmt: { sql, args: (args || []).map(typedArg) },\n })),\n { type: \"close\" },\n ];\n\n const res = await fetch(`${base}/v2/pipeline`, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${TURSO_AUTH_TOKEN}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({ requests }),\n });\n\n if (!res.ok) {\n throw new Error(`Turso HTTP ${res.status}: ${await res.text()}`);\n }\n return res.json();\n}\n\nasync function ensureTable() {\n await tursoPipeline([\n {\n sql: `CREATE TABLE IF NOT EXISTS warfare_events (\n id TEXT PRIMARY KEY,\n title TEXT,\n status TEXT,\n location TEXT,\n date TEXT,\n longitude REAL,\n latitude REAL,\n athumbnail TEXT,\n reference TEXT\n )`,\n },\n {\n sql: `CREATE TABLE IF NOT EXISTS run_lock (\n id INTEGER PRIMARY KEY CHECK (id = 1),\n locked_until INTEGER\n )`,\n },\n ]);\n}\n\n// FIX: Val Town's cron overlap behavior isn't publicly documented, so rather\n// than assume it never overlaps, this lock makes it not matter. Any run\n// checks a single Turso row before touching Browserless at all; if a prior\n// run's lock hasn't expired, this run bails out instead of opening a second\n// Browserless session and contending for your 2-concurrency cap.\nasync function acquireRunLock() {\n const now = Date.now();\n const out = await tursoPipeline([\n { sql: `SELECT locked_until FROM run_lock WHERE id = 1` },\n ]);\n const rows = out?.results?.[0]?.response?.result?.rows || [];\n const lockedUntil = rows[0]?.[0]?.value ? Number(rows[0][0].value) : 0;\n\n if (lockedUntil > now) {\n console.warn(\n `Run lock still held until ${\n new Date(lockedUntil).toISOString()\n }; skipping this tick.`,\n );\n return false;\n }\n\n // Lock covers this run's full possible lifetime (Val Town's hard kill) plus\n // a small margin for Browserless's own session teardown.\n const newLockedUntil = now + RUN_BUDGET_MS + 10_000;\n await tursoPipeline([\n {\n sql: `INSERT INTO run_lock (id, locked_until) VALUES (1, ?) ` +\n `ON CONFLICT(id) DO UPDATE SET locked_until = excluded.locked_until`,\n args: [newLockedUntil],\n },\n ]);\n return true;\n}\n\nasync function pendingSublinks(sublinks) {\n if (sublinks.length === 0) return [];\n const placeholders = sublinks.map(() => \"?\").join(\",\");\n const out = await tursoPipeline([\n {\n sql: `SELECT id FROM warfare_events WHERE id IN (${placeholders})`,\n args: sublinks,\n },\n ]);\n const rows = out?.results?.[0]?.response?.result?.rows || [];\n const known = new Set(rows.map((r) => r[0]?.value));\n return sublinks.filter((l) => !known.has(l));\n}\n\n// ---------------------------------------------------------------------------\n// Date extraction (unchanged — unit-tested offline, all cases pass)\n// ---------------------------------------------------------------------------\nconst MONTHS = {\n january: \"01\",\n february: \"02\",\n march: \"03\",\n april: \"04\",\n may: \"05\",\n june: \"06\",\n july: \"07\",\n august: \"08\",\n september: \"09\",\n october: \"10\",\n november: \"11\",\n december: \"12\",\n};\n\nfunction extractLiveUAMapDate(url) {\n const match = url.match(/\\/(\\d{4})\\/(\\d{1,2})-([a-z]+)-\\d+-/i);\n if (!match) return null;\n const [, year, day, monthName] = match;\n const mm = MONTHS[monthName.toLowerCase()];\n return mm ? `${year}-${mm}-${String(day).padStart(2, \"0\")}` : null;\n}\n\n// ---------------------------------------------------------------------------\n// Parse ONE sublink's HTML -> GeoJSON Feature\n// FIX: log when the default coordinate fallback is used, so bad extractions\n// are visible instead of silently landing at a fixed point.\n// ---------------------------------------------------------------------------\nconst dom = new DOMParser();\n\nfunction parseFeatureHtml(sublink, subHtml) {\n const subdoc = dom.parseFromString(subHtml, \"text/html\");\n\n let lat = 48.8829;\n let lng = 31.181;\n let usedDefaultCoords = true;\n\n const eventScriptMatch = subHtml.match(\n /document\\.addEventListener\\(['\"]DOMContentLoaded['\"][\\s\\S]*?\\);/,\n );\n if (eventScriptMatch) {\n const latMatch = eventScriptMatch[0].match(/lat\\s*=\\s*([0-9.-]+)/);\n const lngMatch = eventScriptMatch[0].match(/lng\\s*=\\s*([0-9.-]+)/);\n if (latMatch) {\n lat = parseFloat(latMatch[1]);\n usedDefaultCoords = false;\n }\n if (lngMatch) {\n lng = parseFloat(lngMatch[1]);\n usedDefaultCoords = false;\n }\n }\n if (usedDefaultCoords) {\n console.warn(`No lat/lng found for ${sublink}; using default coordinates.`);\n }\n\n const infozone = subdoc.querySelector(\".popup-text\");\n const status = infozone?.querySelector(\".bgma\")?.getAttribute(\"data-src\") ||\n null;\n const title = infozone?.querySelector(\"h2\")?.textContent?.trim() || null;\n const locationText =\n infozone?.querySelector(\".tagas strong\")?.textContent?.trim() || null;\n const thumbnail =\n infozone?.querySelector(\".popup_video + * img\")?.getAttribute(\"src\") ||\n null;\n const reference =\n subdoc.querySelector(\".source-link\")?.getAttribute(\"href\") || null;\n\n return {\n type: \"Feature\",\n geometry: { type: \"Point\", coordinates: [lng, lat] },\n properties: {\n references: [sublink, reference].filter(Boolean),\n title,\n status,\n location: locationText,\n thumbnail,\n date: extractLiveUAMapDate(sublink),\n },\n };\n}\n\n// ---------------------------------------------------------------------------\n// Translate + upsert ONE feature into Turso (unchanged)\n// ---------------------------------------------------------------------------\nasync function saveFeature(feature) {\n const { properties, geometry } = feature;\n const [lng, lat] = geometry.coordinates;\n const eventId = properties.references[0] || crypto.randomUUID();\n\n let translatedTitle = properties.title;\n if (translatedTitle) {\n try {\n translatedTitle = await translateToId(properties.title);\n } catch (err) {\n console.error(\"Translation failed for title:\", properties.title, err);\n }\n }\n\n await tursoPipeline([\n {\n sql: `INSERT OR REPLACE INTO warfare_events\n (id, title, status, location, date, longitude, latitude, athumbnail, reference)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n eventId,\n translatedTitle,\n properties.status,\n properties.location,\n properties.date,\n lng,\n lat,\n properties.thumbnail,\n properties.references[1] || properties.references[0] || null,\n ],\n },\n ]);\n}\n\n// ---------------------------------------------------------------------------\n// Helpers\n// ---------------------------------------------------------------------------\nfunction cleanDataLink(raw) {\n if (!raw) return null;\n let v = raw.trim();\n v = v.replace(/\\\\(.)/g, \"$1\");\n v = v.replace(/^[\"']+|[\"']+$/g, \"\").trim();\n return v || null;\n}\n\n// ---------------------------------------------------------------------------\n// PHASE A — /unblock: solve Cloudflare once, harvest sublinks + a live\n// browser session.\n// FIX: UNBLOCK_TIMEOUT_MS dropped from 120s to 45s, and the 408-retry\n// removed. At 120s + a retry, a single Phase A call could take up to ~260s\n// — nearly 5x your stated 55s cron budget. If your scheduler's actual\n// wall-clock limit is anywhere near 55-60s, invocations were very likely\n// getting killed mid-/unblock and overlapping with the next scheduled run,\n// which is the most likely explanation for \"peak concurrency 2/2\" and your\n// 19 rejected requests (you only get 2 concurrent sessions on the free\n// tier). If you genuinely need up to 120s for CF to solve, this needs to\n// live in a background/long-running task, not a tight cron budget — worth\n// checking your Deno Deploy invocation type before tuning further.\n// ---------------------------------------------------------------------------\nasync function initSessionAndExtractLinks() {\n // FIX: Browserless's own docs — \"By default, all requests will go through\n // a random node in the proxy pool\" — confirm that without proxySticky,\n // EVERY request (even within the same connected session) can rotate to a\n // different residential IP node. That means the 21 in-page handover\n // fetches were likely each burning a fresh residential IP assignment\n // instead of reusing the one Cloudflare already cleared — probably the\n // single biggest proxy-unit cost in this whole pipeline, and also a\n // reliability risk (a mismatched IP is exactly what could make Cloudflare\n // re-challenge a \"cleared\" session).\n const queryParams = new URLSearchParams({\n token: BROWSERLESS_TOKEN,\n proxy: PROXY_NETWORK,\n proxyCountry: PROXY_COUNTRY,\n proxySticky: \"true\",\n timeout: String(UNBLOCK_TIMEOUT_MS),\n }).toString();\n\n const unblockURL =\n `https://production-sfo.browserless.io/unblock?${queryParams}`;\n\n const body = {\n url: TARGET_URL,\n content: true,\n browserWSEndpoint: true,\n cookies: false,\n screenshot: false,\n ttl: UNBLOCK_TTL_MS,\n waitForSelector: {\n selector: \"[data-link]\",\n timeout: 15000,\n },\n };\n\n console.log(\"Requesting main page via Browserless /unblock…\");\n\n let res;\n try {\n res = await fetch(unblockURL, {\n method: \"POST\",\n headers: { \"Content-Type\": \"application/json\" },\n body: JSON.stringify(body),\n signal: AbortSignal.timeout(UNBLOCK_TIMEOUT_MS + 5_000),\n });\n } catch (err) {\n console.error(`/unblock request failed (${err.message}); skipping run.`);\n return null;\n }\n\n if (!res.ok) {\n console.error(`/unblock HTTP ${res.status}: ${await res.text()}`);\n return null;\n }\n\n const payload = await res.json();\n const html = payload.content ?? \"\";\n const browserWSEndpoint = payload.browserWSEndpoint ?? null;\n\n console.log(\n `Unblock response: html length=${html.length} ` +\n `browserWSEndpoint=${browserWSEndpoint ? \"yes\" : \"no\"} ` +\n `(${elapsed()}ms elapsed)`,\n );\n\n if (!html) {\n console.error(\"/unblock returned no content; skipping this run.\");\n return null;\n }\n if (!browserWSEndpoint) {\n console.error(\n \"/unblock returned no browserWSEndpoint; cannot continue to Phase C.\",\n );\n return null;\n }\n\n const doc = dom.parseFromString(html, \"text/html\");\n const sublinks = [\n ...new Set(\n [...doc.querySelectorAll(\"[data-link]\")]\n .map((e) => cleanDataLink(e.getAttribute(\"data-link\")))\n .filter(Boolean),\n ),\n ];\n\n if (sublinks.length === 0) {\n const pageTitle = doc.querySelector(\"title\")?.textContent?.trim() || null;\n console.warn(\n `No [data-link] matches. page title=\"${pageTitle}\". ` +\n `If this looks like a Cloudflare or error page, the unblock ` +\n `did not clear the challenge — check the Browserless dashboard.`,\n );\n }\n\n console.log(\n `Extracted ${sublinks.length} sublinks from the unblocked main page ` +\n `(${elapsed()}ms elapsed).`,\n );\n\n return { sublinks, browserWSEndpoint, queryParams };\n}\n\n// ---------------------------------------------------------------------------\n// PHASE C — Connect Puppeteer to the cleared session.\n//\n// Sublink #1: real page.goto() navigation — this is what actually lands\n// the page's document.location on liveuamap.com, so subsequent same-origin\n// fetches aren't blocked by CORS.\n//\n// Sublinks #2..N (\"the handover\"): page.evaluate(fetch(...)) run FROM\n// INSIDE the already-cleared page. Same cookie jar, same IP, same TLS\n// fingerprint Cloudflare already accepted — but no CSS/JS/image loading,\n// no navigation lifecycle, no waitForSelector. This is the expensive part\n// you're seeing in proxy/time units, cut down to just the raw HTML bytes.\n//\n// CAVEAT — please verify on one real sublink before trusting this in prod:\n// this only works if the fields parseFeatureHtml() reads (.popup-text, the\n// inline lat/lng script, etc.) are present in the server-rendered HTML\n// rather than injected by client-side JS after load. Since the original\n// code already gets what it needs from page.content() after just\n// domcontentloaded (not networkidle / full JS execution), that's a good\n// sign it's server-rendered — but I can't confirm this without hitting the\n// live site, which this sandbox can't do. Quick check: log both\n// `await page.content()` (after goto) and the raw `fetch()` text for the\n// same sublink once, and diff them for the fields you actually use.\n// ---------------------------------------------------------------------------\nasync function scrapeSublinksWithPuppeteer(\n pending,\n browserWSEndpoint,\n queryParams,\n) {\n if (pending.length === 0) return { saved: 0, skipped: 0, failed: 0 };\n\n let browser;\n try {\n const sep = browserWSEndpoint.includes(\"?\") ? \"&\" : \"?\"; // FIX: avoid double \"?\"\n browser = await puppeteer.connect({\n browserWSEndpoint: `${browserWSEndpoint}${sep}${queryParams}`,\n });\n } catch (err) {\n console.error(`Puppeteer connect failed: ${err.message}`);\n return { saved: 0, skipped: pending.length, failed: 0 };\n }\n\n const page = await browser.newPage();\n\n // FIX (hypothesis, unverified live): /unblock solves Cloudflare inside a\n // real tab that loaded the live liveuamap.com homepage — a page with\n // continuous background activity (live-update polling, map tiles, etc).\n // Phase C never touched that tab before; if Browserless keeps it open for\n // the session's ttl, it can generate proxy traffic for the whole run\n // independent of how many sublinks you actually scrape. Closing every\n // page except the one we're about to use rules this out. The log line\n // below is the actual confirmation: if it reports >0 on your next run,\n // this was real.\n const existingPages = await browser.pages();\n const leftovers = existingPages.filter((p) => p !== page);\n if (leftovers.length > 0) {\n console.log(\n `Closing ${leftovers.length} leftover page(s) from the /unblock session.`,\n );\n await Promise.all(\n leftovers.map((p) =>\n p.close().catch((err) =>\n console.warn(`Failed to close leftover page: ${err.message}`)\n )\n ),\n );\n } else {\n console.log(\"No leftover pages found in the cleared session.\");\n }\n\n // FIX: block heavy asset types on the one real navigation. parseFeatureHtml\n // only ever reads raw HTML (.popup-text, an inline lat/lng script) — never\n // rendered images/fonts/CSS/media — so these are pure bandwidth with zero\n // effect on correctness. Left enabled for the whole session (not just i===0)\n // since it's harmless for the later in-page fetch() calls too — those never\n // request these resource types in the first place.\n const BLOCKED_RESOURCE_TYPES = new Set([\n \"image\",\n \"stylesheet\",\n \"font\",\n \"media\",\n \"other\",\n ]);\n await page.setRequestInterception(true);\n page.on(\"request\", (req) => {\n if (BLOCKED_RESOURCE_TYPES.has(req.resourceType())) {\n req.abort().catch(() => {});\n } else {\n req.continue().catch(() => {});\n }\n });\n\n let saved = 0;\n let skipped = 0;\n let failed = 0;\n let i = 0;\n\n for (; i < pending.length; i++) {\n if (timeLeft() < HEADROOM_MS) break;\n\n const sublink = pending[i];\n\n try {\n let subHtml;\n\n if (i === 0) {\n // Real navigation once, to establish origin for same-origin fetches after.\n await page.goto(sublink, {\n waitUntil: \"domcontentloaded\",\n timeout: PUPPETEER_NAV_TIMEOUT,\n });\n await page.waitForSelector(\".popup-text\", {\n timeout: PUPPETEER_WAIT_TIMEOUT,\n });\n subHtml = await page.content();\n } else {\n // Handover: cheap in-page fetch, same cleared session.\n subHtml = await page.evaluate(\n async (url, timeoutMs) => {\n const controller = new AbortController();\n const t = setTimeout(() => controller.abort(), timeoutMs);\n try {\n const res = await fetch(url, {\n credentials: \"include\",\n signal: controller.signal,\n });\n if (!res.ok) throw new Error(`HTTP ${res.status}`);\n return await res.text();\n } finally {\n clearTimeout(t);\n }\n },\n sublink,\n PUPPETEER_FETCH_TIMEOUT,\n );\n }\n\n const feature = parseFeatureHtml(sublink, subHtml);\n await saveFeature(feature);\n saved++;\n console.log(`Saved [${saved}] ${feature.properties.title ?? sublink}`);\n } catch (err) {\n if (/timeout|Timeout|abort/i.test(err.message)) {\n skipped++;\n console.warn(`Skipped (timeout) ${sublink}: ${err.message}`);\n } else {\n failed++;\n console.error(`Parse/save failed: ${sublink} — ${err.message}`);\n }\n }\n }\n\n try {\n await browser.close();\n } catch (err) {\n console.warn(`Puppeteer close failed (non-fatal): ${err.message}`);\n }\n\n return { saved, skipped, failed, processed: i };\n}\n\n// ---------------------------------------------------------------------------\n// Main flow (unchanged)\n// ---------------------------------------------------------------------------\nasync function run() {\n await ensureTable();\n\n const gotLock = await acquireRunLock();\n if (!gotLock) return;\n\n const pool = await initSessionAndExtractLinks();\n if (!pool || pool.sublinks.length === 0) {\n console.warn(\"No sublinks extracted from /unblock; nothing to do.\");\n return;\n }\n\n const pending = await pendingSublinks(pool.sublinks);\n console.log(\n `Main page: ${pool.sublinks.length} sublinks, ` +\n `${pool.sublinks.length - pending.length} already stored, ` +\n `${pending.length} pending (${elapsed()}ms elapsed)`,\n );\n\n const { saved, skipped, failed, processed } =\n await scrapeSublinksWithPuppeteer(\n pending,\n pool.browserWSEndpoint,\n pool.queryParams,\n );\n\n const deferred = pending.length - (processed ?? 0);\n\n console.log(\n `Done in ${elapsed()}ms: ${saved} saved, ${skipped} skipped (timeout), ` +\n `${failed} parse/save failed, ${deferred} deferred to next run.`,\n );\n}\n\n// ---------------------------------------------------------------------------\n// Cron entrypoint\n// ---------------------------------------------------------------------------\nexport default async function () {\n if (!TURSO_URL || !TURSO_AUTH_TOKEN) {\n throw new Error(\"Missing TURSO_URL / TURSO_AUTH_TOKEN env vars\");\n }\n if (!BROWSERLESS_TOKEN) {\n throw new Error(\"Missing BROWSERLESS_TOKEN env var\");\n }\n\n try {\n await run();\n } catch (error) {\n console.error(\"Pipeline failed:\", error);\n throw error;\n }\n}","label",[133],"4d2ece01-2d75-488d-80d4-8ae766cde047","ukmtoEvent.js","7e654b18-a9b2-11f1-9377-1607ee4eb77e",154,["D",1788982196432],10,"import { translateToId } from \"./scripts/translate.js\";\n// ---------------------------------------------------------------------------\n// UKMTO incident feed -> Indonesian-translated events -> Turso.\n// Fetches https://sccd.royalnavy.mod.uk/api/ukmto/all (array of incidents),\n// compares against existing records in Turso to skip unchanged items,\n// translates new/changed incidentTypeName + otherDetails into Indonesian,\n// then upserts into Turso in safe chunks.\n// ---------------------------------------------------------------------------\nlet fetchQueue = Promise.resolve();\n\nconst Url = \"https://sccd.royalnavy.mod.uk/api/ukmto/all\";\n\nconst HeaderS = {\n \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64)\",\n Accept: \"application/json\",\n};\n\nconst UkmtoData = {\n type: \"ukmto\",\n parent: null, // Untouched master data\n temp: null, // Current filtered view for the map\n};\n\n// ---------------------------------------------------------------------------\n// Turso client via its HTTP API (POST /v2/pipeline). Credentials from env vars.\n// ---------------------------------------------------------------------------\nconst TURSO_URL = Deno.env.get(\"TURSO_URL\");\nconst TURSO_AUTH_TOKEN = Deno.env.get(\"TURSO_AUTH_TOKEN\");\n\nif (!TURSO_URL || !TURSO_AUTH_TOKEN) {\n throw new Error(\"Missing TURSO_URL / TURSO_AUTH_TOKEN env vars\");\n}\nconst TURSO_BASE = TURSO_URL.replace(/\\/+$/, \"\");\n\n// Turso's v2/pipeline arg format: integer/text values are JSON STRINGS;\n// float values are JSON NUMBERS. See https://docs.turso.tech/sdk/http/quickstart\nfunction typedArg(value) {\n if (value === null || value === undefined) {\n return { type: \"null\", value: null };\n }\n if (typeof value === \"number\") {\n return Number.isInteger(value)\n ? { type: \"integer\", value: String(value) }\n : { type: \"float\", value };\n }\n return { type: \"text\", value: String(value) };\n}\n\nasync function tursoPipeline(stmts) {\n const requests = [\n ...stmts.map(({ sql, args }) => ({\n type: \"execute\",\n stmt: { sql, args: (args || []).map(typedArg) },\n })),\n { type: \"close\" },\n ];\n const res = await fetch(`${TURSO_BASE}/v2/pipeline`, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${TURSO_AUTH_TOKEN}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({ requests }),\n });\n if (!res.ok) {\n throw new Error(`Turso HTTP ${res.status}: ${await res.text()}`);\n }\n return res.json();\n}\n\n// ---------------------------------------------------------------------------\n// Map a raw UKMTO incident to the fields the map consumes.\n// ---------------------------------------------------------------------------\nfunction mapIncident(inc) {\n return {\n incidentTypeName: inc.incidentTypeName,\n incidentNumber: inc.incidentNumber,\n incidentIssuer: inc.incidentIssuer,\n place: inc.place,\n sitecoreId: inc.sitecoreId,\n pinColour: inc.pinColour,\n locationLatitudeDDDMMSS: inc.locationLatitudeDDDMMSS,\n locationLongitudeDDDMMSS: inc.locationLongitudeDDDMMSS,\n utcDateOfIncident: inc.utcDateOfIncident,\n utcDateCreated: inc.utcDateCreated,\n // data-lng / data-lat attributes on map markers\n dataLng: inc.locationLongitude,\n dataLat: inc.locationLatitude,\n otherDetails: inc.otherDetails,\n };\n}\n\n// ---------------------------------------------------------------------------\n// Create table, fetch existing rows for comparison, translate only changes,\n// and upsert in chunks.\n// ---------------------------------------------------------------------------\nasync function saveToDb(incidents) {\n // 1. Ensure table exists\n await tursoPipeline([\n {\n sql: `CREATE TABLE IF NOT EXISTS ukmto_events (\n incidentNumber INTEGER PRIMARY KEY,\n incidentTypeName TEXT,\n incidentTypeNameId TEXT,\n incidentIssuer TEXT,\n place TEXT,\n sitecoreId TEXT,\n pinColour TEXT,\n locationLatitudeDDDMMSS TEXT,\n locationLongitudeDDDMMSS TEXT,\n utcDateOfIncident TEXT,\n utcDateCreated TEXT,\n longitude REAL,\n latitude REAL,\n otherDetails TEXT,\n otherDetailsId TEXT\n )`,\n },\n ]);\n\n // 2. Fetch existing records from Turso to compare against\n let existingMap = new Map();\n try {\n const res = await tursoPipeline([\n {\n sql:\n `SELECT incidentNumber, incidentTypeName, otherDetails, incidentTypeNameId, otherDetailsId FROM ukmto_events`,\n },\n ]);\n const rows = res.results?.[0]?.response?.result?.rows || [];\n for (const row of rows) {\n const incNum = row[0]?.value;\n if (incNum !== undefined && incNum !== null) {\n existingMap.set(Number(incNum), {\n incidentTypeName: row[1]?.value ?? null,\n otherDetails: row[2]?.value ?? null,\n incidentTypeNameId: row[3]?.value ?? null,\n otherDetailsId: row[4]?.value ?? null,\n });\n }\n }\n } catch (err) {\n console.log(\n \"Could not fetch existing rows, proceeding with full sync:\",\n err,\n );\n }\n\n const insertStmts = [];\n let translatedCount = 0;\n let skippedCount = 0;\n\n for (const inc of incidents) {\n const existing = existingMap.get(inc.incidentNumber);\n\n let nameId;\n let detailsId;\n\n // Check if record already exists and text content hasn't changed\n if (\n existing &&\n existing.incidentTypeName === inc.incidentTypeName &&\n existing.otherDetails === inc.otherDetails\n ) {\n // Content is identical! Reuse existing translations and skip inserting.\n nameId = existing.incidentTypeNameId;\n detailsId = existing.otherDetailsId;\n skippedCount++;\n continue;\n }\n\n // If it's new or text changed, translate the text\n try {\n nameId = inc.incidentTypeName\n ? await translateToId(inc.incidentTypeName)\n : inc.incidentTypeName;\n detailsId = inc.otherDetails\n ? await translateToId(inc.otherDetails)\n : inc.otherDetails;\n translatedCount++;\n } catch (err) {\n console.error(\"Translation failed for incident\", inc.incidentNumber, err);\n nameId = inc.incidentTypeName;\n detailsId = inc.otherDetails;\n }\n\n insertStmts.push({\n sql: `INSERT OR REPLACE INTO ukmto_events\n (incidentNumber, incidentTypeName, incidentTypeNameId, incidentIssuer,\n place, sitecoreId, pinColour, locationLatitudeDDDMMSS,\n locationLongitudeDDDMMSS, utcDateOfIncident, utcDateCreated,\n longitude, latitude, otherDetails, otherDetailsId)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n inc.incidentNumber,\n inc.incidentTypeName,\n nameId,\n inc.incidentIssuer,\n inc.place,\n inc.sitecoreId,\n inc.pinColour,\n inc.locationLatitudeDDDMMSS,\n inc.locationLongitudeDDDMMSS,\n inc.utcDateOfIncident,\n inc.utcDateCreated,\n inc.dataLng,\n inc.dataLat,\n inc.otherDetails,\n detailsId,\n ],\n });\n }\n\n // 3. Send inserts in chunks of 10 to avoid payload size/timeout limits\n if (insertStmts.length > 0) {\n const CHUNK_SIZE = 10;\n for (let i = 0; i < insertStmts.length; i += CHUNK_SIZE) {\n const chunk = insertStmts.slice(i, i + CHUNK_SIZE);\n await tursoPipeline([...chunk]);\n }\n }\n\n console.log(\n `Sync complete. Total incoming: ${incidents.length} | Skipped (unchanged): ${skippedCount} | Translated & Updated: ${translatedCount}`,\n );\n}\n\nexport default async function handler() {\n await fetchAndPopulate(Url, HeaderS, UkmtoData);\n}\n\nfunction fetchAndPopulate(url, headerS, constData) {\n fetchQueue = fetchQueue.then(async () => {\n try {\n const response = await fetch(url, { method: \"GET\", headers: headerS });\n if (!response.ok) {\n throw new Error(`UKMTO HTTP ${response.status}`);\n }\n\n const raw = await response.json();\n const mapped = Array.isArray(raw) ? raw.map(mapIncident) : [];\n\n console.log(`Fetched ${mapped.length} UKMTO incidents.`);\n\n // Persist (create table + smart check & translate + upsert)\n await saveToDb(mapped);\n\n constData.parent = mapped;\n constData.temp = [...mapped];\n } catch (error) {\n console.error(\"UKMTO fetch failed:\", error);\n }\n });\n return fetchQueue;\n}\n\nexport { mapIncident, UkmtoData };",[145],"18c6bbf2-42b6-4dc8-9bcd-e81d1e308098","easaEvent.js","b21fc016-a9bf-11f1-8cb4-1607ee4eb77e",151,["D",1788942414030],33,"// ---------------------------------------------------------------------------\n// EASA Conflict Zone Information Bulletins (CZIB) -> JSON -> Turso (Val Town)\n// ---------------------------------------------------------------------------\nimport { DOMParser } from \"npm:linkedom\";\n\nconst MAIN_URL = \"https://www.easa.europa.eu/en/domains/air-operations/czibs\";\n\nconst HEADERS = {\n \"User-Agent\":\n \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36\",\n Accept: \"text/html,application/json,text/plain,*/*\",\n};\n\nconst dom = new DOMParser();\n\nfunction clean(s) {\n return (s || \"\").replace(/\\s+/g, \" \").trim();\n}\n\nfunction isDetailHref(h) {\n if (!h) return false;\n if (!h.includes(\"/czibs/\")) return false;\n if (/czibs\\/(information|map|export|feed)/.test(h)) return false;\n return true;\n}\n\nasync function listActiveRows() {\n const res = await fetch(MAIN_URL, { headers: HEADERS });\n if (!res.ok) throw new Error(`overview HTTP ${res.status}`);\n const html = await res.text();\n const doc = dom.parseFromString(html, \"text/html\");\n\n const rows = doc.querySelectorAll(\n \"#status-div-classeasa-status-field-status-active--div-classeasa-status-field-status-active-field-itemdiv-classcz-status-cz-status-activeactivedivdivdiv tbody tr\",\n );\n const active = [];\n\n for (const row of rows) {\n const link = Array.from(row.querySelectorAll(\"a\")).find((a) =>\n isDetailHref(a.getAttribute(\"href\"))\n );\n\n if (!link) continue;\n\n const detailUrl = new URL(link.getAttribute(\"href\"), MAIN_URL).href;\n const dateStart = clean(\n row.querySelector(\n \"td.views-field.views-field-field-easa-conflict-zone-up-date\",\n )?.textContent,\n );\n const dateEnd = clean(\n row.querySelector(\n \"td.views-field.views-field-field-easa-conflict-zone-until\",\n )?.textContent,\n );\n\n active.push({ detailUrl, dateStart, dateEnd });\n }\n return active;\n}\n\nasync function scrapeDetail(item) {\n try {\n const res = await fetch(item.detailUrl, { headers: HEADERS });\n if (!res.ok) {\n console.error(`detail HTTP ${res.status}: ${item.detailUrl}`);\n return null;\n }\n const html = await res.text();\n const doc = dom.parseFromString(html, \"text/html\");\n\n const fieldValue = (selector) =>\n clean(doc.querySelector(selector)?.textContent);\n\n const czibNumber =\n fieldValue(\".field--name-field-easa-czib-number .field__items div\") ||\n fieldValue(\".field-name-field-easa-czib-number .field-items div\") ||\n fieldValue(\".field-name-field-easa-czib-number div div\");\n\n const publicator = fieldValue(\".czib_ref_publications .field__items div\") ||\n fieldValue(\".czib_ref_publications .field-items div\") ||\n fieldValue(\".czib_ref_publications div div div\");\n\n const airspace = fieldValue(\n \".field--name-field-easa-cz-aff-air .field__items div\",\n ) || fieldValue(\n \".field-name-field-easa-cz-aff-air .field-items div\",\n );\n\n const description = fieldValue(\n \".field--name-field-easa-cz-description .field__items div\",\n ) || fieldValue(\n \".field-name-field-easa-cz-description .field-items div\",\n );\n\n const recommendation = fieldValue(\n \".field--name-field-easa-cz-recommendations .field__items div\",\n ) || fieldValue(\n \".field-name-field-easa-cz-recommendations .field-items div\",\n );\n\n const csvHref =\n doc.querySelector('a[href*=\"conflict-zone-node-export\"]')?.getAttribute(\n \"href\",\n ) || \"\";\n const nid = (csvHref.match(/(\\d+)\\/?$/) || [])[1] || null;\n\n let coordinate = null;\n if (nid) {\n try {\n const apiRes = await fetch(\n `https://www.easa.europa.eu/en/api/maps/czib-node/${nid}`,\n { headers: HEADERS },\n );\n if (apiRes.ok) coordinate = await apiRes.json();\n } catch (err) {\n console.error(\"Coordinate fetch failed for\", item.detailUrl, err);\n }\n }\n\n return {\n nid,\n czibNumber,\n publicator,\n airspace,\n description,\n recommendation,\n coordinate,\n };\n } catch (err) {\n console.error(`Failed parsing detail page ${item.detailUrl}:`, err.message);\n return null;\n }\n}\n\nfunction buildRecord(item, detail) {\n if (!detail) return null;\n return {\n nid: detail.nid || null,\n status: \"Active\",\n czibNumber: detail.czibNumber || null,\n dateStart: item.dateStart || null,\n dateEnd: item.dateEnd || null,\n airspace: detail.airspace || null,\n publicator: detail.publicator || null,\n description: detail.description || null,\n recommendation: detail.recommendation || null,\n coordinate: detail.coordinate,\n detailUrl: item.detailUrl,\n };\n}\n\nasync function mapLimit(items, limit, fn) {\n const results = new Array(items.length);\n let i = 0;\n const workers = Array.from(\n { length: Math.min(limit, items.length) },\n async () => {\n while (i < items.length) {\n const idx = i++;\n results[idx] = await fn(items[idx], idx);\n }\n },\n );\n await Promise.all(workers);\n return results;\n}\n\nconst TURSO_BASE = (Deno.env.get(\"TURSO_URL\") || \"\").replace(/\\/+$/, \"\");\nconst TURSO_AUTH_TOKEN = Deno.env.get(\"TURSO_AUTH_TOKEN\");\n\nif (!TURSO_BASE || !TURSO_AUTH_TOKEN) {\n throw new Error(\n \"Missing TURSO_URL / TURSO_AUTH_TOKEN env vars in Val Town secrets\",\n );\n}\n\nfunction typedArg(value) {\n if (value === null || value === undefined) {\n return { type: \"null\", value: null };\n }\n if (typeof value === \"number\") {\n return { type: Number.isInteger(value) ? \"integer\" : \"float\", value };\n }\n return { type: \"text\", value: String(value) };\n}\n\nfunction typedFloat(value) {\n if (value === null || value === undefined) {\n return { type: \"null\", value: null };\n }\n const num = Number(value);\n if (isNaN(num)) return { type: \"null\", value: null };\n return { type: \"float\", value: num };\n}\n\nasync function tursoPipeline(stmts) {\n const requests = [\n ...stmts.map(({ sql, args }) => ({\n type: \"execute\",\n stmt: { sql, args: args || [] },\n })),\n { type: \"close\" },\n ];\n const res = await fetch(`${TURSO_BASE}/v2/pipeline`, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${TURSO_AUTH_TOKEN}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({ requests }),\n });\n if (!res.ok) throw new Error(`Turso HTTP ${res.status}: ${await res.text()}`);\n return res.json();\n}\n\nasync function saveToDb(records) {\n await tursoPipeline([\n {\n sql: `CREATE TABLE IF NOT EXISTS easa_czibs (\n nid TEXT PRIMARY KEY,\n status TEXT,\n czib_number TEXT,\n date_start TEXT,\n date_end TEXT,\n airspace TEXT,\n publicator TEXT, \n description TEXT, \n recommendation TEXT,\n latitude REAL NULL, \n longitude REAL NULL,\n detail_url TEXT\n )`,\n },\n ]);\n\n const insertStmts = records.map((r) => ({\n sql: `INSERT OR REPLACE INTO easa_czibs\n (nid, status, czib_number, date_start, date_end, airspace,\n publicator, description, recommendation,\n latitude, longitude, detail_url)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n typedArg(r.nid || crypto.randomUUID()),\n typedArg(r.status),\n typedArg(r.czibNumber),\n typedArg(r.dateStart),\n typedArg(r.dateEnd),\n typedArg(r.airspace),\n typedArg(r.publicator ?? \"\"),\n typedArg(r.description ?? \"\"),\n typedArg(r.recommendation ?? \"\"),\n typedFloat(r.coordinate ? r.coordinate.lat : null),\n typedFloat(r.coordinate ? r.coordinate.lng : null),\n typedArg(r.detailUrl),\n ],\n }));\n\n await tursoPipeline(insertStmts);\n}\n\nexport default async function handler() {\n try {\n const rows = await listActiveRows();\n console.log(`Found ${rows.length} active CZIB(s).`);\n\n const details = await mapLimit(rows, 4, (item) => scrapeDetail(item));\n\n const records = [];\n for (let i = 0; i < details.length; i++) {\n const rec = buildRecord(rows[i], details[i]);\n if (rec) records.push(rec);\n }\n\n await saveToDb(records);\n console.log(`Successfully synced ${records.length} CZIB(s) to Turso.`);\n return JSON.stringify(records, null, 2);\n } catch (err) {\n console.error(\"CRITICAL ERROR in handler:\", err.message, err.stack);\n throw err;\n }\n}",[153],"cfc1ae4a-ee38-45f7-b9ab-22eae6147b0d","commonEvent.js","fcc295ea-aa22-11f1-9f3a-1607ee4eb77e",131,["D",1788723482797],22,"// ---------------------------------------------------------------------------\n// Upstash Redis GeoJSON -> Turso (Table: commonevents)\n// Column updated to use 'reference' (singular) to avoid SQL reserved words.\n// ---------------------------------------------------------------------------\n\nconst UPSTASH_URL =\n \"https://informed-stag-162043.upstash.io/json.get/mapEvent/\";\nconst UPSTASH_BASE = \"https://informed-stag-162043.upstash.io\";\nconst UPSTASH_AUTH =\n \"Bearer gQAAAAAAAnj7AAIgcDJmMDZiMDhkM2U2OTM0NTQzYTM3ZTVkMDM0ZWJmNTkxMQ\";\n\nconst TURSO_BASE = (Deno.env.get(\"TURSO_URL\") || \"\").replace(/\\/+$/, \"\");\nconst TURSO_AUTH_TOKEN = Deno.env.get(\"TURSO_AUTH_TOKEN\");\n\nif (!TURSO_BASE || !TURSO_AUTH_TOKEN) {\n throw new Error(\"Missing TURSO_URL / TURSO_AUTH_TOKEN env vars\");\n}\n\nfunction clean(s) {\n return (s || \"\").replace(/\\s+/g, \" \").trim();\n}\n\nfunction hashId(title, date, location) {\n const s = `${title || \"\"}|${date || \"\"}|${location || \"\"}`;\n let h = 0;\n for (let i = 0; i < s.length; i++) {\n h = (Math.imul(31, h) + s.charCodeAt(i)) | 0;\n }\n return (h >>> 0).toString(36);\n}\n\nfunction typedArg(value) {\n if (value === null || value === undefined) {\n return { type: \"null\", value: null };\n }\n if (typeof value === \"number\") {\n return { type: Number.isInteger(value) ? \"integer\" : \"float\", value };\n }\n return { type: \"text\", value: String(value) };\n}\n\nfunction typedFloat(value) {\n if (value === null || value === undefined || Number.isNaN(Number(value))) {\n return { type: \"null\", value: null };\n }\n return { type: \"float\", value: Number(value) };\n}\n\nasync function tursoPipeline(stmts) {\n const requests = [\n ...stmts.map(({ sql, args }) => ({\n type: \"execute\",\n stmt: { sql, args: args || [] },\n })),\n { type: \"close\" },\n ];\n\n const res = await fetch(`${TURSO_BASE}/v2/pipeline`, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${TURSO_AUTH_TOKEN}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({ requests }),\n });\n\n if (!res.ok) {\n const errText = await res.text();\n throw new Error(`Turso HTTP ${res.status}: ${errText}`);\n }\n\n const data = await res.json();\n\n // Statement-level failures must abort the run — otherwise a partial/failed\n // Turso write could still trigger the Upstash clear.\n if (data && Array.isArray(data.results)) {\n const errs = data.results\n .map((result, idx) => (result.type === \"error\" ? { idx, result } : null))\n .filter(Boolean);\n if (errs.length > 0) {\n throw new Error(\n `Turso statement error(s): ${errs\n .map(({ idx, result }) => `[${idx}] ${result.error}`)\n .join(\"; \")}`,\n );\n }\n }\n\n return data;\n}\n\n// ---------------------------------------------------------------------------\n// Fetch, Clean, and Handle Malformed/Truncated JSON Gracefully\n// ---------------------------------------------------------------------------\nasync function fetchAndParseEvents() {\n const res = await fetch(UPSTASH_URL, {\n headers: { Authorization: UPSTASH_AUTH },\n });\n if (!res.ok) {\n throw new Error(`Upstash HTTP ${res.status}: ${await res.text()}`);\n }\n\n const body = await res.json();\n let rawData = body.result;\n\n let parsed;\n try {\n parsed = typeof rawData === \"string\" ? JSON.parse(rawData) : rawData;\n } catch (e) {\n console.warn(\n \"JSON parse failed (possibly truncated). Attempting safe recovery...\",\n );\n try {\n const lastValidIdx = rawData.lastIndexOf(\"}\");\n if (lastValidIdx > 0) {\n const fixedJson = rawData.substring(0, lastValidIdx + 1) + \"]}\";\n parsed = JSON.parse(fixedJson);\n } else {\n throw e;\n }\n } catch (innerErr) {\n throw new Error(\n `Failed to parse or repair Upstash payload: ${innerErr.message}`,\n );\n }\n }\n\n const features = Array.isArray(parsed?.features) ? parsed.features : [];\n\n return features.map((f) => {\n const p = f?.properties || {};\n const coords = Array.isArray(f?.geometry?.coordinates)\n ? f.geometry.coordinates\n : [null, null];\n\n const lng = coords[0] != null ? Number(coords[0]) : null;\n const lat = coords[1] != null ? Number(coords[1]) : null;\n\n return {\n id: hashId(p.title, p.date, p.location),\n title: clean(p.title),\n crime: clean(p.crime),\n date: clean(p.date),\n location: clean(p.location),\n paragraph: clean(p.paragraph),\n thumbnail: p.thumbnail || null,\n latitude: !isNaN(lat) ? lat : null,\n longitude: !isNaN(lng) ? lng : null,\n channel: p.channel ? JSON.stringify(p.channel) : null,\n reference: p.references ? JSON.stringify(p.references) : null,\n };\n });\n}\n\n// ---------------------------------------------------------------------------\n// Save records to Turso\n// ---------------------------------------------------------------------------\nasync function saveToDb(records) {\n // 1. Create table using 'reference' column name\n console.log(\"Ensuring table 'commonevents' exists...\");\n await tursoPipeline([\n {\n sql: `CREATE TABLE IF NOT EXISTS commonevents (\n id TEXT PRIMARY KEY,\n title TEXT,\n crime TEXT,\n date TEXT,\n location TEXT,\n paragraph TEXT,\n thumbnail TEXT,\n latitude REAL,\n longitude REAL,\n channel TEXT,\n reference TEXT,\n synced_at TEXT\n )`,\n },\n ]);\n\n // 2. Prepare insert statements using 'reference'\n const now = new Date().toISOString();\n const insertStmts = records.map((r) => ({\n sql: `INSERT OR REPLACE INTO commonevents \n (id, title, crime, date, location, paragraph, thumbnail, latitude, longitude, channel, reference, synced_at) \n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n typedArg(r.id),\n typedArg(r.title),\n typedArg(r.crime),\n typedArg(r.date),\n typedArg(r.location),\n typedArg(r.paragraph),\n typedArg(r.thumbnail),\n typedFloat(r.latitude),\n typedFloat(r.longitude),\n typedArg(r.channel),\n typedArg(r.reference),\n typedArg(now),\n ],\n }));\n\n // 3. Execute inserts in batches of 10\n console.log(`Inserting ${insertStmts.length} records into Turso...`);\n const BATCH_SIZE = 10;\n for (let i = 0; i < insertStmts.length; i += BATCH_SIZE) {\n const batch = insertStmts.slice(i, i + BATCH_SIZE);\n await tursoPipeline(batch);\n }\n}\n\n// ---------------------------------------------------------------------------\n// Empty the Upstash mapEvent features after a successful Turso sync.\n// Sets features to [] (keeps the rest of the document intact).\n// ---------------------------------------------------------------------------\nasync function clearUpstash() {\n const res = await fetch(`${UPSTASH_BASE}/json.set/mapEvent/$.features`, {\n method: \"POST\",\n headers: {\n Authorization: UPSTASH_AUTH,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify([]),\n });\n if (!res.ok) {\n throw new Error(`Upstash SET HTTP ${res.status}: ${await res.text()}`);\n }\n return res.json();\n}\n\n// ---------------------------------------------------------------------------\n// Main Entry Point\n// ---------------------------------------------------------------------------\nexport default async function handler() {\n const records = await fetchAndParseEvents();\n console.log(`Successfully parsed ${records.length} event(s) from Upstash.`);\n\n await saveToDb(records);\n console.log(\n `Successfully synced ${records.length} record(s) to Turso (table: commonevents).`,\n );\n\n // Only clear the Upstash source if the Turso write succeeded above.\n await clearUpstash();\n console.log(\"Cleared mapEvent from Upstash.\");\n\n return { success: true, total: records.length };\n}",[161],"28dcaac6-769b-40a0-8019-7ba853c08b21","scripts","ca191255-a940-4a18-832b-d6f7739bfbc1",174,["D",1789073099638],1,"directory",[169],"pulls","branches","forks","envVars",{"_183":185,"_186":61},"connections",{"_183":61,"_184":61},"count","unfilledCount",6,"emptyCount",{"_43":168,"_32":169,"_30":170,"_135":-5,"_41":42,"_40":171,"_137":25,"_38":223,"_13":173,"_34":174,"_36":-5,"_24":49,"_142":-5,"_195":224,"_27":225},{"_43":215,"_32":216,"_30":217,"_135":170,"_41":42,"_40":218,"_137":25,"_38":219,"_13":61,"_34":35,"_36":220,"_24":49,"_142":-5,"_195":221,"_27":222},{"_43":207,"_32":208,"_30":209,"_135":170,"_41":42,"_40":210,"_137":25,"_38":211,"_13":61,"_34":35,"_36":212,"_24":49,"_142":-5,"_195":213,"_27":214},{"_43":160,"_32":161,"_30":162,"_135":-5,"_41":42,"_40":163,"_137":25,"_38":204,"_13":165,"_34":140,"_36":166,"_24":49,"_142":-5,"_195":205,"_27":206},{"_43":152,"_32":153,"_30":154,"_135":-5,"_41":42,"_40":155,"_137":25,"_38":201,"_13":157,"_34":140,"_36":158,"_24":49,"_142":-5,"_195":202,"_27":203},{"_43":144,"_32":145,"_30":146,"_135":-5,"_41":42,"_40":147,"_137":25,"_38":198,"_13":149,"_34":140,"_36":150,"_24":49,"_142":-5,"_195":199,"_27":200},{"_43":132,"_32":133,"_30":134,"_135":-5,"_41":42,"_40":136,"_137":49,"_38":194,"_13":139,"_34":140,"_36":141,"_24":49,"_142":-5,"_195":196,"_27":197},["D",1789201435968],"children",[],[133],["D",1788982196432],[],[145],["D",1788942414030],[],[153],["D",1788723482797],[],[161],"a8ce5f8a-de5f-4003-bc1d-bd8ba884f0ea","zenrows-status.js","97a8e3a2-ac8a-11f1-9bec-1607ee4eb77e",168,["D",1788984667491],"// Prints your ZenRows plan status: credits used, period end, concurrency limit.\n// Run this when warEvent.js logs AUTH004 (credits exhausted) or AUTH006\n// (concurrency limit) to see where you stand. Does not consume credits.\nconst key = Deno.env.get(\"ZENROWS_API_KEY\");\nif (!key) throw new Error(\"Missing ZENROWS_API_KEY env var\");\n\nconst res = await fetch(\"https://api.zenrows.com/v1/subscriptions/self/details\", {\n headers: { \"X-API-Key\": key },\n});\nconst body = await res.text();\nconsole.log(`HTTP ${res.status}`);\ntry {\n console.log(JSON.stringify(JSON.parse(body), null, 2));\n} catch {\n console.log(body);\n}\n",[],[169,208],"ea265c60-9e86-4983-b94e-704638316918","translate.js","e5e91c2e-a9c1-11f1-bcf6-1607ee4eb77e",78,["D",1788678567608],"// ---------------------------------------------------------------------------\n// Shared EN -> ID translator used by every event scraper in this val.\n//\n// Primary: NVIDIA Riva \"nvidia/riva-translate-4b-instruct-v2\"\n// (OpenAI-compatible chat completions at integrate.api.nvidia.com).\n// Fallback: free MyMemory API, then the original English text as a last resort\n// so a transient API issue never fails or corrupts a run.\n//\n// The Riva instruct model must have the target language named in the *user*\n// message (otherwise it can default to another language, e.g. French).\n//\n// Requires the NVIDIA_API_KEY env var (set once on the val, shared by all\n// files). If it is missing, translation falls back to MyMemory/English.\n// ---------------------------------------------------------------------------\nconst NVIDIA_API_KEY = Deno.env.get(\"NVIDIA_API_KEY\");\nconst NVIDIA_BASE = \"https://integrate.api.nvidia.com/v1/chat/completions\";\nconst NVIDIA_MODEL = \"nvidia/riva-translate-4b-instruct-v2\";\n\nlet lastTranslate = 0; // MyMemory fallback throttle (~2 req/s)\n\n// Riva has no 500-char cap, but chunk large text so it stays well within the\n// model's input/output token limits (long EASA descriptions).\nconst CHUNK_MAX = 1600;\n\nfunction splitChunks(text) {\n const chunks = [];\n let rest = text.trim();\n while (rest.length > CHUNK_MAX) {\n let cut = rest.lastIndexOf(\". \", CHUNK_MAX);\n if (cut < CHUNK_MAX * 0.5) cut = rest.lastIndexOf(\" \", CHUNK_MAX);\n if (cut < CHUNK_MAX * 0.5) cut = CHUNK_MAX; // no break point -> hard cut\n chunks.push(rest.slice(0, cut + (rest[cut] === \" \" ? 0 : 1)));\n rest = rest.slice(cut + 1).trim();\n }\n if (rest) chunks.push(rest);\n return chunks;\n}\n\n// Primary: NVIDIA Riva instruct translation model.\nasync function rivaOnce(text) {\n const key = NVIDIA_API_KEY;\n if (!key) throw new Error(\"NVIDIA_API_KEY not set\");\n const res = await fetch(NVIDIA_BASE, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${key}`,\n \"Content-Type\": \"application/json\",\n Accept: \"application/json\",\n },\n body: JSON.stringify({\n model: NVIDIA_MODEL,\n messages: [\n { role: \"system\", content: \"You are a translation engine.\" },\n {\n role: \"user\",\n content: `Translate the following from English to Indonesian:\\n${text}`,\n },\n ],\n max_tokens: 2048,\n temperature: 0,\n }),\n });\n if (!res.ok) throw new Error(`Riva HTTP ${res.status}`);\n const data = await res.json();\n const out = data?.choices?.[0]?.message?.content?.trim();\n if (!out || /MYMEMORY WARNING|QUERY LENGTH/i.test(out)) {\n throw new Error(\"bad Riva output\");\n }\n return out;\n}\n\n// Fallback: MyMemory. Its free daily quota can be exhausted, in which case it\n// returns a warning instead of a translation — we treat that as a failure and\n// drop through to the original English rather than persisting the warning.\nasync function mymemoryOnce(text) {\n const elapsed = Date.now() - lastTranslate;\n if (elapsed < 500) {\n await new Promise((r) => setTimeout(r, 500 - elapsed));\n }\n lastTranslate = Date.now();\n\n const url =\n \"https://api.mymemory.translated.net/get?q=\" +\n encodeURIComponent(text) +\n \"&langpair=en|id\";\n const res = await fetch(url);\n if (!res.ok) throw new Error(`MyMemory HTTP ${res.status}`);\n const data = await res.json();\n const translated = data?.responseData?.translatedText;\n if (\n !translated ||\n /MYMEMORY WARNING|QUERY LENGTH LIMIT EXCEEDED|NO QUERY|INVALID PAIR/i.test(\n translated,\n )\n ) {\n return text;\n }\n return translated;\n}\n\nasync function translateOnce(text) {\n try {\n return await rivaOnce(text);\n } catch (err) {\n console.error(\"Riva failed, falling back to MyMemory:\", err.message);\n try {\n return await mymemoryOnce(text);\n } catch (err2) {\n console.error(\"MyMemory fallback failed, keeping original:\", err2.message);\n return text;\n }\n }\n}\n\nexport async function translateToId(text) {\n if (!text) return text;\n const chunks = splitChunks(text);\n if (chunks.length === 1) return await translateOnce(chunks[0]);\n const parts = [];\n for (const c of chunks) parts.push(await translateOnce(c));\n return parts.join(\" \");\n}\n",[],[169,216],["D",1789073099638],[188,189],[169],["D",1788316219773],["D",1789458074143],{"_98":97,"_105":101},{"_98":97,"_99":97,"_100":101,"_18":55,"_102":103,"_104":49},{"_13":87,"_32":88,"_43":42,"_89":44,"_90":-5,"_91":-5,"_92":-5,"_93":-5,"_94":-5,"_95":45,"_96":97},"ctx",{"_80":-5,"_124":264,"_265":266,"_267":268,"_269":49,"_270":49,"_271":49,"_272":25,"_273":-5,"_274":-7,"_275":-7,"_276":61,"_277":49,"_278":49,"_279":-5,"_280":49,"_281":49,"_282":49,"_283":-5},"ENV",{"_235":236,"_237":238,"_239":88,"_240":241,"_242":243,"_244":245,"_246":247,"_248":249,"_250":251,"_252":253,"_254":255,"_256":257,"_258":259,"_260":261,"_262":263},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v223