[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_241":242,"_243":244},"routes/_app.x.$handle.$name",{"_98":97,"_32":45,"_106":88,"_27":53,"_13":14,"_107":-5,"_11":108,"_109":110,"_111":112,"_113":114,"_115":116,"_20":-5,"_117":-5,"_118":-5,"_119":25,"_120":121,"_122":25,"_123":-5,"_124":-5,"_125":126},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_43":44,"_32":45,"_38":46,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":53,"_54":-5,"_18":55,"_56":-5,"_57":25,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":65,"_66":67,"_68":69,"_70":-5,"_71":72,"_73":72,"_74":75,"_76":75,"_77":75,"_78":72,"_79":72,"_80":81,"_82":83,"_84":85,"_86":25},"version",231,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_33":34,"_35":37,"_38":39,"_40":14,"_41":42,"_24":25},"previous",{"_30":31,"_32":28,"_33":34,"_35":36},"deleted",false,"deletedHere","path","warEvent2.js","latestPath","fileId","3387b950-af43-11f1-a26d-1607ee4eb77e","name","fileType","script","content","// iranwarmap-sync.ts\n// Run: deno run --allow-net --allow-env iranwarmap-sync.ts\n\nimport { createClient } from \"npm:@libsql/client@^0.14.0\";\nimport { translateToId } from \"./scripts/translate.js\";\n\n// ---------------------------------------------------------------------------\n// iranwarmap.com HTML -> parsed events -> recategorized + translated -> Turso.\n// ---------------------------------------------------------------------------\nconst Url = \"https://iranwarmap.com\";\n\nconst HeaderS = {\n \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) \" +\n \"AppleWebKit/537.36 (KHTML, like Gecko) \" +\n \"Chrome/138.0.0.0 Safari/537.36\",\n Accept: \"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\",\n \"Accept-Language\": \"en-US,en;q=0.9\",\n};\n\nconst IranData = {\n type: \"iranwarmap\",\n parent: null, // raw events\n temp: null, // filtered view for the map\n};\n\n// Recategorization of source `type` -> new cluster bucket\nconst TYPE_MAP = {\n airstrike: \"bomb-1\",\n artillery_strike: \"bomb-1\",\n missile_launch: \"bomb-1\",\n naval_incident: \"bomb-1\",\n\n nuclear: \"elect-1\",\n sanctions: \"elect-1\",\n\n diplomatic: \"speech-10\",\n\n other: \"phone-2\",\n\n casualty_report: \"dead-2\",\n\n \"missile_launch|drone_strike\": \"shahed-1\",\n\n explosion: \"fires-1\",\n\n ground_operation: \"ak-1\",\n mobilization: \"ak-1\",\n\n drone_strike: \"drone-2\",\n\n air_defense: \"aa-2\",\n};\n\n// ---------------------------------------------------------------------------\n// Turso client\n// ---------------------------------------------------------------------------\nconst TURSO_URL = Deno.env.get(\"TURSO_URL\");\nconst TURSO_AUTH_TOKEN = Deno.env.get(\"TURSO_AUTH_TOKEN\");\n\nif (!TURSO_URL || !TURSO_AUTH_TOKEN) {\n throw new Error(\"Missing TURSO_URL / TURSO_AUTH_TOKEN env vars\");\n}\n\nconst client = createClient({\n url: TURSO_URL,\n authToken: TURSO_AUTH_TOKEN,\n});\n\n// ---------------------------------------------------------------------------\n// Parse initialEvents out of the Next.js RSC flight payload embedded in HTML.\n// ---------------------------------------------------------------------------\nfunction extractInitialEvents(html) {\n const re = /self\\.__next_f\\.push\$\\[1,(\"(?:[^\"\\\\]|\\\\.)*\")\\]\$/gs;\n let payload = \"\";\n let m;\n while ((m = re.exec(html)) !== null) {\n payload += JSON.parse(m[1]);\n }\n\n const key = '\"initialEvents\":';\n const start = payload.indexOf(key);\n if (start === -1) throw new Error(\"initialEvents not found in payload\");\n\n const i = start + key.length;\n let depth = 0;\n let inStr = false;\n let esc = false;\n\n for (let j = i; j < payload.length; j++) {\n const c = payload[j];\n if (inStr) {\n if (esc) esc = false;\n else if (c === \"\\\\\") esc = true;\n else if (c === '\"') inStr = false;\n } else {\n if (c === '\"') inStr = true;\n else if (c === \"[\") depth++;\n else if (c === \"]\") {\n depth--;\n if (depth === 0) {\n return JSON.parse(payload.slice(i, j + 1));\n }\n }\n }\n }\n throw new Error(\"Unbalanced initialEvents array\");\n}\n\n// ---------------------------------------------------------------------------\n// Map a raw event to the row shape we store in Turso.\n// ---------------------------------------------------------------------------\nfunction mapEvent(e) {\n return {\n id: e.id,\n title: e.title,\n type: e.type,\n status: TYPE_MAP[e.type] ?? \"phone-2\", // recategorized\n location: e.locationName ?? null,\n date: e.timestamp ?? null,\n longitude: typeof e.lng === \"number\" ? e.lng : null,\n latitude: typeof e.lat === \"number\" ? e.lat : null,\n athumbnail: null,\n reference: e.sourceUrl ?? null,\n };\n}\n\n// ---------------------------------------------------------------------------\n// Create table, compare, translate only changes, upsert into warfare_events.\n// ---------------------------------------------------------------------------\nasync function saveToDb(events) {\n // 1. Ensure table exists\n await client.execute(`\n CREATE TABLE IF NOT EXISTS warfare_events (\n id TEXT PRIMARY KEY,\n title TEXT,\n titleId TEXT,\n status TEXT,\n location TEXT,\n date TEXT,\n longitude REAL,\n latitude REAL,\n athumbnail TEXT,\n reference TEXT\n )\n `);\n\n // 2. Load existing rows for comparison\n const existingMap = new Map();\n try {\n const res = await client.execute(\n `SELECT id, title, titleId, status, location, date\n FROM warfare_events`,\n );\n for (const row of res.rows) {\n if (row.id !== null && row.id !== undefined) {\n existingMap.set(String(row.id), {\n title: row.title ?? null,\n titleId: row.titleId ?? null,\n status: row.status ?? null,\n location: row.location ?? null,\n date: row.date ?? null,\n });\n }\n }\n } catch (err) {\n console.log(\"Could not fetch existing rows, full sync:\", err);\n }\n\n const stmts = [];\n let translatedCount = 0;\n let skippedCount = 0;\n\n // 3. Translate only when new or changed, then build upserts\n for (const ev of events) {\n const existing = existingMap.get(String(ev.id));\n\n // Unchanged → skip entirely (existing translation still valid)\n if (\n existing &&\n existing.title === ev.title &&\n existing.status === ev.status &&\n existing.location === ev.location &&\n existing.date === ev.date\n ) {\n skippedCount++;\n continue;\n }\n\n // ---- Translate BEFORE it goes to Turso ----\n let titleId = ev.title;\n try {\n if (ev.title) {\n titleId = await translateToId(ev.title);\n }\n translatedCount++;\n } catch (err) {\n console.error(\"Translation failed for event\", ev.id, err);\n // fall back to original title if translation fails\n titleId = ev.title;\n }\n\n stmts.push({\n sql: `INSERT OR REPLACE INTO warfare_events\n (id, title, titleId, status, location, date,\n longitude, latitude, athumbnail, reference)\n VALUES (?,?,?,?,?,?,?,?,?,?)`,\n args: [\n ev.id,\n ev.title,\n titleId,\n ev.status,\n ev.location,\n ev.date,\n ev.longitude,\n ev.latitude,\n ev.athumbnail,\n ev.reference,\n ],\n });\n }\n\n // 4. Upsert in chunks\n if (stmts.length > 0) {\n const CHUNK_SIZE = 50;\n for (let i = 0; i < stmts.length; i += CHUNK_SIZE) {\n await client.batch(stmts.slice(i, i + CHUNK_SIZE), \"write\");\n }\n }\n\n console.log(\n `Sync complete. Total incoming: ${events.length} | ` +\n `Skipped (unchanged): ${skippedCount} | ` +\n `Translated & Updated: ${translatedCount}`,\n );\n}\n\n// ---------------------------------------------------------------------------\n// Fetch + populate (serialized via fetchQueue, same as UKMTO)\n// ---------------------------------------------------------------------------\nlet fetchQueue = Promise.resolve();\n\nfunction fetchAndPopulate(url, headerS, constData) {\n fetchQueue = fetchQueue.then(async () => {\n try {\n const response = await fetch(url, { method: \"GET\", headers: headerS });\n if (!response.ok) {\n throw new Error(`iranwarmap HTTP ${response.status}`);\n }\n\n const html = await response.text();\n const rawEvents = extractInitialEvents(html);\n const mapped = rawEvents.map(mapEvent);\n\n console.log(`Fetched ${mapped.length} iranwarmap events.`);\n\n // Translate happens inside saveToDb, before each INSERT.\n await saveToDb(mapped);\n\n constData.parent = mapped;\n constData.temp = [...mapped];\n } catch (error) {\n console.error(\"iranwarmap fetch failed:\", error);\n }\n });\n return fetchQueue;\n}\n\nexport default async function handler() {\n await fetchAndPopulate(Url, HeaderS, IranData);\n}\n\nexport { extractInitialEvents, IranData, mapEvent };","// iranwarmap-sync.ts\n// Run: deno run --allow-net --allow-env iranwarmap-sync.ts\n\nimport { createClient } from \"npm:@libsql/client@^0.14.0\";\nimport { translateToId } from \"./scripts/translate.js\";\n\n// ---------------------------------------------------------------------------\n// iranwarmap.com HTML -> parsed events -> recategorized + translated -> Turso.\n// ---------------------------------------------------------------------------\nconst Url = \"https://iranwarmap.com\";\n\nconst HeaderS = {\n \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) \" +\n \"AppleWebKit/537.36 (KHTML, like Gecko) \" +\n \"Chrome/138.0.0.0 Safari/537.36\",\n Accept: \"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\",\n \"Accept-Language\": \"en-US,en;q=0.9\",\n};\n\nconst IranData = {\n type: \"iranwarmap\",\n parent: null,\n temp: null,\n};\n\nconst TYPE_MAP = {\n airstrike: \"bomb-1\",\n artillery_strike: \"bomb-1\",\n missile_launch: \"bomb-1\",\n naval_incident: \"bomb-1\",\n nuclear: \"elect-1\",\n sanctions: \"elect-1\",\n diplomatic: \"speech-10\",\n other: \"phone-2\",\n casualty_report: \"dead-2\",\n \"missile_launch|drone_strike\": \"shahed-1\",\n explosion: \"fires-1\",\n ground_operation: \"ak-1\",\n mobilization: \"ak-1\",\n drone_strike: \"drone-2\",\n air_defense: \"aa-2\",\n};\n\n// ---------------------------------------------------------------------------\n// Turso client\n// ---------------------------------------------------------------------------\nconsole.log(\"[boot] reading env vars...\");\nconst TURSO_URL = Deno.env.get(\"TURSO_URL\");\nconst TURSO_AUTH_TOKEN = Deno.env.get(\"TURSO_AUTH_TOKEN\");\n\nif (!TURSO_URL || !TURSO_AUTH_TOKEN) {\n console.error(\n \"[boot] Missing env vars. TURSO_URL=%s TURSO_AUTH_TOKEN=%s\",\n TURSO_URL ? \"set\" : \"MISSING\",\n TURSO_AUTH_TOKEN ? \"set\" : \"MISSING\",\n );\n throw new Error(\"Missing TURSO_URL / TURSO_AUTH_TOKEN env vars\");\n}\nconsole.log(\"[boot] env vars OK. TURSO_URL=%s\", TURSO_URL);\n\nconst client = createClient({\n url: TURSO_URL,\n authToken: TURSO_AUTH_TOKEN,\n});\nconsole.log(\"[boot] Turso client created.\");\n\n// ---------------------------------------------------------------------------\n// Parse initialEvents out of the Next.js RSC flight payload embedded in HTML.\n// ---------------------------------------------------------------------------\nfunction extractInitialEvents(html) {\n console.log(\"[parse] HTML length = %d chars\", html.length);\n\n const re = /self\\.__next_f\\.push\$\\[1,(\"(?:[^\"\\\\]|\\\\.)*\")\\]\$/gs;\n let payload = \"\";\n let m;\n let chunkCount = 0;\n while ((m = re.exec(html)) !== null) {\n try {\n payload += JSON.parse(m[1]);\n chunkCount++;\n } catch (err) {\n console.warn(\"[parse] failed to decode one RSC chunk:\", err);\n }\n }\n console.log(\n \"[parse] RSC chunks decoded = %d, payload length = %d\",\n chunkCount,\n payload.length,\n );\n\n if (payload.length === 0) {\n throw new Error(\n \"[parse] No RSC chunks found. Page structure may have changed.\",\n );\n }\n\n const key = '\"initialEvents\":';\n const start = payload.indexOf(key);\n console.log(\"[parse] indexOf(%s) = %d\", key, start);\n if (start === -1) {\n // Dump a small window around where we'd expect it for debugging\n const hint = payload.indexOf(\"initialEvents\");\n console.warn(\"[parse] 'initialEvents' substring at\", hint);\n console.warn(\n \"[parse] payload preview:\",\n payload.slice(Math.max(0, hint - 100), hint + 200),\n );\n throw new Error(\"initialEvents not found in payload\");\n }\n\n const i = start + key.length;\n let depth = 0;\n let inStr = false;\n let esc = false;\n\n for (let j = i; j < payload.length; j++) {\n const c = payload[j];\n if (inStr) {\n if (esc) esc = false;\n else if (c === \"\\\\\") esc = true;\n else if (c === '\"') inStr = false;\n } else {\n if (c === '\"') inStr = true;\n else if (c === \"[\") depth++;\n else if (c === \"]\") {\n depth--;\n if (depth === 0) {\n const arr = JSON.parse(payload.slice(i, j + 1));\n console.log(\"[parse] initialEvents length = %d\", arr.length);\n return arr;\n }\n }\n }\n }\n throw new Error(\"Unbalanced initialEvents array\");\n}\n\n// ---------------------------------------------------------------------------\n// Map a raw event to the row shape we store in Turso.\n// ---------------------------------------------------------------------------\nfunction mapEvent(e) {\n return {\n id: e.id,\n title: e.title,\n type: e.type,\n status: TYPE_MAP[e.type] ?? \"phone-2\",\n location: e.locationName ?? null,\n date: e.timestamp ?? null,\n longitude: typeof e.lng === \"number\" ? e.lng : null,\n latitude: typeof e.lat === \"number\" ? e.lat : null,\n athumbnail: null,\n reference: e.sourceUrl ?? null,\n };\n}\n\n// ---------------------------------------------------------------------------\n// Create table, compare, translate only changes, upsert into warfare_events.\n// ---------------------------------------------------------------------------\nasync function saveToDb(events) {\n console.log(\"[db] saveToDb start, incoming = %d\", events.length);\n\n console.log(\"[db] ensuring table warfare_events exists...\");\n await client.execute(`\n CREATE TABLE IF NOT EXISTS warfare_events (\n id TEXT PRIMARY KEY,\n title TEXT,\n titleId TEXT,\n status TEXT,\n location TEXT,\n date TEXT,\n longitude REAL,\n latitude REAL,\n athumbnail TEXT,\n reference TEXT\n )\n `);\n console.log(\"[db] table OK.\");\n\n console.log(\"[db] loading existing rows...\");\n const existingMap = new Map();\n try {\n const res = await client.execute(\n `SELECT id, title, titleId, status, location, date\n FROM warfare_events`,\n );\n console.log(\"[db] existing rows fetched = %d\", res.rows.length);\n for (const row of res.rows) {\n if (row.id !== null && row.id !== undefined) {\n existingMap.set(String(row.id), {\n title: row.title ?? null,\n titleId: row.titleId ?? null,\n status: row.status ?? null,\n location: row.location ?? null,\n date: row.date ?? null,\n });\n }\n }\n console.log(\"[db] existingMap size = %d\", existingMap.size);\n } catch (err) {\n console.warn(\"[db] could not load existing rows, full sync:\", err);\n }\n\n const stmts = [];\n let translatedCount = 0;\n let skippedCount = 0;\n\n console.log(\"[loop] processing events...\");\n for (const ev of events) {\n const existing = existingMap.get(String(ev.id));\n\n if (\n existing &&\n existing.title === ev.title &&\n existing.status === ev.status &&\n existing.location === ev.location &&\n existing.date === ev.date\n ) {\n skippedCount++;\n continue;\n }\n\n console.log(\n \"[loop] translating id=%s type=%s status=%s title=%s\",\n ev.id,\n ev.type,\n ev.status,\n ev.title?.slice(0, 60),\n );\n\n let titleId = ev.title;\n try {\n if (ev.title) {\n titleId = await translateToId(ev.title);\n console.log(\"[loop] -> titleId=%s\", String(titleId).slice(0, 60));\n }\n translatedCount++;\n } catch (err) {\n console.error(\"[loop] translation failed for\", ev.id, err);\n titleId = ev.title;\n }\n\n stmts.push({\n sql: `INSERT OR REPLACE INTO warfare_events\n (id, title, titleId, status, location, date,\n longitude, latitude, athumbnail, reference)\n VALUES (?,?,?,?,?,?,?,?,?,?)`,\n args: [\n ev.id,\n ev.title,\n titleId,\n ev.status,\n ev.location,\n ev.date,\n ev.longitude,\n ev.latitude,\n ev.athumbnail,\n ev.reference,\n ],\n });\n }\n\n console.log(\n \"[loop] done. stmts=%d skipped=%d translated=%d\",\n stmts.length,\n skippedCount,\n translatedCount,\n );\n\n if (stmts.length > 0) {\n const CHUNK_SIZE = 50;\n console.log(\"[db] upserting in chunks of %d...\", CHUNK_SIZE);\n for (let i = 0; i < stmts.length; i += CHUNK_SIZE) {\n const chunk = stmts.slice(i, i + CHUNK_SIZE);\n try {\n await client.batch(chunk, \"write\");\n console.log(\n \"[db] upserted %d/%d\",\n Math.min(i + chunk.length, stmts.length),\n stmts.length,\n );\n } catch (err) {\n console.error(\"[db] chunk %d failed:\", i, err);\n throw err;\n }\n }\n } else {\n console.log(\"[db] nothing to upsert.\");\n }\n\n console.log(\n \"[done] Total incoming: %d | Skipped: %d | Translated & Updated: %d\",\n events.length,\n skippedCount,\n translatedCount,\n );\n}\n\n// ---------------------------------------------------------------------------\n// Fetch + populate (serialized via fetchQueue, same as UKMTO)\n// ---------------------------------------------------------------------------\nlet fetchQueue = Promise.resolve();\n\nfunction fetchAndPopulate(url, headerS, constData) {\n console.log(\"[fetch] queued fetch for %s\", url);\n fetchQueue = fetchQueue.then(async () => {\n try {\n console.log(\"[fetch] GET %s\", url);\n const response = await fetch(url, { method: \"GET\", headers: headerS });\n console.log(\"[fetch] status = %d\", response.status);\n if (!response.ok) {\n throw new Error(`iranwarmap HTTP ${response.status}`);\n }\n\n const html = await response.text();\n console.log(\"[fetch] html received = %d chars\", html.length);\n\n const rawEvents = extractInitialEvents(html);\n console.log(\"[map] mapping %d raw events...\", rawEvents.length);\n const mapped = rawEvents.map(mapEvent);\n console.log(\"[map] mapped = %d\", mapped.length);\n\n await saveToDb(mapped);\n\n constData.parent = mapped;\n constData.temp = [...mapped];\n console.log(\n \"[fetch] done. constData.parent.length = %d\",\n constData.parent.length,\n );\n } catch (error) {\n console.error(\"[fetch] FAILED:\", error);\n throw error;\n }\n });\n return fetchQueue;\n}\n\nexport default async function handler() {\n console.log(\"[handler] invoked\");\n await fetchAndPopulate(Url, HeaderS, IranData);\n}\n\nexport { extractInitialEvents, IranData, mapEvent };\n\n// ---------------------------------------------------------------------------\n// Run directly if executed as a script (this is what was missing!)\n// ---------------------------------------------------------------------------\nif (import.meta.main) {\n console.log(\"[main] running as script...\");\n handler()\n .then(() => console.log(\"[main] finished OK\"))\n .catch((err) => {\n console.error(\"[main] failed:\", err);\n Deno.exit(1);\n });\n}","createdAt",["D",1789283991480],"branchVersion","branchId","1dc3dd44-cc65-414f-8e4a-735d1ac2a5b5","id","feed5b42-8d51-40be-9982-13fc813b4b7c","warData",["D",1788316219773],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","","imageUrl","0762e08b-0f1b-495a-871c-4fba2da9304a","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","019f5a38-1aa2-74ee-aaf5-8eac040acbf4","tursoName","feed5b428d5140be998213fc813b4b7c","tursoHostname","feed5b428d5140be998213fc813b4b7c-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789458074143],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_98":97,"_105":101},"org",{"_98":97,"_99":97,"_100":101,"_18":55,"_102":103,"_104":25},"branch",{"_13":87,"_32":88,"_43":42,"_89":44,"_90":-5,"_91":-5,"_92":-5,"_93":-5,"_94":-5,"_95":45,"_96":97},"isOwned",237,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","mustedpual","handle","displayName","avatar","https://avatars.githubusercontent.com/u/157880878?v=4","tier","free","canRestrictVals","profileImageUrl","branchName","adoptOrgId",{"_43":44,"_32":45,"_38":236,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":53,"_54":-5,"_18":55,"_56":-5,"_57":25,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":65,"_66":67,"_68":237,"_70":-5,"_71":72,"_73":72,"_74":75,"_76":75,"_77":75,"_78":72,"_79":72,"_80":238,"_82":239,"_84":240,"_86":25},"tree",[193,196,197,198,199,200],"files",[193,194,195,196,197,198,199,200],"counts",{"_182":61,"_183":179,"_184":61,"_185":186,"_187":188},"contents",[127,128,129,130,131,132],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_43":174,"_32":175,"_30":176,"_134":-5,"_41":42,"_40":177,"_135":153,"_38":178,"_13":179,"_33":180,"_35":-5,"_24":25,"_138":-5,"_27":181},{"_43":166,"_32":167,"_30":168,"_134":-5,"_41":42,"_40":169,"_135":153,"_38":170,"_13":171,"_33":146,"_35":172,"_24":25,"_138":-5,"_27":173},{"_43":158,"_32":159,"_30":160,"_134":-5,"_41":42,"_40":161,"_135":153,"_38":162,"_13":163,"_33":146,"_35":164,"_24":25,"_138":-5,"_27":165},{"_43":149,"_32":150,"_30":151,"_134":-5,"_41":42,"_40":152,"_135":153,"_38":154,"_13":155,"_33":146,"_35":156,"_24":25,"_138":-5,"_27":157},{"_43":140,"_32":141,"_30":142,"_134":-5,"_41":42,"_40":143,"_135":25,"_38":144,"_13":145,"_33":146,"_35":147,"_24":25,"_138":-5,"_27":148},{"_43":133,"_32":28,"_30":31,"_134":-5,"_41":42,"_40":14,"_135":25,"_38":136,"_13":137,"_33":34,"_35":37,"_24":25,"_138":-5,"_27":139},"3287a0e1-8647-42de-9e48-568ad824a26e","parentId","latest",["D",1789283991480],3,"label",[28],"fc0909dd-c4bb-4729-a9a5-de6c24848122","warEvent.js","d75184f6-a8f1-11f1-abed-1607ee4eb77e",227,["D",1789279607071],84,"interval","import { DOMParser } from \"npm:linkedom\";\nimport puppeteer from \"npm:puppeteer-core\";\nimport { translateToId } from \"./scripts/translate.js\";\n\n// ---------------------------------------------------------------------------\n// Configuration & Credentials\n// ---------------------------------------------------------------------------\nconst TURSO_URL = Deno.env.get(\"TURSO_URL\");\nconst TURSO_AUTH_TOKEN = Deno.env.get(\"TURSO_AUTH_TOKEN\");\nconst BROWSERLESS_TOKEN = Deno.env.get(\"BROWSERLESS_TOKEN\");\nconst BROWSERLESS_TOKENB = Deno.env.get(\"BROWSERLESS_TOKENB\"); // backup\n\n// Primary first, then backup. Filter out any that aren't configured so a\n// missing backup token just degrades to single-token behavior.\nconst BROWSERLESS_TOKENS = [\n { label: \"BROWSERLESS_TOKEN\", value: BROWSERLESS_TOKEN },\n { label: \"BROWSERLESS_TOKENB\", value: BROWSERLESS_TOKENB },\n].filter((t) => Boolean(t.value));\n\n// ---------------------------------------------------------------------------\n// Architecture (v2 — reuse-first, fetch-only where possible)\n//\n// PHASE A — POST /unblock once on liveuamap.com.\n// • Solves Cloudflare in a real browser session (costs proxy+captcha units).\n// • Returns main-page HTML (→ extract sublinks) AND a live\n// browserWSEndpoint (→ keep the cleared session alive).\n// • Tries BROWSERLESS_TOKEN first; if that token is exhausted (HTTP\n// 402/429/403 or a quota message in the body), retries with\n// BROWSERLESS_TOKENB automatically. The winning token's queryParams\n// are carried forward so Phase C connects with the SAME token.\n//\n// PHASE B — Turso lookup: which sublinks are not yet stored.\n//\n// PHASE C — Connect Puppeteer to the browserWSEndpoint from Phase A and\n// REUSE the page /unblock already cleared, instead of opening a new one.\n// Every sublink is first tried as an in-page `fetch()` from that page —\n// same cookies, same IP, same TLS fingerprint Cloudflare already\n// accepted, no asset loading, no navigation lifecycle. A real\n// page.goto() is now only a *fallback* used when fetch() fails (e.g. a\n// page that needs a real navigation/JS to populate, or a rare\n// re-challenge), not something forced on sublink #1.\n//\n// ASSUMPTION THIS RELIES ON (unverified against the live site — confirm\n// on your next real run via the log lines marked \"ASSUMPTION CHECK\"\n// below): Browserless's Cloudflare clearance from /unblock is scoped to\n// the browser context / sticky proxy session, not to the specific tab\n// object. If that's false, reusing the /unblock page instead of opening\n// a fresh one won't matter either way (harmless), but you'd expect fetch()\n// on the very first sublink to fail or get re-challenged — watch for that.\n// ---------------------------------------------------------------------------\n\nconst TARGET_URL = \"https://liveuamap.com\";\nconst PROXY_NETWORK = \"residential\"; // datacenter IPs get pre-flagged\nconst PROXY_COUNTRY = \"de\";\n\nconst RUN_BUDGET_MS = 55_000; // total cron budget\nconst HEADROOM_MS = 10_000; // stop scraping when this remains\nconst UNBLOCK_TIMEOUT_MS = 45_000;\nconst UNBLOCK_TTL_MS = 65_000; // Val Town free tier hard-kills the val at 60s;\n// the Browserless session should die shortly after too, not squat on one of\n// your 2 concurrency slots.\nconst PUPPETEER_NAV_TIMEOUT = 15_000; // fallback goto() navigation timeout\nconst PUPPETEER_WAIT_TIMEOUT = 10_000; // fallback waitForSelector timeout\nconst PUPPETEER_FETCH_TIMEOUT = 8_000; // in-page fetch timeout (the cheap path)\n\nconst START = Date.now();\nconst elapsed = () => Date.now() - START;\nconst timeLeft = () => RUN_BUDGET_MS - elapsed();\n\n// ---------------------------------------------------------------------------\n// Turso helpers (unchanged)\n// ---------------------------------------------------------------------------\nfunction typedArg(value) {\n if (value === null || value === undefined) {\n return { type: \"null\", value: null };\n }\n if (typeof value === \"number\") {\n // Turso's pipeline API rejects integers sent as JSON numbers — it wants\n // them as strings (to preserve 64-bit precision). Floats are fine as-is.\n return Number.isInteger(value)\n ? { type: \"integer\", value: String(value) }\n : { type: \"float\", value };\n }\n return { type: \"text\", value: String(value) };\n}\n\nasync function tursoPipeline(stmts) {\n const base = TURSO_URL.replace(/\\/+$/, \"\");\n const requests = [\n ...stmts.map(({ sql, args }) => ({\n type: \"execute\",\n stmt: { sql, args: (args || []).map(typedArg) },\n })),\n { type: \"close\" },\n ];\n\n const res = await fetch(`${base}/v2/pipeline`, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${TURSO_AUTH_TOKEN}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({ requests }),\n });\n\n if (!res.ok) {\n throw new Error(`Turso HTTP ${res.status}: ${await res.text()}`);\n }\n return res.json();\n}\n\nasync function ensureTable() {\n await tursoPipeline([\n {\n sql: `CREATE TABLE IF NOT EXISTS warfare_events (\n id TEXT PRIMARY KEY,\n title TEXT,\n status TEXT,\n location TEXT,\n date TEXT,\n longitude REAL,\n latitude REAL,\n athumbnail TEXT,\n reference TEXT\n )`,\n },\n {\n sql: `CREATE TABLE IF NOT EXISTS run_lock (\n id INTEGER PRIMARY KEY CHECK (id = 1),\n locked_until INTEGER\n )`,\n },\n ]);\n}\n\n// Val Town's cron overlap behavior isn't publicly documented, so rather than\n// assume it never overlaps, this lock makes it not matter. Any run checks a\n// single Turso row before touching Browserless at all; if a prior run's lock\n// hasn't expired, this run bails out instead of opening a second Browserless\n// session and contending for your 2-concurrency cap.\nasync function acquireRunLock() {\n const now = Date.now();\n const out = await tursoPipeline([\n { sql: `SELECT locked_until FROM run_lock WHERE id = 1` },\n ]);\n const rows = out?.results?.[0]?.response?.result?.rows || [];\n const lockedUntil = rows[0]?.[0]?.value ? Number(rows[0][0].value) : 0;\n\n if (lockedUntil > now) {\n console.warn(\n `Run lock still held until ${\n new Date(lockedUntil).toISOString()\n }; skipping this tick.`,\n );\n return false;\n }\n\n const newLockedUntil = now + RUN_BUDGET_MS + 10_000;\n await tursoPipeline([\n {\n sql: `INSERT INTO run_lock (id, locked_until) VALUES (1, ?) ` +\n `ON CONFLICT(id) DO UPDATE SET locked_until = excluded.locked_until`,\n args: [newLockedUntil],\n },\n ]);\n return true;\n}\n\nasync function pendingSublinks(sublinks) {\n if (sublinks.length === 0) return [];\n const placeholders = sublinks.map(() => \"?\").join(\",\");\n const out = await tursoPipeline([\n {\n sql: `SELECT id FROM warfare_events WHERE id IN (${placeholders})`,\n args: sublinks,\n },\n ]);\n const rows = out?.results?.[0]?.response?.result?.rows || [];\n const known = new Set(rows.map((r) => r[0]?.value));\n return sublinks.filter((l) => !known.has(l));\n}\n\n// ---------------------------------------------------------------------------\n// Date extraction (unchanged)\n// ---------------------------------------------------------------------------\nconst MONTHS = {\n january: \"01\",\n february: \"02\",\n march: \"03\",\n april: \"04\",\n may: \"05\",\n june: \"06\",\n july: \"07\",\n august: \"08\",\n september: \"09\",\n october: \"10\",\n november: \"11\",\n december: \"12\",\n};\n\nfunction extractLiveUAMapDate(url) {\n const match = url.match(/\\/(\\d{4})\\/(\\d{1,2})-([a-z]+)-\\d+-/i);\n if (!match) return null;\n const [, year, day, monthName] = match;\n const mm = MONTHS[monthName.toLowerCase()];\n return mm ? `${year}-${mm}-${String(day).padStart(2, \"0\")}` : null;\n}\n\n// ---------------------------------------------------------------------------\n// Parse ONE sublink's HTML -> GeoJSON Feature (unchanged)\n// ---------------------------------------------------------------------------\nconst dom = new DOMParser();\n\nfunction parseFeatureHtml(sublink, subHtml) {\n const subdoc = dom.parseFromString(subHtml, \"text/html\");\n\n let lat = 48.8829;\n let lng = 31.181;\n let usedDefaultCoords = true;\n\n const eventScriptMatch = subHtml.match(\n /document\\.addEventListener\$['\"]DOMContentLoaded['\"][\\s\\S]*?\$;/,\n );\n if (eventScriptMatch) {\n const latMatch = eventScriptMatch[0].match(/lat\\s*=\\s*([0-9.-]+)/);\n const lngMatch = eventScriptMatch[0].match(/lng\\s*=\\s*([0-9.-]+)/);\n if (latMatch) {\n lat = parseFloat(latMatch[1]);\n usedDefaultCoords = false;\n }\n if (lngMatch) {\n lng = parseFloat(lngMatch[1]);\n usedDefaultCoords = false;\n }\n }\n if (usedDefaultCoords) {\n console.warn(`No lat/lng found for ${sublink}; using default coordinates.`);\n }\n\n const infozone = subdoc.querySelector(\".popup-text\");\n const status = infozone?.querySelector(\".bgma\")?.getAttribute(\"data-src\") ||\n null;\n const title = infozone?.querySelector(\"h2\")?.textContent?.trim() || null;\n const locationText =\n infozone?.querySelector(\".tagas strong\")?.textContent?.trim() || null;\n const thumbnail =\n infozone?.querySelector(\".popup_video + * img\")?.getAttribute(\"src\") ||\n null;\n const reference =\n subdoc.querySelector(\".source-link\")?.getAttribute(\"href\") || null;\n\n return {\n type: \"Feature\",\n geometry: { type: \"Point\", coordinates: [lng, lat] },\n properties: {\n references: [sublink, reference].filter(Boolean),\n title,\n status,\n location: locationText,\n thumbnail,\n date: extractLiveUAMapDate(sublink),\n },\n };\n}\n\n// ---------------------------------------------------------------------------\n// Translate + upsert ONE feature into Turso (unchanged)\n// ---------------------------------------------------------------------------\nasync function saveFeature(feature) {\n const { properties, geometry } = feature;\n const [lng, lat] = geometry.coordinates;\n const eventId = properties.references[0] || crypto.randomUUID();\n\n let translatedTitle = properties.title;\n if (translatedTitle) {\n try {\n translatedTitle = await translateToId(properties.title);\n } catch (err) {\n console.error(\"Translation failed for title:\", properties.title, err);\n }\n }\n\n await tursoPipeline([\n {\n sql: `INSERT OR REPLACE INTO warfare_events\n (id, title, status, location, date, longitude, latitude, athumbnail, reference)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n eventId,\n translatedTitle,\n properties.status,\n properties.location,\n properties.date,\n lng,\n lat,\n properties.thumbnail,\n properties.references[1] || properties.references[0] || null,\n ],\n },\n ]);\n}\n\n// ---------------------------------------------------------------------------\n// Helpers\n// ---------------------------------------------------------------------------\nfunction cleanDataLink(raw) {\n if (!raw) return null;\n let v = raw.trim();\n v = v.replace(/\\\\(.)/g, \"$1\");\n v = v.replace(/^[\"']+|[\"']+$/g, \"\").trim();\n return v || null;\n}\n\nfunction buildUnblockQueryParams(token) {\n return new URLSearchParams({\n token,\n proxy: PROXY_NETWORK,\n proxyCountry: PROXY_COUNTRY,\n proxySticky: \"true\",\n timeout: String(UNBLOCK_TIMEOUT_MS),\n }).toString();\n}\n\n// Heuristic: which responses mean \"this token is out of budget\" (worth\n// trying the backup) vs \"this request is malformed / site is down\" (don't\n// burn the backup token on it).\n//\n// 402 = payment required (out of units)\n// 429 = rate limited\n// 403 = sometimes Browserless uses this for token/permission problems\n// Body checks catch the cases where Browserless returns 200/4xx but the\n// payload text says quota/limit/units.\nfunction isTokenExhausted(status, bodyText) {\n if (status === 401 || status === 402 || status === 403 || status === 429) {\n return true;\n }\n const t = (bodyText || \"\").toLowerCase();\n return (\n t.includes(\"units usage limit\") ||\n t.includes(\"usage limit\") ||\n t.includes(\"out of units\") ||\n t.includes(\"quota\") ||\n t.includes(\"upgrade to a paid\") ||\n t.includes(\"rate limit\") ||\n t.includes(\"rate-limit\") ||\n t.includes(\"exceeded\") ||\n t.includes(\"no remaining\")\n );\n}\n\n// ---------------------------------------------------------------------------\n// PHASE A — /unblock: solve Cloudflare once, harvest sublinks + a live\n// browser session. Tries each configured Browserless token in order; the\n// first one that returns a usable response wins, and its queryParams are\n// returned so Phase C connects with the same token.\n// ---------------------------------------------------------------------------\nasync function initSessionAndExtractLinks() {\n if (BROWSERLESS_TOKENS.length === 0) {\n console.error(\n \"No Browserless tokens configured (BROWSERLESS_TOKEN / BROWSERLESS_TOKENB).\",\n );\n return null;\n }\n\n const body = {\n url: TARGET_URL,\n content: true,\n browserWSEndpoint: true,\n cookies: false,\n screenshot: false,\n ttl: UNBLOCK_TTL_MS,\n waitForSelector: {\n selector: \"[data-link]\",\n timeout: 15000,\n },\n };\n\n let lastError = null;\n\n for (let i = 0; i < BROWSERLESS_TOKENS.length; i++) {\n const { label, value: token } = BROWSERLESS_TOKENS[i];\n const isLastToken = i === BROWSERLESS_TOKENS.length - 1;\n const queryParams = buildUnblockQueryParams(token);\n const unblockURL =\n `https://production-sfo.browserless.io/unblock?${queryParams}`;\n\n console.log(\n `Requesting main page via Browserless /unblock using ${label}…`,\n );\n\n let res;\n try {\n res = await fetch(unblockURL, {\n method: \"POST\",\n headers: { \"Content-Type\": \"application/json\" },\n body: JSON.stringify(body),\n signal: AbortSignal.timeout(UNBLOCK_TIMEOUT_MS + 5_000),\n });\n } catch (err) {\n // Network/timeout error — not token exhaustion, but still worth trying\n // the next token since the /unblock attempt cost us nothing (no 200).\n console.error(`/unblock request failed on ${label} (${err.message}).`);\n lastError = err;\n if (isLastToken) break;\n continue;\n }\n\n if (!res.ok) {\n const errText = await res.text();\n if (!isLastToken && isTokenExhausted(res.status, errText)) {\n console.warn(\n `${label} appears exhausted (HTTP ${res.status}): ` +\n `${errText.slice(0, 200)} — trying next token…`,\n );\n lastError = new Error(`HTTP ${res.status}: ${errText.slice(0, 200)}`);\n continue;\n }\n console.error(`/unblock HTTP ${res.status} on ${label}: ${errText}`);\n return null; // non-exhaustion error on the last token — give up\n }\n\n // ----- Success path with this token -----\n const payload = await res.json();\n const html = payload.content ?? \"\";\n const browserWSEndpoint = payload.browserWSEndpoint ?? null;\n\n console.log(\n `Unblock response (${label}): html length=${html.length} ` +\n `browserWSEndpoint=${browserWSEndpoint ? \"yes\" : \"no\"} ` +\n `(${elapsed()}ms elapsed)`,\n );\n\n if (!html) {\n console.error(`/unblock returned no content on ${label}; skipping run.`);\n return null;\n }\n if (!browserWSEndpoint) {\n console.error(\n `/unblock returned no browserWSEndpoint on ${label}; cannot continue to Phase C.`,\n );\n return null;\n }\n\n const doc = dom.parseFromString(html, \"text/html\");\n const sublinks = [\n ...new Set(\n [...doc.querySelectorAll(\"[data-link]\")]\n .map((e) => cleanDataLink(e.getAttribute(\"data-link\")))\n .filter(Boolean),\n ),\n ];\n\n if (sublinks.length === 0) {\n const pageTitle = doc.querySelector(\"title\")?.textContent?.trim() || null;\n console.warn(\n `No [data-link] matches. page title=\"${pageTitle}\". ` +\n `If this looks like a Cloudflare or error page, the unblock ` +\n `did not clear the challenge — check the Browserless dashboard.`,\n );\n }\n\n console.log(\n `Extracted ${sublinks.length} sublinks from the unblocked main page ` +\n `(${elapsed()}ms elapsed) using ${label}.`,\n );\n\n // BYTE INSTRUMENTATION: this is the actual Phase A transfer size (the\n // homepage HTML /unblock rendered and returned), so you can compare it\n // against the Phase C total logged at the end of scrapeSublinksWithPuppeteer\n // and see which phase is really driving your proxy-unit bill instead of\n // guessing from the unit total alone. Note this undercounts Phase A's true\n // proxy bandwidth (it excludes CSS/JS/tile bytes /unblock loaded to solve\n // Cloudflare but didn't return to you) — treat it as a floor, not the full\n // Phase A cost.\n const unblockHtmlBytes = new TextEncoder().encode(html).length;\n console.log(\n `Phase A returned HTML: ${unblockHtmlBytes} bytes ` +\n `(${(unblockHtmlBytes / 1_000_000).toFixed(2)} MB) — proxy-billed ` +\n `bandwidth for the actual page load/render is likely higher than this, ` +\n `since it excludes assets loaded but not returned.`,\n );\n\n // queryParams already carries the winning token, so Phase C connects\n // via puppeteer with that same token — no bookkeeping needed.\n return { sublinks, browserWSEndpoint, queryParams };\n }\n\n console.error(\n `All Browserless tokens exhausted/failed. Last error: ${lastError?.message}`,\n );\n return null;\n}\n\n// ---------------------------------------------------------------------------\n// PHASE C (v2) — Connect Puppeteer to the cleared session and REUSE the\n// existing page instead of opening a new one.\n//\n// Every sublink is tried as an in-page fetch() first — same cookie jar,\n// same IP, same TLS fingerprint Cloudflare already accepted, no asset\n// loading, no navigation lifecycle. A real page.goto() only happens as a\n// per-link fallback if fetch() throws (HTTP error, CORS, timeout, or the\n// page turns out to need client-side JS to populate .popup-text).\n//\n// This removes the old hard rule \"sublink #1 must be a real navigation.\"\n// If the reused page really is still on liveuamap.com's origin (it should\n// be, straight out of /unblock), fetch() should work for #1 too. Watch the\n// \"ASSUMPTION CHECK\" log lines on your first live run to confirm this.\n// ---------------------------------------------------------------------------\nasync function scrapeSublinksWithPuppeteer(\n pending,\n browserWSEndpoint,\n queryParams,\n) {\n if (pending.length === 0) return { saved: 0, skipped: 0, failed: 0 };\n\n let browser;\n try {\n const sep = browserWSEndpoint.includes(\"?\") ? \"&\" : \"?\";\n browser = await puppeteer.connect({\n browserWSEndpoint: `${browserWSEndpoint}${sep}${queryParams}`,\n });\n } catch (err) {\n console.error(`Puppeteer connect failed: ${err.message}`);\n return { saved: 0, skipped: pending.length, failed: 0 };\n }\n\n // Reuse the page /unblock already cleared, instead of opening a fresh tab.\n // Fall back to a new page only if, for some reason, none exist.\n const existingPages = await browser.pages();\n const page = existingPages[0] ?? await browser.newPage();\n\n const extraPages = existingPages.filter((p) => p !== page);\n if (extraPages.length > 0) {\n console.log(`Closing ${extraPages.length} extra page(s) from the session.`);\n await Promise.all(\n extraPages.map((p) =>\n p.close().catch((err) =>\n console.warn(`Failed to close extra page: ${err.message}`)\n )\n ),\n );\n }\n\n // parseFeatureHtml only ever reads raw HTML (.popup-text, an inline\n // lat/lng script) — never rendered images/fonts/CSS/media — so blocking\n // these is pure bandwidth savings with zero effect on correctness. Applies\n // to both fetch() calls and any fallback goto() navigation.\n const BLOCKED_RESOURCE_TYPES = new Set([\n \"image\",\n \"stylesheet\",\n \"font\",\n \"media\",\n \"other\",\n ]);\n await page.setRequestInterception(true);\n page.on(\"request\", (req) => {\n if (BLOCKED_RESOURCE_TYPES.has(req.resourceType())) {\n req.abort().catch(() => {});\n } else {\n req.continue().catch(() => {});\n }\n });\n\n async function fetchInPage(url) {\n return await page.evaluate(\n async (u, timeoutMs) => {\n const controller = new AbortController();\n const t = setTimeout(() => controller.abort(), timeoutMs);\n try {\n const res = await fetch(u, {\n credentials: \"include\",\n signal: controller.signal,\n });\n if (!res.ok) throw new Error(`HTTP ${res.status}`);\n return await res.text();\n } finally {\n clearTimeout(t);\n }\n },\n url,\n PUPPETEER_FETCH_TIMEOUT,\n );\n }\n\n // Sublink content is server-rendered HTML, same as the main page — the\n // only reason any of this goes through a browser at all is to inherit the\n // Cloudflare-cleared session (cookies/IP/fingerprint), not to run JS or\n // wait for client-side rendering. So a failed fetch() is presumed to be\n // transient (network blip, momentary 5xx) rather than \"this page needs a\n // real render\" — worth one quick retry on the SAME cheap path before\n // paying for a full navigation.\n const FETCH_RETRY_DELAY_MS = 400;\n const sleep = (ms) => new Promise((r) => setTimeout(r, ms));\n\n async function fetchWithRetry(url) {\n try {\n return await fetchInPage(url);\n } catch (firstErr) {\n await sleep(FETCH_RETRY_DELAY_MS);\n try {\n return await fetchInPage(url);\n } catch (secondErr) {\n // Both attempts failed on the cheap path — surface the second\n // error, since it's the more recent signal.\n throw secondErr;\n }\n }\n }\n\n // LAST RESORT, not \"the version with real rendering.\" goto() reuses the\n // exact same session/cookies/IP as fetchInPage() — it has no special\n // ability to get past Cloudflare that fetch() lacks. If the session has\n // been re-challenged, goto() is just as likely to land on a challenge\n // page, at much higher cost (full asset pipeline minus what we block,\n // plus JS execution). Its only real value here is for the rare case where\n // a sublink's connection genuinely needs a fresh TCP/TLS handshake that\n // fetch() inside the page context can't cleanly retry.\n async function navigateFallback(url) {\n await page.goto(url, {\n waitUntil: \"domcontentloaded\",\n timeout: PUPPETEER_NAV_TIMEOUT,\n });\n await page.waitForSelector(\".popup-text\", {\n timeout: PUPPETEER_WAIT_TIMEOUT,\n });\n return await page.content();\n }\n\n let saved = 0;\n let skipped = 0;\n let failed = 0;\n let fetchOnlyCount = 0;\n let fallbackCount = 0;\n let fetchBytes = 0; // total HTML bytes pulled via the cheap in-page fetch() path\n let fallbackBytes = 0; // total HTML bytes pulled via the goto() fallback path\n let consecutiveTotalFailures = 0; // both fetch retry AND goto fallback failed\n const BAIL_THRESHOLD = 3; // repeated total failures suggest the whole\n // session got re-challenged, not that each individual sublink is broken —\n // paying for more goto() navigations at that point is throwing money at\n // something a fresh /unblock next tick is far more likely to fix.\n let i = 0;\n\n for (; i < pending.length; i++) {\n if (timeLeft() < HEADROOM_MS) break;\n\n if (consecutiveTotalFailures >= BAIL_THRESHOLD) {\n console.warn(\n `${consecutiveTotalFailures} consecutive total failures (fetch retry ` +\n `+ goto fallback both failed) — likely session re-challenge rather ` +\n `than per-link issues. Bailing on the remaining ${\n pending.length - i\n } sublinks this run instead of paying for more navigations that ` +\n `probably won't succeed either; they'll be picked up next tick.`,\n );\n break;\n }\n\n const sublink = pending[i];\n\n try {\n let subHtml;\n let usedFallback = false;\n\n try {\n subHtml = await fetchWithRetry(sublink);\n fetchOnlyCount++;\n fetchBytes += new TextEncoder().encode(subHtml).length;\n consecutiveTotalFailures = 0;\n } catch (fetchErr) {\n if (i === 0) {\n // ASSUMPTION CHECK: if this fires on the very first sublink, the\n // reused page was NOT already same-origin / CF-cleared the way\n // we assumed — worth investigating before trusting fetch-first\n // for the rest of the run.\n console.warn(\n `ASSUMPTION CHECK: in-page fetch failed on first sublink even ` +\n `after retry (${fetchErr.message}); falling back to goto(). ` +\n `This may mean the reused page wasn't already cleared/same-origin.`,\n );\n }\n try {\n subHtml = await navigateFallback(sublink);\n usedFallback = true;\n fallbackCount++;\n fallbackBytes += new TextEncoder().encode(subHtml).length;\n consecutiveTotalFailures = 0;\n } catch (gotoErr) {\n consecutiveTotalFailures++;\n throw gotoErr;\n }\n }\n\n const feature = parseFeatureHtml(sublink, subHtml);\n await saveFeature(feature);\n saved++;\n console.log(\n `Saved [${saved}] ${feature.properties.title ?? sublink}` +\n (usedFallback ? \" (via goto fallback)\" : \"\"),\n );\n } catch (err) {\n if (/timeout|Timeout|abort/i.test(err.message)) {\n skipped++;\n console.warn(`Skipped (timeout) ${sublink}: ${err.message}`);\n } else {\n failed++;\n console.error(`Parse/save failed: ${sublink} — ${err.message}`);\n }\n }\n }\n\n const phaseCTotalBytes = fetchBytes + fallbackBytes;\n console.log(\n `Fetch-only: ${fetchOnlyCount} calls, ${fetchBytes} bytes ` +\n `(${(fetchBytes / 1_000_000).toFixed(2)} MB). ` +\n `Goto fallback: ${fallbackCount} calls, ${fallbackBytes} bytes ` +\n `(${(fallbackBytes / 1_000_000).toFixed(2)} MB). ` +\n `Phase C total returned: ${phaseCTotalBytes} bytes ` +\n `(${(phaseCTotalBytes / 1_000_000).toFixed(2)} MB). ` +\n `Note: like the Phase A figure, this undercounts true fallback ` +\n `bandwidth — goto() loads more than what page.content() returns, ` +\n `even with images/CSS/fonts/media blocked (JS execution and any ` +\n `same-origin XHRs the page fires still count). Compare this line's ` +\n `MB total against the Phase A \"returned HTML\" line logged earlier in ` +\n `this run to see which phase actually dominates.`,\n );\n\n try {\n await browser.close();\n } catch (err) {\n console.warn(`Puppeteer close failed (non-fatal): ${err.message}`);\n }\n\n return { saved, skipped, failed, processed: i };\n}\n\n// ---------------------------------------------------------------------------\n// Main flow (unchanged)\n// ---------------------------------------------------------------------------\nasync function run() {\n await ensureTable();\n\n const gotLock = await acquireRunLock();\n if (!gotLock) return;\n\n const pool = await initSessionAndExtractLinks();\n if (!pool || pool.sublinks.length === 0) {\n console.warn(\"No sublinks extracted from /unblock; nothing to do.\");\n return;\n }\n\n const pending = await pendingSublinks(pool.sublinks);\n console.log(\n `Main page: ${pool.sublinks.length} sublinks, ` +\n `${pool.sublinks.length - pending.length} already stored, ` +\n `${pending.length} pending (${elapsed()}ms elapsed)`,\n );\n\n const { saved, skipped, failed, processed } =\n await scrapeSublinksWithPuppeteer(\n pending,\n pool.browserWSEndpoint,\n pool.queryParams,\n );\n\n const deferred = pending.length - (processed ?? 0);\n\n console.log(\n `Done in ${elapsed()}ms: ${saved} saved, ${skipped} skipped (timeout), ` +\n `${failed} parse/save failed, ${deferred} deferred to next run.`,\n );\n}\n\n// ---------------------------------------------------------------------------\n// Cron entrypoint\n// ---------------------------------------------------------------------------\nexport default async function () {\n if (!TURSO_URL || !TURSO_AUTH_TOKEN) {\n throw new Error(\"Missing TURSO_URL / TURSO_AUTH_TOKEN env vars\");\n }\n if (BROWSERLESS_TOKENS.length === 0) {\n throw new Error(\n \"Missing BROWSERLESS_TOKEN and BROWSERLESS_TOKENB env vars (need at least one)\",\n );\n }\n\n try {\n await run();\n } catch (error) {\n console.error(\"Pipeline failed:\", error);\n throw error;\n }\n}",[141],"4d2ece01-2d75-488d-80d4-8ae766cde047","ukmtoEvent.js","7e654b18-a9b2-11f1-9377-1607ee4eb77e",154,true,["D",1788982196432],10,"import { translateToId } from \"./scripts/translate.js\";\n// ---------------------------------------------------------------------------\n// UKMTO incident feed -> Indonesian-translated events -> Turso.\n// Fetches https://sccd.royalnavy.mod.uk/api/ukmto/all (array of incidents),\n// compares against existing records in Turso to skip unchanged items,\n// translates new/changed incidentTypeName + otherDetails into Indonesian,\n// then upserts into Turso in safe chunks.\n// ---------------------------------------------------------------------------\nlet fetchQueue = Promise.resolve();\n\nconst Url = \"https://sccd.royalnavy.mod.uk/api/ukmto/all\";\n\nconst HeaderS = {\n \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64)\",\n Accept: \"application/json\",\n};\n\nconst UkmtoData = {\n type: \"ukmto\",\n parent: null, // Untouched master data\n temp: null, // Current filtered view for the map\n};\n\n// ---------------------------------------------------------------------------\n// Turso client via its HTTP API (POST /v2/pipeline). Credentials from env vars.\n// ---------------------------------------------------------------------------\nconst TURSO_URL = Deno.env.get(\"TURSO_URL\");\nconst TURSO_AUTH_TOKEN = Deno.env.get(\"TURSO_AUTH_TOKEN\");\n\nif (!TURSO_URL || !TURSO_AUTH_TOKEN) {\n throw new Error(\"Missing TURSO_URL / TURSO_AUTH_TOKEN env vars\");\n}\nconst TURSO_BASE = TURSO_URL.replace(/\\/+$/, \"\");\n\n// Turso's v2/pipeline arg format: integer/text values are JSON STRINGS;\n// float values are JSON NUMBERS. See https://docs.turso.tech/sdk/http/quickstart\nfunction typedArg(value) {\n if (value === null || value === undefined) {\n return { type: \"null\", value: null };\n }\n if (typeof value === \"number\") {\n return Number.isInteger(value)\n ? { type: \"integer\", value: String(value) }\n : { type: \"float\", value };\n }\n return { type: \"text\", value: String(value) };\n}\n\nasync function tursoPipeline(stmts) {\n const requests = [\n ...stmts.map(({ sql, args }) => ({\n type: \"execute\",\n stmt: { sql, args: (args || []).map(typedArg) },\n })),\n { type: \"close\" },\n ];\n const res = await fetch(`${TURSO_BASE}/v2/pipeline`, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${TURSO_AUTH_TOKEN}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({ requests }),\n });\n if (!res.ok) {\n throw new Error(`Turso HTTP ${res.status}: ${await res.text()}`);\n }\n return res.json();\n}\n\n// ---------------------------------------------------------------------------\n// Map a raw UKMTO incident to the fields the map consumes.\n// ---------------------------------------------------------------------------\nfunction mapIncident(inc) {\n return {\n incidentTypeName: inc.incidentTypeName,\n incidentNumber: inc.incidentNumber,\n incidentIssuer: inc.incidentIssuer,\n place: inc.place,\n sitecoreId: inc.sitecoreId,\n pinColour: inc.pinColour,\n locationLatitudeDDDMMSS: inc.locationLatitudeDDDMMSS,\n locationLongitudeDDDMMSS: inc.locationLongitudeDDDMMSS,\n utcDateOfIncident: inc.utcDateOfIncident,\n utcDateCreated: inc.utcDateCreated,\n // data-lng / data-lat attributes on map markers\n dataLng: inc.locationLongitude,\n dataLat: inc.locationLatitude,\n otherDetails: inc.otherDetails,\n };\n}\n\n// ---------------------------------------------------------------------------\n// Create table, fetch existing rows for comparison, translate only changes,\n// and upsert in chunks.\n// ---------------------------------------------------------------------------\nasync function saveToDb(incidents) {\n // 1. Ensure table exists\n await tursoPipeline([\n {\n sql: `CREATE TABLE IF NOT EXISTS ukmto_events (\n incidentNumber INTEGER PRIMARY KEY,\n incidentTypeName TEXT,\n incidentTypeNameId TEXT,\n incidentIssuer TEXT,\n place TEXT,\n sitecoreId TEXT,\n pinColour TEXT,\n locationLatitudeDDDMMSS TEXT,\n locationLongitudeDDDMMSS TEXT,\n utcDateOfIncident TEXT,\n utcDateCreated TEXT,\n longitude REAL,\n latitude REAL,\n otherDetails TEXT,\n otherDetailsId TEXT\n )`,\n },\n ]);\n\n // 2. Fetch existing records from Turso to compare against\n let existingMap = new Map();\n try {\n const res = await tursoPipeline([\n {\n sql:\n `SELECT incidentNumber, incidentTypeName, otherDetails, incidentTypeNameId, otherDetailsId FROM ukmto_events`,\n },\n ]);\n const rows = res.results?.[0]?.response?.result?.rows || [];\n for (const row of rows) {\n const incNum = row[0]?.value;\n if (incNum !== undefined && incNum !== null) {\n existingMap.set(Number(incNum), {\n incidentTypeName: row[1]?.value ?? null,\n otherDetails: row[2]?.value ?? null,\n incidentTypeNameId: row[3]?.value ?? null,\n otherDetailsId: row[4]?.value ?? null,\n });\n }\n }\n } catch (err) {\n console.log(\n \"Could not fetch existing rows, proceeding with full sync:\",\n err,\n );\n }\n\n const insertStmts = [];\n let translatedCount = 0;\n let skippedCount = 0;\n\n for (const inc of incidents) {\n const existing = existingMap.get(inc.incidentNumber);\n\n let nameId;\n let detailsId;\n\n // Check if record already exists and text content hasn't changed\n if (\n existing &&\n existing.incidentTypeName === inc.incidentTypeName &&\n existing.otherDetails === inc.otherDetails\n ) {\n // Content is identical! Reuse existing translations and skip inserting.\n nameId = existing.incidentTypeNameId;\n detailsId = existing.otherDetailsId;\n skippedCount++;\n continue;\n }\n\n // If it's new or text changed, translate the text\n try {\n nameId = inc.incidentTypeName\n ? await translateToId(inc.incidentTypeName)\n : inc.incidentTypeName;\n detailsId = inc.otherDetails\n ? await translateToId(inc.otherDetails)\n : inc.otherDetails;\n translatedCount++;\n } catch (err) {\n console.error(\"Translation failed for incident\", inc.incidentNumber, err);\n nameId = inc.incidentTypeName;\n detailsId = inc.otherDetails;\n }\n\n insertStmts.push({\n sql: `INSERT OR REPLACE INTO ukmto_events\n (incidentNumber, incidentTypeName, incidentTypeNameId, incidentIssuer,\n place, sitecoreId, pinColour, locationLatitudeDDDMMSS,\n locationLongitudeDDDMMSS, utcDateOfIncident, utcDateCreated,\n longitude, latitude, otherDetails, otherDetailsId)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n inc.incidentNumber,\n inc.incidentTypeName,\n nameId,\n inc.incidentIssuer,\n inc.place,\n inc.sitecoreId,\n inc.pinColour,\n inc.locationLatitudeDDDMMSS,\n inc.locationLongitudeDDDMMSS,\n inc.utcDateOfIncident,\n inc.utcDateCreated,\n inc.dataLng,\n inc.dataLat,\n inc.otherDetails,\n detailsId,\n ],\n });\n }\n\n // 3. Send inserts in chunks of 10 to avoid payload size/timeout limits\n if (insertStmts.length > 0) {\n const CHUNK_SIZE = 10;\n for (let i = 0; i < insertStmts.length; i += CHUNK_SIZE) {\n const chunk = insertStmts.slice(i, i + CHUNK_SIZE);\n await tursoPipeline([...chunk]);\n }\n }\n\n console.log(\n `Sync complete. Total incoming: ${incidents.length} | Skipped (unchanged): ${skippedCount} | Translated & Updated: ${translatedCount}`,\n );\n}\n\nexport default async function handler() {\n await fetchAndPopulate(Url, HeaderS, UkmtoData);\n}\n\nfunction fetchAndPopulate(url, headerS, constData) {\n fetchQueue = fetchQueue.then(async () => {\n try {\n const response = await fetch(url, { method: \"GET\", headers: headerS });\n if (!response.ok) {\n throw new Error(`UKMTO HTTP ${response.status}`);\n }\n\n const raw = await response.json();\n const mapped = Array.isArray(raw) ? raw.map(mapIncident) : [];\n\n console.log(`Fetched ${mapped.length} UKMTO incidents.`);\n\n // Persist (create table + smart check & translate + upsert)\n await saveToDb(mapped);\n\n constData.parent = mapped;\n constData.temp = [...mapped];\n } catch (error) {\n console.error(\"UKMTO fetch failed:\", error);\n }\n });\n return fetchQueue;\n}\n\nexport { mapIncident, UkmtoData };",[150],"18c6bbf2-42b6-4dc8-9bcd-e81d1e308098","easaEvent.js","b21fc016-a9bf-11f1-8cb4-1607ee4eb77e",151,["D",1788942414030],33,"// ---------------------------------------------------------------------------\n// EASA Conflict Zone Information Bulletins (CZIB) -> JSON -> Turso (Val Town)\n// ---------------------------------------------------------------------------\nimport { DOMParser } from \"npm:linkedom\";\n\nconst MAIN_URL = \"https://www.easa.europa.eu/en/domains/air-operations/czibs\";\n\nconst HEADERS = {\n \"User-Agent\":\n \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36\",\n Accept: \"text/html,application/json,text/plain,*/*\",\n};\n\nconst dom = new DOMParser();\n\nfunction clean(s) {\n return (s || \"\").replace(/\\s+/g, \" \").trim();\n}\n\nfunction isDetailHref(h) {\n if (!h) return false;\n if (!h.includes(\"/czibs/\")) return false;\n if (/czibs\\/(information|map|export|feed)/.test(h)) return false;\n return true;\n}\n\nasync function listActiveRows() {\n const res = await fetch(MAIN_URL, { headers: HEADERS });\n if (!res.ok) throw new Error(`overview HTTP ${res.status}`);\n const html = await res.text();\n const doc = dom.parseFromString(html, \"text/html\");\n\n const rows = doc.querySelectorAll(\n \"#status-div-classeasa-status-field-status-active--div-classeasa-status-field-status-active-field-itemdiv-classcz-status-cz-status-activeactivedivdivdiv tbody tr\",\n );\n const active = [];\n\n for (const row of rows) {\n const link = Array.from(row.querySelectorAll(\"a\")).find((a) =>\n isDetailHref(a.getAttribute(\"href\"))\n );\n\n if (!link) continue;\n\n const detailUrl = new URL(link.getAttribute(\"href\"), MAIN_URL).href;\n const dateStart = clean(\n row.querySelector(\n \"td.views-field.views-field-field-easa-conflict-zone-up-date\",\n )?.textContent,\n );\n const dateEnd = clean(\n row.querySelector(\n \"td.views-field.views-field-field-easa-conflict-zone-until\",\n )?.textContent,\n );\n\n active.push({ detailUrl, dateStart, dateEnd });\n }\n return active;\n}\n\nasync function scrapeDetail(item) {\n try {\n const res = await fetch(item.detailUrl, { headers: HEADERS });\n if (!res.ok) {\n console.error(`detail HTTP ${res.status}: ${item.detailUrl}`);\n return null;\n }\n const html = await res.text();\n const doc = dom.parseFromString(html, \"text/html\");\n\n const fieldValue = (selector) =>\n clean(doc.querySelector(selector)?.textContent);\n\n const czibNumber =\n fieldValue(\".field--name-field-easa-czib-number .field__items div\") ||\n fieldValue(\".field-name-field-easa-czib-number .field-items div\") ||\n fieldValue(\".field-name-field-easa-czib-number div div\");\n\n const publicator = fieldValue(\".czib_ref_publications .field__items div\") ||\n fieldValue(\".czib_ref_publications .field-items div\") ||\n fieldValue(\".czib_ref_publications div div div\");\n\n const airspace = fieldValue(\n \".field--name-field-easa-cz-aff-air .field__items div\",\n ) || fieldValue(\n \".field-name-field-easa-cz-aff-air .field-items div\",\n );\n\n const description = fieldValue(\n \".field--name-field-easa-cz-description .field__items div\",\n ) || fieldValue(\n \".field-name-field-easa-cz-description .field-items div\",\n );\n\n const recommendation = fieldValue(\n \".field--name-field-easa-cz-recommendations .field__items div\",\n ) || fieldValue(\n \".field-name-field-easa-cz-recommendations .field-items div\",\n );\n\n const csvHref =\n doc.querySelector('a[href*=\"conflict-zone-node-export\"]')?.getAttribute(\n \"href\",\n ) || \"\";\n const nid = (csvHref.match(/(\\d+)\\/?$/) || [])[1] || null;\n\n let coordinate = null;\n if (nid) {\n try {\n const apiRes = await fetch(\n `https://www.easa.europa.eu/en/api/maps/czib-node/${nid}`,\n { headers: HEADERS },\n );\n if (apiRes.ok) coordinate = await apiRes.json();\n } catch (err) {\n console.error(\"Coordinate fetch failed for\", item.detailUrl, err);\n }\n }\n\n return {\n nid,\n czibNumber,\n publicator,\n airspace,\n description,\n recommendation,\n coordinate,\n };\n } catch (err) {\n console.error(`Failed parsing detail page ${item.detailUrl}:`, err.message);\n return null;\n }\n}\n\nfunction buildRecord(item, detail) {\n if (!detail) return null;\n return {\n nid: detail.nid || null,\n status: \"Active\",\n czibNumber: detail.czibNumber || null,\n dateStart: item.dateStart || null,\n dateEnd: item.dateEnd || null,\n airspace: detail.airspace || null,\n publicator: detail.publicator || null,\n description: detail.description || null,\n recommendation: detail.recommendation || null,\n coordinate: detail.coordinate,\n detailUrl: item.detailUrl,\n };\n}\n\nasync function mapLimit(items, limit, fn) {\n const results = new Array(items.length);\n let i = 0;\n const workers = Array.from(\n { length: Math.min(limit, items.length) },\n async () => {\n while (i < items.length) {\n const idx = i++;\n results[idx] = await fn(items[idx], idx);\n }\n },\n );\n await Promise.all(workers);\n return results;\n}\n\nconst TURSO_BASE = (Deno.env.get(\"TURSO_URL\") || \"\").replace(/\\/+$/, \"\");\nconst TURSO_AUTH_TOKEN = Deno.env.get(\"TURSO_AUTH_TOKEN\");\n\nif (!TURSO_BASE || !TURSO_AUTH_TOKEN) {\n throw new Error(\n \"Missing TURSO_URL / TURSO_AUTH_TOKEN env vars in Val Town secrets\",\n );\n}\n\nfunction typedArg(value) {\n if (value === null || value === undefined) {\n return { type: \"null\", value: null };\n }\n if (typeof value === \"number\") {\n return { type: Number.isInteger(value) ? \"integer\" : \"float\", value };\n }\n return { type: \"text\", value: String(value) };\n}\n\nfunction typedFloat(value) {\n if (value === null || value === undefined) {\n return { type: \"null\", value: null };\n }\n const num = Number(value);\n if (isNaN(num)) return { type: \"null\", value: null };\n return { type: \"float\", value: num };\n}\n\nasync function tursoPipeline(stmts) {\n const requests = [\n ...stmts.map(({ sql, args }) => ({\n type: \"execute\",\n stmt: { sql, args: args || [] },\n })),\n { type: \"close\" },\n ];\n const res = await fetch(`${TURSO_BASE}/v2/pipeline`, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${TURSO_AUTH_TOKEN}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({ requests }),\n });\n if (!res.ok) throw new Error(`Turso HTTP ${res.status}: ${await res.text()}`);\n return res.json();\n}\n\nasync function saveToDb(records) {\n await tursoPipeline([\n {\n sql: `CREATE TABLE IF NOT EXISTS easa_czibs (\n nid TEXT PRIMARY KEY,\n status TEXT,\n czib_number TEXT,\n date_start TEXT,\n date_end TEXT,\n airspace TEXT,\n publicator TEXT, \n description TEXT, \n recommendation TEXT,\n latitude REAL NULL, \n longitude REAL NULL,\n detail_url TEXT\n )`,\n },\n ]);\n\n const insertStmts = records.map((r) => ({\n sql: `INSERT OR REPLACE INTO easa_czibs\n (nid, status, czib_number, date_start, date_end, airspace,\n publicator, description, recommendation,\n latitude, longitude, detail_url)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n typedArg(r.nid || crypto.randomUUID()),\n typedArg(r.status),\n typedArg(r.czibNumber),\n typedArg(r.dateStart),\n typedArg(r.dateEnd),\n typedArg(r.airspace),\n typedArg(r.publicator ?? \"\"),\n typedArg(r.description ?? \"\"),\n typedArg(r.recommendation ?? \"\"),\n typedFloat(r.coordinate ? r.coordinate.lat : null),\n typedFloat(r.coordinate ? r.coordinate.lng : null),\n typedArg(r.detailUrl),\n ],\n }));\n\n await tursoPipeline(insertStmts);\n}\n\nexport default async function handler() {\n try {\n const rows = await listActiveRows();\n console.log(`Found ${rows.length} active CZIB(s).`);\n\n const details = await mapLimit(rows, 4, (item) => scrapeDetail(item));\n\n const records = [];\n for (let i = 0; i < details.length; i++) {\n const rec = buildRecord(rows[i], details[i]);\n if (rec) records.push(rec);\n }\n\n await saveToDb(records);\n console.log(`Successfully synced ${records.length} CZIB(s) to Turso.`);\n return JSON.stringify(records, null, 2);\n } catch (err) {\n console.error(\"CRITICAL ERROR in handler:\", err.message, err.stack);\n throw err;\n }\n}",[159],"cfc1ae4a-ee38-45f7-b9ab-22eae6147b0d","commonEvent.js","fcc295ea-aa22-11f1-9f3a-1607ee4eb77e",131,["D",1788723482797],22,"// ---------------------------------------------------------------------------\n// Upstash Redis GeoJSON -> Turso (Table: commonevents)\n// Column updated to use 'reference' (singular) to avoid SQL reserved words.\n// ---------------------------------------------------------------------------\n\nconst UPSTASH_URL =\n \"https://informed-stag-162043.upstash.io/json.get/mapEvent/\";\nconst UPSTASH_BASE = \"https://informed-stag-162043.upstash.io\";\nconst UPSTASH_AUTH =\n \"Bearer gQAAAAAAAnj7AAIgcDJmMDZiMDhkM2U2OTM0NTQzYTM3ZTVkMDM0ZWJmNTkxMQ\";\n\nconst TURSO_BASE = (Deno.env.get(\"TURSO_URL\") || \"\").replace(/\\/+$/, \"\");\nconst TURSO_AUTH_TOKEN = Deno.env.get(\"TURSO_AUTH_TOKEN\");\n\nif (!TURSO_BASE || !TURSO_AUTH_TOKEN) {\n throw new Error(\"Missing TURSO_URL / TURSO_AUTH_TOKEN env vars\");\n}\n\nfunction clean(s) {\n return (s || \"\").replace(/\\s+/g, \" \").trim();\n}\n\nfunction hashId(title, date, location) {\n const s = `${title || \"\"}|${date || \"\"}|${location || \"\"}`;\n let h = 0;\n for (let i = 0; i < s.length; i++) {\n h = (Math.imul(31, h) + s.charCodeAt(i)) | 0;\n }\n return (h >>> 0).toString(36);\n}\n\nfunction typedArg(value) {\n if (value === null || value === undefined) {\n return { type: \"null\", value: null };\n }\n if (typeof value === \"number\") {\n return { type: Number.isInteger(value) ? \"integer\" : \"float\", value };\n }\n return { type: \"text\", value: String(value) };\n}\n\nfunction typedFloat(value) {\n if (value === null || value === undefined || Number.isNaN(Number(value))) {\n return { type: \"null\", value: null };\n }\n return { type: \"float\", value: Number(value) };\n}\n\nasync function tursoPipeline(stmts) {\n const requests = [\n ...stmts.map(({ sql, args }) => ({\n type: \"execute\",\n stmt: { sql, args: args || [] },\n })),\n { type: \"close\" },\n ];\n\n const res = await fetch(`${TURSO_BASE}/v2/pipeline`, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${TURSO_AUTH_TOKEN}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({ requests }),\n });\n\n if (!res.ok) {\n const errText = await res.text();\n throw new Error(`Turso HTTP ${res.status}: ${errText}`);\n }\n\n const data = await res.json();\n\n // Statement-level failures must abort the run — otherwise a partial/failed\n // Turso write could still trigger the Upstash clear.\n if (data && Array.isArray(data.results)) {\n const errs = data.results\n .map((result, idx) => (result.type === \"error\" ? { idx, result } : null))\n .filter(Boolean);\n if (errs.length > 0) {\n throw new Error(\n `Turso statement error(s): ${errs\n .map(({ idx, result }) => `[${idx}] ${result.error}`)\n .join(\"; \")}`,\n );\n }\n }\n\n return data;\n}\n\n// ---------------------------------------------------------------------------\n// Fetch, Clean, and Handle Malformed/Truncated JSON Gracefully\n// ---------------------------------------------------------------------------\nasync function fetchAndParseEvents() {\n const res = await fetch(UPSTASH_URL, {\n headers: { Authorization: UPSTASH_AUTH },\n });\n if (!res.ok) {\n throw new Error(`Upstash HTTP ${res.status}: ${await res.text()}`);\n }\n\n const body = await res.json();\n let rawData = body.result;\n\n let parsed;\n try {\n parsed = typeof rawData === \"string\" ? JSON.parse(rawData) : rawData;\n } catch (e) {\n console.warn(\n \"JSON parse failed (possibly truncated). Attempting safe recovery...\",\n );\n try {\n const lastValidIdx = rawData.lastIndexOf(\"}\");\n if (lastValidIdx > 0) {\n const fixedJson = rawData.substring(0, lastValidIdx + 1) + \"]}\";\n parsed = JSON.parse(fixedJson);\n } else {\n throw e;\n }\n } catch (innerErr) {\n throw new Error(\n `Failed to parse or repair Upstash payload: ${innerErr.message}`,\n );\n }\n }\n\n const features = Array.isArray(parsed?.features) ? parsed.features : [];\n\n return features.map((f) => {\n const p = f?.properties || {};\n const coords = Array.isArray(f?.geometry?.coordinates)\n ? f.geometry.coordinates\n : [null, null];\n\n const lng = coords[0] != null ? Number(coords[0]) : null;\n const lat = coords[1] != null ? Number(coords[1]) : null;\n\n return {\n id: hashId(p.title, p.date, p.location),\n title: clean(p.title),\n crime: clean(p.crime),\n date: clean(p.date),\n location: clean(p.location),\n paragraph: clean(p.paragraph),\n thumbnail: p.thumbnail || null,\n latitude: !isNaN(lat) ? lat : null,\n longitude: !isNaN(lng) ? lng : null,\n channel: p.channel ? JSON.stringify(p.channel) : null,\n reference: p.references ? JSON.stringify(p.references) : null,\n };\n });\n}\n\n// ---------------------------------------------------------------------------\n// Save records to Turso\n// ---------------------------------------------------------------------------\nasync function saveToDb(records) {\n // 1. Create table using 'reference' column name\n console.log(\"Ensuring table 'commonevents' exists...\");\n await tursoPipeline([\n {\n sql: `CREATE TABLE IF NOT EXISTS commonevents (\n id TEXT PRIMARY KEY,\n title TEXT,\n crime TEXT,\n date TEXT,\n location TEXT,\n paragraph TEXT,\n thumbnail TEXT,\n latitude REAL,\n longitude REAL,\n channel TEXT,\n reference TEXT,\n synced_at TEXT\n )`,\n },\n ]);\n\n // 2. Prepare insert statements using 'reference'\n const now = new Date().toISOString();\n const insertStmts = records.map((r) => ({\n sql: `INSERT OR REPLACE INTO commonevents \n (id, title, crime, date, location, paragraph, thumbnail, latitude, longitude, channel, reference, synced_at) \n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n typedArg(r.id),\n typedArg(r.title),\n typedArg(r.crime),\n typedArg(r.date),\n typedArg(r.location),\n typedArg(r.paragraph),\n typedArg(r.thumbnail),\n typedFloat(r.latitude),\n typedFloat(r.longitude),\n typedArg(r.channel),\n typedArg(r.reference),\n typedArg(now),\n ],\n }));\n\n // 3. Execute inserts in batches of 10\n console.log(`Inserting ${insertStmts.length} records into Turso...`);\n const BATCH_SIZE = 10;\n for (let i = 0; i < insertStmts.length; i += BATCH_SIZE) {\n const batch = insertStmts.slice(i, i + BATCH_SIZE);\n await tursoPipeline(batch);\n }\n}\n\n// ---------------------------------------------------------------------------\n// Empty the Upstash mapEvent features after a successful Turso sync.\n// Sets features to [] (keeps the rest of the document intact).\n// ---------------------------------------------------------------------------\nasync function clearUpstash() {\n const res = await fetch(`${UPSTASH_BASE}/json.set/mapEvent/$.features`, {\n method: \"POST\",\n headers: {\n Authorization: UPSTASH_AUTH,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify([]),\n });\n if (!res.ok) {\n throw new Error(`Upstash SET HTTP ${res.status}: ${await res.text()}`);\n }\n return res.json();\n}\n\n// ---------------------------------------------------------------------------\n// Main Entry Point\n// ---------------------------------------------------------------------------\nexport default async function handler() {\n const records = await fetchAndParseEvents();\n console.log(`Successfully parsed ${records.length} event(s) from Upstash.`);\n\n await saveToDb(records);\n console.log(\n `Successfully synced ${records.length} record(s) to Turso (table: commonevents).`,\n );\n\n // Only clear the Upstash source if the Turso write succeeded above.\n await clearUpstash();\n console.log(\"Cleared mapEvent from Upstash.\");\n\n return { success: true, total: records.length };\n}",[167],"28dcaac6-769b-40a0-8019-7ba853c08b21","scripts","ca191255-a940-4a18-832b-d6f7739bfbc1",174,["D",1789073099638],1,"directory",[175],"pulls","branches","forks","envVars",{"_189":191,"_192":61},"connections",{"_189":61,"_190":61},"count","unfilledCount",6,"emptyCount",{"_43":174,"_32":175,"_30":176,"_134":-5,"_41":42,"_40":177,"_135":153,"_38":233,"_13":179,"_33":180,"_35":-5,"_24":25,"_138":-5,"_202":234,"_27":235},{"_43":225,"_32":226,"_30":227,"_134":176,"_41":42,"_40":228,"_135":153,"_38":229,"_13":61,"_33":34,"_35":230,"_24":25,"_138":-5,"_202":231,"_27":232},{"_43":217,"_32":218,"_30":219,"_134":176,"_41":42,"_40":220,"_135":153,"_38":221,"_13":61,"_33":34,"_35":222,"_24":25,"_138":-5,"_202":223,"_27":224},{"_43":166,"_32":167,"_30":168,"_134":-5,"_41":42,"_40":169,"_135":153,"_38":214,"_13":171,"_33":146,"_35":172,"_24":25,"_138":-5,"_202":215,"_27":216},{"_43":158,"_32":159,"_30":160,"_134":-5,"_41":42,"_40":161,"_135":153,"_38":211,"_13":163,"_33":146,"_35":164,"_24":25,"_138":-5,"_202":212,"_27":213},{"_43":149,"_32":150,"_30":151,"_134":-5,"_41":42,"_40":152,"_135":153,"_38":208,"_13":155,"_33":146,"_35":156,"_24":25,"_138":-5,"_202":209,"_27":210},{"_43":140,"_32":141,"_30":142,"_134":-5,"_41":42,"_40":143,"_135":25,"_38":205,"_13":145,"_33":146,"_35":147,"_24":25,"_138":-5,"_202":206,"_27":207},{"_43":133,"_32":28,"_30":31,"_134":-5,"_41":42,"_40":14,"_135":25,"_38":201,"_13":137,"_33":34,"_35":37,"_24":25,"_138":-5,"_202":203,"_27":204},["D",1789283991480],"children",[],[28],["D",1789279607071],[],[141],["D",1788982196432],[],[150],["D",1788942414030],[],[159],["D",1788723482797],[],[167],"a8ce5f8a-de5f-4003-bc1d-bd8ba884f0ea","zenrows-status.js","97a8e3a2-ac8a-11f1-9bec-1607ee4eb77e",168,["D",1788984667491],"// Prints your ZenRows plan status: credits used, period end, concurrency limit.\n// Run this when warEvent.js logs AUTH004 (credits exhausted) or AUTH006\n// (concurrency limit) to see where you stand. Does not consume credits.\nconst key = Deno.env.get(\"ZENROWS_API_KEY\");\nif (!key) throw new Error(\"Missing ZENROWS_API_KEY env var\");\n\nconst res = await fetch(\"https://api.zenrows.com/v1/subscriptions/self/details\", {\n headers: { \"X-API-Key\": key },\n});\nconst body = await res.text();\nconsole.log(`HTTP ${res.status}`);\ntry {\n console.log(JSON.stringify(JSON.parse(body), null, 2));\n} catch {\n console.log(body);\n}\n",[],[175,218],"ea265c60-9e86-4983-b94e-704638316918","translate.js","e5e91c2e-a9c1-11f1-bcf6-1607ee4eb77e",78,["D",1788678567608],"// ---------------------------------------------------------------------------\n// Shared EN -> ID translator used by every event scraper in this val.\n//\n// Primary: NVIDIA Riva \"nvidia/riva-translate-4b-instruct-v2\"\n// (OpenAI-compatible chat completions at integrate.api.nvidia.com).\n// Fallback: free MyMemory API, then the original English text as a last resort\n// so a transient API issue never fails or corrupts a run.\n//\n// The Riva instruct model must have the target language named in the *user*\n// message (otherwise it can default to another language, e.g. French).\n//\n// Requires the NVIDIA_API_KEY env var (set once on the val, shared by all\n// files). If it is missing, translation falls back to MyMemory/English.\n// ---------------------------------------------------------------------------\nconst NVIDIA_API_KEY = Deno.env.get(\"NVIDIA_API_KEY\");\nconst NVIDIA_BASE = \"https://integrate.api.nvidia.com/v1/chat/completions\";\nconst NVIDIA_MODEL = \"nvidia/riva-translate-4b-instruct-v2\";\n\nlet lastTranslate = 0; // MyMemory fallback throttle (~2 req/s)\n\n// Riva has no 500-char cap, but chunk large text so it stays well within the\n// model's input/output token limits (long EASA descriptions).\nconst CHUNK_MAX = 1600;\n\nfunction splitChunks(text) {\n const chunks = [];\n let rest = text.trim();\n while (rest.length > CHUNK_MAX) {\n let cut = rest.lastIndexOf(\". \", CHUNK_MAX);\n if (cut < CHUNK_MAX * 0.5) cut = rest.lastIndexOf(\" \", CHUNK_MAX);\n if (cut < CHUNK_MAX * 0.5) cut = CHUNK_MAX; // no break point -> hard cut\n chunks.push(rest.slice(0, cut + (rest[cut] === \" \" ? 0 : 1)));\n rest = rest.slice(cut + 1).trim();\n }\n if (rest) chunks.push(rest);\n return chunks;\n}\n\n// Primary: NVIDIA Riva instruct translation model.\nasync function rivaOnce(text) {\n const key = NVIDIA_API_KEY;\n if (!key) throw new Error(\"NVIDIA_API_KEY not set\");\n const res = await fetch(NVIDIA_BASE, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${key}`,\n \"Content-Type\": \"application/json\",\n Accept: \"application/json\",\n },\n body: JSON.stringify({\n model: NVIDIA_MODEL,\n messages: [\n { role: \"system\", content: \"You are a translation engine.\" },\n {\n role: \"user\",\n content: `Translate the following from English to Indonesian:\\n${text}`,\n },\n ],\n max_tokens: 2048,\n temperature: 0,\n }),\n });\n if (!res.ok) throw new Error(`Riva HTTP ${res.status}`);\n const data = await res.json();\n const out = data?.choices?.[0]?.message?.content?.trim();\n if (!out || /MYMEMORY WARNING|QUERY LENGTH/i.test(out)) {\n throw new Error(\"bad Riva output\");\n }\n return out;\n}\n\n// Fallback: MyMemory. Its free daily quota can be exhausted, in which case it\n// returns a warning instead of a translation — we treat that as a failure and\n// drop through to the original English rather than persisting the warning.\nasync function mymemoryOnce(text) {\n const elapsed = Date.now() - lastTranslate;\n if (elapsed < 500) {\n await new Promise((r) => setTimeout(r, 500 - elapsed));\n }\n lastTranslate = Date.now();\n\n const url =\n \"https://api.mymemory.translated.net/get?q=\" +\n encodeURIComponent(text) +\n \"&langpair=en|id\";\n const res = await fetch(url);\n if (!res.ok) throw new Error(`MyMemory HTTP ${res.status}`);\n const data = await res.json();\n const translated = data?.responseData?.translatedText;\n if (\n !translated ||\n /MYMEMORY WARNING|QUERY LENGTH LIMIT EXCEEDED|NO QUERY|INVALID PAIR/i.test(\n translated,\n )\n ) {\n return text;\n }\n return translated;\n}\n\nasync function translateOnce(text) {\n try {\n return await rivaOnce(text);\n } catch (err) {\n console.error(\"Riva failed, falling back to MyMemory:\", err.message);\n try {\n return await mymemoryOnce(text);\n } catch (err2) {\n console.error(\"MyMemory fallback failed, keeping original:\", err2.message);\n return text;\n }\n }\n}\n\nexport async function translateToId(text) {\n if (!text) return text;\n const chunks = splitChunks(text);\n if (chunks.length === 1) return await translateOnce(chunks[0]);\n const parts = [];\n for (const c of chunks) parts.push(await translateOnce(c));\n return parts.join(\" \");\n}\n",[],[175,226],["D",1789073099638],[194,195],[175],["D",1788316219773],["D",1789458074143],{"_98":97,"_105":101},{"_98":97,"_99":97,"_100":101,"_18":55,"_102":103,"_104":25},{"_13":87,"_32":88,"_43":42,"_89":44,"_90":-5,"_91":-5,"_92":-5,"_93":-5,"_94":-5,"_95":45,"_96":97},"ctx",{"_80":-5,"_124":274,"_275":276,"_277":278,"_279":25,"_280":25,"_281":25,"_282":153,"_283":-5,"_284":-7,"_285":-7,"_286":61,"_287":25,"_288":25,"_289":-5,"_290":25,"_291":25,"_292":25,"_293":-5},"ENV",{"_245":246,"_247":248,"_249":88,"_250":251,"_252":253,"_254":255,"_256":257,"_258":259,"_260":261,"_262":263,"_264":265,"_266":267,"_268":269,"_270":271,"_272":273},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v231