[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_228":229,"_230":231},"routes/_app.x.$handle.$name",{"_98":97,"_32":45,"_106":88,"_27":53,"_13":14,"_107":-5,"_11":108,"_109":110,"_111":112,"_113":114,"_115":116,"_20":-5,"_117":-5,"_118":-5,"_119":25,"_120":121,"_122":25,"_123":-5,"_124":-5,"_125":126},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_43":44,"_32":45,"_38":46,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":53,"_54":-5,"_18":55,"_56":-5,"_57":25,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":65,"_66":67,"_68":69,"_70":-5,"_71":72,"_73":72,"_74":75,"_76":75,"_77":75,"_78":72,"_79":72,"_80":81,"_82":83,"_84":85,"_86":25},"version",202,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_33":34,"_35":37,"_38":39,"_40":14,"_41":42,"_24":25},"previous",{"_30":31,"_32":28,"_33":34,"_35":36},"deleted",false,"deletedHere","path","warEvent.js","latestPath","fileId","d75184f6-a8f1-11f1-abed-1607ee4eb77e","name","fileType","interval","content","import { DOMParser } from \"npm:linkedom\";\nimport { translateToId } from \"./scripts/translate.js\";\n\n// ---------------------------------------------------------------------------\n// Configuration & Credentials\n// ---------------------------------------------------------------------------\nconst TURSO_URL = Deno.env.get(\"TURSO_URL\");\nconst TURSO_AUTH_TOKEN = Deno.env.get(\"TURSO_AUTH_TOKEN\");\nconst ZENROWS_API_KEY = Deno.env.get(\"ZENROWS_API_KEY\");\n// NOTE: this token was pasted in plain text in the original request. Move it\n// to a Val Town env var (`BROWSERLESS_TOKEN`) instead of leaving it hardcoded\n// — the fallback here only exists so the script keeps working until you do.\nconst BROWSERLESS_TOKEN = Deno.env.get(\"BROWSERLESS_TOKEN\") ??\n \"2VEWf6Pl4hsPdohe482b2e288738d6dc962e3fea6e68dd00c\";\n\n// ---------------------------------------------------------------------------\n// Concept (two-phase pipeline, no WebSocket/CDP anywhere)\n//\n// PHASE A — one plain HTTPS call to ZenRows' scraper API (js_render=true),\n// used to get past Cloudflare and harvest a reusable session:\n// 1. GET https://api.zenrows.com/v1/?url=...&js_render=true&json_response=true\n// (wait_for=\"[data-link]\" makes ZenRows wait for the DOM to be ready\n// before returning, same job the old MutationObserver did)\n// 2. Parse the returned HTML for [data-link] sublinks\n// 3. Take the cookies/headers ZenRows returns alongside the HTML → \"pool\"\n// 4. Nothing to close — it's a single stateless request, so the ZenRows\n// run ends the moment the response lands\n//\n// PHASE B — Turso lookup: which sublinks are not yet stored → pending list.\n//\n// PHASE C — Browserless /smart-scrape (also plain HTTPS), once per pending\n// sublink, reusing the cookie/header pool harvested in Phase A:\n// 5. POST /smart-scrape { url, formats: [\"html\"], cookies, headers }\n// 6. Take the returned html — request completes, nothing to close\n// 7. Parse → translate → Turso insert\n// ---------------------------------------------------------------------------\nconst PROXY_COUNTRY = \"de\";\nconst RUN_BUDGET_MS = 55_000;\nconst HEADROOM_MS = 10_000;\nconst ZENROWS_TIMEOUT_MS = 45_000;\nconst BROWSERLESS_TIMEOUT_MS = 60_000;\n\nconst TARGET_URL = \"https://liveuamap.com\";\nconst START = Date.now();\nconst elapsed = () => Date.now() - START;\nconst timeLeft = () => RUN_BUDGET_MS - elapsed();\n\n// ---------------------------------------------------------------------------\n// Turso helpers\n// ---------------------------------------------------------------------------\nfunction typedArg(value) {\n if (value === null || value === undefined) {\n return { type: \"null\", value: null };\n }\n if (typeof value === \"number\") {\n return { type: Number.isInteger(value) ? \"integer\" : \"float\", value };\n }\n return { type: \"text\", value: String(value) };\n}\n\nasync function tursoPipeline(stmts) {\n const base = TURSO_URL.replace(/\\/+$/, \"\");\n const requests = [\n ...stmts.map(({ sql, args }) => ({\n type: \"execute\",\n stmt: { sql, args: (args || []).map(typedArg) },\n })),\n { type: \"close\" },\n ];\n\n const res = await fetch(`${base}/v2/pipeline`, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${TURSO_AUTH_TOKEN}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({ requests }),\n });\n\n if (!res.ok) {\n throw new Error(`Turso HTTP ${res.status}: ${await res.text()}`);\n }\n return res.json();\n}\n\nasync function ensureTable() {\n await tursoPipeline([\n {\n sql: `CREATE TABLE IF NOT EXISTS warfare_events (\n id TEXT PRIMARY KEY,\n title TEXT,\n status TEXT,\n location TEXT,\n date TEXT,\n longitude REAL,\n latitude REAL,\n athumbnail TEXT,\n reference TEXT\n )`,\n },\n ]);\n}\n\nasync function pendingSublinks(sublinks) {\n if (sublinks.length === 0) return [];\n const placeholders = sublinks.map(() => \"?\").join(\",\");\n const out = await tursoPipeline([\n {\n sql: `SELECT id FROM warfare_events WHERE id IN (${placeholders})`,\n args: sublinks,\n },\n ]);\n const rows = out?.results?.[0]?.response?.result?.rows || [];\n const known = new Set(rows.map((r) => r[0]?.value));\n return sublinks.filter((l) => !known.has(l));\n}\n\n// ---------------------------------------------------------------------------\n// Extract date like \"2024/24-april-...\" -> \"2024-04-24\"\n// ---------------------------------------------------------------------------\nconst MONTHS = {\n january: \"01\",\n february: \"02\",\n march: \"03\",\n april: \"04\",\n may: \"05\",\n june: \"06\",\n july: \"07\",\n august: \"08\",\n september: \"09\",\n october: \"10\",\n november: \"11\",\n december: \"12\",\n};\n\nfunction extractLiveUAMapDate(url) {\n const match = url.match(/\\/(\\d{4})\\/(\\d{1,2})-([a-z]+)-\\d+-/i);\n if (!match) return null;\n const [, year, day, monthName] = match;\n const mm = MONTHS[monthName.toLowerCase()];\n return mm ? `${year}-${mm}-${String(day).padStart(2, \"0\")}` : null;\n}\n\n// ---------------------------------------------------------------------------\n// Parse ONE sublink's HTML -> GeoJSON Feature. Pure/sync, no network.\n// ---------------------------------------------------------------------------\nconst dom = new DOMParser();\n\nfunction parseFeatureHtml(sublink, subHtml) {\n const subdoc = dom.parseFromString(subHtml, \"text/html\");\n\n let lat = 48.8829;\n let lng = 31.181;\n\n const eventScriptMatch = subHtml.match(\n /document\\.addEventListener\$['\"]DOMContentLoaded['\"][\\s\\S]*?\$;/,\n );\n if (eventScriptMatch) {\n const latMatch = eventScriptMatch[0].match(/lat\\s*=\\s*([0-9.-]+)/);\n const lngMatch = eventScriptMatch[0].match(/lng\\s*=\\s*([0-9.-]+)/);\n if (latMatch) lat = parseFloat(latMatch[1]);\n if (lngMatch) lng = parseFloat(lngMatch[1]);\n }\n\n const infozone = subdoc.querySelector(\".popup-text\");\n const status = infozone?.querySelector(\".bgma\")?.getAttribute(\"data-src\") ||\n null;\n const title = infozone?.querySelector(\"h2\")?.textContent?.trim() || null;\n const locationText =\n infozone?.querySelector(\".tagas strong\")?.textContent?.trim() || null;\n const thumbnail =\n infozone?.querySelector(\".popup_video + * img\")?.getAttribute(\"src\") ||\n null;\n const reference =\n subdoc.querySelector(\".source-link\")?.getAttribute(\"href\") || null;\n\n return {\n type: \"Feature\",\n geometry: { type: \"Point\", coordinates: [lng, lat] },\n properties: {\n references: [sublink, reference].filter(Boolean),\n title,\n status,\n location: locationText,\n thumbnail,\n date: extractLiveUAMapDate(sublink),\n },\n };\n}\n\n// ---------------------------------------------------------------------------\n// Translate + upsert ONE feature into Turso\n// ---------------------------------------------------------------------------\nasync function saveFeature(feature) {\n const { properties, geometry } = feature;\n const [lng, lat] = geometry.coordinates;\n const eventId = properties.references[0] || crypto.randomUUID();\n\n let translatedTitle = properties.title;\n if (translatedTitle) {\n try {\n translatedTitle = await translateToId(properties.title);\n } catch (err) {\n console.error(\"Translation failed for title:\", properties.title, err);\n }\n }\n\n await tursoPipeline([\n {\n sql: `INSERT OR REPLACE INTO warfare_events\n (id, title, status, location, date, longitude, latitude, athumbnail, reference)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n eventId,\n translatedTitle,\n properties.status,\n properties.location,\n properties.date,\n lng,\n lat,\n properties.thumbnail,\n properties.references[1] || properties.references[0] || null,\n ],\n },\n ]);\n}\n\n// ---------------------------------------------------------------------------\n// Cookie/header pool normalization — ZenRows' json_response and Browserless\n// don't necessarily agree on cookie shape, so coerce whatever comes back\n// into { name, value, domain, path, expires, httpOnly, secure, sameSite }.\n// ---------------------------------------------------------------------------\nfunction parseSetCookieString(line) {\n const [pair] = String(line).split(\";\");\n const idx = pair.indexOf(\"=\");\n if (idx === -1) return null;\n return {\n name: pair.slice(0, idx).trim(),\n value: pair.slice(idx + 1).trim(),\n };\n}\n\nfunction normalizeCookies(rawCookies, fallbackDomain) {\n let entries = [];\n\n if (Array.isArray(rawCookies)) {\n entries = rawCookies\n .map((c) =>\n typeof c === \"string\"\n ? parseSetCookieString(c)\n : {\n name: c.name ?? c.key,\n value: c.value,\n domain: c.domain,\n path: c.path,\n expires: c.expires,\n httpOnly: c.httpOnly,\n secure: c.secure,\n sameSite: c.sameSite,\n }\n )\n .filter(Boolean);\n } else if (rawCookies && typeof rawCookies === \"object\") {\n entries = Object.entries(rawCookies).map(([name, value]) => ({\n name,\n value: String(value),\n }));\n }\n\n return entries\n .filter((c) => c.name)\n .map((c) => ({ domain: fallbackDomain, path: \"/\", ...c }));\n}\n\nfunction cleanDataLink(raw) {\n if (!raw) return null;\n let v = raw.trim();\n // The site emits this attribute JSON-escaped (e.g. \\\"https:\\/\\/...\\\"),\n // so undo any backslash-escaping first (\\/ -> /, \\\" -> \", etc.)...\n v = v.replace(/\\\\(.)/g, \"$1\");\n // ...then strip any quote characters left wrapping the value.\n v = v.replace(/^[\"']+|[\"']+$/g, \"\").trim();\n return v || null;\n}\n\n// ---------------------------------------------------------------------------\n// PHASE A — ZenRows scraper API via plain fetch (no WebSocket/CDP): clear\n// Cloudflare once, harvest sublinks + a cookie/header pool.\n// ---------------------------------------------------------------------------\nasync function initSessionAndExtractLinks() {\n const endpoint = new URL(\"https://api.zenrows.com/v1/\");\n endpoint.searchParams.set(\"url\", TARGET_URL);\n endpoint.searchParams.set(\"apikey\", ZENROWS_API_KEY);\n endpoint.searchParams.set(\"js_render\", \"true\");\n endpoint.searchParams.set(\"premium_proxy\", \"true\");\n endpoint.searchParams.set(\"proxy_country\", PROXY_COUNTRY);\n endpoint.searchParams.set(\"wait_for\", \"[data-link]\");\n // json_response=true wraps the render result as JSON — { html, headers,\n // cookies, ... } — instead of just the raw HTML body, so we can pull the\n // cookies/headers pool out alongside the page. Confirm the exact field\n // names against your ZenRows plan's docs; this code tolerates a few\n // reasonable shapes below.\n endpoint.searchParams.set(\"json_response\", \"true\");\n\n console.log(\"Requesting main page via ZenRows…\");\n let res;\n try {\n res = await fetch(endpoint, {\n signal: AbortSignal.timeout(ZENROWS_TIMEOUT_MS),\n });\n } catch (err) {\n console.error(\n `ZenRows request failed (${err.message}); skipping this run.`,\n );\n return null;\n }\n\n if (!res.ok) {\n console.error(`ZenRows HTTP ${res.status}: ${await res.text()}`);\n return null;\n }\n\n const rawBody = await res.text();\n const contentType = res.headers.get(\"content-type\") || \"\";\n console.log(\n `ZenRows response: status=${res.status} content-type=\"${contentType}\"`,\n );\n\n let payload;\n try {\n // ZenRows sometimes labels this response text/plain even though the\n // body is actually the json_response JSON wrapper, so parse first and\n // only fall back to \"it's raw HTML\" if that genuinely fails.\n payload = JSON.parse(rawBody);\n console.log(\n \"ZenRows body parsed as JSON. payload keys:\",\n Object.keys(payload),\n );\n } catch {\n payload = { html: rawBody };\n console.log(\"ZenRows body was not JSON; treating it as raw HTML.\");\n }\n\n const html = payload.html ?? payload.data?.html ?? \"\";\n console.log(\n `ZenRows html length=${html.length} snippet=` +\n JSON.stringify(html.slice(0, 400)),\n );\n if (!html) {\n console.error(\"ZenRows response had no html field; skipping this run.\");\n return null;\n }\n\n const doc = dom.parseFromString(html, \"text/html\");\n const pageTitle = doc.querySelector(\"title\")?.textContent?.trim() || null;\n const sublinks = [\n ...new Set(\n [...doc.querySelectorAll(\"[data-link]\")]\n .map((e) => cleanDataLink(e.getAttribute(\"data-link\")))\n .filter(Boolean),\n ),\n ];\n\n if (sublinks.length === 0) {\n // Diagnostics only — helps tell \"wrong selector\" apart from \"wrong page\".\n const anyFeedler = doc.querySelectorAll(\"#feedler\").length;\n const anySourcees = doc.querySelectorAll(\".sourcees\").length;\n const bodyLen = doc.querySelector(\"body\")?.textContent?.length ?? 0;\n console.warn(\n `No [data-link] matches. page title=\"${pageTitle}\" ` +\n `#feedler found=${anyFeedler} .sourcees found=${anySourcees} ` +\n `body text length=${bodyLen}. ` +\n `If title/body look like a Cloudflare or error page rather than the ` +\n `real site, js_render/premium_proxy/wait_for aren't clearing the ` +\n `challenge — check your ZenRows dashboard logs for this request.`,\n );\n }\n\n const targetHost = new URL(TARGET_URL).hostname;\n const rawCookies = payload.cookies ?? payload.data?.cookies ?? [];\n const cookies = normalizeCookies(rawCookies, targetHost);\n\n const rawHeaders = payload.headers ?? payload.data?.headers ?? {};\n const headers = typeof rawHeaders === \"object\" && rawHeaders !== null\n ? rawHeaders\n : {};\n const userAgent = headers[\"User-Agent\"] ?? headers[\"user-agent\"] ?? null;\n\n console.log(\n `Extracted ${sublinks.length} sublinks, ${cookies.length} cookies ` +\n `(${elapsed()}ms elapsed). ZenRows call finished — nothing left to close.`,\n );\n\n return { sublinks, cookies, headers, userAgent };\n}\n\n// ---------------------------------------------------------------------------\n// PHASE C — Browserless /smart-scrape: one stateless HTTPS call per pending\n// sublink, reusing the cookie/header pool from Phase A.\n// ---------------------------------------------------------------------------\nasync function fetchHtmlViaBrowserless(url, pool) {\n const endpoint =\n `https://production-sfo.browserless.io/smart-scrape?timeout=${BROWSERLESS_TIMEOUT_MS}&token=${BROWSERLESS_TOKEN}`;\n\n const body = {\n url,\n formats: [\"html\"],\n };\n if (pool?.cookies?.length) body.cookies = pool.cookies;\n if (pool?.userAgent) {\n body.headers = { ...pool.headers, \"User-Agent\": pool.userAgent };\n } else if (pool?.headers && Object.keys(pool.headers).length) {\n body.headers = pool.headers;\n }\n\n const res = await fetch(endpoint, {\n method: \"POST\",\n headers: { \"Content-Type\": \"application/json\" },\n body: JSON.stringify(body),\n signal: AbortSignal.timeout(BROWSERLESS_TIMEOUT_MS + 5_000),\n });\n\n if (!res.ok) {\n throw new Error(`Browserless HTTP ${res.status}: ${await res.text()}`);\n }\n\n const data = await res.json();\n // Response shape isn't confirmed here — check it against your Browserless\n // account's smart-scrape docs and adjust these lookups if the html payload\n // lives somewhere else in the JSON.\n const html = data?.data?.html ??\n data?.data?.[0]?.html ??\n data?.html ??\n data?.content ??\n null;\n\n if (typeof html !== \"string\" || html.length === 0) {\n throw new Error(\n `No html field found in Browserless response: ${\n JSON.stringify(data).slice(0, 300)\n }`,\n );\n }\n return html;\n}\n\n// ---------------------------------------------------------------------------\n// Main flow\n// ---------------------------------------------------------------------------\nasync function run() {\n await ensureTable();\n\n // --- Phase A: ZenRows init — links + cookie/header pool, one HTTP call ---\n const pool = await initSessionAndExtractLinks();\n if (!pool || pool.sublinks.length === 0) {\n console.warn(\n \"No sublinks extracted from the ZenRows phase; nothing to do.\",\n );\n return;\n }\n\n // --- Phase B: Turso — which sublinks are already stored? ---\n const pending = await pendingSublinks(pool.sublinks);\n console.log(\n `Main page: ${pool.sublinks.length} sublinks, ${\n pool.sublinks.length - pending.length\n } already stored, ${pending.length} pending (${elapsed()}ms elapsed)`,\n );\n\n // --- Phase C: per-sublink Browserless smart-scrape, then parse/translate/save ---\n let saved = 0;\n let fetchSkipped = 0;\n let saveFailed = 0;\n let i = 0;\n for (; i < pending.length; i++) {\n if (timeLeft() < HEADROOM_MS) break;\n\n const sublink = pending[i];\n\n let html;\n try {\n html = await fetchHtmlViaBrowserless(sublink, pool);\n } catch (err) {\n fetchSkipped++;\n console.warn(\n `Browserless fetch failed for ${sublink} (${err.message}) — skipping.`,\n );\n continue;\n }\n\n try {\n const feature = parseFeatureHtml(sublink, html);\n await saveFeature(feature);\n saved++;\n console.log(`Saved [${saved}] ${feature.properties.title ?? sublink}`);\n } catch (error) {\n saveFailed++;\n console.error(\"Parse/save failed:\", sublink, error.message);\n }\n }\n\n const deferred = pending.length - i;\n\n console.log(\n `Done in ${elapsed()}ms: ${saved} saved, ${fetchSkipped} skipped (browserless fetch failed), ` +\n `${saveFailed} parse/save failed, ${deferred} deferred to next run.`,\n );\n}\n\n// ---------------------------------------------------------------------------\n// Cron entrypoint\n// ---------------------------------------------------------------------------\nexport default async function () {\n if (!TURSO_URL || !TURSO_AUTH_TOKEN) {\n throw new Error(\"Missing TURSO_URL / TURSO_AUTH_TOKEN env vars\");\n }\n if (!ZENROWS_API_KEY) {\n throw new Error(\"Missing ZENROWS_API_KEY env var\");\n }\n if (!BROWSERLESS_TOKEN) {\n throw new Error(\"Missing BROWSERLESS_TOKEN env var\");\n }\n\n try {\n await run();\n } catch (error) {\n console.error(\"Pipeline failed:\", error);\n throw error;\n }\n}","import { DOMParser } from \"npm:linkedom\";\nimport { translateToId } from \"./scripts/translate.js\";\n\n// ---------------------------------------------------------------------------\n// Configuration & Credentials\n// ---------------------------------------------------------------------------\nconst TURSO_URL = Deno.env.get(\"TURSO_URL\");\nconst TURSO_AUTH_TOKEN = Deno.env.get(\"TURSO_AUTH_TOKEN\");\nconst ZENROWS_API_KEY = Deno.env.get(\"ZENROWS_API_KEY\");\n// NOTE: this token was pasted in plain text in the original request. Move it\n// to a Val Town env var (`BROWSERLESS_TOKEN`) instead of leaving it hardcoded\n// — the fallback here only exists so the script keeps working until you do.\nconst BROWSERLESS_TOKEN = Deno.env.get(\"BROWSERLESS_TOKEN\") ??\n \"2VEWf6Pl4hsPdohe482b2e288738d6dc962e3fea6e68dd00c\";\n\n// ---------------------------------------------------------------------------\n// Concept (two-phase pipeline, no WebSocket/CDP anywhere)\n//\n// PHASE A — one plain HTTPS call to ZenRows' scraper API (js_render=true),\n// used to get past Cloudflare and harvest a reusable session:\n// 1. GET https://api.zenrows.com/v1/?url=...&js_render=true&json_response=true\n// (wait_for=\"[data-link]\" makes ZenRows wait for the DOM to be ready\n// before returning, same job the old MutationObserver did)\n// 2. Parse the returned HTML for [data-link] sublinks\n// 3. Take the cookies/headers ZenRows returns alongside the HTML → \"pool\"\n// 4. Nothing to close — it's a single stateless request, so the ZenRows\n// run ends the moment the response lands\n//\n// PHASE B — Turso lookup: which sublinks are not yet stored → pending list.\n//\n// PHASE C — Browserless /smart-scrape (also plain HTTPS), once per pending\n// sublink, reusing the cookie/header pool harvested in Phase A:\n// 5. POST /smart-scrape { url, formats: [\"html\"], cookies, headers }\n// 6. Take the returned html — request completes, nothing to close\n// 7. Parse → translate → Turso insert\n// ---------------------------------------------------------------------------\nconst PROXY_COUNTRY = \"de\";\nconst RUN_BUDGET_MS = 55_000;\nconst HEADROOM_MS = 10_000;\nconst ZENROWS_TIMEOUT_MS = 45_000;\nconst BROWSERLESS_TIMEOUT_MS = 60_000;\n\nconst TARGET_URL = \"https://liveuamap.com\";\nconst START = Date.now();\nconst elapsed = () => Date.now() - START;\nconst timeLeft = () => RUN_BUDGET_MS - elapsed();\n\n// ---------------------------------------------------------------------------\n// Turso helpers\n// ---------------------------------------------------------------------------\nfunction typedArg(value) {\n if (value === null || value === undefined) {\n return { type: \"null\", value: null };\n }\n if (typeof value === \"number\") {\n return { type: Number.isInteger(value) ? \"integer\" : \"float\", value };\n }\n return { type: \"text\", value: String(value) };\n}\n\nasync function tursoPipeline(stmts) {\n const base = TURSO_URL.replace(/\\/+$/, \"\");\n const requests = [\n ...stmts.map(({ sql, args }) => ({\n type: \"execute\",\n stmt: { sql, args: (args || []).map(typedArg) },\n })),\n { type: \"close\" },\n ];\n\n const res = await fetch(`${base}/v2/pipeline`, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${TURSO_AUTH_TOKEN}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({ requests }),\n });\n\n if (!res.ok) {\n throw new Error(`Turso HTTP ${res.status}: ${await res.text()}`);\n }\n return res.json();\n}\n\nasync function ensureTable() {\n await tursoPipeline([\n {\n sql: `CREATE TABLE IF NOT EXISTS warfare_events (\n id TEXT PRIMARY KEY,\n title TEXT,\n status TEXT,\n location TEXT,\n date TEXT,\n longitude REAL,\n latitude REAL,\n athumbnail TEXT,\n reference TEXT\n )`,\n },\n ]);\n}\n\nasync function pendingSublinks(sublinks) {\n if (sublinks.length === 0) return [];\n const placeholders = sublinks.map(() => \"?\").join(\",\");\n const out = await tursoPipeline([\n {\n sql: `SELECT id FROM warfare_events WHERE id IN (${placeholders})`,\n args: sublinks,\n },\n ]);\n const rows = out?.results?.[0]?.response?.result?.rows || [];\n const known = new Set(rows.map((r) => r[0]?.value));\n return sublinks.filter((l) => !known.has(l));\n}\n\n// ---------------------------------------------------------------------------\n// Extract date like \"2024/24-april-...\" -> \"2024-04-24\"\n// ---------------------------------------------------------------------------\nconst MONTHS = {\n january: \"01\",\n february: \"02\",\n march: \"03\",\n april: \"04\",\n may: \"05\",\n june: \"06\",\n july: \"07\",\n august: \"08\",\n september: \"09\",\n october: \"10\",\n november: \"11\",\n december: \"12\",\n};\n\nfunction extractLiveUAMapDate(url) {\n const match = url.match(/\\/(\\d{4})\\/(\\d{1,2})-([a-z]+)-\\d+-/i);\n if (!match) return null;\n const [, year, day, monthName] = match;\n const mm = MONTHS[monthName.toLowerCase()];\n return mm ? `${year}-${mm}-${String(day).padStart(2, \"0\")}` : null;\n}\n\n// ---------------------------------------------------------------------------\n// Parse ONE sublink's HTML -> GeoJSON Feature. Pure/sync, no network.\n// ---------------------------------------------------------------------------\nconst dom = new DOMParser();\n\nfunction parseFeatureHtml(sublink, subHtml) {\n const subdoc = dom.parseFromString(subHtml, \"text/html\");\n\n let lat = 48.8829;\n let lng = 31.181;\n\n const eventScriptMatch = subHtml.match(\n /document\\.addEventListener\$['\"]DOMContentLoaded['\"][\\s\\S]*?\$;/,\n );\n if (eventScriptMatch) {\n const latMatch = eventScriptMatch[0].match(/lat\\s*=\\s*([0-9.-]+)/);\n const lngMatch = eventScriptMatch[0].match(/lng\\s*=\\s*([0-9.-]+)/);\n if (latMatch) lat = parseFloat(latMatch[1]);\n if (lngMatch) lng = parseFloat(lngMatch[1]);\n }\n\n const infozone = subdoc.querySelector(\".popup-text\");\n const status = infozone?.querySelector(\".bgma\")?.getAttribute(\"data-src\") ||\n null;\n const title = infozone?.querySelector(\"h2\")?.textContent?.trim() || null;\n const locationText =\n infozone?.querySelector(\".tagas strong\")?.textContent?.trim() || null;\n const thumbnail =\n infozone?.querySelector(\".popup_video + * img\")?.getAttribute(\"src\") ||\n null;\n const reference =\n subdoc.querySelector(\".source-link\")?.getAttribute(\"href\") || null;\n\n return {\n type: \"Feature\",\n geometry: { type: \"Point\", coordinates: [lng, lat] },\n properties: {\n references: [sublink, reference].filter(Boolean),\n title,\n status,\n location: locationText,\n thumbnail,\n date: extractLiveUAMapDate(sublink),\n },\n };\n}\n\n// ---------------------------------------------------------------------------\n// Translate + upsert ONE feature into Turso\n// ---------------------------------------------------------------------------\nasync function saveFeature(feature) {\n const { properties, geometry } = feature;\n const [lng, lat] = geometry.coordinates;\n const eventId = properties.references[0] || crypto.randomUUID();\n\n let translatedTitle = properties.title;\n if (translatedTitle) {\n try {\n translatedTitle = await translateToId(properties.title);\n } catch (err) {\n console.error(\"Translation failed for title:\", properties.title, err);\n }\n }\n\n await tursoPipeline([\n {\n sql: `INSERT OR REPLACE INTO warfare_events\n (id, title, status, location, date, longitude, latitude, athumbnail, reference)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n eventId,\n translatedTitle,\n properties.status,\n properties.location,\n properties.date,\n lng,\n lat,\n properties.thumbnail,\n properties.references[1] || properties.references[0] || null,\n ],\n },\n ]);\n}\n\n// ---------------------------------------------------------------------------\n// Cookie/header pool normalization — ZenRows' json_response and Browserless\n// don't necessarily agree on cookie shape, so coerce whatever comes back\n// into { name, value, domain, path, expires, httpOnly, secure, sameSite }.\n// ---------------------------------------------------------------------------\nfunction parseSetCookieString(line) {\n const [pair] = String(line).split(\";\");\n const idx = pair.indexOf(\"=\");\n if (idx === -1) return null;\n return {\n name: pair.slice(0, idx).trim(),\n value: pair.slice(idx + 1).trim(),\n };\n}\n\nfunction normalizeCookies(rawCookies, fallbackDomain) {\n let entries = [];\n\n if (Array.isArray(rawCookies)) {\n entries = rawCookies\n .map((c) =>\n typeof c === \"string\"\n ? parseSetCookieString(c)\n : {\n name: c.name ?? c.key,\n value: c.value,\n domain: c.domain,\n path: c.path,\n expires: c.expires,\n httpOnly: c.httpOnly,\n secure: c.secure,\n sameSite: c.sameSite,\n }\n )\n .filter(Boolean);\n } else if (rawCookies && typeof rawCookies === \"object\") {\n entries = Object.entries(rawCookies).map(([name, value]) => ({\n name,\n value: String(value),\n }));\n }\n\n return entries\n .filter((c) => c.name)\n .map((c) => ({ domain: fallbackDomain, path: \"/\", ...c }));\n}\n\nfunction cleanDataLink(raw) {\n if (!raw) return null;\n let v = raw.trim();\n // The site emits this attribute JSON-escaped (e.g. \\\"https:\\/\\/...\\\"),\n // so undo any backslash-escaping first (\\/ -> /, \\\" -> \", etc.)...\n v = v.replace(/\\\\(.)/g, \"$1\");\n // ...then strip any quote characters left wrapping the value.\n v = v.replace(/^[\"']+|[\"']+$/g, \"\").trim();\n return v || null;\n}\n\n// ---------------------------------------------------------------------------\n// PHASE A — ZenRows scraper API via plain fetch (no WebSocket/CDP): clear\n// Cloudflare once, harvest sublinks + a cookie/header pool.\n// ---------------------------------------------------------------------------\nasync function initSessionAndExtractLinks() {\n const endpoint = new URL(\"https://api.zenrows.com/v1/\");\n endpoint.searchParams.set(\"url\", TARGET_URL);\n endpoint.searchParams.set(\"apikey\", ZENROWS_API_KEY);\n endpoint.searchParams.set(\"js_render\", \"true\");\n endpoint.searchParams.set(\"premium_proxy\", \"true\");\n endpoint.searchParams.set(\"proxy_country\", PROXY_COUNTRY);\n endpoint.searchParams.set(\"wait_for\", \"[data-link]\");\n // json_response=true wraps the render result as JSON — { html, headers,\n // cookies, ... } — instead of just the raw HTML body, so we can pull the\n // cookies/headers pool out alongside the page. Confirm the exact field\n // names against your ZenRows plan's docs; this code tolerates a few\n // reasonable shapes below.\n endpoint.searchParams.set(\"json_response\", \"true\");\n // json_response alone only returns { html, xhr, js_instructions_report }.\n // Cookies/headers need to be explicitly requested via `outputs`.\n endpoint.searchParams.set(\"outputs\", \"headers,cookies\");\n\n console.log(\"Requesting main page via ZenRows…\");\n let res;\n try {\n res = await fetch(endpoint, {\n signal: AbortSignal.timeout(ZENROWS_TIMEOUT_MS),\n });\n } catch (err) {\n console.error(\n `ZenRows request failed (${err.message}); skipping this run.`,\n );\n return null;\n }\n\n if (!res.ok) {\n console.error(`ZenRows HTTP ${res.status}: ${await res.text()}`);\n return null;\n }\n\n const rawBody = await res.text();\n const contentType = res.headers.get(\"content-type\") || \"\";\n console.log(\n `ZenRows response: status=${res.status} content-type=\"${contentType}\"`,\n );\n\n let payload;\n try {\n // ZenRows sometimes labels this response text/plain even though the\n // body is actually the json_response JSON wrapper, so parse first and\n // only fall back to \"it's raw HTML\" if that genuinely fails.\n payload = JSON.parse(rawBody);\n console.log(\n \"ZenRows body parsed as JSON. payload keys:\",\n Object.keys(payload),\n );\n } catch {\n payload = { html: rawBody };\n console.log(\"ZenRows body was not JSON; treating it as raw HTML.\");\n }\n\n const html = payload.html ?? payload.data?.html ?? \"\";\n console.log(\n `ZenRows html length=${html.length} snippet=` +\n JSON.stringify(html.slice(0, 400)),\n );\n if (!html) {\n console.error(\"ZenRows response had no html field; skipping this run.\");\n return null;\n }\n\n const doc = dom.parseFromString(html, \"text/html\");\n const pageTitle = doc.querySelector(\"title\")?.textContent?.trim() || null;\n const sublinks = [\n ...new Set(\n [...doc.querySelectorAll(\"[data-link]\")]\n .map((e) => cleanDataLink(e.getAttribute(\"data-link\")))\n .filter(Boolean),\n ),\n ];\n\n if (sublinks.length === 0) {\n // Diagnostics only — helps tell \"wrong selector\" apart from \"wrong page\".\n const anyFeedler = doc.querySelectorAll(\"#feedler\").length;\n const anySourcees = doc.querySelectorAll(\".sourcees\").length;\n const bodyLen = doc.querySelector(\"body\")?.textContent?.length ?? 0;\n console.warn(\n `No [data-link] matches. page title=\"${pageTitle}\" ` +\n `#feedler found=${anyFeedler} .sourcees found=${anySourcees} ` +\n `body text length=${bodyLen}. ` +\n `If title/body look like a Cloudflare or error page rather than the ` +\n `real site, js_render/premium_proxy/wait_for aren't clearing the ` +\n `challenge — check your ZenRows dashboard logs for this request.`,\n );\n }\n\n const targetHost = new URL(TARGET_URL).hostname;\n const rawCookies = payload.cookies ?? payload.data?.cookies ?? [];\n const cookies = normalizeCookies(rawCookies, targetHost);\n\n const rawHeaders = payload.headers ?? payload.data?.headers ?? {};\n const headers = typeof rawHeaders === \"object\" && rawHeaders !== null\n ? rawHeaders\n : {};\n const userAgent = headers[\"User-Agent\"] ?? headers[\"user-agent\"] ?? null;\n\n console.log(\n `Extracted ${sublinks.length} sublinks, ${cookies.length} cookies ` +\n `(${elapsed()}ms elapsed). ZenRows call finished — nothing left to close.`,\n );\n\n return { sublinks, cookies, headers, userAgent };\n}\n\n// ---------------------------------------------------------------------------\n// PHASE C — Browserless /smart-scrape: one stateless HTTPS call per pending\n// sublink, reusing the cookie/header pool from Phase A.\n// ---------------------------------------------------------------------------\nasync function fetchHtmlViaBrowserless(url, pool) {\n const endpoint =\n `https://production-sfo.browserless.io/smart-scrape?timeout=${BROWSERLESS_TIMEOUT_MS}&token=${BROWSERLESS_TOKEN}`;\n\n const body = {\n url,\n formats: [\"html\"],\n };\n if (pool?.cookies?.length) body.cookies = pool.cookies;\n if (pool?.userAgent) {\n body.headers = { ...pool.headers, \"User-Agent\": pool.userAgent };\n } else if (pool?.headers && Object.keys(pool.headers).length) {\n body.headers = pool.headers;\n }\n\n const res = await fetch(endpoint, {\n method: \"POST\",\n headers: { \"Content-Type\": \"application/json\" },\n body: JSON.stringify(body),\n signal: AbortSignal.timeout(BROWSERLESS_TIMEOUT_MS + 5_000),\n });\n\n if (!res.ok) {\n throw new Error(`Browserless HTTP ${res.status}: ${await res.text()}`);\n }\n\n const data = await res.json();\n // Response shape isn't confirmed here — check it against your Browserless\n // account's smart-scrape docs and adjust these lookups if the html payload\n // lives somewhere else in the JSON.\n const html = data?.data?.html ??\n data?.data?.[0]?.html ??\n data?.html ??\n data?.content ??\n null;\n\n if (typeof html !== \"string\" || html.length === 0) {\n throw new Error(\n `No html field found in Browserless response: ${\n JSON.stringify(data).slice(0, 300)\n }`,\n );\n }\n return html;\n}\n\n// ---------------------------------------------------------------------------\n// Main flow\n// ---------------------------------------------------------------------------\nasync function run() {\n await ensureTable();\n\n // --- Phase A: ZenRows init — links + cookie/header pool, one HTTP call ---\n const pool = await initSessionAndExtractLinks();\n if (!pool || pool.sublinks.length === 0) {\n console.warn(\n \"No sublinks extracted from the ZenRows phase; nothing to do.\",\n );\n return;\n }\n\n // --- Phase B: Turso — which sublinks are already stored? ---\n const pending = await pendingSublinks(pool.sublinks);\n console.log(\n `Main page: ${pool.sublinks.length} sublinks, ${\n pool.sublinks.length - pending.length\n } already stored, ${pending.length} pending (${elapsed()}ms elapsed)`,\n );\n\n // --- Phase C: per-sublink Browserless smart-scrape, then parse/translate/save ---\n let saved = 0;\n let fetchSkipped = 0;\n let saveFailed = 0;\n let i = 0;\n for (; i < pending.length; i++) {\n if (timeLeft() < HEADROOM_MS) break;\n\n const sublink = pending[i];\n\n let html;\n try {\n html = await fetchHtmlViaBrowserless(sublink, pool);\n } catch (err) {\n fetchSkipped++;\n console.warn(\n `Browserless fetch failed for ${sublink} (${err.message}) — skipping.`,\n );\n continue;\n }\n\n try {\n const feature = parseFeatureHtml(sublink, html);\n await saveFeature(feature);\n saved++;\n console.log(`Saved [${saved}] ${feature.properties.title ?? sublink}`);\n } catch (error) {\n saveFailed++;\n console.error(\"Parse/save failed:\", sublink, error.message);\n }\n }\n\n const deferred = pending.length - i;\n\n console.log(\n `Done in ${elapsed()}ms: ${saved} saved, ${fetchSkipped} skipped (browserless fetch failed), ` +\n `${saveFailed} parse/save failed, ${deferred} deferred to next run.`,\n );\n}\n\n// ---------------------------------------------------------------------------\n// Cron entrypoint\n// ---------------------------------------------------------------------------\nexport default async function () {\n if (!TURSO_URL || !TURSO_AUTH_TOKEN) {\n throw new Error(\"Missing TURSO_URL / TURSO_AUTH_TOKEN env vars\");\n }\n if (!ZENROWS_API_KEY) {\n throw new Error(\"Missing ZENROWS_API_KEY env var\");\n }\n if (!BROWSERLESS_TOKEN) {\n throw new Error(\"Missing BROWSERLESS_TOKEN env var\");\n }\n\n try {\n await run();\n } catch (error) {\n console.error(\"Pipeline failed:\", error);\n throw error;\n }\n}","createdAt",["D",1789098987119],"branchVersion","branchId","1dc3dd44-cc65-414f-8e4a-735d1ac2a5b5","id","feed5b42-8d51-40be-9982-13fc813b4b7c","warData",["D",1788316219773],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","","imageUrl","0762e08b-0f1b-495a-871c-4fba2da9304a","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","019f5a38-1aa2-74ee-aaf5-8eac040acbf4","tursoName","feed5b428d5140be998213fc813b4b7c","tursoHostname","feed5b428d5140be998213fc813b4b7c-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789458074143],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_98":97,"_105":101},"org",{"_98":97,"_99":97,"_100":101,"_18":55,"_102":103,"_104":25},"branch",{"_13":87,"_32":88,"_43":42,"_89":44,"_90":-5,"_91":-5,"_92":-5,"_93":-5,"_94":-5,"_95":45,"_96":97},"isOwned",237,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","mustedpual","handle","displayName","avatar","https://avatars.githubusercontent.com/u/157880878?v=4","tier","free","canRestrictVals","profileImageUrl","branchName","adoptOrgId",{"_43":44,"_32":45,"_38":223,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":53,"_54":-5,"_18":55,"_56":-5,"_57":25,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":65,"_66":67,"_68":224,"_70":-5,"_71":72,"_73":72,"_74":75,"_76":75,"_77":75,"_78":72,"_79":72,"_80":225,"_82":226,"_84":227,"_86":25},"tree",[183,186,187,188,189],"files",[183,184,185,186,187,188,189],"counts",{"_172":61,"_173":169,"_174":61,"_175":176,"_177":178},"contents",[127,128,129,130,131],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_43":164,"_32":165,"_30":166,"_133":-5,"_41":42,"_40":167,"_134":143,"_38":168,"_13":169,"_33":170,"_35":-5,"_24":25,"_137":-5,"_27":171},{"_43":156,"_32":157,"_30":158,"_133":-5,"_41":42,"_40":159,"_134":143,"_38":160,"_13":161,"_33":34,"_35":162,"_24":25,"_137":-5,"_27":163},{"_43":148,"_32":149,"_30":150,"_133":-5,"_41":42,"_40":151,"_134":143,"_38":152,"_13":153,"_33":34,"_35":154,"_24":25,"_137":-5,"_27":155},{"_43":139,"_32":140,"_30":141,"_133":-5,"_41":42,"_40":142,"_134":143,"_38":144,"_13":145,"_33":34,"_35":146,"_24":25,"_137":-5,"_27":147},{"_43":132,"_32":28,"_30":31,"_133":-5,"_41":42,"_40":14,"_134":25,"_38":135,"_13":136,"_33":34,"_35":37,"_24":25,"_137":-5,"_27":138},"486746f4-41c9-4e69-acee-fc7fe7c63ed1","parentId","latest",["D",1789098987119],69,"label",[28],"4d2ece01-2d75-488d-80d4-8ae766cde047","ukmtoEvent.js","7e654b18-a9b2-11f1-9377-1607ee4eb77e",154,true,["D",1788982196432],10,"import { translateToId } from \"./scripts/translate.js\";\n// ---------------------------------------------------------------------------\n// UKMTO incident feed -> Indonesian-translated events -> Turso.\n// Fetches https://sccd.royalnavy.mod.uk/api/ukmto/all (array of incidents),\n// compares against existing records in Turso to skip unchanged items,\n// translates new/changed incidentTypeName + otherDetails into Indonesian,\n// then upserts into Turso in safe chunks.\n// ---------------------------------------------------------------------------\nlet fetchQueue = Promise.resolve();\n\nconst Url = \"https://sccd.royalnavy.mod.uk/api/ukmto/all\";\n\nconst HeaderS = {\n \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64)\",\n Accept: \"application/json\",\n};\n\nconst UkmtoData = {\n type: \"ukmto\",\n parent: null, // Untouched master data\n temp: null, // Current filtered view for the map\n};\n\n// ---------------------------------------------------------------------------\n// Turso client via its HTTP API (POST /v2/pipeline). Credentials from env vars.\n// ---------------------------------------------------------------------------\nconst TURSO_URL = Deno.env.get(\"TURSO_URL\");\nconst TURSO_AUTH_TOKEN = Deno.env.get(\"TURSO_AUTH_TOKEN\");\n\nif (!TURSO_URL || !TURSO_AUTH_TOKEN) {\n throw new Error(\"Missing TURSO_URL / TURSO_AUTH_TOKEN env vars\");\n}\nconst TURSO_BASE = TURSO_URL.replace(/\\/+$/, \"\");\n\n// Turso's v2/pipeline arg format: integer/text values are JSON STRINGS;\n// float values are JSON NUMBERS. See https://docs.turso.tech/sdk/http/quickstart\nfunction typedArg(value) {\n if (value === null || value === undefined) {\n return { type: \"null\", value: null };\n }\n if (typeof value === \"number\") {\n return Number.isInteger(value)\n ? { type: \"integer\", value: String(value) }\n : { type: \"float\", value };\n }\n return { type: \"text\", value: String(value) };\n}\n\nasync function tursoPipeline(stmts) {\n const requests = [\n ...stmts.map(({ sql, args }) => ({\n type: \"execute\",\n stmt: { sql, args: (args || []).map(typedArg) },\n })),\n { type: \"close\" },\n ];\n const res = await fetch(`${TURSO_BASE}/v2/pipeline`, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${TURSO_AUTH_TOKEN}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({ requests }),\n });\n if (!res.ok) {\n throw new Error(`Turso HTTP ${res.status}: ${await res.text()}`);\n }\n return res.json();\n}\n\n// ---------------------------------------------------------------------------\n// Map a raw UKMTO incident to the fields the map consumes.\n// ---------------------------------------------------------------------------\nfunction mapIncident(inc) {\n return {\n incidentTypeName: inc.incidentTypeName,\n incidentNumber: inc.incidentNumber,\n incidentIssuer: inc.incidentIssuer,\n place: inc.place,\n sitecoreId: inc.sitecoreId,\n pinColour: inc.pinColour,\n locationLatitudeDDDMMSS: inc.locationLatitudeDDDMMSS,\n locationLongitudeDDDMMSS: inc.locationLongitudeDDDMMSS,\n utcDateOfIncident: inc.utcDateOfIncident,\n utcDateCreated: inc.utcDateCreated,\n // data-lng / data-lat attributes on map markers\n dataLng: inc.locationLongitude,\n dataLat: inc.locationLatitude,\n otherDetails: inc.otherDetails,\n };\n}\n\n// ---------------------------------------------------------------------------\n// Create table, fetch existing rows for comparison, translate only changes,\n// and upsert in chunks.\n// ---------------------------------------------------------------------------\nasync function saveToDb(incidents) {\n // 1. Ensure table exists\n await tursoPipeline([\n {\n sql: `CREATE TABLE IF NOT EXISTS ukmto_events (\n incidentNumber INTEGER PRIMARY KEY,\n incidentTypeName TEXT,\n incidentTypeNameId TEXT,\n incidentIssuer TEXT,\n place TEXT,\n sitecoreId TEXT,\n pinColour TEXT,\n locationLatitudeDDDMMSS TEXT,\n locationLongitudeDDDMMSS TEXT,\n utcDateOfIncident TEXT,\n utcDateCreated TEXT,\n longitude REAL,\n latitude REAL,\n otherDetails TEXT,\n otherDetailsId TEXT\n )`,\n },\n ]);\n\n // 2. Fetch existing records from Turso to compare against\n let existingMap = new Map();\n try {\n const res = await tursoPipeline([\n {\n sql:\n `SELECT incidentNumber, incidentTypeName, otherDetails, incidentTypeNameId, otherDetailsId FROM ukmto_events`,\n },\n ]);\n const rows = res.results?.[0]?.response?.result?.rows || [];\n for (const row of rows) {\n const incNum = row[0]?.value;\n if (incNum !== undefined && incNum !== null) {\n existingMap.set(Number(incNum), {\n incidentTypeName: row[1]?.value ?? null,\n otherDetails: row[2]?.value ?? null,\n incidentTypeNameId: row[3]?.value ?? null,\n otherDetailsId: row[4]?.value ?? null,\n });\n }\n }\n } catch (err) {\n console.log(\n \"Could not fetch existing rows, proceeding with full sync:\",\n err,\n );\n }\n\n const insertStmts = [];\n let translatedCount = 0;\n let skippedCount = 0;\n\n for (const inc of incidents) {\n const existing = existingMap.get(inc.incidentNumber);\n\n let nameId;\n let detailsId;\n\n // Check if record already exists and text content hasn't changed\n if (\n existing &&\n existing.incidentTypeName === inc.incidentTypeName &&\n existing.otherDetails === inc.otherDetails\n ) {\n // Content is identical! Reuse existing translations and skip inserting.\n nameId = existing.incidentTypeNameId;\n detailsId = existing.otherDetailsId;\n skippedCount++;\n continue;\n }\n\n // If it's new or text changed, translate the text\n try {\n nameId = inc.incidentTypeName\n ? await translateToId(inc.incidentTypeName)\n : inc.incidentTypeName;\n detailsId = inc.otherDetails\n ? await translateToId(inc.otherDetails)\n : inc.otherDetails;\n translatedCount++;\n } catch (err) {\n console.error(\"Translation failed for incident\", inc.incidentNumber, err);\n nameId = inc.incidentTypeName;\n detailsId = inc.otherDetails;\n }\n\n insertStmts.push({\n sql: `INSERT OR REPLACE INTO ukmto_events\n (incidentNumber, incidentTypeName, incidentTypeNameId, incidentIssuer,\n place, sitecoreId, pinColour, locationLatitudeDDDMMSS,\n locationLongitudeDDDMMSS, utcDateOfIncident, utcDateCreated,\n longitude, latitude, otherDetails, otherDetailsId)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n inc.incidentNumber,\n inc.incidentTypeName,\n nameId,\n inc.incidentIssuer,\n inc.place,\n inc.sitecoreId,\n inc.pinColour,\n inc.locationLatitudeDDDMMSS,\n inc.locationLongitudeDDDMMSS,\n inc.utcDateOfIncident,\n inc.utcDateCreated,\n inc.dataLng,\n inc.dataLat,\n inc.otherDetails,\n detailsId,\n ],\n });\n }\n\n // 3. Send inserts in chunks of 10 to avoid payload size/timeout limits\n if (insertStmts.length > 0) {\n const CHUNK_SIZE = 10;\n for (let i = 0; i < insertStmts.length; i += CHUNK_SIZE) {\n const chunk = insertStmts.slice(i, i + CHUNK_SIZE);\n await tursoPipeline([...chunk]);\n }\n }\n\n console.log(\n `Sync complete. Total incoming: ${incidents.length} | Skipped (unchanged): ${skippedCount} | Translated & Updated: ${translatedCount}`,\n );\n}\n\nexport default async function handler() {\n await fetchAndPopulate(Url, HeaderS, UkmtoData);\n}\n\nfunction fetchAndPopulate(url, headerS, constData) {\n fetchQueue = fetchQueue.then(async () => {\n try {\n const response = await fetch(url, { method: \"GET\", headers: headerS });\n if (!response.ok) {\n throw new Error(`UKMTO HTTP ${response.status}`);\n }\n\n const raw = await response.json();\n const mapped = Array.isArray(raw) ? raw.map(mapIncident) : [];\n\n console.log(`Fetched ${mapped.length} UKMTO incidents.`);\n\n // Persist (create table + smart check & translate + upsert)\n await saveToDb(mapped);\n\n constData.parent = mapped;\n constData.temp = [...mapped];\n } catch (error) {\n console.error(\"UKMTO fetch failed:\", error);\n }\n });\n return fetchQueue;\n}\n\nexport { mapIncident, UkmtoData };",[140],"18c6bbf2-42b6-4dc8-9bcd-e81d1e308098","easaEvent.js","b21fc016-a9bf-11f1-8cb4-1607ee4eb77e",151,["D",1788942414030],33,"// ---------------------------------------------------------------------------\n// EASA Conflict Zone Information Bulletins (CZIB) -> JSON -> Turso (Val Town)\n// ---------------------------------------------------------------------------\nimport { DOMParser } from \"npm:linkedom\";\n\nconst MAIN_URL = \"https://www.easa.europa.eu/en/domains/air-operations/czibs\";\n\nconst HEADERS = {\n \"User-Agent\":\n \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36\",\n Accept: \"text/html,application/json,text/plain,*/*\",\n};\n\nconst dom = new DOMParser();\n\nfunction clean(s) {\n return (s || \"\").replace(/\\s+/g, \" \").trim();\n}\n\nfunction isDetailHref(h) {\n if (!h) return false;\n if (!h.includes(\"/czibs/\")) return false;\n if (/czibs\\/(information|map|export|feed)/.test(h)) return false;\n return true;\n}\n\nasync function listActiveRows() {\n const res = await fetch(MAIN_URL, { headers: HEADERS });\n if (!res.ok) throw new Error(`overview HTTP ${res.status}`);\n const html = await res.text();\n const doc = dom.parseFromString(html, \"text/html\");\n\n const rows = doc.querySelectorAll(\n \"#status-div-classeasa-status-field-status-active--div-classeasa-status-field-status-active-field-itemdiv-classcz-status-cz-status-activeactivedivdivdiv tbody tr\",\n );\n const active = [];\n\n for (const row of rows) {\n const link = Array.from(row.querySelectorAll(\"a\")).find((a) =>\n isDetailHref(a.getAttribute(\"href\"))\n );\n\n if (!link) continue;\n\n const detailUrl = new URL(link.getAttribute(\"href\"), MAIN_URL).href;\n const dateStart = clean(\n row.querySelector(\n \"td.views-field.views-field-field-easa-conflict-zone-up-date\",\n )?.textContent,\n );\n const dateEnd = clean(\n row.querySelector(\n \"td.views-field.views-field-field-easa-conflict-zone-until\",\n )?.textContent,\n );\n\n active.push({ detailUrl, dateStart, dateEnd });\n }\n return active;\n}\n\nasync function scrapeDetail(item) {\n try {\n const res = await fetch(item.detailUrl, { headers: HEADERS });\n if (!res.ok) {\n console.error(`detail HTTP ${res.status}: ${item.detailUrl}`);\n return null;\n }\n const html = await res.text();\n const doc = dom.parseFromString(html, \"text/html\");\n\n const fieldValue = (selector) =>\n clean(doc.querySelector(selector)?.textContent);\n\n const czibNumber =\n fieldValue(\".field--name-field-easa-czib-number .field__items div\") ||\n fieldValue(\".field-name-field-easa-czib-number .field-items div\") ||\n fieldValue(\".field-name-field-easa-czib-number div div\");\n\n const publicator = fieldValue(\".czib_ref_publications .field__items div\") ||\n fieldValue(\".czib_ref_publications .field-items div\") ||\n fieldValue(\".czib_ref_publications div div div\");\n\n const airspace = fieldValue(\n \".field--name-field-easa-cz-aff-air .field__items div\",\n ) || fieldValue(\n \".field-name-field-easa-cz-aff-air .field-items div\",\n );\n\n const description = fieldValue(\n \".field--name-field-easa-cz-description .field__items div\",\n ) || fieldValue(\n \".field-name-field-easa-cz-description .field-items div\",\n );\n\n const recommendation = fieldValue(\n \".field--name-field-easa-cz-recommendations .field__items div\",\n ) || fieldValue(\n \".field-name-field-easa-cz-recommendations .field-items div\",\n );\n\n const csvHref =\n doc.querySelector('a[href*=\"conflict-zone-node-export\"]')?.getAttribute(\n \"href\",\n ) || \"\";\n const nid = (csvHref.match(/(\\d+)\\/?$/) || [])[1] || null;\n\n let coordinate = null;\n if (nid) {\n try {\n const apiRes = await fetch(\n `https://www.easa.europa.eu/en/api/maps/czib-node/${nid}`,\n { headers: HEADERS },\n );\n if (apiRes.ok) coordinate = await apiRes.json();\n } catch (err) {\n console.error(\"Coordinate fetch failed for\", item.detailUrl, err);\n }\n }\n\n return {\n nid,\n czibNumber,\n publicator,\n airspace,\n description,\n recommendation,\n coordinate,\n };\n } catch (err) {\n console.error(`Failed parsing detail page ${item.detailUrl}:`, err.message);\n return null;\n }\n}\n\nfunction buildRecord(item, detail) {\n if (!detail) return null;\n return {\n nid: detail.nid || null,\n status: \"Active\",\n czibNumber: detail.czibNumber || null,\n dateStart: item.dateStart || null,\n dateEnd: item.dateEnd || null,\n airspace: detail.airspace || null,\n publicator: detail.publicator || null,\n description: detail.description || null,\n recommendation: detail.recommendation || null,\n coordinate: detail.coordinate,\n detailUrl: item.detailUrl,\n };\n}\n\nasync function mapLimit(items, limit, fn) {\n const results = new Array(items.length);\n let i = 0;\n const workers = Array.from(\n { length: Math.min(limit, items.length) },\n async () => {\n while (i < items.length) {\n const idx = i++;\n results[idx] = await fn(items[idx], idx);\n }\n },\n );\n await Promise.all(workers);\n return results;\n}\n\nconst TURSO_BASE = (Deno.env.get(\"TURSO_URL\") || \"\").replace(/\\/+$/, \"\");\nconst TURSO_AUTH_TOKEN = Deno.env.get(\"TURSO_AUTH_TOKEN\");\n\nif (!TURSO_BASE || !TURSO_AUTH_TOKEN) {\n throw new Error(\n \"Missing TURSO_URL / TURSO_AUTH_TOKEN env vars in Val Town secrets\",\n );\n}\n\nfunction typedArg(value) {\n if (value === null || value === undefined) {\n return { type: \"null\", value: null };\n }\n if (typeof value === \"number\") {\n return { type: Number.isInteger(value) ? \"integer\" : \"float\", value };\n }\n return { type: \"text\", value: String(value) };\n}\n\nfunction typedFloat(value) {\n if (value === null || value === undefined) {\n return { type: \"null\", value: null };\n }\n const num = Number(value);\n if (isNaN(num)) return { type: \"null\", value: null };\n return { type: \"float\", value: num };\n}\n\nasync function tursoPipeline(stmts) {\n const requests = [\n ...stmts.map(({ sql, args }) => ({\n type: \"execute\",\n stmt: { sql, args: args || [] },\n })),\n { type: \"close\" },\n ];\n const res = await fetch(`${TURSO_BASE}/v2/pipeline`, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${TURSO_AUTH_TOKEN}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({ requests }),\n });\n if (!res.ok) throw new Error(`Turso HTTP ${res.status}: ${await res.text()}`);\n return res.json();\n}\n\nasync function saveToDb(records) {\n await tursoPipeline([\n {\n sql: `CREATE TABLE IF NOT EXISTS easa_czibs (\n nid TEXT PRIMARY KEY,\n status TEXT,\n czib_number TEXT,\n date_start TEXT,\n date_end TEXT,\n airspace TEXT,\n publicator TEXT, \n description TEXT, \n recommendation TEXT,\n latitude REAL NULL, \n longitude REAL NULL,\n detail_url TEXT\n )`,\n },\n ]);\n\n const insertStmts = records.map((r) => ({\n sql: `INSERT OR REPLACE INTO easa_czibs\n (nid, status, czib_number, date_start, date_end, airspace,\n publicator, description, recommendation,\n latitude, longitude, detail_url)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n typedArg(r.nid || crypto.randomUUID()),\n typedArg(r.status),\n typedArg(r.czibNumber),\n typedArg(r.dateStart),\n typedArg(r.dateEnd),\n typedArg(r.airspace),\n typedArg(r.publicator ?? \"\"),\n typedArg(r.description ?? \"\"),\n typedArg(r.recommendation ?? \"\"),\n typedFloat(r.coordinate ? r.coordinate.lat : null),\n typedFloat(r.coordinate ? r.coordinate.lng : null),\n typedArg(r.detailUrl),\n ],\n }));\n\n await tursoPipeline(insertStmts);\n}\n\nexport default async function handler() {\n try {\n const rows = await listActiveRows();\n console.log(`Found ${rows.length} active CZIB(s).`);\n\n const details = await mapLimit(rows, 4, (item) => scrapeDetail(item));\n\n const records = [];\n for (let i = 0; i < details.length; i++) {\n const rec = buildRecord(rows[i], details[i]);\n if (rec) records.push(rec);\n }\n\n await saveToDb(records);\n console.log(`Successfully synced ${records.length} CZIB(s) to Turso.`);\n return JSON.stringify(records, null, 2);\n } catch (err) {\n console.error(\"CRITICAL ERROR in handler:\", err.message, err.stack);\n throw err;\n }\n}",[149],"cfc1ae4a-ee38-45f7-b9ab-22eae6147b0d","commonEvent.js","fcc295ea-aa22-11f1-9f3a-1607ee4eb77e",131,["D",1788723482797],22,"// ---------------------------------------------------------------------------\n// Upstash Redis GeoJSON -> Turso (Table: commonevents)\n// Column updated to use 'reference' (singular) to avoid SQL reserved words.\n// ---------------------------------------------------------------------------\n\nconst UPSTASH_URL =\n \"https://informed-stag-162043.upstash.io/json.get/mapEvent/\";\nconst UPSTASH_BASE = \"https://informed-stag-162043.upstash.io\";\nconst UPSTASH_AUTH =\n \"Bearer gQAAAAAAAnj7AAIgcDJmMDZiMDhkM2U2OTM0NTQzYTM3ZTVkMDM0ZWJmNTkxMQ\";\n\nconst TURSO_BASE = (Deno.env.get(\"TURSO_URL\") || \"\").replace(/\\/+$/, \"\");\nconst TURSO_AUTH_TOKEN = Deno.env.get(\"TURSO_AUTH_TOKEN\");\n\nif (!TURSO_BASE || !TURSO_AUTH_TOKEN) {\n throw new Error(\"Missing TURSO_URL / TURSO_AUTH_TOKEN env vars\");\n}\n\nfunction clean(s) {\n return (s || \"\").replace(/\\s+/g, \" \").trim();\n}\n\nfunction hashId(title, date, location) {\n const s = `${title || \"\"}|${date || \"\"}|${location || \"\"}`;\n let h = 0;\n for (let i = 0; i < s.length; i++) {\n h = (Math.imul(31, h) + s.charCodeAt(i)) | 0;\n }\n return (h >>> 0).toString(36);\n}\n\nfunction typedArg(value) {\n if (value === null || value === undefined) {\n return { type: \"null\", value: null };\n }\n if (typeof value === \"number\") {\n return { type: Number.isInteger(value) ? \"integer\" : \"float\", value };\n }\n return { type: \"text\", value: String(value) };\n}\n\nfunction typedFloat(value) {\n if (value === null || value === undefined || Number.isNaN(Number(value))) {\n return { type: \"null\", value: null };\n }\n return { type: \"float\", value: Number(value) };\n}\n\nasync function tursoPipeline(stmts) {\n const requests = [\n ...stmts.map(({ sql, args }) => ({\n type: \"execute\",\n stmt: { sql, args: args || [] },\n })),\n { type: \"close\" },\n ];\n\n const res = await fetch(`${TURSO_BASE}/v2/pipeline`, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${TURSO_AUTH_TOKEN}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({ requests }),\n });\n\n if (!res.ok) {\n const errText = await res.text();\n throw new Error(`Turso HTTP ${res.status}: ${errText}`);\n }\n\n const data = await res.json();\n\n // Statement-level failures must abort the run — otherwise a partial/failed\n // Turso write could still trigger the Upstash clear.\n if (data && Array.isArray(data.results)) {\n const errs = data.results\n .map((result, idx) => (result.type === \"error\" ? { idx, result } : null))\n .filter(Boolean);\n if (errs.length > 0) {\n throw new Error(\n `Turso statement error(s): ${errs\n .map(({ idx, result }) => `[${idx}] ${result.error}`)\n .join(\"; \")}`,\n );\n }\n }\n\n return data;\n}\n\n// ---------------------------------------------------------------------------\n// Fetch, Clean, and Handle Malformed/Truncated JSON Gracefully\n// ---------------------------------------------------------------------------\nasync function fetchAndParseEvents() {\n const res = await fetch(UPSTASH_URL, {\n headers: { Authorization: UPSTASH_AUTH },\n });\n if (!res.ok) {\n throw new Error(`Upstash HTTP ${res.status}: ${await res.text()}`);\n }\n\n const body = await res.json();\n let rawData = body.result;\n\n let parsed;\n try {\n parsed = typeof rawData === \"string\" ? JSON.parse(rawData) : rawData;\n } catch (e) {\n console.warn(\n \"JSON parse failed (possibly truncated). Attempting safe recovery...\",\n );\n try {\n const lastValidIdx = rawData.lastIndexOf(\"}\");\n if (lastValidIdx > 0) {\n const fixedJson = rawData.substring(0, lastValidIdx + 1) + \"]}\";\n parsed = JSON.parse(fixedJson);\n } else {\n throw e;\n }\n } catch (innerErr) {\n throw new Error(\n `Failed to parse or repair Upstash payload: ${innerErr.message}`,\n );\n }\n }\n\n const features = Array.isArray(parsed?.features) ? parsed.features : [];\n\n return features.map((f) => {\n const p = f?.properties || {};\n const coords = Array.isArray(f?.geometry?.coordinates)\n ? f.geometry.coordinates\n : [null, null];\n\n const lng = coords[0] != null ? Number(coords[0]) : null;\n const lat = coords[1] != null ? Number(coords[1]) : null;\n\n return {\n id: hashId(p.title, p.date, p.location),\n title: clean(p.title),\n crime: clean(p.crime),\n date: clean(p.date),\n location: clean(p.location),\n paragraph: clean(p.paragraph),\n thumbnail: p.thumbnail || null,\n latitude: !isNaN(lat) ? lat : null,\n longitude: !isNaN(lng) ? lng : null,\n channel: p.channel ? JSON.stringify(p.channel) : null,\n reference: p.references ? JSON.stringify(p.references) : null,\n };\n });\n}\n\n// ---------------------------------------------------------------------------\n// Save records to Turso\n// ---------------------------------------------------------------------------\nasync function saveToDb(records) {\n // 1. Create table using 'reference' column name\n console.log(\"Ensuring table 'commonevents' exists...\");\n await tursoPipeline([\n {\n sql: `CREATE TABLE IF NOT EXISTS commonevents (\n id TEXT PRIMARY KEY,\n title TEXT,\n crime TEXT,\n date TEXT,\n location TEXT,\n paragraph TEXT,\n thumbnail TEXT,\n latitude REAL,\n longitude REAL,\n channel TEXT,\n reference TEXT,\n synced_at TEXT\n )`,\n },\n ]);\n\n // 2. Prepare insert statements using 'reference'\n const now = new Date().toISOString();\n const insertStmts = records.map((r) => ({\n sql: `INSERT OR REPLACE INTO commonevents \n (id, title, crime, date, location, paragraph, thumbnail, latitude, longitude, channel, reference, synced_at) \n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n typedArg(r.id),\n typedArg(r.title),\n typedArg(r.crime),\n typedArg(r.date),\n typedArg(r.location),\n typedArg(r.paragraph),\n typedArg(r.thumbnail),\n typedFloat(r.latitude),\n typedFloat(r.longitude),\n typedArg(r.channel),\n typedArg(r.reference),\n typedArg(now),\n ],\n }));\n\n // 3. Execute inserts in batches of 10\n console.log(`Inserting ${insertStmts.length} records into Turso...`);\n const BATCH_SIZE = 10;\n for (let i = 0; i < insertStmts.length; i += BATCH_SIZE) {\n const batch = insertStmts.slice(i, i + BATCH_SIZE);\n await tursoPipeline(batch);\n }\n}\n\n// ---------------------------------------------------------------------------\n// Empty the Upstash mapEvent features after a successful Turso sync.\n// Sets features to [] (keeps the rest of the document intact).\n// ---------------------------------------------------------------------------\nasync function clearUpstash() {\n const res = await fetch(`${UPSTASH_BASE}/json.set/mapEvent/$.features`, {\n method: \"POST\",\n headers: {\n Authorization: UPSTASH_AUTH,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify([]),\n });\n if (!res.ok) {\n throw new Error(`Upstash SET HTTP ${res.status}: ${await res.text()}`);\n }\n return res.json();\n}\n\n// ---------------------------------------------------------------------------\n// Main Entry Point\n// ---------------------------------------------------------------------------\nexport default async function handler() {\n const records = await fetchAndParseEvents();\n console.log(`Successfully parsed ${records.length} event(s) from Upstash.`);\n\n await saveToDb(records);\n console.log(\n `Successfully synced ${records.length} record(s) to Turso (table: commonevents).`,\n );\n\n // Only clear the Upstash source if the Turso write succeeded above.\n await clearUpstash();\n console.log(\"Cleared mapEvent from Upstash.\");\n\n return { success: true, total: records.length };\n}",[157],"28dcaac6-769b-40a0-8019-7ba853c08b21","scripts","ca191255-a940-4a18-832b-d6f7739bfbc1",174,["D",1789073099638],1,"directory",[165],"pulls","branches","forks","envVars",{"_179":181,"_182":61},"connections",{"_179":61,"_180":61},"count","unfilledCount",6,"emptyCount",{"_43":164,"_32":165,"_30":166,"_133":-5,"_41":42,"_40":167,"_134":143,"_38":220,"_13":169,"_33":170,"_35":-5,"_24":25,"_137":-5,"_191":221,"_27":222},{"_43":212,"_32":213,"_30":214,"_133":166,"_41":42,"_40":215,"_134":143,"_38":216,"_13":61,"_33":208,"_35":217,"_24":25,"_137":-5,"_191":218,"_27":219},{"_43":203,"_32":204,"_30":205,"_133":166,"_41":42,"_40":206,"_134":143,"_38":207,"_13":61,"_33":208,"_35":209,"_24":25,"_137":-5,"_191":210,"_27":211},{"_43":156,"_32":157,"_30":158,"_133":-5,"_41":42,"_40":159,"_134":143,"_38":200,"_13":161,"_33":34,"_35":162,"_24":25,"_137":-5,"_191":201,"_27":202},{"_43":148,"_32":149,"_30":150,"_133":-5,"_41":42,"_40":151,"_134":143,"_38":197,"_13":153,"_33":34,"_35":154,"_24":25,"_137":-5,"_191":198,"_27":199},{"_43":139,"_32":140,"_30":141,"_133":-5,"_41":42,"_40":142,"_134":143,"_38":194,"_13":145,"_33":34,"_35":146,"_24":25,"_137":-5,"_191":195,"_27":196},{"_43":132,"_32":28,"_30":31,"_133":-5,"_41":42,"_40":14,"_134":25,"_38":190,"_13":136,"_33":34,"_35":37,"_24":25,"_137":-5,"_191":192,"_27":193},["D",1789098987119],"children",[],[28],["D",1788982196432],[],[140],["D",1788942414030],[],[149],["D",1788723482797],[],[157],"a8ce5f8a-de5f-4003-bc1d-bd8ba884f0ea","zenrows-status.js","97a8e3a2-ac8a-11f1-9bec-1607ee4eb77e",168,["D",1788984667491],"script","// Prints your ZenRows plan status: credits used, period end, concurrency limit.\n// Run this when warEvent.js logs AUTH004 (credits exhausted) or AUTH006\n// (concurrency limit) to see where you stand. Does not consume credits.\nconst key = Deno.env.get(\"ZENROWS_API_KEY\");\nif (!key) throw new Error(\"Missing ZENROWS_API_KEY env var\");\n\nconst res = await fetch(\"https://api.zenrows.com/v1/subscriptions/self/details\", {\n headers: { \"X-API-Key\": key },\n});\nconst body = await res.text();\nconsole.log(`HTTP ${res.status}`);\ntry {\n console.log(JSON.stringify(JSON.parse(body), null, 2));\n} catch {\n console.log(body);\n}\n",[],[165,204],"ea265c60-9e86-4983-b94e-704638316918","translate.js","e5e91c2e-a9c1-11f1-bcf6-1607ee4eb77e",78,["D",1788678567608],"// ---------------------------------------------------------------------------\n// Shared EN -> ID translator used by every event scraper in this val.\n//\n// Primary: NVIDIA Riva \"nvidia/riva-translate-4b-instruct-v2\"\n// (OpenAI-compatible chat completions at integrate.api.nvidia.com).\n// Fallback: free MyMemory API, then the original English text as a last resort\n// so a transient API issue never fails or corrupts a run.\n//\n// The Riva instruct model must have the target language named in the *user*\n// message (otherwise it can default to another language, e.g. French).\n//\n// Requires the NVIDIA_API_KEY env var (set once on the val, shared by all\n// files). If it is missing, translation falls back to MyMemory/English.\n// ---------------------------------------------------------------------------\nconst NVIDIA_API_KEY = Deno.env.get(\"NVIDIA_API_KEY\");\nconst NVIDIA_BASE = \"https://integrate.api.nvidia.com/v1/chat/completions\";\nconst NVIDIA_MODEL = \"nvidia/riva-translate-4b-instruct-v2\";\n\nlet lastTranslate = 0; // MyMemory fallback throttle (~2 req/s)\n\n// Riva has no 500-char cap, but chunk large text so it stays well within the\n// model's input/output token limits (long EASA descriptions).\nconst CHUNK_MAX = 1600;\n\nfunction splitChunks(text) {\n const chunks = [];\n let rest = text.trim();\n while (rest.length > CHUNK_MAX) {\n let cut = rest.lastIndexOf(\". \", CHUNK_MAX);\n if (cut < CHUNK_MAX * 0.5) cut = rest.lastIndexOf(\" \", CHUNK_MAX);\n if (cut < CHUNK_MAX * 0.5) cut = CHUNK_MAX; // no break point -> hard cut\n chunks.push(rest.slice(0, cut + (rest[cut] === \" \" ? 0 : 1)));\n rest = rest.slice(cut + 1).trim();\n }\n if (rest) chunks.push(rest);\n return chunks;\n}\n\n// Primary: NVIDIA Riva instruct translation model.\nasync function rivaOnce(text) {\n const key = NVIDIA_API_KEY;\n if (!key) throw new Error(\"NVIDIA_API_KEY not set\");\n const res = await fetch(NVIDIA_BASE, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${key}`,\n \"Content-Type\": \"application/json\",\n Accept: \"application/json\",\n },\n body: JSON.stringify({\n model: NVIDIA_MODEL,\n messages: [\n { role: \"system\", content: \"You are a translation engine.\" },\n {\n role: \"user\",\n content: `Translate the following from English to Indonesian:\\n${text}`,\n },\n ],\n max_tokens: 2048,\n temperature: 0,\n }),\n });\n if (!res.ok) throw new Error(`Riva HTTP ${res.status}`);\n const data = await res.json();\n const out = data?.choices?.[0]?.message?.content?.trim();\n if (!out || /MYMEMORY WARNING|QUERY LENGTH/i.test(out)) {\n throw new Error(\"bad Riva output\");\n }\n return out;\n}\n\n// Fallback: MyMemory. Its free daily quota can be exhausted, in which case it\n// returns a warning instead of a translation — we treat that as a failure and\n// drop through to the original English rather than persisting the warning.\nasync function mymemoryOnce(text) {\n const elapsed = Date.now() - lastTranslate;\n if (elapsed < 500) {\n await new Promise((r) => setTimeout(r, 500 - elapsed));\n }\n lastTranslate = Date.now();\n\n const url =\n \"https://api.mymemory.translated.net/get?q=\" +\n encodeURIComponent(text) +\n \"&langpair=en|id\";\n const res = await fetch(url);\n if (!res.ok) throw new Error(`MyMemory HTTP ${res.status}`);\n const data = await res.json();\n const translated = data?.responseData?.translatedText;\n if (\n !translated ||\n /MYMEMORY WARNING|QUERY LENGTH LIMIT EXCEEDED|NO QUERY|INVALID PAIR/i.test(\n translated,\n )\n ) {\n return text;\n }\n return translated;\n}\n\nasync function translateOnce(text) {\n try {\n return await rivaOnce(text);\n } catch (err) {\n console.error(\"Riva failed, falling back to MyMemory:\", err.message);\n try {\n return await mymemoryOnce(text);\n } catch (err2) {\n console.error(\"MyMemory fallback failed, keeping original:\", err2.message);\n return text;\n }\n }\n}\n\nexport async function translateToId(text) {\n if (!text) return text;\n const chunks = splitChunks(text);\n if (chunks.length === 1) return await translateOnce(chunks[0]);\n const parts = [];\n for (const c of chunks) parts.push(await translateOnce(c));\n return parts.join(\" \");\n}\n",[],[165,213],["D",1789073099638],[184,185],[165],["D",1788316219773],["D",1789458074143],{"_98":97,"_105":101},{"_98":97,"_99":97,"_100":101,"_18":55,"_102":103,"_104":25},{"_13":87,"_32":88,"_43":42,"_89":44,"_90":-5,"_91":-5,"_92":-5,"_93":-5,"_94":-5,"_95":45,"_96":97},"ctx",{"_80":-5,"_124":261,"_262":263,"_264":265,"_266":25,"_267":25,"_268":25,"_269":143,"_270":-5,"_271":-7,"_272":-7,"_273":61,"_274":25,"_275":25,"_276":-5,"_277":25,"_278":25,"_279":25,"_280":-5},"ENV",{"_232":233,"_234":235,"_236":88,"_237":238,"_239":240,"_241":242,"_243":244,"_245":246,"_247":248,"_249":250,"_251":252,"_253":254,"_255":256,"_257":258,"_259":260},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v202