[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_232":233,"_234":235},"routes/_app.x.$handle.$name",{"_99":98,"_32":45,"_107":89,"_27":54,"_13":14,"_108":-5,"_11":109,"_110":111,"_112":113,"_114":115,"_116":117,"_20":-5,"_118":-5,"_119":-5,"_120":49,"_121":122,"_123":49,"_124":-5,"_125":-5,"_126":127},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_43":44,"_32":45,"_38":46,"_47":-5,"_48":49,"_50":51,"_52":51,"_53":54,"_55":-5,"_18":56,"_57":-5,"_58":49,"_59":-5,"_60":-5,"_61":62,"_63":64,"_65":66,"_67":68,"_69":70,"_71":-5,"_72":73,"_74":73,"_75":76,"_77":76,"_78":76,"_79":73,"_80":73,"_81":82,"_83":84,"_85":86,"_87":49},"version",216,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":33,"_34":35,"_36":37,"_38":39,"_40":14,"_41":42,"_24":25},"previous",{"_30":31,"_32":33,"_34":35,"_36":37},"deleted",true,"deletedHere","path","scripts/testFetch.js","latestPath","fileId","61db7516-ae77-11f1-8b72-1607ee4eb77e","name","testFetch.js","fileType","script","content","// config.js or index.js\nasync function fetchProtectedData() {\n const targetUrl = \"https://liveuamap.com/\";\n\n // 1. Paste your exact cookie string from Thorium here\n const rawCookies =\n `_ga=GA1.1.1500622894.1787906667; __cmpconsentx101583=CQppEKgQppEKgAfYvBENCuFsAP_gAEPgAAigLTtR_G_fbWlj-TL3abskeYxf99hr5sQxBgLIgk4BzDrW5JwG12ESJEyShiAKmRIAqnBBIQFtGAjURUABIIAVJSDsYESUgRNIJuBAgFMRQwNQCEhvmYthGQCRwkp3cwZgmRsM6Vru2NxAw5gjHUS1IqQlUJgdCYMoCQmsZBKS87IU9-x-v4vw9MboEm0eSVn9tQtp4xcIABAAJAAgAAAgBAAABAAABICAAAAAAAAAAgAAAAAAAAAIK0gAmGhUQAlgQCBAICEECABQVhABQIAgAACAggIATBgQ5AwAVWESAEACAAAIAQAAggABAAAJAAhEAEABAIAQIAAoAAQAAAAIAGBAABABQgAQAAgOgYpAAQCBQAJEYQBggQgAIBAC2FAAQBAgDACkAKAAQIgYCAAAAAAoAAAA8LAQEkBKhIIAsILoAEAAAACMEQBBIQYAAAAABAAgAAAgBAAABAAAAAAAAAAAAAAAAgAA.ILTtR_G_fbWlj-TL3abskeYxf99hr5sQxBgLIgk4BzDrW5JwG12ESJEyShiAKmRIAqnBBIQFtGAjURUABIIAVJSDsYESUgRNIJuBAgFMRQwNQCEhvmYthGQCRwkp3cwZgmRsM6Vru2NxAw5gjHUS1IqQlUJgdCYMoCQmsZBKS87IU9-x-v4vw9MboEm0eSVn9tQtp4xcIABAAJAAgAAAgBAAABAAABICAAAAAAAAAAgAAAAAAAAAIA; __cmpcccx101583=aCQprOhNAABp9rnjD3mZzYaYxmXPXvGrDzYMdPGMOPHDdeMsGGmd41poYwYaYMGmWaNNaZnbDJ14xo1qxZrTA48YGLQ0MYmMMsMDTI1YweGywGrVrTI1jWQxq1MGWjMoamTTQxXbLxmAPDznwTzk9Hg-eCNTWlgxr3o1YYZWg0OHjQxaa1NLBjVaZqYNDValkc8arQ1lneqMCaExGEmi1iIYxMAZLkPvWUwrQwmJad5NRhZDBhguS2JZEwtMrUYLQYgwWADJBEXg; _pubcid=7e11df84-c3d5-4795-a592-b200035e5f46; _sharedID=46cf4fb4-16eb-46a3-bbbd-6ec8c0a92815; _cc_id=47cc492d091c68d70cb6b4f2a3b42c14; pbjs-unifiedid=%7B%22TDID%22%3A%2259b7994b-489f-4e10-81fb-5a1c37d1242e%22%2C%22TDID_LOOKUP%22%3A%22TRUE%22%2C%22TDID_CREATED_AT%22%3A%222026-07-28T08%3A44%3A43%22%7D; pbjs-unifiedid_cst=15sj7A%3D%3D; usprivacy=1---; hb_insticator_uid=9e4cd1f4-7a14-408c-be23-1fc19b211a01; _cq_duid=1.1788132305.iJhiC9DqltPowVRr; _cq_session=1.1788132305219.YB34YYjPcMNLtlaV.1788132305218; cf_clearance=eI2lbGYQqGBcBORXwXEgFvOlrXgsTHePj3bAKKuv9CE-1788937193-1.2.1.1-aO1HLCzt5cN3uqJi6pVveT0.KiDXFALzRprUpIEossfmgKmy277Ai50igEqELXkhAfugqpcXmczXxEHKkHHHV7BNAuOf0w7ariBA8nz7WQbs5ttYQ9ESc3j88gCqJHn5_53YCJQQZSz7ZTaEdqm5YU10I_NW_SYT.ASn81adZgEahh3LG4iNyoUXMa_242xJ5uqFfjZ.Tab8yxLaTnskitW1LWeCgyEFDjYesPi3XS9pxX.lukogYfDrsmQYc5CnaSkr3CnFRVR6hOwIBCkTx3OdyqRGumh2ChSjWWx9DZ3GnZqW.nsPOSEShDIVOw6gmyycFRKHseyx_Pnx7ACk5X5xyXmaCu6rYrr04jd6kZG.tuvWa8wgAXtcB3y0RrIKho8JrdEJQR.gQHGsF_vyhN6IYoiRSQIvKBlCkQUqd8iMPyJUN6snapLz97c9cP426yP2bGtfq8OHnQEXFYcx6dP49KBL3HLuqEVG0FEf.YNeg5qnqS4S34F.UmDSS0UKbXOvS5NdTHbVL5faJj_gfg; panoramaId_expiry=1789547495672; panoramaId=753e7c03cdcd1ddeae15e584d439185ca02cbd4392c360d862d5f90c70ff68e9; panoramaIdType=panoDevice; connectId={\"ttl\":86400000,\"lastUsed\":1789072799133,\"lastSynced\":1789072799133}; __qca=I0-2027128112-1789192031579; FCCDCF=%5Bnull%2Cnull%2Cnull%2Cnull%2Cnull%2Cnull%2C%5B%5B32%2C%22%5B%5C%222bb512bc-dcc1-4fcf-9950-92a88409f448%5C%22%2C%5B1787906791%2C740000000%5D%5D%22%5D%5D%5D; cto_bidid=L_yjeV9kVTFsT050JTJCZXc5MXF6eGYlMkZrRVVVS3VkdmdKQXd6U2ZBbiUyQjdMN1hrQ0sxU1JmVWpMVnk3Qk5vMWJMOWR0ZTRLRUNPRVVvc1E4NWVNWUg0dUV4RDY0NktmYlhMbyUyRnNkck1wa3d5N3NHJTJCUVklM0Q; _awl=2.1789196023.5-d1f77eb05ce7a936844f0eda072fbeb0-6763652d617369612d6561737431-0; FCNEC=%5B%5B%22AKsRol-nZaOdPyTNM2BvNbN8y8TSNjKXTDGVErsmje1eRFaPnHmpwrqjc6lPmRsTjlhJp1Cbzb3sEdqsArNNwYPNXs45718-lk7VoxfbpnEJwbCOyFiP7B4RiJC2Xr-k0vjVrhMFq1Ywdcs8BzGFSqiFhLejZcScLA%3D%3D%22%5D%5D; __gads=ID=418147eeb9dfa99f:T=1787906683:RT=1789196304:S=ALNI_MZ6THTIjlQDK3SsTGCAvKJPO-pMXg; __gpi=UID=0000154dee31726b:T=1787906683:RT=1789196304:S=ALNI_Ma9d2qGoc4NUyrMn9O4-bAGV9G0gA; __eoi=ID=ecc24654f734adb4:T=1787906683:RT=1789196304:S=AA-AfjajM6sCZgypw4UjCTgLtDrH; cto_bundle=3_a2qV9pVFdHNFJWJTJGQUp6S1BHQTFaSFJDQ1NJbDJnaTBkNWhIellIOTQzYWJKZzZlMndXMGxLVldLa0xuN2lCSiUyQlVrUEZUMHZJTkJ4bnRMWFAxN0VSV2wzN2d2aTFuYXlJNG5Jejk4VXY5ekN0U0xjeXlOcENaMmQyJTJCT1NPTXNqRTdMQmpqJTJGOTRYemdZVDBRU1ZKa0UzaVA2USUzRCUzRA; _ga_3RF9DY81PT=GS2.1.s1789192025$o32$g1$t1789196317$j26$l0$h0; _sharedID_cst=kSylLAssaw%3D%3D`;\n\n // 2. Paste your exact Thorium User-Agent string here\n const userAgent =\n `Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36`;\n\n try {\n const response = await fetch(targetUrl, {\n method: \"GET\",\n headers: {\n \"Cookie\": rawCookies,\n \"User-Agent\": userAgent,\n \"Accept\": \"application/json, text/plain, */*\",\n \"Accept-Language\": \"en-US,en;q=0.9\",\n },\n });\n\n // Check if Cloudflare blocked it (will return 403 or a 503 challenge page)\n if (!response.ok) {\n throw new Error(\n `HTTP Error: ${response.status} - ${response.statusText}`,\n );\n }\n\n const data = await response.json(); // swap to response.text() if it's not JSON\n console.log(\"Successfully fetched data:\", data);\n } catch (error) {\n console.error(\"Fetch failed:\", error.message);\n console.log(\"Tip: Check if your cf_clearance cookie expired in Thorium!\");\n }\n}\n\nfetchProtectedData();","createdAt",["D",1789196681800],"branchVersion","branchId","1dc3dd44-cc65-414f-8e4a-735d1ac2a5b5","id","feed5b42-8d51-40be-9982-13fc813b4b7c","warData",["D",1788316219773],"deletedAt","legacyEnvVars",false,"privacy","public","httpPrivacy","description","","imageUrl","0762e08b-0f1b-495a-871c-4fba2da9304a","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","019f5a38-1aa2-74ee-aaf5-8eac040acbf4","tursoName","feed5b428d5140be998213fc813b4b7c","tursoHostname","feed5b428d5140be998213fc813b4b7c-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789458074143],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_99":98,"_106":102},"org",{"_99":98,"_100":98,"_101":102,"_18":56,"_103":104,"_105":49},"branch",{"_13":88,"_32":89,"_43":42,"_90":44,"_91":-5,"_92":-5,"_93":-5,"_94":-5,"_95":-5,"_96":45,"_97":98},"isOwned",237,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","mustedpual","handle","displayName","avatar","https://avatars.githubusercontent.com/u/157880878?v=4","tier","free","canRestrictVals","profileImageUrl","branchName","adoptOrgId",{"_43":44,"_32":45,"_38":227,"_47":-5,"_48":49,"_50":51,"_52":51,"_53":54,"_55":-5,"_18":56,"_57":-5,"_58":49,"_59":-5,"_60":-5,"_61":62,"_63":64,"_65":66,"_67":68,"_69":228,"_71":-5,"_72":73,"_74":73,"_75":76,"_77":76,"_78":76,"_79":73,"_80":73,"_81":229,"_83":230,"_85":231,"_87":49},"tree",[188,191,192,193,194],"files",[188,189,190,191,192,193,194],"counts",{"_177":62,"_178":174,"_179":62,"_180":181,"_182":183},"contents",[128,129,130,131,132],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_43":169,"_32":170,"_30":171,"_136":-5,"_41":42,"_40":172,"_138":25,"_38":173,"_13":174,"_34":175,"_36":-5,"_24":49,"_143":-5,"_27":176},{"_43":161,"_32":162,"_30":163,"_136":-5,"_41":42,"_40":164,"_138":25,"_38":165,"_13":166,"_34":141,"_36":167,"_24":49,"_143":-5,"_27":168},{"_43":153,"_32":154,"_30":155,"_136":-5,"_41":42,"_40":156,"_138":25,"_38":157,"_13":158,"_34":141,"_36":159,"_24":49,"_143":-5,"_27":160},{"_43":145,"_32":146,"_30":147,"_136":-5,"_41":42,"_40":148,"_138":25,"_38":149,"_13":150,"_34":141,"_36":151,"_24":49,"_143":-5,"_27":152},{"_43":133,"_32":134,"_30":135,"_136":-5,"_41":42,"_40":137,"_138":49,"_38":139,"_13":140,"_34":141,"_36":142,"_24":49,"_143":-5,"_27":144},"1ae58406-3e1e-402c-b5b1-922140e20c0d","warEvent.js","d75184f6-a8f1-11f1-abed-1607ee4eb77e","parentId",209,"latest",["D",1789100841683],76,"interval","import { DOMParser } from \"npm:linkedom\";\nimport puppeteer from \"npm:puppeteer-core\";\nimport { translateToId } from \"./scripts/translate.js\";\n\n// ---------------------------------------------------------------------------\n// Configuration & Credentials\n// ---------------------------------------------------------------------------\nconst TURSO_URL = Deno.env.get(\"TURSO_URL\");\nconst TURSO_AUTH_TOKEN = Deno.env.get(\"TURSO_AUTH_TOKEN\");\nconst BROWSERLESS_TOKEN = Deno.env.get(\"BROWSERLESS_TOKEN\");\n\n// ---------------------------------------------------------------------------\n// Architecture (all-Browserless, no ZenRows)\n//\n// PHASE A — POST /unblock once on liveuamap.com.\n// • Solves Cloudflare in a real browser session.\n// • Returns main-page HTML (→ extract sublinks) AND a live\n// browserWSEndpoint (→ keep the cleared session alive).\n//\n// PHASE B — Turso lookup: which sublinks are not yet stored.\n//\n// PHASE C — Connect Puppeteer to the browserWSEndpoint from Phase A.\n// Navigate every pending sublink inside the SAME browser session.\n// Cloudflare sees the same fingerprint + IP that already passed,\n// so no new challenge is triggered. Parse → translate → Turso.\n// ---------------------------------------------------------------------------\n\nconst TARGET_URL = \"https://liveuamap.com\";\nconst PROXY_NETWORK = \"residential\"; // datacenter IPs get pre-flagged\nconst PROXY_COUNTRY = \"de\";\n\nconst RUN_BUDGET_MS = 55_000; // total cron budget\nconst HEADROOM_MS = 10_000; // stop scraping when this remains\nconst UNBLOCK_TIMEOUT_MS = 120_000; // /unblock may take 30–60s to solve CF\nconst UNBLOCK_TTL_MS = 120_000; // how long the WS session stays open\nconst PUPPETEER_NAV_TIMEOUT = 15_000; // per-sublink navigation timeout\nconst PUPPETEER_WAIT_TIMEOUT = 10_000; // waitForSelector timeout\n\nconst START = Date.now();\nconst elapsed = () => Date.now() - START;\nconst timeLeft = () => RUN_BUDGET_MS - elapsed();\n\n// ---------------------------------------------------------------------------\n// Turso helpers (unchanged)\n// ---------------------------------------------------------------------------\nfunction typedArg(value) {\n if (value === null || value === undefined) {\n return { type: \"null\", value: null };\n }\n if (typeof value === \"number\") {\n return { type: Number.isInteger(value) ? \"integer\" : \"float\", value };\n }\n return { type: \"text\", value: String(value) };\n}\n\nasync function tursoPipeline(stmts) {\n const base = TURSO_URL.replace(/\\/+$/, \"\");\n const requests = [\n ...stmts.map(({ sql, args }) => ({\n type: \"execute\",\n stmt: { sql, args: (args || []).map(typedArg) },\n })),\n { type: \"close\" },\n ];\n\n const res = await fetch(`${base}/v2/pipeline`, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${TURSO_AUTH_TOKEN}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({ requests }),\n });\n\n if (!res.ok) {\n throw new Error(`Turso HTTP ${res.status}: ${await res.text()}`);\n }\n return res.json();\n}\n\nasync function ensureTable() {\n await tursoPipeline([\n {\n sql: `CREATE TABLE IF NOT EXISTS warfare_events (\n id TEXT PRIMARY KEY,\n title TEXT,\n status TEXT,\n location TEXT,\n date TEXT,\n longitude REAL,\n latitude REAL,\n athumbnail TEXT,\n reference TEXT\n )`,\n },\n ]);\n}\n\nasync function pendingSublinks(sublinks) {\n if (sublinks.length === 0) return [];\n const placeholders = sublinks.map(() => \"?\").join(\",\");\n const out = await tursoPipeline([\n {\n sql: `SELECT id FROM warfare_events WHERE id IN (${placeholders})`,\n args: sublinks,\n },\n ]);\n const rows = out?.results?.[0]?.response?.result?.rows || [];\n const known = new Set(rows.map((r) => r[0]?.value));\n return sublinks.filter((l) => !known.has(l));\n}\n\n// ---------------------------------------------------------------------------\n// Date extraction (unchanged)\n// ---------------------------------------------------------------------------\nconst MONTHS = {\n january: \"01\",\n february: \"02\",\n march: \"03\",\n april: \"04\",\n may: \"05\",\n june: \"06\",\n july: \"07\",\n august: \"08\",\n september: \"09\",\n october: \"10\",\n november: \"11\",\n december: \"12\",\n};\n\nfunction extractLiveUAMapDate(url) {\n const match = url.match(/\\/(\\d{4})\\/(\\d{1,2})-([a-z]+)-\\d+-/i);\n if (!match) return null;\n const [, year, day, monthName] = match;\n const mm = MONTHS[monthName.toLowerCase()];\n return mm ? `${year}-${mm}-${String(day).padStart(2, \"0\")}` : null;\n}\n\n// ---------------------------------------------------------------------------\n// Parse ONE sublink's HTML -> GeoJSON Feature (unchanged)\n// ---------------------------------------------------------------------------\nconst dom = new DOMParser();\n\nfunction parseFeatureHtml(sublink, subHtml) {\n const subdoc = dom.parseFromString(subHtml, \"text/html\");\n\n let lat = 48.8829;\n let lng = 31.181;\n\n const eventScriptMatch = subHtml.match(\n /document\\.addEventListener\\(['\"]DOMContentLoaded['\"][\\s\\S]*?\\);/,\n );\n if (eventScriptMatch) {\n const latMatch = eventScriptMatch[0].match(/lat\\s*=\\s*([0-9.-]+)/);\n const lngMatch = eventScriptMatch[0].match(/lng\\s*=\\s*([0-9.-]+)/);\n if (latMatch) lat = parseFloat(latMatch[1]);\n if (lngMatch) lng = parseFloat(lngMatch[1]);\n }\n\n const infozone = subdoc.querySelector(\".popup-text\");\n const status = infozone?.querySelector(\".bgma\")?.getAttribute(\"data-src\") ||\n null;\n const title = infozone?.querySelector(\"h2\")?.textContent?.trim() || null;\n const locationText =\n infozone?.querySelector(\".tagas strong\")?.textContent?.trim() || null;\n const thumbnail =\n infozone?.querySelector(\".popup_video + * img\")?.getAttribute(\"src\") ||\n null;\n const reference =\n subdoc.querySelector(\".source-link\")?.getAttribute(\"href\") || null;\n\n return {\n type: \"Feature\",\n geometry: { type: \"Point\", coordinates: [lng, lat] },\n properties: {\n references: [sublink, reference].filter(Boolean),\n title,\n status,\n location: locationText,\n thumbnail,\n date: extractLiveUAMapDate(sublink),\n },\n };\n}\n\n// ---------------------------------------------------------------------------\n// Translate + upsert ONE feature into Turso (unchanged)\n// ---------------------------------------------------------------------------\nasync function saveFeature(feature) {\n const { properties, geometry } = feature;\n const [lng, lat] = geometry.coordinates;\n const eventId = properties.references[0] || crypto.randomUUID();\n\n let translatedTitle = properties.title;\n if (translatedTitle) {\n try {\n translatedTitle = await translateToId(properties.title);\n } catch (err) {\n console.error(\"Translation failed for title:\", properties.title, err);\n }\n }\n\n await tursoPipeline([\n {\n sql: `INSERT OR REPLACE INTO warfare_events\n (id, title, status, location, date, longitude, latitude, athumbnail, reference)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n eventId,\n translatedTitle,\n properties.status,\n properties.location,\n properties.date,\n lng,\n lat,\n properties.thumbnail,\n properties.references[1] || properties.references[0] || null,\n ],\n },\n ]);\n}\n\n// ---------------------------------------------------------------------------\n// Helpers\n// ---------------------------------------------------------------------------\nfunction cleanDataLink(raw) {\n if (!raw) return null;\n let v = raw.trim();\n v = v.replace(/\\\\(.)/g, \"$1\");\n v = v.replace(/^[\"']+|[\"']+$/g, \"\").trim();\n return v || null;\n}\n\n// ---------------------------------------------------------------------------\n// PHASE A — /unblock: solve Cloudflare once, harvest sublinks + a live\n// browser session. Retries once on HTTP 408 (Cloudflare solve took too long).\n// ---------------------------------------------------------------------------\nasync function initSessionAndExtractLinks() {\n const queryParams = new URLSearchParams({\n token: BROWSERLESS_TOKEN,\n proxy: PROXY_NETWORK,\n proxyCountry: PROXY_COUNTRY,\n timeout: String(UNBLOCK_TIMEOUT_MS),\n }).toString();\n\n const unblockURL =\n `https://production-sfo.browserless.io/unblock?${queryParams}`;\n\n const body = {\n url: TARGET_URL,\n content: true,\n browserWSEndpoint: true,\n cookies: false,\n screenshot: false,\n ttl: UNBLOCK_TTL_MS,\n waitForSelector: { // ← ADD THIS\n selector: \"[data-link]\",\n timeout: 15000,\n },\n };\n\n console.log(\"Requesting main page via Browserless /unblock…\");\n\n let res;\n for (let attempt = 1; attempt <= 2; attempt++) {\n try {\n res = await fetch(unblockURL, {\n method: \"POST\",\n headers: { \"Content-Type\": \"application/json\" },\n body: JSON.stringify(body),\n signal: AbortSignal.timeout(UNBLOCK_TIMEOUT_MS + 10_000),\n });\n } catch (err) {\n console.error(`/unblock request failed (${err.message}); skipping run.`);\n return null;\n }\n\n // 408 = Cloudflare challenge didn't finish within the timeout.\n if (res.status === 408 && attempt === 1) {\n console.warn(\"/unblock returned 408; retrying once…\");\n continue;\n }\n break;\n }\n\n if (!res.ok) {\n console.error(`/unblock HTTP ${res.status}: ${await res.text()}`);\n return null;\n }\n\n const payload = await res.json();\n const html = payload.content ?? \"\";\n const browserWSEndpoint = payload.browserWSEndpoint ?? null;\n\n console.log(\n `Unblock response: html length=${html.length} ` +\n `browserWSEndpoint=${browserWSEndpoint ? \"yes\" : \"no\"} ` +\n `(${elapsed()}ms elapsed)`,\n );\n\n if (!html) {\n console.error(\"/unblock returned no content; skipping this run.\");\n return null;\n }\n if (!browserWSEndpoint) {\n console.error(\n \"/unblock returned no browserWSEndpoint; cannot continue to Phase C.\",\n );\n return null;\n }\n\n const doc = dom.parseFromString(html, \"text/html\");\n const sublinks = [\n ...new Set(\n [...doc.querySelectorAll(\"[data-link]\")]\n .map((e) => cleanDataLink(e.getAttribute(\"data-link\")))\n .filter(Boolean),\n ),\n ];\n\n if (sublinks.length === 0) {\n const pageTitle = doc.querySelector(\"title\")?.textContent?.trim() || null;\n console.warn(\n `No [data-link] matches. page title=\"${pageTitle}\". ` +\n `If this looks like a Cloudflare or error page, the unblock ` +\n `did not clear the challenge — check the Browserless dashboard.`,\n );\n }\n\n console.log(\n `Extracted ${sublinks.length} sublinks from the unblocked main page ` +\n `(${elapsed()}ms elapsed).`,\n );\n\n return { sublinks, browserWSEndpoint, queryParams };\n}\n\n// ---------------------------------------------------------------------------\n// PHASE C — Connect Puppeteer to the cleared session and scrape every\n// pending sublink inside it. No new Cloudflare challenge should appear.\n// ---------------------------------------------------------------------------\nasync function scrapeSublinksWithPuppeteer(\n pending,\n browserWSEndpoint,\n queryParams,\n) {\n if (pending.length === 0) return { saved: 0, skipped: 0, failed: 0 };\n\n let browser;\n try {\n browser = await puppeteer.connect({\n browserWSEndpoint: `${browserWSEndpoint}?${queryParams}`,\n });\n } catch (err) {\n console.error(`Puppeteer connect failed: ${err.message}`);\n return { saved: 0, skipped: pending.length, failed: 0 };\n }\n\n const page = await browser.newPage();\n\n let saved = 0;\n let skipped = 0;\n let failed = 0;\n let i = 0;\n\n for (; i < pending.length; i++) {\n if (timeLeft() < HEADROOM_MS) break;\n\n const sublink = pending[i];\n\n try {\n await page.goto(sublink, {\n waitUntil: \"domcontentloaded\",\n timeout: PUPPETEER_NAV_TIMEOUT,\n });\n\n // The sublink detail lives in .popup-text — not [data-link] (that's\n // the main feed selector). Waiting on the wrong selector was one of\n // the causes of the old /smart-scrape timeouts.\n await page.waitForSelector(\".popup-text\", {\n timeout: PUPPETEER_WAIT_TIMEOUT,\n });\n\n const subHtml = await page.content();\n\n const feature = parseFeatureHtml(sublink, subHtml);\n await saveFeature(feature);\n saved++;\n console.log(`Saved [${saved}] ${feature.properties.title ?? sublink}`);\n } catch (err) {\n // Distinguish nav timeout / wait timeout from parse errors.\n if (/timeout|Timeout/i.test(err.message)) {\n skipped++;\n console.warn(`Skipped (timeout) ${sublink}: ${err.message}`);\n } else {\n failed++;\n console.error(`Parse/save failed: ${sublink} — ${err.message}`);\n }\n }\n }\n\n // Close the browser explicitly — this ends the session and stops billing.\n try {\n await browser.close();\n } catch (err) {\n console.warn(`Puppeteer close failed (non-fatal): ${err.message}`);\n }\n\n return { saved, skipped, failed, processed: i };\n}\n\n// ---------------------------------------------------------------------------\n// Main flow\n// ---------------------------------------------------------------------------\nasync function run() {\n await ensureTable();\n\n // --- Phase A: /unblock — solve CF once, get sublinks + live WS session ---\n const pool = await initSessionAndExtractLinks();\n if (!pool || pool.sublinks.length === 0) {\n console.warn(\"No sublinks extracted from /unblock; nothing to do.\");\n return;\n }\n\n // --- Phase B: Turso — which sublinks are already stored? ---\n const pending = await pendingSublinks(pool.sublinks);\n console.log(\n `Main page: ${pool.sublinks.length} sublinks, ` +\n `${pool.sublinks.length - pending.length} already stored, ` +\n `${pending.length} pending (${elapsed()}ms elapsed)`,\n );\n\n // --- Phase C: Puppeteer inside the cleared session ---\n const { saved, skipped, failed, processed } =\n await scrapeSublinksWithPuppeteer(\n pending,\n pool.browserWSEndpoint,\n pool.queryParams,\n );\n\n const deferred = pending.length - (processed ?? 0);\n\n console.log(\n `Done in ${elapsed()}ms: ${saved} saved, ${skipped} skipped (timeout), ` +\n `${failed} parse/save failed, ${deferred} deferred to next run.`,\n );\n}\n\n// ---------------------------------------------------------------------------\n// Cron entrypoint\n// ---------------------------------------------------------------------------\nexport default async function () {\n if (!TURSO_URL || !TURSO_AUTH_TOKEN) {\n throw new Error(\"Missing TURSO_URL / TURSO_AUTH_TOKEN env vars\");\n }\n if (!BROWSERLESS_TOKEN) {\n throw new Error(\"Missing BROWSERLESS_TOKEN env var\");\n }\n\n try {\n await run();\n } catch (error) {\n console.error(\"Pipeline failed:\", error);\n throw error;\n }\n}","label",[134],"4d2ece01-2d75-488d-80d4-8ae766cde047","ukmtoEvent.js","7e654b18-a9b2-11f1-9377-1607ee4eb77e",154,["D",1788982196432],10,"import { translateToId } from \"./scripts/translate.js\";\n// ---------------------------------------------------------------------------\n// UKMTO incident feed -> Indonesian-translated events -> Turso.\n// Fetches https://sccd.royalnavy.mod.uk/api/ukmto/all (array of incidents),\n// compares against existing records in Turso to skip unchanged items,\n// translates new/changed incidentTypeName + otherDetails into Indonesian,\n// then upserts into Turso in safe chunks.\n// ---------------------------------------------------------------------------\nlet fetchQueue = Promise.resolve();\n\nconst Url = \"https://sccd.royalnavy.mod.uk/api/ukmto/all\";\n\nconst HeaderS = {\n \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64)\",\n Accept: \"application/json\",\n};\n\nconst UkmtoData = {\n type: \"ukmto\",\n parent: null, // Untouched master data\n temp: null, // Current filtered view for the map\n};\n\n// ---------------------------------------------------------------------------\n// Turso client via its HTTP API (POST /v2/pipeline). Credentials from env vars.\n// ---------------------------------------------------------------------------\nconst TURSO_URL = Deno.env.get(\"TURSO_URL\");\nconst TURSO_AUTH_TOKEN = Deno.env.get(\"TURSO_AUTH_TOKEN\");\n\nif (!TURSO_URL || !TURSO_AUTH_TOKEN) {\n throw new Error(\"Missing TURSO_URL / TURSO_AUTH_TOKEN env vars\");\n}\nconst TURSO_BASE = TURSO_URL.replace(/\\/+$/, \"\");\n\n// Turso's v2/pipeline arg format: integer/text values are JSON STRINGS;\n// float values are JSON NUMBERS. See https://docs.turso.tech/sdk/http/quickstart\nfunction typedArg(value) {\n if (value === null || value === undefined) {\n return { type: \"null\", value: null };\n }\n if (typeof value === \"number\") {\n return Number.isInteger(value)\n ? { type: \"integer\", value: String(value) }\n : { type: \"float\", value };\n }\n return { type: \"text\", value: String(value) };\n}\n\nasync function tursoPipeline(stmts) {\n const requests = [\n ...stmts.map(({ sql, args }) => ({\n type: \"execute\",\n stmt: { sql, args: (args || []).map(typedArg) },\n })),\n { type: \"close\" },\n ];\n const res = await fetch(`${TURSO_BASE}/v2/pipeline`, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${TURSO_AUTH_TOKEN}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({ requests }),\n });\n if (!res.ok) {\n throw new Error(`Turso HTTP ${res.status}: ${await res.text()}`);\n }\n return res.json();\n}\n\n// ---------------------------------------------------------------------------\n// Map a raw UKMTO incident to the fields the map consumes.\n// ---------------------------------------------------------------------------\nfunction mapIncident(inc) {\n return {\n incidentTypeName: inc.incidentTypeName,\n incidentNumber: inc.incidentNumber,\n incidentIssuer: inc.incidentIssuer,\n place: inc.place,\n sitecoreId: inc.sitecoreId,\n pinColour: inc.pinColour,\n locationLatitudeDDDMMSS: inc.locationLatitudeDDDMMSS,\n locationLongitudeDDDMMSS: inc.locationLongitudeDDDMMSS,\n utcDateOfIncident: inc.utcDateOfIncident,\n utcDateCreated: inc.utcDateCreated,\n // data-lng / data-lat attributes on map markers\n dataLng: inc.locationLongitude,\n dataLat: inc.locationLatitude,\n otherDetails: inc.otherDetails,\n };\n}\n\n// ---------------------------------------------------------------------------\n// Create table, fetch existing rows for comparison, translate only changes,\n// and upsert in chunks.\n// ---------------------------------------------------------------------------\nasync function saveToDb(incidents) {\n // 1. Ensure table exists\n await tursoPipeline([\n {\n sql: `CREATE TABLE IF NOT EXISTS ukmto_events (\n incidentNumber INTEGER PRIMARY KEY,\n incidentTypeName TEXT,\n incidentTypeNameId TEXT,\n incidentIssuer TEXT,\n place TEXT,\n sitecoreId TEXT,\n pinColour TEXT,\n locationLatitudeDDDMMSS TEXT,\n locationLongitudeDDDMMSS TEXT,\n utcDateOfIncident TEXT,\n utcDateCreated TEXT,\n longitude REAL,\n latitude REAL,\n otherDetails TEXT,\n otherDetailsId TEXT\n )`,\n },\n ]);\n\n // 2. Fetch existing records from Turso to compare against\n let existingMap = new Map();\n try {\n const res = await tursoPipeline([\n {\n sql:\n `SELECT incidentNumber, incidentTypeName, otherDetails, incidentTypeNameId, otherDetailsId FROM ukmto_events`,\n },\n ]);\n const rows = res.results?.[0]?.response?.result?.rows || [];\n for (const row of rows) {\n const incNum = row[0]?.value;\n if (incNum !== undefined && incNum !== null) {\n existingMap.set(Number(incNum), {\n incidentTypeName: row[1]?.value ?? null,\n otherDetails: row[2]?.value ?? null,\n incidentTypeNameId: row[3]?.value ?? null,\n otherDetailsId: row[4]?.value ?? null,\n });\n }\n }\n } catch (err) {\n console.log(\n \"Could not fetch existing rows, proceeding with full sync:\",\n err,\n );\n }\n\n const insertStmts = [];\n let translatedCount = 0;\n let skippedCount = 0;\n\n for (const inc of incidents) {\n const existing = existingMap.get(inc.incidentNumber);\n\n let nameId;\n let detailsId;\n\n // Check if record already exists and text content hasn't changed\n if (\n existing &&\n existing.incidentTypeName === inc.incidentTypeName &&\n existing.otherDetails === inc.otherDetails\n ) {\n // Content is identical! Reuse existing translations and skip inserting.\n nameId = existing.incidentTypeNameId;\n detailsId = existing.otherDetailsId;\n skippedCount++;\n continue;\n }\n\n // If it's new or text changed, translate the text\n try {\n nameId = inc.incidentTypeName\n ? await translateToId(inc.incidentTypeName)\n : inc.incidentTypeName;\n detailsId = inc.otherDetails\n ? await translateToId(inc.otherDetails)\n : inc.otherDetails;\n translatedCount++;\n } catch (err) {\n console.error(\"Translation failed for incident\", inc.incidentNumber, err);\n nameId = inc.incidentTypeName;\n detailsId = inc.otherDetails;\n }\n\n insertStmts.push({\n sql: `INSERT OR REPLACE INTO ukmto_events\n (incidentNumber, incidentTypeName, incidentTypeNameId, incidentIssuer,\n place, sitecoreId, pinColour, locationLatitudeDDDMMSS,\n locationLongitudeDDDMMSS, utcDateOfIncident, utcDateCreated,\n longitude, latitude, otherDetails, otherDetailsId)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n inc.incidentNumber,\n inc.incidentTypeName,\n nameId,\n inc.incidentIssuer,\n inc.place,\n inc.sitecoreId,\n inc.pinColour,\n inc.locationLatitudeDDDMMSS,\n inc.locationLongitudeDDDMMSS,\n inc.utcDateOfIncident,\n inc.utcDateCreated,\n inc.dataLng,\n inc.dataLat,\n inc.otherDetails,\n detailsId,\n ],\n });\n }\n\n // 3. Send inserts in chunks of 10 to avoid payload size/timeout limits\n if (insertStmts.length > 0) {\n const CHUNK_SIZE = 10;\n for (let i = 0; i < insertStmts.length; i += CHUNK_SIZE) {\n const chunk = insertStmts.slice(i, i + CHUNK_SIZE);\n await tursoPipeline([...chunk]);\n }\n }\n\n console.log(\n `Sync complete. Total incoming: ${incidents.length} | Skipped (unchanged): ${skippedCount} | Translated & Updated: ${translatedCount}`,\n );\n}\n\nexport default async function handler() {\n await fetchAndPopulate(Url, HeaderS, UkmtoData);\n}\n\nfunction fetchAndPopulate(url, headerS, constData) {\n fetchQueue = fetchQueue.then(async () => {\n try {\n const response = await fetch(url, { method: \"GET\", headers: headerS });\n if (!response.ok) {\n throw new Error(`UKMTO HTTP ${response.status}`);\n }\n\n const raw = await response.json();\n const mapped = Array.isArray(raw) ? raw.map(mapIncident) : [];\n\n console.log(`Fetched ${mapped.length} UKMTO incidents.`);\n\n // Persist (create table + smart check & translate + upsert)\n await saveToDb(mapped);\n\n constData.parent = mapped;\n constData.temp = [...mapped];\n } catch (error) {\n console.error(\"UKMTO fetch failed:\", error);\n }\n });\n return fetchQueue;\n}\n\nexport { mapIncident, UkmtoData };",[146],"18c6bbf2-42b6-4dc8-9bcd-e81d1e308098","easaEvent.js","b21fc016-a9bf-11f1-8cb4-1607ee4eb77e",151,["D",1788942414030],33,"// ---------------------------------------------------------------------------\n// EASA Conflict Zone Information Bulletins (CZIB) -> JSON -> Turso (Val Town)\n// ---------------------------------------------------------------------------\nimport { DOMParser } from \"npm:linkedom\";\n\nconst MAIN_URL = \"https://www.easa.europa.eu/en/domains/air-operations/czibs\";\n\nconst HEADERS = {\n \"User-Agent\":\n \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36\",\n Accept: \"text/html,application/json,text/plain,*/*\",\n};\n\nconst dom = new DOMParser();\n\nfunction clean(s) {\n return (s || \"\").replace(/\\s+/g, \" \").trim();\n}\n\nfunction isDetailHref(h) {\n if (!h) return false;\n if (!h.includes(\"/czibs/\")) return false;\n if (/czibs\\/(information|map|export|feed)/.test(h)) return false;\n return true;\n}\n\nasync function listActiveRows() {\n const res = await fetch(MAIN_URL, { headers: HEADERS });\n if (!res.ok) throw new Error(`overview HTTP ${res.status}`);\n const html = await res.text();\n const doc = dom.parseFromString(html, \"text/html\");\n\n const rows = doc.querySelectorAll(\n \"#status-div-classeasa-status-field-status-active--div-classeasa-status-field-status-active-field-itemdiv-classcz-status-cz-status-activeactivedivdivdiv tbody tr\",\n );\n const active = [];\n\n for (const row of rows) {\n const link = Array.from(row.querySelectorAll(\"a\")).find((a) =>\n isDetailHref(a.getAttribute(\"href\"))\n );\n\n if (!link) continue;\n\n const detailUrl = new URL(link.getAttribute(\"href\"), MAIN_URL).href;\n const dateStart = clean(\n row.querySelector(\n \"td.views-field.views-field-field-easa-conflict-zone-up-date\",\n )?.textContent,\n );\n const dateEnd = clean(\n row.querySelector(\n \"td.views-field.views-field-field-easa-conflict-zone-until\",\n )?.textContent,\n );\n\n active.push({ detailUrl, dateStart, dateEnd });\n }\n return active;\n}\n\nasync function scrapeDetail(item) {\n try {\n const res = await fetch(item.detailUrl, { headers: HEADERS });\n if (!res.ok) {\n console.error(`detail HTTP ${res.status}: ${item.detailUrl}`);\n return null;\n }\n const html = await res.text();\n const doc = dom.parseFromString(html, \"text/html\");\n\n const fieldValue = (selector) =>\n clean(doc.querySelector(selector)?.textContent);\n\n const czibNumber =\n fieldValue(\".field--name-field-easa-czib-number .field__items div\") ||\n fieldValue(\".field-name-field-easa-czib-number .field-items div\") ||\n fieldValue(\".field-name-field-easa-czib-number div div\");\n\n const publicator = fieldValue(\".czib_ref_publications .field__items div\") ||\n fieldValue(\".czib_ref_publications .field-items div\") ||\n fieldValue(\".czib_ref_publications div div div\");\n\n const airspace = fieldValue(\n \".field--name-field-easa-cz-aff-air .field__items div\",\n ) || fieldValue(\n \".field-name-field-easa-cz-aff-air .field-items div\",\n );\n\n const description = fieldValue(\n \".field--name-field-easa-cz-description .field__items div\",\n ) || fieldValue(\n \".field-name-field-easa-cz-description .field-items div\",\n );\n\n const recommendation = fieldValue(\n \".field--name-field-easa-cz-recommendations .field__items div\",\n ) || fieldValue(\n \".field-name-field-easa-cz-recommendations .field-items div\",\n );\n\n const csvHref =\n doc.querySelector('a[href*=\"conflict-zone-node-export\"]')?.getAttribute(\n \"href\",\n ) || \"\";\n const nid = (csvHref.match(/(\\d+)\\/?$/) || [])[1] || null;\n\n let coordinate = null;\n if (nid) {\n try {\n const apiRes = await fetch(\n `https://www.easa.europa.eu/en/api/maps/czib-node/${nid}`,\n { headers: HEADERS },\n );\n if (apiRes.ok) coordinate = await apiRes.json();\n } catch (err) {\n console.error(\"Coordinate fetch failed for\", item.detailUrl, err);\n }\n }\n\n return {\n nid,\n czibNumber,\n publicator,\n airspace,\n description,\n recommendation,\n coordinate,\n };\n } catch (err) {\n console.error(`Failed parsing detail page ${item.detailUrl}:`, err.message);\n return null;\n }\n}\n\nfunction buildRecord(item, detail) {\n if (!detail) return null;\n return {\n nid: detail.nid || null,\n status: \"Active\",\n czibNumber: detail.czibNumber || null,\n dateStart: item.dateStart || null,\n dateEnd: item.dateEnd || null,\n airspace: detail.airspace || null,\n publicator: detail.publicator || null,\n description: detail.description || null,\n recommendation: detail.recommendation || null,\n coordinate: detail.coordinate,\n detailUrl: item.detailUrl,\n };\n}\n\nasync function mapLimit(items, limit, fn) {\n const results = new Array(items.length);\n let i = 0;\n const workers = Array.from(\n { length: Math.min(limit, items.length) },\n async () => {\n while (i < items.length) {\n const idx = i++;\n results[idx] = await fn(items[idx], idx);\n }\n },\n );\n await Promise.all(workers);\n return results;\n}\n\nconst TURSO_BASE = (Deno.env.get(\"TURSO_URL\") || \"\").replace(/\\/+$/, \"\");\nconst TURSO_AUTH_TOKEN = Deno.env.get(\"TURSO_AUTH_TOKEN\");\n\nif (!TURSO_BASE || !TURSO_AUTH_TOKEN) {\n throw new Error(\n \"Missing TURSO_URL / TURSO_AUTH_TOKEN env vars in Val Town secrets\",\n );\n}\n\nfunction typedArg(value) {\n if (value === null || value === undefined) {\n return { type: \"null\", value: null };\n }\n if (typeof value === \"number\") {\n return { type: Number.isInteger(value) ? \"integer\" : \"float\", value };\n }\n return { type: \"text\", value: String(value) };\n}\n\nfunction typedFloat(value) {\n if (value === null || value === undefined) {\n return { type: \"null\", value: null };\n }\n const num = Number(value);\n if (isNaN(num)) return { type: \"null\", value: null };\n return { type: \"float\", value: num };\n}\n\nasync function tursoPipeline(stmts) {\n const requests = [\n ...stmts.map(({ sql, args }) => ({\n type: \"execute\",\n stmt: { sql, args: args || [] },\n })),\n { type: \"close\" },\n ];\n const res = await fetch(`${TURSO_BASE}/v2/pipeline`, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${TURSO_AUTH_TOKEN}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({ requests }),\n });\n if (!res.ok) throw new Error(`Turso HTTP ${res.status}: ${await res.text()}`);\n return res.json();\n}\n\nasync function saveToDb(records) {\n await tursoPipeline([\n {\n sql: `CREATE TABLE IF NOT EXISTS easa_czibs (\n nid TEXT PRIMARY KEY,\n status TEXT,\n czib_number TEXT,\n date_start TEXT,\n date_end TEXT,\n airspace TEXT,\n publicator TEXT, \n description TEXT, \n recommendation TEXT,\n latitude REAL NULL, \n longitude REAL NULL,\n detail_url TEXT\n )`,\n },\n ]);\n\n const insertStmts = records.map((r) => ({\n sql: `INSERT OR REPLACE INTO easa_czibs\n (nid, status, czib_number, date_start, date_end, airspace,\n publicator, description, recommendation,\n latitude, longitude, detail_url)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n typedArg(r.nid || crypto.randomUUID()),\n typedArg(r.status),\n typedArg(r.czibNumber),\n typedArg(r.dateStart),\n typedArg(r.dateEnd),\n typedArg(r.airspace),\n typedArg(r.publicator ?? \"\"),\n typedArg(r.description ?? \"\"),\n typedArg(r.recommendation ?? \"\"),\n typedFloat(r.coordinate ? r.coordinate.lat : null),\n typedFloat(r.coordinate ? r.coordinate.lng : null),\n typedArg(r.detailUrl),\n ],\n }));\n\n await tursoPipeline(insertStmts);\n}\n\nexport default async function handler() {\n try {\n const rows = await listActiveRows();\n console.log(`Found ${rows.length} active CZIB(s).`);\n\n const details = await mapLimit(rows, 4, (item) => scrapeDetail(item));\n\n const records = [];\n for (let i = 0; i < details.length; i++) {\n const rec = buildRecord(rows[i], details[i]);\n if (rec) records.push(rec);\n }\n\n await saveToDb(records);\n console.log(`Successfully synced ${records.length} CZIB(s) to Turso.`);\n return JSON.stringify(records, null, 2);\n } catch (err) {\n console.error(\"CRITICAL ERROR in handler:\", err.message, err.stack);\n throw err;\n }\n}",[154],"cfc1ae4a-ee38-45f7-b9ab-22eae6147b0d","commonEvent.js","fcc295ea-aa22-11f1-9f3a-1607ee4eb77e",131,["D",1788723482797],22,"// ---------------------------------------------------------------------------\n// Upstash Redis GeoJSON -> Turso (Table: commonevents)\n// Column updated to use 'reference' (singular) to avoid SQL reserved words.\n// ---------------------------------------------------------------------------\n\nconst UPSTASH_URL =\n \"https://informed-stag-162043.upstash.io/json.get/mapEvent/\";\nconst UPSTASH_BASE = \"https://informed-stag-162043.upstash.io\";\nconst UPSTASH_AUTH =\n \"Bearer gQAAAAAAAnj7AAIgcDJmMDZiMDhkM2U2OTM0NTQzYTM3ZTVkMDM0ZWJmNTkxMQ\";\n\nconst TURSO_BASE = (Deno.env.get(\"TURSO_URL\") || \"\").replace(/\\/+$/, \"\");\nconst TURSO_AUTH_TOKEN = Deno.env.get(\"TURSO_AUTH_TOKEN\");\n\nif (!TURSO_BASE || !TURSO_AUTH_TOKEN) {\n throw new Error(\"Missing TURSO_URL / TURSO_AUTH_TOKEN env vars\");\n}\n\nfunction clean(s) {\n return (s || \"\").replace(/\\s+/g, \" \").trim();\n}\n\nfunction hashId(title, date, location) {\n const s = `${title || \"\"}|${date || \"\"}|${location || \"\"}`;\n let h = 0;\n for (let i = 0; i < s.length; i++) {\n h = (Math.imul(31, h) + s.charCodeAt(i)) | 0;\n }\n return (h >>> 0).toString(36);\n}\n\nfunction typedArg(value) {\n if (value === null || value === undefined) {\n return { type: \"null\", value: null };\n }\n if (typeof value === \"number\") {\n return { type: Number.isInteger(value) ? \"integer\" : \"float\", value };\n }\n return { type: \"text\", value: String(value) };\n}\n\nfunction typedFloat(value) {\n if (value === null || value === undefined || Number.isNaN(Number(value))) {\n return { type: \"null\", value: null };\n }\n return { type: \"float\", value: Number(value) };\n}\n\nasync function tursoPipeline(stmts) {\n const requests = [\n ...stmts.map(({ sql, args }) => ({\n type: \"execute\",\n stmt: { sql, args: args || [] },\n })),\n { type: \"close\" },\n ];\n\n const res = await fetch(`${TURSO_BASE}/v2/pipeline`, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${TURSO_AUTH_TOKEN}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify({ requests }),\n });\n\n if (!res.ok) {\n const errText = await res.text();\n throw new Error(`Turso HTTP ${res.status}: ${errText}`);\n }\n\n const data = await res.json();\n\n // Statement-level failures must abort the run — otherwise a partial/failed\n // Turso write could still trigger the Upstash clear.\n if (data && Array.isArray(data.results)) {\n const errs = data.results\n .map((result, idx) => (result.type === \"error\" ? { idx, result } : null))\n .filter(Boolean);\n if (errs.length > 0) {\n throw new Error(\n `Turso statement error(s): ${errs\n .map(({ idx, result }) => `[${idx}] ${result.error}`)\n .join(\"; \")}`,\n );\n }\n }\n\n return data;\n}\n\n// ---------------------------------------------------------------------------\n// Fetch, Clean, and Handle Malformed/Truncated JSON Gracefully\n// ---------------------------------------------------------------------------\nasync function fetchAndParseEvents() {\n const res = await fetch(UPSTASH_URL, {\n headers: { Authorization: UPSTASH_AUTH },\n });\n if (!res.ok) {\n throw new Error(`Upstash HTTP ${res.status}: ${await res.text()}`);\n }\n\n const body = await res.json();\n let rawData = body.result;\n\n let parsed;\n try {\n parsed = typeof rawData === \"string\" ? JSON.parse(rawData) : rawData;\n } catch (e) {\n console.warn(\n \"JSON parse failed (possibly truncated). Attempting safe recovery...\",\n );\n try {\n const lastValidIdx = rawData.lastIndexOf(\"}\");\n if (lastValidIdx > 0) {\n const fixedJson = rawData.substring(0, lastValidIdx + 1) + \"]}\";\n parsed = JSON.parse(fixedJson);\n } else {\n throw e;\n }\n } catch (innerErr) {\n throw new Error(\n `Failed to parse or repair Upstash payload: ${innerErr.message}`,\n );\n }\n }\n\n const features = Array.isArray(parsed?.features) ? parsed.features : [];\n\n return features.map((f) => {\n const p = f?.properties || {};\n const coords = Array.isArray(f?.geometry?.coordinates)\n ? f.geometry.coordinates\n : [null, null];\n\n const lng = coords[0] != null ? Number(coords[0]) : null;\n const lat = coords[1] != null ? Number(coords[1]) : null;\n\n return {\n id: hashId(p.title, p.date, p.location),\n title: clean(p.title),\n crime: clean(p.crime),\n date: clean(p.date),\n location: clean(p.location),\n paragraph: clean(p.paragraph),\n thumbnail: p.thumbnail || null,\n latitude: !isNaN(lat) ? lat : null,\n longitude: !isNaN(lng) ? lng : null,\n channel: p.channel ? JSON.stringify(p.channel) : null,\n reference: p.references ? JSON.stringify(p.references) : null,\n };\n });\n}\n\n// ---------------------------------------------------------------------------\n// Save records to Turso\n// ---------------------------------------------------------------------------\nasync function saveToDb(records) {\n // 1. Create table using 'reference' column name\n console.log(\"Ensuring table 'commonevents' exists...\");\n await tursoPipeline([\n {\n sql: `CREATE TABLE IF NOT EXISTS commonevents (\n id TEXT PRIMARY KEY,\n title TEXT,\n crime TEXT,\n date TEXT,\n location TEXT,\n paragraph TEXT,\n thumbnail TEXT,\n latitude REAL,\n longitude REAL,\n channel TEXT,\n reference TEXT,\n synced_at TEXT\n )`,\n },\n ]);\n\n // 2. Prepare insert statements using 'reference'\n const now = new Date().toISOString();\n const insertStmts = records.map((r) => ({\n sql: `INSERT OR REPLACE INTO commonevents \n (id, title, crime, date, location, paragraph, thumbnail, latitude, longitude, channel, reference, synced_at) \n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n typedArg(r.id),\n typedArg(r.title),\n typedArg(r.crime),\n typedArg(r.date),\n typedArg(r.location),\n typedArg(r.paragraph),\n typedArg(r.thumbnail),\n typedFloat(r.latitude),\n typedFloat(r.longitude),\n typedArg(r.channel),\n typedArg(r.reference),\n typedArg(now),\n ],\n }));\n\n // 3. Execute inserts in batches of 10\n console.log(`Inserting ${insertStmts.length} records into Turso...`);\n const BATCH_SIZE = 10;\n for (let i = 0; i < insertStmts.length; i += BATCH_SIZE) {\n const batch = insertStmts.slice(i, i + BATCH_SIZE);\n await tursoPipeline(batch);\n }\n}\n\n// ---------------------------------------------------------------------------\n// Empty the Upstash mapEvent features after a successful Turso sync.\n// Sets features to [] (keeps the rest of the document intact).\n// ---------------------------------------------------------------------------\nasync function clearUpstash() {\n const res = await fetch(`${UPSTASH_BASE}/json.set/mapEvent/$.features`, {\n method: \"POST\",\n headers: {\n Authorization: UPSTASH_AUTH,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify([]),\n });\n if (!res.ok) {\n throw new Error(`Upstash SET HTTP ${res.status}: ${await res.text()}`);\n }\n return res.json();\n}\n\n// ---------------------------------------------------------------------------\n// Main Entry Point\n// ---------------------------------------------------------------------------\nexport default async function handler() {\n const records = await fetchAndParseEvents();\n console.log(`Successfully parsed ${records.length} event(s) from Upstash.`);\n\n await saveToDb(records);\n console.log(\n `Successfully synced ${records.length} record(s) to Turso (table: commonevents).`,\n );\n\n // Only clear the Upstash source if the Turso write succeeded above.\n await clearUpstash();\n console.log(\"Cleared mapEvent from Upstash.\");\n\n return { success: true, total: records.length };\n}",[162],"28dcaac6-769b-40a0-8019-7ba853c08b21","scripts","ca191255-a940-4a18-832b-d6f7739bfbc1",174,["D",1789073099638],1,"directory",[170],"pulls","branches","forks","envVars",{"_184":186,"_187":62},"connections",{"_184":62,"_185":62},"count","unfilledCount",6,"emptyCount",{"_43":169,"_32":170,"_30":171,"_136":-5,"_41":42,"_40":172,"_138":25,"_38":224,"_13":174,"_34":175,"_36":-5,"_24":49,"_143":-5,"_196":225,"_27":226},{"_43":216,"_32":217,"_30":218,"_136":171,"_41":42,"_40":219,"_138":25,"_38":220,"_13":62,"_34":35,"_36":221,"_24":49,"_143":-5,"_196":222,"_27":223},{"_43":208,"_32":209,"_30":210,"_136":171,"_41":42,"_40":211,"_138":25,"_38":212,"_13":62,"_34":35,"_36":213,"_24":49,"_143":-5,"_196":214,"_27":215},{"_43":161,"_32":162,"_30":163,"_136":-5,"_41":42,"_40":164,"_138":25,"_38":205,"_13":166,"_34":141,"_36":167,"_24":49,"_143":-5,"_196":206,"_27":207},{"_43":153,"_32":154,"_30":155,"_136":-5,"_41":42,"_40":156,"_138":25,"_38":202,"_13":158,"_34":141,"_36":159,"_24":49,"_143":-5,"_196":203,"_27":204},{"_43":145,"_32":146,"_30":147,"_136":-5,"_41":42,"_40":148,"_138":25,"_38":199,"_13":150,"_34":141,"_36":151,"_24":49,"_143":-5,"_196":200,"_27":201},{"_43":133,"_32":134,"_30":135,"_136":-5,"_41":42,"_40":137,"_138":49,"_38":195,"_13":140,"_34":141,"_36":142,"_24":49,"_143":-5,"_196":197,"_27":198},["D",1789100841683],"children",[],[134],["D",1788982196432],[],[146],["D",1788942414030],[],[154],["D",1788723482797],[],[162],"a8ce5f8a-de5f-4003-bc1d-bd8ba884f0ea","zenrows-status.js","97a8e3a2-ac8a-11f1-9bec-1607ee4eb77e",168,["D",1788984667491],"// Prints your ZenRows plan status: credits used, period end, concurrency limit.\n// Run this when warEvent.js logs AUTH004 (credits exhausted) or AUTH006\n// (concurrency limit) to see where you stand. Does not consume credits.\nconst key = Deno.env.get(\"ZENROWS_API_KEY\");\nif (!key) throw new Error(\"Missing ZENROWS_API_KEY env var\");\n\nconst res = await fetch(\"https://api.zenrows.com/v1/subscriptions/self/details\", {\n headers: { \"X-API-Key\": key },\n});\nconst body = await res.text();\nconsole.log(`HTTP ${res.status}`);\ntry {\n console.log(JSON.stringify(JSON.parse(body), null, 2));\n} catch {\n console.log(body);\n}\n",[],[170,209],"ea265c60-9e86-4983-b94e-704638316918","translate.js","e5e91c2e-a9c1-11f1-bcf6-1607ee4eb77e",78,["D",1788678567608],"// ---------------------------------------------------------------------------\n// Shared EN -> ID translator used by every event scraper in this val.\n//\n// Primary: NVIDIA Riva \"nvidia/riva-translate-4b-instruct-v2\"\n// (OpenAI-compatible chat completions at integrate.api.nvidia.com).\n// Fallback: free MyMemory API, then the original English text as a last resort\n// so a transient API issue never fails or corrupts a run.\n//\n// The Riva instruct model must have the target language named in the *user*\n// message (otherwise it can default to another language, e.g. French).\n//\n// Requires the NVIDIA_API_KEY env var (set once on the val, shared by all\n// files). If it is missing, translation falls back to MyMemory/English.\n// ---------------------------------------------------------------------------\nconst NVIDIA_API_KEY = Deno.env.get(\"NVIDIA_API_KEY\");\nconst NVIDIA_BASE = \"https://integrate.api.nvidia.com/v1/chat/completions\";\nconst NVIDIA_MODEL = \"nvidia/riva-translate-4b-instruct-v2\";\n\nlet lastTranslate = 0; // MyMemory fallback throttle (~2 req/s)\n\n// Riva has no 500-char cap, but chunk large text so it stays well within the\n// model's input/output token limits (long EASA descriptions).\nconst CHUNK_MAX = 1600;\n\nfunction splitChunks(text) {\n const chunks = [];\n let rest = text.trim();\n while (rest.length > CHUNK_MAX) {\n let cut = rest.lastIndexOf(\". \", CHUNK_MAX);\n if (cut < CHUNK_MAX * 0.5) cut = rest.lastIndexOf(\" \", CHUNK_MAX);\n if (cut < CHUNK_MAX * 0.5) cut = CHUNK_MAX; // no break point -> hard cut\n chunks.push(rest.slice(0, cut + (rest[cut] === \" \" ? 0 : 1)));\n rest = rest.slice(cut + 1).trim();\n }\n if (rest) chunks.push(rest);\n return chunks;\n}\n\n// Primary: NVIDIA Riva instruct translation model.\nasync function rivaOnce(text) {\n const key = NVIDIA_API_KEY;\n if (!key) throw new Error(\"NVIDIA_API_KEY not set\");\n const res = await fetch(NVIDIA_BASE, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${key}`,\n \"Content-Type\": \"application/json\",\n Accept: \"application/json\",\n },\n body: JSON.stringify({\n model: NVIDIA_MODEL,\n messages: [\n { role: \"system\", content: \"You are a translation engine.\" },\n {\n role: \"user\",\n content: `Translate the following from English to Indonesian:\\n${text}`,\n },\n ],\n max_tokens: 2048,\n temperature: 0,\n }),\n });\n if (!res.ok) throw new Error(`Riva HTTP ${res.status}`);\n const data = await res.json();\n const out = data?.choices?.[0]?.message?.content?.trim();\n if (!out || /MYMEMORY WARNING|QUERY LENGTH/i.test(out)) {\n throw new Error(\"bad Riva output\");\n }\n return out;\n}\n\n// Fallback: MyMemory. Its free daily quota can be exhausted, in which case it\n// returns a warning instead of a translation — we treat that as a failure and\n// drop through to the original English rather than persisting the warning.\nasync function mymemoryOnce(text) {\n const elapsed = Date.now() - lastTranslate;\n if (elapsed < 500) {\n await new Promise((r) => setTimeout(r, 500 - elapsed));\n }\n lastTranslate = Date.now();\n\n const url =\n \"https://api.mymemory.translated.net/get?q=\" +\n encodeURIComponent(text) +\n \"&langpair=en|id\";\n const res = await fetch(url);\n if (!res.ok) throw new Error(`MyMemory HTTP ${res.status}`);\n const data = await res.json();\n const translated = data?.responseData?.translatedText;\n if (\n !translated ||\n /MYMEMORY WARNING|QUERY LENGTH LIMIT EXCEEDED|NO QUERY|INVALID PAIR/i.test(\n translated,\n )\n ) {\n return text;\n }\n return translated;\n}\n\nasync function translateOnce(text) {\n try {\n return await rivaOnce(text);\n } catch (err) {\n console.error(\"Riva failed, falling back to MyMemory:\", err.message);\n try {\n return await mymemoryOnce(text);\n } catch (err2) {\n console.error(\"MyMemory fallback failed, keeping original:\", err2.message);\n return text;\n }\n }\n}\n\nexport async function translateToId(text) {\n if (!text) return text;\n const chunks = splitChunks(text);\n if (chunks.length === 1) return await translateOnce(chunks[0]);\n const parts = [];\n for (const c of chunks) parts.push(await translateOnce(c));\n return parts.join(\" \");\n}\n",[],[170,217],["D",1789073099638],[189,190],[170],["D",1788316219773],["D",1789458074143],{"_99":98,"_106":102},{"_99":98,"_100":98,"_101":102,"_18":56,"_103":104,"_105":49},{"_13":88,"_32":89,"_43":42,"_90":44,"_91":-5,"_92":-5,"_93":-5,"_94":-5,"_95":-5,"_96":45,"_97":98},"ctx",{"_81":-5,"_125":265,"_266":267,"_268":269,"_270":49,"_271":49,"_272":49,"_273":25,"_274":-5,"_275":-7,"_276":-7,"_277":62,"_278":49,"_279":49,"_280":-5,"_281":49,"_282":49,"_283":49,"_284":-5},"ENV",{"_236":237,"_238":239,"_240":89,"_241":242,"_243":244,"_245":246,"_247":248,"_249":250,"_251":252,"_253":254,"_255":256,"_257":258,"_259":260,"_261":262,"_263":264},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v216