[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8},"actionData","errors","root",{"_105":106,"_107":108},"routes/embed.x.$handle.$name.$",{"_9":10,"_11":12,"_13":14,"_15":16,"_17":18,"_19":20,"_21":-5},"handle","jfkll","name","iv-pivot-modeling","path","synchistory.ts","project",{"_22":45,"_11":12,"_33":46,"_47":-5,"_48":42,"_49":50,"_51":50,"_52":53,"_54":-5,"_55":56,"_57":58,"_59":32,"_60":-5,"_61":-5,"_62":63,"_64":65,"_66":67,"_68":69,"_70":71,"_72":-5,"_73":74,"_75":74,"_76":77,"_78":77,"_79":77,"_80":74,"_81":74,"_82":83,"_84":85,"_86":87,"_88":42},"file",{"_22":23,"_11":14,"_24":25,"_26":-5,"_27":28,"_29":30,"_31":32,"_33":34,"_35":36,"_37":38,"_39":40,"_41":42,"_43":-5,"_13":44},"permalink","https://www.val.town/x/jfkll/iv-pivot-modeling/code/synchistory.ts","extension","id","a3e5daa8-1508-473e-890a-314147f6960b","fileId","0e439a08-a2d0-11f1-b608-1607ee4eb77e","parentId","branchId","23a25aad-3858-4c07-b83c-fbef6bbef586","branchVersion",11,"latest",true,"createdAt",["D",1787915016823],"version",3,"fileType","http","content","/**\n * syncHistory — pulls NQ session history from Databento into the rangeLadder store.\n *\n * Deploy as a SECOND Val.town HTTP val alongside rangeLadder. It writes the same\n * blob key, so rangeLadder picks the data up with no change to that val or the JAR.\n *\n * Set DATABENTO_API_KEY in Val.town environment variables (key starts with \"db-\").\n *\n * ROUTES\n * GET / status + stored history summary\n * GET /cost price the backfill BEFORE running it (free, no data charge)\n * POST /sync fetch + rebuild the window {days?: 30, symbol?: \"NQ.v.0\"}\n * POST /sync/daily fetch just the last session and append (cron target)\n *\n * WHY ohlcv-1m AND NOT ohlcv-1h:\n * the session boundaries are 17:00 / 02:00 / 08:30 / 15:00 CT. 08:30 is not on an\n * hour mark, so hourly bars cannot resolve the RTH open or the Europe/RTH split.\n * Minute bars are ~1,380 records per session — a few MB for a full backfill.\n *\n * SYMBOLOGY: stype_in=continuous with \"NQ.v.0\" = front month by volume, which\n * matches how the model was fitted (dominant contract by volume, roll included).\n */\n\nimport { blob } from \"https://esm.town/v/std/blob\";\n\nconst KEY = \"rangeLadder:history:v1\"; // shared with rangeLadder val\nconst KEEP = 45;\nconst DATASET = \"GLBX.MDP3\";\nconst DEFAULT_SYMBOL = \"NQ.v.0\";\nconst HIST = \"https://hist.databento.com/v0\";\n\n// Session boundaries, minute-of-day, America/Chicago.\nconst ASIA_START = 17 * 60,\n EU_START = 2 * 60,\n RTH_START = 8 * 60 + 30,\n RTH_END = 15 * 60;\n\ntype Day = { date: string; rthRange: number; rthOpen: number };\n\n// ------------------------------------------------------------- CT helpers --\n\n/**\n * Offset lookup is cached per UTC hour: the America/Chicago offset only changes\n * at DST boundaries, so recomputing it per record would be wasted work on ~30k rows.\n */\nconst offsetCache = new Map();\nconst CT_FMT = new Intl.DateTimeFormat(\"en-US\", {\n timeZone: \"America/Chicago\",\n hour12: false,\n year: \"numeric\",\n month: \"2-digit\",\n day: \"2-digit\",\n hour: \"2-digit\",\n minute: \"2-digit\",\n});\n\nfunction ctParts(ms: number) {\n const hourKey = Math.floor(ms / 3_600_000);\n let off = offsetCache.get(hourKey);\n if (off === undefined) {\n const p = CT_FMT.formatToParts(new Date(ms));\n const g = (t: string) => Number(p.find((x) => x.type === t)!.value);\n // Reconstruct the CT wall-clock instant as if it were UTC, then difference.\n const asUtc = Date.UTC(\n g(\"year\"),\n g(\"month\") - 1,\n g(\"day\"),\n g(\"hour\") % 24,\n g(\"minute\"),\n );\n off = asUtc - Math.floor(ms / 60_000) * 60_000;\n offsetCache.set(hourKey, off);\n }\n const local = new Date(ms + off);\n return {\n y: local.getUTCFullYear(),\n m: local.getUTCMonth(),\n d: local.getUTCDate(),\n mod: local.getUTCHours() * 60 + local.getUTCMinutes(),\n };\n}\n\n/** Trade date: rolls at 17:00 CT, weekend stamps pushed to Monday. */\nexport function tradeDate(ms: number): { date: string; mod: number } {\n const { y, m, d, mod } = ctParts(ms);\n const dt = new Date(Date.UTC(y, m, d));\n if (mod >= ASIA_START) dt.setUTCDate(dt.getUTCDate() + 1);\n const wd = dt.getUTCDay();\n if (wd === 6) dt.setUTCDate(dt.getUTCDate() + 2);\n else if (wd === 0) dt.setUTCDate(dt.getUTCDate() + 1);\n return { date: dt.toISOString().slice(0, 10), mod };\n}\n\nexport function sessionOf(mod: number): \"RTH\" | \"ASIA\" | \"EU\" | \"POST\" {\n if (mod >= RTH_START && mod < RTH_END) return \"RTH\";\n if (mod >= ASIA_START || mod < EU_START) return \"ASIA\";\n if (mod >= EU_START && mod < RTH_START) return \"EU\";\n return \"POST\";\n}\n\n// ------------------------------------------------------------- parsing ----\n\n/**\n * Databento JSON-lines. Prices may arrive as fixed-point 1e-9 integers (default)\n * or as decimals when pretty_px is honoured, so scale defensively: no real NQ\n * price approaches 1e9, which makes the magnitude an unambiguous discriminator.\n */\nfunction px(v: unknown): number {\n const n = typeof v === \"string\" ? Number(v) : (v as number);\n if (!Number.isFinite(n)) return NaN;\n return Math.abs(n) > 1e9 ? n / 1e9 : n;\n}\n\nexport type Bar = { ts: number; o: number; h: number; l: number };\n\nexport function parseBars(body: string): Bar[] {\n const out: Bar[] = [];\n for (const line of body.split(\"\\n\")) {\n const s = line.trim();\n if (!s) continue;\n let r: any;\n try {\n r = JSON.parse(s);\n } catch {\n continue;\n }\n const tsRaw = r.hd?.ts_event ?? r.ts_event;\n if (tsRaw === undefined) continue;\n // ts_event is nanoseconds since epoch (string or number).\n const ts = Math.floor(Number(tsRaw) / 1e6);\n const o = px(r.open), h = px(r.high), l = px(r.low);\n if (!Number.isFinite(ts) || !Number.isFinite(h) || !Number.isFinite(l)) {\n continue;\n }\n // OHLCV bars with no trades report zeros; they'd wreck a session low.\n if (h <= 0 || l <= 0) continue;\n out.push({ ts, o, h, l });\n }\n return out;\n}\n\n/** Fold minute bars into per-trade-date session stats. */\nexport function foldSessions(bars: Bar[]) {\n type S = { rthOpen: number; rthHi: number; rthLo: number; rthBars: number };\n const map = new Map();\n for (const b of bars) {\n const { date, mod } = tradeDate(b.ts);\n if (sessionOf(mod) !== \"RTH\") continue;\n let s = map.get(date);\n if (!s) {\n s = { rthOpen: 0, rthHi: 0, rthLo: 0, rthBars: 0 };\n map.set(date, s);\n }\n if (s.rthBars === 0) {\n s.rthOpen = b.o;\n s.rthHi = b.h;\n s.rthLo = b.l;\n } else {\n s.rthHi = Math.max(s.rthHi, b.h);\n s.rthLo = Math.min(s.rthLo, b.l);\n }\n s.rthBars++;\n }\n const days: Day[] = [];\n for (const [date, s] of map) {\n // Half-days (~210 min) and partial pulls would bias the range distribution\n // the model was fitted on, so require most of a 390-minute session.\n if (s.rthBars < 300 || s.rthHi <= s.rthLo) continue;\n days.push({\n date,\n rthRange: +(s.rthHi - s.rthLo).toFixed(2),\n rthOpen: +s.rthOpen.toFixed(2),\n });\n }\n return days.sort((a, b) => (a.date < b.date ? 1 : -1));\n}\n\nfunction normalise(days: Day[]): Day[] {\n const seen = new Map();\n for (const d of days) {\n if (!/^\\d{4}-\\d{2}-\\d{2}$/.test(d?.date ?? \"\")) continue;\n if (!(d.rthRange > 0) || !(d.rthOpen > 0)) continue;\n if (seen.has(d.date)) continue; // first wins = newest/authoritative\n seen.set(d.date, d);\n }\n return [...seen.values()].sort((a, b) => (a.date < b.date ? 1 : -1)).slice(\n 0,\n KEEP,\n );\n}\n\n// ------------------------------------------------------------- databento ---\n\nfunction auth(): string {\n const key = Deno.env.get(\"DATABENTO_API_KEY\");\n if (!key) throw new Error(\"DATABENTO_API_KEY not set in Val.town env vars\");\n return \"Basic \" + btoa(key + \":\");\n}\n\nasync function db(\n method: string,\n params: Record,\n): Promise {\n const r = await fetch(`${HIST}/${method}`, {\n method: \"POST\",\n headers: {\n Authorization: auth(),\n \"content-type\": \"application/x-www-form-urlencoded\",\n },\n body: new URLSearchParams(params).toString(),\n });\n const text = await r.text();\n if (!r.ok) {\n throw new Error(`databento ${method} ${r.status}: ${text.slice(0, 300)}`);\n }\n return text;\n}\n\nfunction range(days: number) {\n const end = new Date();\n const start = new Date(end.getTime() - days * 86_400_000);\n return {\n start: start.toISOString().slice(0, 10),\n end: end.toISOString().slice(0, 10),\n };\n}\n\nasync function fetchBars(symbol: string, days: number): Promise {\n const { start, end } = range(days);\n const body = await db(\"timeseries.get_range\", {\n dataset: DATASET,\n symbols: symbol,\n schema: \"ohlcv-1m\",\n stype_in: \"continuous\",\n start,\n end,\n encoding: \"json\",\n compression: \"none\",\n });\n return parseBars(body);\n}\n\n// ---------------------------------------------------------------- routes ---\n\nexport default async function (req: Request): Promise {\n const json = (o: unknown, s = 200) =>\n new Response(JSON.stringify(o, null, 1), {\n status: s,\n headers: { \"content-type\": \"application/json\" },\n });\n const path = new URL(req.url).pathname.replace(/\\/+$/, \"\") || \"/\";\n\n try {\n if (req.method === \"GET\" && path === \"/\") {\n const h = ((await blob.getJSON(KEY)) as Day[]) ?? [];\n return json({\n ok: true,\n service: \"syncHistory\",\n dataset: DATASET,\n schema: \"ohlcv-1m\",\n keyConfigured: !!Deno.env.get(\"DATABENTO_API_KEY\"),\n stored: h.length,\n newest: h[0]?.date ?? null,\n oldest: h[h.length - 1]?.date ?? null,\n });\n }\n\n // Price the pull before spending anything. Metadata calls are not billed.\n if (req.method === \"GET\" && path === \"/cost\") {\n const u = new URL(req.url);\n const days = Number(u.searchParams.get(\"days\") ?? 30);\n const symbol = u.searchParams.get(\"symbol\") ?? DEFAULT_SYMBOL;\n const { start, end } = range(days);\n const cost = await db(\"metadata.get_cost\", {\n dataset: DATASET,\n symbols: symbol,\n schema: \"ohlcv-1m\",\n stype_in: \"continuous\",\n start,\n end,\n });\n const size = await db(\"metadata.get_billable_size\", {\n dataset: DATASET,\n symbols: symbol,\n schema: \"ohlcv-1m\",\n stype_in: \"continuous\",\n start,\n end,\n });\n return json({\n ok: true,\n symbol,\n days,\n start,\n end,\n costUsd: Number(cost),\n billableBytes: Number(size),\n });\n }\n\n if (req.method === \"POST\" && (path === \"/sync\" || path === \"/sync/daily\")) {\n const b = await req.json().catch(() => ({}));\n const symbol = b.symbol ?? DEFAULT_SYMBOL;\n // Daily top-up needs a few days of slack for weekends and holidays.\n const days = path === \"/sync/daily\" ? (b.days ?? 5) : (b.days ?? 32);\n\n const bars = await fetchBars(symbol, days);\n const fresh = foldSessions(bars);\n if (!fresh.length) {\n return json({\n ok: false,\n error: \"no complete RTH sessions returned\",\n bars: bars.length,\n }, 422);\n }\n\n const existing = ((await blob.getJSON(KEY)) as Day[]) ?? [];\n // Fresh first so a corrected session overrides the stored one.\n const merged = normalise([...fresh, ...existing]);\n await blob.setJSON(KEY, merged);\n\n return json({\n ok: true,\n mode: path === \"/sync/daily\" ? \"daily\" : \"backfill\",\n symbol,\n barsFetched: bars.length,\n sessionsParsed: fresh.length,\n stored: merged.length,\n newest: merged[0]?.date,\n oldest: merged[merged.length - 1]?.date,\n ready: merged.length >= 22,\n });\n }\n\n return json(\n { ok: false, error: `no route for ${req.method} ${path}` },\n 404,\n );\n } catch (e) {\n return json({\n ok: false,\n error: String(e instanceof Error ? e.message : e),\n }, 500);\n }\n}","deleted",false,"label",[14],"e3512a25-a0a2-40f6-af2a-4b242fe63ecd",["D",1787910058981],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","","imageUrl","userId","edb0ee8b-6eb5-4033-8065-92fa86e65418","pinnedFileId","931869b8-a2c4-11f1-a829-1607ee4eb77e","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","019f98c4-b792-72f1-83e7-7d98b5b52c8e","tursoName","e3512a25a0a240f6af2a4b242fe63ecd","tursoHostname","e3512a25a0a240f6af2a4b242fe63ecd-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789479081816],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_9":10,"_104":53},"org",{"_9":10,"_99":53,"_100":53,"_55":56,"_101":102,"_103":42},"branch",{"_35":89,"_11":90,"_22":28,"_91":45,"_92":-5,"_93":-5,"_94":-5,"_95":-5,"_96":-5,"_97":12,"_98":10},"isOwned",16,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","displayName","avatar","tier","free","canRestrictVals","profileImageUrl","ctx",{"_82":-5,"_138":139,"_140":141,"_142":143,"_144":42,"_145":42,"_146":42,"_147":32,"_148":-5,"_149":-7,"_150":-7,"_151":63,"_152":42,"_153":42,"_154":-5,"_155":42,"_156":42,"_157":42,"_158":-5},"ENV",{"_109":110,"_111":112,"_113":90,"_114":115,"_116":117,"_118":119,"_120":121,"_122":123,"_124":125,"_126":127,"_128":129,"_130":131,"_132":133,"_134":135,"_136":137},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town","orgs",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]