[{"_1":2,"_170":-5,"_171":-5},"loaderData",{"_3":4,"_92":93},"root",{"_5":6,"_35":36,"_65":66},"ctx",{"_7":8,"_23":-5,"_24":25,"_26":27,"_28":20,"_29":20,"_30":20,"_31":32,"_33":20,"_34":20},"auth",{"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20,"_21":22},"sessionClaims","sessionId","sessionStatus","userId","actor","orgId","orgRole","orgSlug","orgPermissions","factorVerificationAge","isAuthenticated",false,"tokenType","session_token","user","preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","notificationCount",0,"flagDenols","isTeamOrgMember","ENV",{"_37":38,"_39":40,"_41":42,"_43":44,"_45":46,"_47":48,"_49":50,"_51":52,"_53":54,"_55":56,"_57":58,"_59":60,"_61":62,"_63":64},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","VAL_RUNTIME_NAME","prod","RENDER_GIT_COMMIT","5c349ef9e7d7333839a0634dd567aae2d49623a6","RENDER_GIT_BRANCH","main","CLERK_PUBLISHABLE_KEY","pk_live_Y2xlcmsudmFsLnRvd24k","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","API_URL","https://api.val.town","clerkState",{"_67":68},"__internal_clerk_state",{"_69":70,"_71":46,"_72":73,"_74":73,"_75":20,"_76":73,"_77":73,"_78":73,"_79":73,"_80":81,"_90":20,"_91":20},"__clerk_ssr_state",{"_21":22,"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20},"__publishableKey","__proxyUrl","","__domain","__isSatellite","__signInUrl","__signUpUrl","__afterSignInUrl","__afterSignUpUrl","__clerk_debug",{"_82":20,"_19":20,"_83":73,"_84":85,"_86":73,"_87":46,"_88":20,"_89":73},"isSignedIn","proxyUrl","reason","session-token-and-uat-missing","message","publishableKey","isSatellite","domain","__telemetryDisabled","__telemetryDebug","routes/embed.x.$handle.$name.$",{"_94":95,"_96":97,"_98":99,"_100":101,"_149":150,"_167":168,"_169":-5},"handle","dvdsgl","name","glide","path","main.tsx","project",{"_102":103,"_96":97,"_104":105,"_106":-5,"_107":108,"_109":110,"_111":-5,"_112":-5,"_12":113,"_114":-5,"_115":108,"_116":-5,"_117":-5,"_118":32,"_14":119,"_120":121,"_122":121,"_123":124,"_125":124,"_126":124,"_127":121,"_128":121,"_23":129,"_132":133,"_135":136,"_148":20},"id","425b461c-275a-11f0-86ff-569c3dd06744","createdAt",["D",1746192906228],"deletedAt","legacyEnvVars",true,"privacy","public","description","imageUrl","3b461902-9486-4c07-8b66-43eb3951c998","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount","e2a9c5ea-7139-4786-a331-3447349f1dc2","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm",{"_94":95,"_130":131},"profileImageUrl","https://images.clerk.dev/oauth_github/img_2P4kUJCVmzhr86JK45fuJloIUzo.jpeg","org",{"_94":95,"_134":131,"_12":113},"avatar","branch",{"_137":138,"_96":44,"_102":139,"_140":103,"_141":-5,"_142":-5,"_143":-5,"_144":-5,"_145":-5,"_146":97,"_147":95},"version",300,"425b9a72-275a-11f0-86ff-569c3dd06744","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","isOwned","file",{"_102":151,"_96":99,"_152":153,"_154":-5,"_155":139,"_156":138,"_157":108,"_104":158,"_137":159,"_160":161,"_162":163,"_164":20,"_165":-5,"_98":166},"4a26c556-275a-11f0-86ff-569c3dd06744","fileId","425f83d0-275a-11f0-86ff-569c3dd06744","parentId","branchId","branchVersion","latest",["D",1722293521402],299,"fileType","script","content","import { withTimeLog } from \"https://esm.town/v/dvdsgl/effect\";\nimport { Chunk, Context, Data, Effect, Layer, Logger, LogLevel, Option, Stream } from \"npm:effect\";\nimport { UnknownException } from \"npm:effect/Cause\";\nimport { v4 as uuidv4 } from \"npm:uuid\";\n\nconst baseUrl = `https://api.glideapps.com`;\n\ntype Stash = { $stashID: string };\n\ntype PutResult = { data: { rowIDs: string[] } };\n\nexport type GlideInfo = { token: string };\n\nexport class Glide extends Context.Tag(\"Glide\")() {}\n\nexport class GlideError extends Data.TaggedError(\"GlideError\")<{\n type?: string;\n message: string;\n}> {}\n\nexport const layer = (info: GlideInfo) => Layer.succeed(Glide, Glide.of(info));\n\nconst getGlide: Effect.Effect = Effect.gen(function*() {\n const provided = yield* Effect.serviceOption(Glide);\n\n if (Option.isSome(provided)) return provided.value;\n\n const env = {\n token: Deno.env.get(\"glide\"),\n };\n\n if (typeof env.token === \"string\") return env;\n\n yield* new GlideError({\n message: \"No Glide layer provided, and 'glide' secret not found in environment\",\n });\n});\n\nfunction removeUnsupportedColumnTypes(row: any): any {\n const pr = { ...row };\n for (const [k, v] of Object.entries(pr)) {\n // Glide does not accept arrays or objects yet\n if (Array.isArray(v) || typeof v === \"object\") {\n delete pr[k];\n }\n }\n return pr;\n}\n\n// Handles calling fetch, checking for API errors\nfunction fetchGlide(\n method: \"GET\" | \"POST\" | \"PUT\",\n url: string,\n body: unknown,\n): Effect.Effect {\n return Effect.gen(function*() {\n const { token } = yield* getGlide;\n const result = yield* Effect.tryPromise(() =>\n fetch(url, {\n method,\n headers: {\n Authorization: `Bearer ${token}`,\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify(body),\n })\n ).pipe(\n Effect.tap(Effect.annotateCurrentSpan({ method, url })),\n Effect.withLogSpan(\"fetchGlide\"),\n );\n\n if (!result.ok) {\n yield* new GlideError({ message: result.statusText });\n }\n const json = yield* Effect.tryPromise(() => result.json());\n if (\"error\" in json) {\n yield* new GlideError(json.error);\n }\n return json;\n }).pipe(\n Effect.catchTag(\n \"UnknownException\",\n () => new GlideError({ message: \"Could not fetch\" }),\n ),\n );\n}\n\nexport const putTable = (\n id: string,\n rows: Stash | any[],\n): Effect.Effect =>\n Effect.gen(function*() {\n // Some tables have a prefix, e.g. \"native-table-\" but this API\n // does not accept it.\n id = id.replace(\"native-table-\", \"\");\n\n const url = `${baseUrl}/tables/${id}?onSchemaError=updateSchema`;\n\n yield* Effect.logDebug(`putTable ${url}`);\n\n if (Array.isArray(rows)) {\n rows = rows.map(removeUnsupportedColumnTypes);\n }\n const results = yield* fetchGlide(\"PUT\", url, {\n rows,\n schema: { columns: [] },\n });\n yield* Effect.logDebug(\"putTable succeeded\");\n\n return results;\n }).pipe(withTimeLog(`putTable`));\n\nexport const makeStash = (): Stash => ({ $stashID: uuidv4() });\n\nexport const postStash = (\n stash: string | Stash,\n rows: any[],\n serial: string | number,\n): Effect.Effect =>\n Effect.gen(function*() {\n const id = typeof stash === \"string\" ? stash : stash.$stashID;\n\n rows = rows.map(removeUnsupportedColumnTypes);\n const url = `${baseUrl}/stashes/${id}/${serial}`;\n yield* Effect.logDebug(`postStash ${serial}`);\n yield* fetchGlide(\"POST\", url, rows);\n return { $stashID: id };\n }).pipe(\n // Sometimes the API just fails\n Effect.retry({ times: 3 }),\n );\n\ntype Import = {\n table: string;\n getRows: () => AsyncGenerator;\n};\n\nasync function* bufferAsyncIterable(\n asyncIterable: AsyncIterable,\n max = 1_000,\n): AsyncGenerator {\n let buffer: T[] = [];\n\n for await (let chunk of asyncIterable) {\n if (!Array.isArray(chunk)) chunk = [chunk];\n buffer.push(...chunk);\n while (buffer.length >= max) yield buffer.splice(0, max);\n }\n\n // Yield any remaining elements in the buffer\n if (buffer.length > 0) yield buffer;\n}\n\nexport function importTableEffect(props: Import) {\n const { table, getRows } = props;\n\n return Effect.gen(function*() {\n yield* Effect.logDebug(`ingest start`);\n\n const stash = makeStash();\n const stream = Stream.fromAsyncIterable(\n // We buffer the rows into larger chunks\n // because many APIs only fetch small chunks (e.g. 100)\n // records at a time.\n bufferAsyncIterable(getRows()),\n (e) => new Error(String(e)),\n );\n\n let totalRows = 0;\n let serial = 0;\n yield* Stream.runForEach(stream, (rows) =>\n Effect.gen(function*() {\n totalRows += rows.length;\n yield* Effect.logDebug(`ingest ${totalRows} total rows`);\n yield* postStash(stash, rows, serial++);\n }));\n\n return yield* putTable(table, stash);\n }).pipe(withTimeLog(`ingest`));\n}\n\n// Runs an injestion job.\nexport async function importTable(\n props: Import & { debug?: boolean },\n) {\n const program = importTableEffect(props).pipe(\n Logger.withMinimumLogLevel(props.debug ? LogLevel.Debug : LogLevel.Info),\n );\n return await Effect.runPromise(program);\n}","deleted","label",[99],"permalink","https://www.val.town/x/dvdsgl/glide/code/main.tsx","extension","actionData","errors"]