[{"_1":2,"_183":-5,"_184":-5},"loaderData",{"_3":4,"_95":96},"root",{"_5":6,"_36":37,"_68":69},"ctx",{"_7":8,"_23":-5,"_24":25,"_26":27,"_28":20,"_29":20,"_30":20,"_31":-5,"_32":33,"_34":20,"_35":20},"auth",{"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20,"_21":22},"sessionClaims","sessionId","sessionStatus","userId","actor","orgId","orgRole","orgSlug","orgPermissions","factorVerificationAge","isAuthenticated",false,"tokenType","session_token","user","preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesChatPanelState","notificationCount",0,"flagDenols","isBusinessOrgMember","ENV",{"_38":39,"_40":41,"_42":43,"_44":45,"_46":47,"_48":49,"_50":51,"_52":53,"_54":55,"_56":57,"_58":59,"_60":61,"_62":63,"_64":65,"_66":67},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","VAL_RUNTIME_NAME","prod","RENDER_GIT_COMMIT","c82f8f7f511b45da3b6daf08ba8af1c21e3c5623","RENDER_GIT_BRANCH","main","CLERK_PUBLISHABLE_KEY","pk_live_Y2xlcmsudmFsLnRvd24k","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","API_URL","https://api.val.town","clerkState",{"_70":71},"__internal_clerk_state",{"_72":73,"_74":47,"_75":76,"_77":76,"_78":20,"_79":76,"_80":76,"_81":76,"_82":76,"_83":84,"_93":20,"_94":20},"__clerk_ssr_state",{"_21":22,"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20},"__publishableKey","__proxyUrl","","__domain","__isSatellite","__signInUrl","__signUpUrl","__afterSignInUrl","__afterSignUpUrl","__clerk_debug",{"_85":20,"_19":20,"_86":76,"_87":88,"_89":76,"_90":47,"_91":20,"_92":76},"isSignedIn","proxyUrl","reason","session-token-and-uat-missing","message","publishableKey","isSatellite","domain","__telemetryDisabled","__telemetryDebug","routes/embed.x.$handle.$name.$",{"_97":98,"_99":100,"_101":102,"_103":104,"_156":157,"_180":181,"_182":-5},"handle","lupinarashi","name","Arashis-Global-Scraper","path","Main/src/fetchstepsql.tsx","project",{"_105":106,"_99":100,"_107":108,"_109":-5,"_110":20,"_111":112,"_113":-5,"_114":-5,"_12":115,"_116":-5,"_117":20,"_118":-5,"_119":-5,"_120":33,"_14":121,"_122":-5,"_123":-5,"_124":-5,"_125":126,"_127":126,"_128":129,"_130":129,"_131":129,"_132":126,"_133":126,"_23":134,"_137":138,"_142":143,"_155":20},"id","055b1d20-2d92-11f1-bd5a-42dde27851f2","createdAt",["D",1775024060352],"deletedAt","legacyEnvVars","privacy","public","description","imageUrl","019d34f5-a118-71aa-86be-a3b05ce7b06e","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount","019d34f5-a118-71aa-86be-a6dc6eeabba9","tursoName","tursoHostname","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm",{"_97":98,"_135":136},"profileImageUrl","https://img.clerk.com/eyJ0eXBlIjoiZGVmYXVsdCIsImlpZCI6Imluc18yTm10cGhQUklwU0tqZHNiOHNnNGd2ZjJURFUiLCJyaWQiOiJ1c2VyXzNCWndldUNFUURoY3B1Uzk1d2lQOVhDTE8xaCJ9","org",{"_97":98,"_139":136,"_12":115,"_140":141},"avatar","tier","free","branch",{"_144":145,"_99":45,"_105":146,"_147":106,"_148":-5,"_149":-5,"_150":-5,"_151":-5,"_152":-5,"_153":100,"_154":98},"version",58,"055bbfa0-2d92-11f1-bd5a-42dde27851f2","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","isOwned","file",{"_105":158,"_99":159,"_160":161,"_162":163,"_164":146,"_165":166,"_167":168,"_107":169,"_144":170,"_171":172,"_173":174,"_175":20,"_176":-5,"_101":177},"c31e5890-2d92-11f1-b71d-42dde27851f2","fetchstepsql.tsx","fileId","b59c6dba-2d92-11f1-a6f8-42dde27851f2","parentId","1ea982e4-2d92-11f1-9d8c-42dde27851f2","branchId","branchVersion",26,"latest",true,["D",1775024378707],4,"fileType","http","content","import { createClient } from \"npm:@libsql/client/web\";\nimport { Redis } from \"npm:@upstash/redis\";\nimport { SQL_CONFIG, UPSTASH_CONFIG } from \"../customize.js\";\nconst {\n TABLE_NAME,\n DEFAULT_LIMIT,\n MAX_LIMIT,\n MAX_RETRIES,\n BASE_BACKOFF_MS,\n QUERY_TIMEOUT_MS,\n TURSO_DATABASE_URL_ENV,\n TURSO_AUTH_TOKEN_ENV,\n ANIME_FETCH_LIMIT_ENV,\n} = SQL_CONFIG;\ntype AnimeInfoRow = {\n id: string;\n type: \"series\" | \"movies\";\n title: string;\n episodes: number;\n season: number;\n};\ntype FetchResult =\n | {\n ok: true;\n count: number;\n data: AnimeInfoRow[];\n skipped: string[];\n }\n | {\n ok: false;\n error: string;\n details?: string;\n };\nlet cachedClient: ReturnType | null = null;\nexport let animeRows: AnimeInfoRow[] = [];\nexport let id = \"\";\nexport let episodes = 0;\nexport let season = 1;\nexport let animeIds: string[] = [];\nexport let animeEpisodes: number[] = [];\nexport let animeSeasons: number[] = [];\nfunction readEnv(name: string): string {\n const value = Deno.env.get(name)?.trim();\n if (!value) {\n throw new Error(`Missing required environment variable: ${name}`);\n }\n return value;\n}\nfunction applyAnimeState(data: AnimeInfoRow[]) {\n animeRows = data;\n animeIds = data.map((row) => row.id);\n animeEpisodes = data.map((row) => row.episodes);\n animeSeasons = data.map((row) => row.season);\n const first = data[0];\n id = first?.id ?? \"\";\n episodes = first?.episodes ?? 0;\n season = first?.season ?? 1;\n}\nfunction sleep(ms: number): Promise {\n return new Promise((resolve) => setTimeout(resolve, ms));\n}\nfunction isTransientError(err: unknown): boolean {\n const msg = err instanceof Error\n ? `${err.name}: ${err.message}`.toLowerCase()\n : String(err).toLowerCase();\n return (\n msg.includes(\"timeout\") ||\n msg.includes(\"timed out\") ||\n msg.includes(\"network\") ||\n msg.includes(\"fetch\") ||\n msg.includes(\"econnreset\") ||\n msg.includes(\"etimedout\") ||\n msg.includes(\"temporary\") ||\n msg.includes(\"503\") ||\n msg.includes(\"502\") ||\n msg.includes(\"429\")\n );\n}\nfunction toCleanString(value: unknown, field: string): string {\n if (typeof value === \"string\") {\n const v = value.trim();\n if (v.length === 0) {\n throw new Error(`Invalid empty string in field: ${field}`);\n }\n return v;\n }\n if (value === null || value === undefined) {\n throw new Error(`Missing required field: ${field}`);\n }\n const v = String(value).trim();\n if (v.length === 0) {\n throw new Error(`Invalid empty string in field: ${field}`);\n }\n return v;\n}\nfunction toSafeInteger(value: unknown, field: string): number {\n const n = typeof value === \"number\" ? value : Number(value);\n if (!Number.isFinite(n) || !Number.isInteger(n)) {\n throw new Error(`Invalid integer in field: ${field}`);\n }\n return n;\n}\nfunction normalizeType(value: unknown): \"series\" | \"movies\" {\n const raw = toCleanString(value, \"type\").toLowerCase();\n if (\n raw.includes(\"tv\") ||\n raw.includes(\"ova\") ||\n raw.includes(\"ona\") ||\n raw.includes(\"series\") ||\n raw.includes(\"special\")\n ) {\n return \"series\";\n }\n if (\n raw.includes(\"movie\") ||\n raw.includes(\"movies\") ||\n raw.includes(\"music\") ||\n raw.includes(\"musics\")\n ) {\n return \"movies\";\n }\n throw new Error(`Unsupported type value: ${raw}`);\n}\nfunction normalizeRow(row: Record): AnimeInfoRow {\n return {\n id: toCleanString(row.id, \"id\"),\n type: normalizeType(row.type),\n title: toCleanString(row.title, \"title\"),\n episodes: toSafeInteger(row.episodes, \"episodes\"),\n season: toSafeInteger(row.season, \"season\"),\n };\n}\nfunction getClient() {\n if (cachedClient) return cachedClient;\n const url = readEnv(TURSO_DATABASE_URL_ENV);\n const authToken = Deno.env.get(TURSO_AUTH_TOKEN_ENV)?.trim();\n cachedClient = createClient({ url, authToken: authToken || undefined });\n return cachedClient;\n}\ntype UpstashRuntimeConfig = {\n redisUrl: string;\n redisToken: string;\n lastCommittedIdKey: string;\n};\nfunction resolveUpstashConfig(): UpstashRuntimeConfig {\n const urlEnv = toCleanString(\n UPSTASH_CONFIG?.UPSTASH_REDIS_REST_URL_ENV,\n \"UPSTASH_REDIS_REST_URL_ENV\",\n );\n const tokenEnv = toCleanString(\n UPSTASH_CONFIG?.UPSTASH_REDIS_REST_TOKEN_ENV,\n \"UPSTASH_REDIS_REST_TOKEN_ENV\",\n );\n const lastCommittedIdKey = toCleanString(\n UPSTASH_CONFIG?.LAST_COMMITTED_ID_KEY,\n \"LAST_COMMITTED_ID_KEY\",\n );\n const redisUrl = Deno.env.get(urlEnv)?.trim();\n const redisToken = Deno.env.get(tokenEnv)?.trim();\n if (!redisUrl) {\n throw new Error(`Missing Upstash Redis URL in env var: ${urlEnv}`);\n }\n if (!redisToken) {\n throw new Error(`Missing Upstash Redis token in env var: ${tokenEnv}`);\n }\n return {\n redisUrl,\n redisToken,\n lastCommittedIdKey,\n };\n}\nfunction createUpstashClient() {\n const { redisUrl, redisToken } = resolveUpstashConfig();\n return new Redis({\n url: redisUrl,\n token: redisToken,\n });\n}\nasync function readLastCommittedId(\n redis: Redis,\n key: string,\n): Promise {\n const value = await redis.get(key);\n if (typeof value !== \"string\") return null;\n const trimmed = value.trim();\n return trimmed.length > 0 ? trimmed : null;\n}\nasync function executeWithRetry(operation: () => Promise): Promise {\n let lastError: unknown;\n for (let attempt = 1; attempt <= MAX_RETRIES; attempt += 1) {\n try {\n return await operation();\n } catch (err) {\n lastError = err;\n const shouldRetry = attempt < MAX_RETRIES && isTransientError(err);\n if (!shouldRetry) break;\n const backoff = BASE_BACKOFF_MS * 2 ** (attempt - 1);\n console.warn(\n `[sqlfetchstep] transient error on attempt ${attempt}; retrying in ${backoff}ms`,\n err,\n );\n await sleep(backoff);\n }\n }\n throw lastError instanceof Error ? lastError : new Error(String(lastError));\n}\nasync function withTimeout(\n promise: Promise,\n timeoutMs: number,\n): Promise {\n let timeoutId: ReturnType | undefined;\n const timeout = new Promise((_, reject) => {\n timeoutId = setTimeout(() => {\n reject(new Error(`Query timed out after ${timeoutMs}ms`));\n }, timeoutMs);\n });\n try {\n return await Promise.race([promise, timeout]);\n } finally {\n if (timeoutId !== undefined) clearTimeout(timeoutId);\n }\n}\nexport async function fetchAnimeInfo(\n limit = MAX_LIMIT,\n afterId: string | null = null,\n): Promise {\n if (!Number.isFinite(limit) || limit <= 0) {\n return {\n ok: false,\n error: \"Invalid limit\",\n details: `Received: ${String(limit)}`,\n };\n }\n const client = getClient();\n const cursorId = afterId?.trim() ?? \"\";\n const hasCursor = cursorId.length > 0;\n const sql = `\n SELECT id, type, title, episodes, season\n FROM ${TABLE_NAME}\n WHERE id IS NOT NULL\n AND type IS NOT NULL\n AND title IS NOT NULL\n AND episodes IS NOT NULL\n AND season IS NOT NULL\n ${hasCursor ? \"AND id > ?\" : \"\"}\n ORDER BY id ASC\n LIMIT ?\n `;\n const args = hasCursor\n ? [cursorId, Math.min(limit, MAX_LIMIT)]\n : [Math.min(limit, MAX_LIMIT)];\n try {\n const result = await executeWithRetry(() =>\n withTimeout(\n client.execute({ sql, args }),\n QUERY_TIMEOUT_MS,\n )\n );\n const data: AnimeInfoRow[] = [];\n const skipped: string[] = [];\n for (const rawRow of result.rows as unknown as Record[]) {\n try {\n data.push(normalizeRow(rawRow));\n } catch (rowErr) {\n const rawId = rawRow?.id;\n const rowId = typeof rawId === \"string\"\n ? rawId.trim()\n : String(rawId ?? \"\").trim();\n if (rowId) {\n skipped.push(rowId);\n console.warn(`[sqlfetchstep] skipped anime id=${rowId}`, rowErr);\n } else {\n console.warn(\n `[sqlfetchstep] skipped malformed row with no valid id`,\n rowErr,\n rawRow,\n );\n }\n }\n }\n return {\n ok: true,\n count: data.length,\n data,\n skipped,\n };\n } catch (err) {\n const message = err instanceof Error ? err.message : String(err);\n console.error(`[sqlfetchstep] database read failed`, err);\n return {\n ok: false,\n error: \"Failed to fetch anime info from Turso\",\n details: message,\n };\n }\n}\nexport async function fetchAnimeInfoPayload(\n limit = MAX_LIMIT,\n afterId: string | null = null,\n): Promise {\n const result = await fetchAnimeInfo(limit, afterId);\n if (!result.ok) {\n throw new Error(\n `${result.error}${result.details ? `: ${result.details}` : \"\"}`,\n );\n }\n return result.data;\n}\nexport default async function (): Promise {\n try {\n const { lastCommittedIdKey } = resolveUpstashConfig();\n const redis = createUpstashClient();\n const lastCommittedId = await readLastCommittedId(\n redis,\n lastCommittedIdKey,\n );\n const data = await fetchAnimeInfoPayload(MAX_LIMIT, lastCommittedId);\n applyAnimeState(data);\n for (const [index, row] of data.entries()) {\n console.log(\n `[sqlfetchstep] row ${index + 1}/${data.length}: ${\n JSON.stringify(row)\n }`,\n );\n }\n return data;\n } catch (err) {\n console.error(`[sqlfetchstep] fatal error`, err);\n return [];\n }\n}","deleted","label",[178,179,159],"Main","src","permalink","https://www.val.town/x/lupinarashi/Arashis-Global-Scraper/code/Main/src/fetchstepsql.tsx","extension","actionData","errors"]