[{"_1":2,"_182":-5,"_183":-5},"loaderData",{"_3":4,"_95":96},"root",{"_5":6,"_36":37,"_68":69},"ctx",{"_7":8,"_23":-5,"_24":25,"_26":27,"_28":20,"_29":20,"_30":20,"_31":-5,"_32":33,"_34":20,"_35":20},"auth",{"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20,"_21":22},"sessionClaims","sessionId","sessionStatus","userId","actor","orgId","orgRole","orgSlug","orgPermissions","factorVerificationAge","isAuthenticated",false,"tokenType","session_token","user","preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesChatPanelState","notificationCount",0,"flagDenols","isBusinessOrgMember","ENV",{"_38":39,"_40":41,"_42":43,"_44":45,"_46":47,"_48":49,"_50":51,"_52":53,"_54":55,"_56":57,"_58":59,"_60":61,"_62":63,"_64":65,"_66":67},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","VAL_RUNTIME_NAME","prod","RENDER_GIT_COMMIT","c82f8f7f511b45da3b6daf08ba8af1c21e3c5623","RENDER_GIT_BRANCH","main","CLERK_PUBLISHABLE_KEY","pk_live_Y2xlcmsudmFsLnRvd24k","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","API_URL","https://api.val.town","clerkState",{"_70":71},"__internal_clerk_state",{"_72":73,"_74":47,"_75":76,"_77":76,"_78":20,"_79":76,"_80":76,"_81":76,"_82":76,"_83":84,"_93":20,"_94":20},"__clerk_ssr_state",{"_21":22,"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20},"__publishableKey","__proxyUrl","","__domain","__isSatellite","__signInUrl","__signUpUrl","__afterSignInUrl","__afterSignUpUrl","__clerk_debug",{"_85":20,"_19":20,"_86":76,"_87":88,"_89":76,"_90":47,"_91":20,"_92":76},"isSignedIn","proxyUrl","reason","session-token-and-uat-missing","message","publishableKey","isSatellite","domain","__telemetryDisabled","__telemetryDebug","routes/embed.x.$handle.$name.$",{"_97":98,"_99":100,"_101":102,"_103":104,"_156":157,"_179":180,"_181":-5},"handle","lupinarashi","name","Arashis-Global-Scraper","path","Main/src/committer.tsx","project",{"_105":106,"_99":100,"_107":108,"_109":-5,"_110":20,"_111":112,"_113":-5,"_114":-5,"_12":115,"_116":-5,"_117":20,"_118":-5,"_119":-5,"_120":33,"_14":121,"_122":-5,"_123":-5,"_124":-5,"_125":126,"_127":126,"_128":129,"_130":129,"_131":129,"_132":126,"_133":126,"_23":134,"_137":138,"_142":143,"_155":20},"id","055b1d20-2d92-11f1-bd5a-42dde27851f2","createdAt",["D",1775024060352],"deletedAt","legacyEnvVars","privacy","public","description","imageUrl","019d34f5-a118-71aa-86be-a3b05ce7b06e","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount","019d34f5-a118-71aa-86be-a6dc6eeabba9","tursoName","tursoHostname","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm",{"_97":98,"_135":136},"profileImageUrl","https://img.clerk.com/eyJ0eXBlIjoiZGVmYXVsdCIsImlpZCI6Imluc18yTm10cGhQUklwU0tqZHNiOHNnNGd2ZjJURFUiLCJyaWQiOiJ1c2VyXzNCWndldUNFUURoY3B1Uzk1d2lQOVhDTE8xaCJ9","org",{"_97":98,"_139":136,"_12":115,"_140":141},"avatar","tier","free","branch",{"_144":145,"_99":45,"_105":146,"_147":106,"_148":-5,"_149":-5,"_150":-5,"_151":-5,"_152":-5,"_153":100,"_154":98},"version",58,"055bbfa0-2d92-11f1-bd5a-42dde27851f2","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","isOwned","file",{"_105":158,"_99":159,"_160":161,"_162":163,"_164":146,"_165":145,"_166":167,"_107":168,"_144":169,"_170":171,"_172":173,"_174":20,"_175":-5,"_101":176},"aa19379e-2d96-11f1-9bbc-42dde27851f2","committer.tsx","fileId","5ca8a44e-2d92-11f1-96ba-42dde27851f2","parentId","1ea982e4-2d92-11f1-9d8c-42dde27851f2","branchId","branchVersion","latest",true,["D",1775026054708],5,"fileType","http","content","import { createClient } from \"npm:@libsql/client/web\";\nimport { Redis } from \"npm:@upstash/redis\";\nimport * as source from \"./fetchstepsql.tsx\";\nimport extractionEngine from \"./extractionengine.tsx\";\nimport { SQL_CONFIG, UPSTASH_CONFIG } from \"../customize.js\";\ntype ExtractionStatus = \"ok\" | \"timeout\" | \"error\" | \"invalid\";\ntype ExtractionItem = {\n index: number;\n data: string;\n status: ExtractionStatus;\n};\ntype ExtractionPayload = {\n generatedAt: string;\n anime_id?: string;\n animeId?: string;\n season?: number;\n episodes?: number;\n items: ExtractionItem[];\n};\ntype AnimeJob = {\n id: string;\n season: number;\n episodes: number;\n};\ntype CommitRow = {\n anime_id: string;\n season: number;\n episode_number: number;\n episode_title: string | null;\n quality: string;\n language: string;\n server_name: string;\n stream_url: string;\n download_url: string | null;\n created_at: string;\n};\ntype CommitJobResult = {\n anime_id: string;\n season: number;\n episodes: number;\n totalFound: number;\n inserted: number;\n skipped: number;\n items: Array<{\n index: number;\n status: ExtractionStatus;\n data: string;\n }>;\n};\ntype CommitResult = {\n generatedAt: string;\n totalAnime: number;\n totalFound: number;\n inserted: number;\n skipped: number;\n results: CommitJobResult[];\n anime_id?: string;\n season?: number;\n episodes?: number;\n items?: Array<{\n index: number;\n status: ExtractionStatus;\n data: string;\n }>;\n};\nfunction asNonEmptyString(value: unknown): string {\n return String(value ?? \"\").trim();\n}\nfunction isLikelyHttpUrl(value: string): boolean {\n return /^https?:\\/\\//i.test(value.trim());\n}\nfunction clampInt(\n value: unknown,\n fallback: number,\n min = 1,\n max = Number.MAX_SAFE_INTEGER,\n): number {\n const n = Number(value);\n if (!Number.isFinite(n)) return fallback;\n const i = Math.trunc(n);\n return Math.min(max, Math.max(min, i));\n}\nfunction assertSqlIdentifier(name: string, label: string): string {\n const trimmed = name.trim();\n if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(trimmed)) {\n throw new Error(`Invalid ${label}: ${name}`);\n }\n return trimmed;\n}\nfunction assertKvPrefix(name: string, label: string): string {\n const trimmed = name.trim();\n if (!trimmed) {\n throw new Error(`Invalid ${label}: empty value`);\n }\n if (!/^[A-Za-z0-9:_-]+$/.test(trimmed)) {\n throw new Error(`Invalid ${label}: ${name}`);\n }\n return trimmed;\n}\nfunction resolveUpstashConfig() {\n const urlEnv = asNonEmptyString(UPSTASH_CONFIG?.UPSTASH_REDIS_REST_URL_ENV);\n const tokenEnv = asNonEmptyString(\n UPSTASH_CONFIG?.UPSTASH_REDIS_REST_TOKEN_ENV,\n );\n const prefixRaw = asNonEmptyString(UPSTASH_CONFIG?.PROGRESS_KEY_PREFIX);\n if (!urlEnv) {\n throw new Error(\"UPSTASH_CONFIG.UPSTASH_REDIS_REST_URL_ENV is missing.\");\n }\n if (!tokenEnv) {\n throw new Error(\"UPSTASH_CONFIG.UPSTASH_REDIS_REST_TOKEN_ENV is missing.\");\n }\n if (!prefixRaw) {\n throw new Error(\"UPSTASH_CONFIG.PROGRESS_KEY_PREFIX is missing.\");\n }\n const redisUrl = asNonEmptyString(Deno.env.get(urlEnv));\n const redisToken = asNonEmptyString(Deno.env.get(tokenEnv));\n const progressPrefix = assertKvPrefix(prefixRaw, \"PROGRESS_KEY_PREFIX\");\n if (!redisUrl) {\n throw new Error(`Missing Upstash Redis URL in env var: ${urlEnv}`);\n }\n if (!redisToken) {\n throw new Error(`Missing Upstash Redis token in env var: ${tokenEnv}`);\n }\n return {\n redisUrl,\n redisToken,\n progressPrefix,\n currentAnimeKey: `${progressPrefix}:current_anime_id`,\n allProgressPattern: `${progressPrefix}:*`,\n };\n}\nfunction createUpstashClient() {\n const { redisUrl, redisToken } = resolveUpstashConfig();\n return new Redis({\n url: redisUrl,\n token: redisToken,\n });\n}\nasync function clearProgressNamespace(\n redis: Redis,\n pattern: string,\n): Promise {\n let cursor = \"0\";\n let deleted = 0;\n do {\n const [nextCursor, keys] = await redis.scan(cursor, {\n match: pattern,\n count: 100,\n });\n if (keys.length > 0) {\n for (const group of chunk(keys, 1000)) {\n deleted += await redis.unlink(...group);\n }\n }\n cursor = nextCursor;\n } while (cursor !== \"0\");\n return deleted;\n}\nasync function saveCurrentAnimeProgress(\n redis: Redis,\n currentAnimeKey: string,\n allProgressPattern: string,\n animeId: string,\n) {\n await clearProgressNamespace(redis, allProgressPattern);\n await redis.set(currentAnimeKey, animeId);\n}\nfunction getIstTimestamp(now = new Date()): string {\n const ist = new Date(now.getTime() + 5.5 * 60 * 60 * 1000);\n const yyyy = ist.getUTCFullYear();\n const mm = String(ist.getUTCMonth() + 1).padStart(2, \"0\");\n const dd = String(ist.getUTCDate()).padStart(2, \"0\");\n const hh = String(ist.getUTCHours()).padStart(2, \"0\");\n const mi = String(ist.getUTCMinutes()).padStart(2, \"0\");\n const ss = String(ist.getUTCSeconds()).padStart(2, \"0\");\n return `${yyyy}-${mm}-${dd} ${hh}:${mi}:${ss}`;\n}\nfunction isRecord(value: unknown): value is Record {\n return typeof value === \"object\" && value !== null;\n}\nfunction normalizePayload(raw: unknown): ExtractionPayload {\n const payload = isRecord(raw) ? raw : {};\n const itemsRaw = Array.isArray(payload.items) ? payload.items : [];\n const items = itemsRaw.map((item, i) => {\n const rec = isRecord(item) ? item : {};\n const status = rec.status === \"ok\" ||\n rec.status === \"timeout\" ||\n rec.status === \"error\" ||\n rec.status === \"invalid\"\n ? rec.status\n : \"invalid\";\n return {\n index: clampInt(rec.index ?? i + 1, i + 1, 1),\n data: asNonEmptyString(rec.data),\n status,\n };\n });\n return {\n generatedAt: asNonEmptyString(payload.generatedAt) ||\n new Date().toISOString(),\n anime_id: asNonEmptyString(payload.anime_id) || undefined,\n animeId: asNonEmptyString(payload.animeId) || undefined,\n season: Number.isFinite(Number(payload.season))\n ? Math.trunc(Number(payload.season))\n : undefined,\n episodes: Number.isFinite(Number(payload.episodes))\n ? Math.trunc(Number(payload.episodes))\n : undefined,\n items,\n };\n}\nfunction resolveSqlConfig() {\n const databaseUrlEnv = asNonEmptyString(SQL_CONFIG?.TURSO_DATABASE_URL_ENV);\n const authTokenEnv = asNonEmptyString(SQL_CONFIG?.TURSO_AUTH_TOKEN_ENV);\n const tableNameRaw = asNonEmptyString(SQL_CONFIG?.STREAM_TABLE_NAME);\n if (!databaseUrlEnv) {\n throw new Error(\"SQL_CONFIG.TURSO_DATABASE_URL_ENV is missing.\");\n }\n if (!authTokenEnv) {\n throw new Error(\"SQL_CONFIG.TURSO_AUTH_TOKEN_ENV is missing.\");\n }\n if (!tableNameRaw) {\n throw new Error(\"SQL_CONFIG.STREAM_TABLE_NAME is missing.\");\n }\n const databaseUrl = asNonEmptyString(Deno.env.get(databaseUrlEnv));\n const authToken = asNonEmptyString(Deno.env.get(authTokenEnv));\n const tableName = assertSqlIdentifier(tableNameRaw, \"STREAM_TABLE_NAME\");\n if (!databaseUrl) {\n throw new Error(`Missing Turso database URL in env var: ${databaseUrlEnv}`);\n }\n if (!authToken) {\n throw new Error(`Missing Turso auth token in env var: ${authTokenEnv}`);\n }\n return { databaseUrl, authToken, tableName };\n}\nfunction normalizeJob(raw: unknown, fallbackIndex: number): AnimeJob {\n const rec = isRecord(raw) ? raw : {};\n const id = asNonEmptyString(rec.id ?? rec.anime_id ?? rec.animeId);\n if (!id) {\n throw new Error(`Missing anime id at batch index ${fallbackIndex}.`);\n }\n return {\n id,\n season: clampInt(rec.season, 1, 1),\n episodes: clampInt(rec.episodes, 1, 1),\n };\n}\nfunction getSourceJobs(): AnimeJob[] {\n const src = source as Record;\n const explicitJobs = src.jobs ?? src.batch ?? src.queue;\n if (Array.isArray(explicitJobs) && explicitJobs.length > 0) {\n return explicitJobs.map((job, i) => normalizeJob(job, i));\n }\n const ids = src.ids;\n if (Array.isArray(ids) && ids.length > 0) {\n const seasons = Array.isArray(src.seasons) ? src.seasons : [];\n const episodes = Array.isArray(src.episodesList)\n ? src.episodesList\n : Array.isArray(src.episodeCounts)\n ? src.episodeCounts\n : [];\n return ids.map((id, i) =>\n normalizeJob(\n {\n id,\n season: seasons[i] ?? src.season ?? 1,\n episodes: episodes[i] ?? src.episodes ?? 1,\n },\n i,\n )\n );\n }\n const singleId = asNonEmptyString(src.id ?? src.anime_id ?? src.animeId);\n if (!singleId) {\n throw new Error(\n \"fetchstepsql.tsx did not provide a valid anime id or jobs array.\",\n );\n }\n return [\n normalizeJob(\n {\n id: singleId,\n season: src.season ?? 1,\n episodes: src.episodes ?? 1,\n },\n 0,\n ),\n ];\n}\nfunction toCommitRows(\n payload: ExtractionPayload,\n animeId: string,\n season: number,\n): { rows: CommitRow[]; skipped: number } {\n const ordered = [...payload.items].sort((a, b) => a.index - b.index);\n const rows: CommitRow[] = [];\n let skipped = 0;\n for (let i = 0; i < ordered.length; i++) {\n const item = ordered[i];\n const episodeNumber = clampInt(\n item.index > 0 ? item.index : i + 1,\n i + 1,\n 1,\n );\n if (item.status !== \"ok\" || !isLikelyHttpUrl(item.data)) {\n skipped++;\n continue;\n }\n rows.push({\n anime_id: animeId,\n season,\n episode_number: episodeNumber,\n episode_title: null,\n quality: \"1080p\",\n language: \"Hindi / English / Japanese\",\n server_name: \"Aurora-X\",\n stream_url: item.data.trim(),\n download_url: null,\n created_at: getIstTimestamp(),\n });\n }\n return { rows, skipped };\n}\nasync function ensureSchema(\n client: ReturnType,\n tableName: string,\n) {\n await client.execute(`\n CREATE UNIQUE INDEX IF NOT EXISTS idx_${tableName}_anime_season_episode\n ON ${tableName} (anime_id, season, episode_number)\n `);\n}\nfunction chunk(items: T[], size = 50): T[][] {\n const out: T[][] = [];\n for (let i = 0; i < items.length; i += size) {\n out.push(items.slice(i, i + size));\n }\n return out;\n}\nasync function upsertRows(\n client: ReturnType,\n tableName: string,\n rows: CommitRow[],\n) {\n if (rows.length === 0) return;\n const statements = rows.map((row) => ({\n sql: `\n INSERT INTO ${tableName} (\n anime_id,\n season,\n episode_number,\n episode_title,\n quality,\n language,\n server_name,\n stream_url,\n download_url,\n created_at\n )\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n ON CONFLICT(anime_id, season, episode_number) DO UPDATE SET\n episode_title = excluded.episode_title,\n quality = excluded.quality,\n language = excluded.language,\n server_name = excluded.server_name,\n stream_url = excluded.stream_url,\n download_url = excluded.download_url,\n created_at = excluded.created_at\n `,\n args: [\n row.anime_id,\n row.season,\n row.episode_number,\n row.episode_title,\n row.quality,\n row.language,\n row.server_name,\n row.stream_url,\n row.download_url,\n row.created_at,\n ],\n }));\n for (const group of chunk(statements, 50)) {\n await client.batch(group);\n }\n}\nasync function runJob(\n client: ReturnType,\n tableName: string,\n job: AnimeJob,\n progressRedis: Redis,\n progressKey: string,\n progressPattern: string,\n): Promise {\n const rawExtraction =\n await (extractionEngine as (job?: AnimeJob) => Promise)(job);\n const payloadsRaw = Array.isArray(rawExtraction)\n ? rawExtraction\n : isRecord(rawExtraction) && Array.isArray(rawExtraction.results)\n ? rawExtraction.results\n : [rawExtraction];\n const payloads = payloadsRaw.map((entry) => normalizePayload(entry));\n let inserted = 0;\n let skipped = 0;\n let totalFound = 0;\n const collectedItems: CommitJobResult[\"items\"] = [];\n for (const payload of payloads) {\n const animeId = asNonEmptyString(payload.anime_id ?? payload.animeId) ||\n job.id;\n const season = clampInt(payload.season ?? job.season, job.season, 1);\n const { rows, skipped: localSkipped } = toCommitRows(\n payload,\n animeId,\n season,\n );\n await upsertRows(client, tableName, rows);\n inserted += rows.length;\n skipped += localSkipped;\n totalFound += payload.items.length;\n collectedItems.push(\n ...payload.items.map((item) => ({\n index: item.index,\n status: item.status,\n data: item.data,\n })),\n );\n }\n try {\n await saveCurrentAnimeProgress(\n progressRedis,\n progressKey,\n progressPattern,\n job.id,\n );\n } catch (error) {\n console.error(\"Upstash progress update failed after SQL commit:\", {\n animeId: job.id,\n error,\n });\n }\n return {\n anime_id: job.id,\n season: job.season,\n episodes: job.episodes,\n totalFound,\n inserted,\n skipped,\n items: collectedItems,\n };\n}\nexport default async function committer(): Promise {\n const jobs = getSourceJobs();\n const { databaseUrl, authToken, tableName } = resolveSqlConfig();\n const progressRedis = createUpstashClient();\n const { currentAnimeKey, allProgressPattern } = resolveUpstashConfig();\n const client = createClient({\n url: databaseUrl,\n authToken,\n });\n try {\n await ensureSchema(client, tableName);\n const results: CommitJobResult[] = [];\n for (const job of jobs) {\n results.push(\n await runJob(\n client,\n tableName,\n job,\n progressRedis,\n currentAnimeKey,\n allProgressPattern,\n ),\n );\n }\n const aggregate = results.reduce(\n (acc, result) => {\n acc.totalFound += result.totalFound;\n acc.inserted += result.inserted;\n acc.skipped += result.skipped;\n return acc;\n },\n { totalFound: 0, inserted: 0, skipped: 0 },\n );\n const first = results[0];\n return {\n generatedAt: new Date().toISOString(),\n totalAnime: results.length,\n totalFound: aggregate.totalFound,\n inserted: aggregate.inserted,\n skipped: aggregate.skipped,\n results,\n anime_id: first?.anime_id,\n season: first?.season,\n episodes: first?.episodes,\n items: first?.items,\n };\n } finally {\n client.close();\n }\n}","deleted","label",[177,178,159],"Main","src","permalink","https://www.val.town/x/lupinarashi/Arashis-Global-Scraper/code/Main/src/committer.tsx","extension","actionData","errors"]