[{"_1":2,"_183":-5,"_184":-5},"loaderData",{"_3":4,"_95":96},"root",{"_5":6,"_36":37,"_68":69},"ctx",{"_7":8,"_23":-5,"_24":25,"_26":27,"_28":20,"_29":20,"_30":20,"_31":-5,"_32":33,"_34":20,"_35":20},"auth",{"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20,"_21":22},"sessionClaims","sessionId","sessionStatus","userId","actor","orgId","orgRole","orgSlug","orgPermissions","factorVerificationAge","isAuthenticated",false,"tokenType","session_token","user","preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesChatPanelState","notificationCount",0,"flagDenols","isBusinessOrgMember","ENV",{"_38":39,"_40":41,"_42":43,"_44":45,"_46":47,"_48":49,"_50":51,"_52":53,"_54":55,"_56":57,"_58":59,"_60":61,"_62":63,"_64":65,"_66":67},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","VAL_RUNTIME_NAME","prod","RENDER_GIT_COMMIT","c82f8f7f511b45da3b6daf08ba8af1c21e3c5623","RENDER_GIT_BRANCH","main","CLERK_PUBLISHABLE_KEY","pk_live_Y2xlcmsudmFsLnRvd24k","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","API_URL","https://api.val.town","clerkState",{"_70":71},"__internal_clerk_state",{"_72":73,"_74":47,"_75":76,"_77":76,"_78":20,"_79":76,"_80":76,"_81":76,"_82":76,"_83":84,"_93":20,"_94":20},"__clerk_ssr_state",{"_21":22,"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20},"__publishableKey","__proxyUrl","","__domain","__isSatellite","__signInUrl","__signUpUrl","__afterSignInUrl","__afterSignUpUrl","__clerk_debug",{"_85":20,"_19":20,"_86":76,"_87":88,"_89":76,"_90":47,"_91":20,"_92":76},"isSignedIn","proxyUrl","reason","session-token-and-uat-missing","message","publishableKey","isSatellite","domain","__telemetryDisabled","__telemetryDebug","routes/embed.x.$handle.$name.$",{"_97":98,"_99":100,"_101":102,"_103":104,"_156":157,"_180":181,"_182":-5},"handle","lupinarashi","name","Arashis-Global-Scraper","path","Main/src/extractionengine.tsx","project",{"_105":106,"_99":100,"_107":108,"_109":-5,"_110":20,"_111":112,"_113":-5,"_114":-5,"_12":115,"_116":-5,"_117":20,"_118":-5,"_119":-5,"_120":33,"_14":121,"_122":-5,"_123":-5,"_124":-5,"_125":126,"_127":126,"_128":129,"_130":129,"_131":129,"_132":126,"_133":126,"_23":134,"_137":138,"_142":143,"_155":20},"id","055b1d20-2d92-11f1-bd5a-42dde27851f2","createdAt",["D",1775024060352],"deletedAt","legacyEnvVars","privacy","public","description","imageUrl","019d34f5-a118-71aa-86be-a3b05ce7b06e","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount","019d34f5-a118-71aa-86be-a6dc6eeabba9","tursoName","tursoHostname","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm",{"_97":98,"_135":136},"profileImageUrl","https://img.clerk.com/eyJ0eXBlIjoiZGVmYXVsdCIsImlpZCI6Imluc18yTm10cGhQUklwU0tqZHNiOHNnNGd2ZjJURFUiLCJyaWQiOiJ1c2VyXzNCWndldUNFUURoY3B1Uzk1d2lQOVhDTE8xaCJ9","org",{"_97":98,"_139":136,"_12":115,"_140":141},"avatar","tier","free","branch",{"_144":145,"_99":45,"_105":146,"_147":106,"_148":-5,"_149":-5,"_150":-5,"_151":-5,"_152":-5,"_153":100,"_154":98},"version",58,"055bbfa0-2d92-11f1-bd5a-42dde27851f2","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","isOwned","file",{"_105":158,"_99":159,"_160":161,"_162":163,"_164":146,"_165":166,"_167":168,"_107":169,"_144":170,"_171":172,"_173":174,"_175":20,"_176":-5,"_101":177},"2dc49010-2d93-11f1-8432-42dde27851f2","extractionengine.tsx","fileId","0585d6b8-2d93-11f1-b6d4-42dde27851f2","parentId","1ea982e4-2d92-11f1-9d8c-42dde27851f2","branchId","branchVersion",31,"latest",true,["D",1775024557639],4,"fileType","http","content","import { type Browser, chromium, type Request } from \"npm:playwright-core\";\nimport { EXTRACTION_CONFIG } from \"../customize.js\";\ntype InputItem = string | { url: string };\ntype SeasonUrlBundle = {\n season: number;\n episodes: number;\n urls: string[];\n};\ntype SlugEngineBundle = {\n title: string;\n type: \"series\" | \"movies\";\n urls: string[] | SeasonUrlBundle[];\n};\ntype RawInputPayload = {\n season: number;\n episodes: number;\n urls: string[];\n};\ntype ResultItemInternal = {\n index: number;\n url: string;\n data: string;\n status: \"ok\" | \"timeout\" | \"error\" | \"invalid\";\n note?: string;\n matchedRequest?: string;\n};\ntype ResultItem = {\n index: number;\n data: string;\n status: \"ok\" | \"timeout\" | \"error\" | \"invalid\";\n};\ntype UrlTask = {\n index: number;\n url: string;\n};\nfunction toPublicResultItem(item: ResultItemInternal): ResultItem {\n return {\n index: item.index + 1,\n data: item.data,\n status: item.status,\n };\n}\nfunction sleep(ms: number) {\n return new Promise((resolve) => setTimeout(resolve, ms));\n}\nfunction asStringUrl(item: InputItem): string {\n if (typeof item === \"string\") return item.trim();\n return String(item?.url ?? \"\").trim();\n}\nfunction normalizeInput(input: unknown): string[] {\n const arr: InputItem[] = Array.isArray(input) ? (input as InputItem[]) : [];\n return arr.map(asStringUrl).filter(Boolean);\n}\nfunction extractDataValue(requestUrl: string): string | null {\n if (!/index\\.php/i.test(requestUrl) || !/[?&]data=/i.test(requestUrl)) {\n return null;\n }\n try {\n const u = new URL(requestUrl);\n const value = u.searchParams.get(\"data\");\n if (value) return value;\n } catch {\n }\n const match = requestUrl.match(/[?&]data=([^&#]+)/i);\n if (!match?.[1]) return null;\n try {\n return decodeURIComponent(match[1]);\n } catch {\n return match[1];\n }\n}\nfunction isLikelyHttpUrl(value: string): boolean {\n return /^https?:\\/\\//i.test(value);\n}\nfunction clampInt(value: number, min: number, max: number): number {\n const n = Number.isFinite(value) ? Math.trunc(value) : min;\n return Math.min(max, Math.max(min, n));\n}\nfunction getRemoteBrowserEndpoint(): string {\n return (\n Deno.env.get(\"STEEL_WS_ENDPOINT\")?.trim() ||\n Deno.env.get(\"PLAYWRIGHT_WS_ENDPOINT\")?.trim() ||\n Deno.env.get(\"BROWSER_WS_ENDPOINT\")?.trim() ||\n \"\"\n );\n}\nasync function readRawInput(): Promise {\n const mod = await import(\"./slugenginerun.tsx\");\n const bundles = (await mod.default()) as SlugEngineBundle[];\n const urls: string[] = [];\n let firstSeason = 1;\n for (const bundle of bundles) {\n if (!Array.isArray(bundle?.urls)) continue;\n for (const entry of bundle.urls as unknown[]) {\n if (typeof entry === \"string\") {\n const u = entry.trim();\n if (u) urls.push(u);\n continue;\n }\n if (\n entry &&\n typeof entry === \"object\" &&\n Array.isArray((entry as SeasonUrlBundle).urls)\n ) {\n const seasonBundle = entry as SeasonUrlBundle;\n if (urls.length === 0) {\n firstSeason = Number(seasonBundle.season) || 1;\n }\n urls.push(...seasonBundle.urls.filter(Boolean));\n }\n }\n }\n return {\n season: firstSeason,\n episodes: urls.length,\n urls,\n };\n}\nfunction getPoolSettings(totalUrls: number) {\n const cfg = EXTRACTION_CONFIG as {\n browserCount?: number;\n tabsPerBrowser?: number;\n maxBrowserCount?: number;\n maxTabsPerBrowser?: number;\n };\n const maxBrowserCount = clampInt(Number(cfg.maxBrowserCount ?? 6), 1, 6);\n const maxTabsPerBrowser = clampInt(\n Number(cfg.maxTabsPerBrowser ?? 15),\n 1,\n 15,\n );\n const requestedBrowserCount = Number(cfg.browserCount ?? NaN);\n const requestedTabsPerBrowser = Number(cfg.tabsPerBrowser ?? NaN);\n const browserCount =\n Number.isFinite(requestedBrowserCount) && requestedBrowserCount > 0\n ? clampInt(requestedBrowserCount, 1, maxBrowserCount)\n : clampInt(Math.ceil(totalUrls / 14), 1, maxBrowserCount);\n const tabsPerBrowser =\n Number.isFinite(requestedTabsPerBrowser) && requestedTabsPerBrowser > 0\n ? clampInt(requestedTabsPerBrowser, 1, maxTabsPerBrowser)\n : clampInt(Math.ceil(totalUrls / browserCount), 1, maxTabsPerBrowser);\n return {\n browserCount,\n tabsPerBrowser,\n };\n}\nasync function connectBrowser(): Promise {\n const endpoint = getRemoteBrowserEndpoint();\n if (!endpoint) {\n throw new Error(\n \"Missing remote browser endpoint. Set STEEL_WS_ENDPOINT or PLAYWRIGHT_WS_ENDPOINT.\",\n );\n }\n let lastErr: unknown = null;\n for (\n let attempt = 0; attempt <= EXTRACTION_CONFIG.maxRetryConnect; attempt++\n ) {\n try {\n return await chromium.connect(endpoint, {\n timeout: EXTRACTION_CONFIG.connectTimeoutMs,\n });\n } catch (err) {\n lastErr = err;\n try {\n return await chromium.connectOverCDP(endpoint, {\n timeout: EXTRACTION_CONFIG.connectTimeoutMs,\n });\n } catch (err2) {\n lastErr = err2;\n if (attempt < EXTRACTION_CONFIG.maxRetryConnect) {\n await sleep(400 * (attempt + 1));\n continue;\n }\n }\n }\n }\n throw new Error(\n `Unable to connect to remote browser: ${\n String((lastErr as Error)?.message ?? lastErr)\n }`,\n );\n}\nasync function connectBrowserPool(desiredCount: number): Promise {\n const browsers: Browser[] = [];\n for (let i = 0; i < desiredCount; i++) {\n if (i > 0) await sleep(150);\n try {\n const browser = await connectBrowser();\n browsers.push(browser);\n } catch {\n }\n }\n if (browsers.length === 0) {\n throw new Error(\"Unable to connect any browser session.\");\n }\n return browsers;\n}\nfunction distributeRoundRobin(\n urls: string[],\n browserCount: number,\n): UrlTask[][] {\n const shards: UrlTask[][] = Array.from({ length: browserCount }, () => []);\n urls.forEach((url, index) => {\n shards[index % browserCount].push({ url, index });\n });\n return shards;\n}\nasync function inspectOneUrl(\n browser: Browser,\n url: string,\n index: number,\n): Promise {\n if (!isLikelyHttpUrl(url)) {\n return {\n index,\n url,\n data: \"URL NOT FOUND\",\n status: \"invalid\",\n note: \"Invalid URL format\",\n };\n }\n const context = await browser.newContext({\n viewport: EXTRACTION_CONFIG.viewport,\n ignoreHTTPSErrors: true,\n serviceWorkers: \"block\",\n });\n const page = await context.newPage();\n let resolved = false;\n let matchedRequest = \"\";\n let foundData: string | null = null;\n const onRequest = (req: Request) => {\n if (resolved) return;\n const reqUrl = req.url();\n const data = extractDataValue(reqUrl);\n if (data !== null) {\n matchedRequest = reqUrl;\n foundData = data;\n resolved = true;\n }\n };\n page.on(\"request\", onRequest);\n try {\n await page.goto(url, {\n waitUntil: \"domcontentloaded\",\n timeout: EXTRACTION_CONFIG.gotoTimeoutMs,\n }).catch(() => null);\n const deadline = Date.now() + EXTRACTION_CONFIG.perUrlWindowMs;\n while (Date.now() < deadline && !resolved) {\n await sleep(50);\n }\n if (foundData !== null) {\n return {\n index,\n url,\n data: foundData,\n status: \"ok\",\n matchedRequest,\n };\n }\n return {\n index,\n url,\n data: \"URL NOT FOUND\",\n status: \"timeout\",\n note: \"No matching index.php?data=... request within 3 seconds\",\n };\n } catch (err) {\n return {\n index,\n url,\n data: \"URL NOT FOUND\",\n status: \"error\",\n note: String((err as Error)?.message ?? err),\n };\n } finally {\n page.off(\"request\", onRequest);\n await page.close({ runBeforeUnload: false }).catch(() => {});\n await context.close().catch(() => {});\n }\n}\nasync function processShard(\n browser: Browser,\n shard: UrlTask[],\n tabsPerBrowser: number,\n): Promise {\n const results: ResultItem[] = [];\n for (let i = 0; i < shard.length; i += tabsPerBrowser) {\n const wave = shard.slice(i, i + tabsPerBrowser);\n const waveResults = await Promise.all(\n wave.map((task) => inspectOneUrl(browser, task.url, task.index)),\n );\n results.push(...waveResults);\n }\n return results;\n}\nexport default async function extractionEngine() {\n const urls = await readRawInput();\n const payload: {\n generatedAt: string;\n total: number;\n items: ResultItem[];\n } = {\n generatedAt: new Date().toISOString(),\n total: urls.length,\n items: [],\n };\n if (urls.length === 0) {\n const outputPath = Deno.env.get(\"OUTPUT_PATH\")?.trim();\n if (outputPath) {\n await Deno.writeTextFile(outputPath, JSON.stringify(payload, null, 2));\n }\n return payload;\n }\n const { browserCount, tabsPerBrowser } = getPoolSettings(urls.length);\n const browsers = await connectBrowserPool(browserCount);\n try {\n const shards = distributeRoundRobin(urls, browsers.length);\n const shardResults = await Promise.all(\n browsers.map((browser, i) =>\n processShard(browser, shards[i] ?? [], tabsPerBrowser)\n ),\n );\n const ordered = shardResults\n .flat()\n .sort((a, b) => a.index - b.index)\n .map(toPublicResultItem);\n payload.items = ordered;\n const outputPath = Deno.env.get(\"OUTPUT_PATH\")?.trim();\n if (outputPath) {\n await Deno.writeTextFile(outputPath, JSON.stringify(payload, null, 2));\n }\n return payload;\n } finally {\n await Promise.all(\n browsers.map((browser) => browser.close().catch(() => {})),\n );\n }\n}","deleted","label",[178,179,159],"Main","src","permalink","https://www.val.town/x/lupinarashi/Arashis-Global-Scraper/code/Main/src/extractionengine.tsx","extension","actionData","errors"]