[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_157":158,"_159":160},"routes/_app.x.$handle.$name",{"_95":94,"_32":44,"_103":85,"_27":53,"_13":14,"_104":-5,"_11":105,"_106":107,"_108":109,"_110":111,"_112":113,"_20":-5,"_114":-5,"_115":-5,"_116":48,"_117":118,"_119":48,"_120":-5,"_121":-5,"_122":123},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_42":43,"_32":44,"_37":45,"_46":-5,"_47":48,"_49":50,"_51":50,"_52":53,"_54":-5,"_18":55,"_56":-5,"_57":48,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":78,"_79":80,"_81":82,"_83":48},"version",33,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_33":34,"_35":36,"_37":38,"_39":14,"_40":41,"_24":25},"previous",{"_30":31,"_32":28,"_33":34,"_35":36},"deleted",true,"deletedHere","path","worker.ts","latestPath","fileId","c6c27542-7e8e-11f1-b6e6-1607ee4eb77e","name","fileType","http","content","import { createClient } from \"npm:@libsql/client/web\";\nimport { Redis } from \"npm:@upstash/redis\";\n\nimport {\n OFFSET_KEY,\n PLACEHOLDER_BANNER,\n SOURCE_URL,\n TURSO_AUTH_TOKEN,\n TURSO_DATABASE_URL,\n UPSTASH_REDIS_REST_TOKEN,\n UPSTASH_REDIS_REST_URL,\n} from \"./customize.js\";\n\nconst PAGE_SIZE = 32;\nconst BATCHES_PER_INVOCATION = 5;\nconst REQUEST_TIMEOUT_MS = 20_000;\nconst USER_AGENT =\n \"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0\";\n\ntype MangaRow = {\n key: string;\n title: string;\n type: string | null;\n year: number | null;\n banner_1: string;\n banner_2: string | null;\n author: string | null;\n tags: string | null;\n status: string | null;\n};\n\ntype RunItem = {\n offset: number;\n ok: boolean;\n fetched: number;\n stored: number;\n key?: string;\n error?: string;\n};\n\nconst sql = createClient({\n url: TURSO_DATABASE_URL,\n authToken: TURSO_AUTH_TOKEN,\n});\n\nconst redis = new Redis({\n url: UPSTASH_REDIS_REST_URL,\n token: UPSTASH_REDIS_REST_TOKEN,\n});\n\nconst UPSERT_SQL = `\nINSERT INTO neon_manga_library (\n key,\n title,\n type,\n year,\n banner_1,\n banner_2,\n author,\n tags,\n status\n) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)\nON CONFLICT(key) DO UPDATE SET\n title = excluded.title,\n type = excluded.type,\n year = excluded.year,\n banner_1 = excluded.banner_1,\n banner_2 = excluded.banner_2,\n author = excluded.author,\n tags = excluded.tags,\n status = excluded.status\n`;\n\nfunction jsonResponse(body: unknown, init: ResponseInit = {}): Response {\n const headers = new Headers(init.headers);\n headers.set(\"content-type\", \"application/json; charset=utf-8\");\n headers.set(\"access-control-allow-origin\", \"*\");\n headers.set(\"access-control-allow-methods\", \"GET,OPTIONS\");\n headers.set(\"access-control-allow-headers\", \"content-type, authorization\");\n return new Response(JSON.stringify(body, null, 2), { ...init, headers });\n}\n\nfunction normalizeText(value: unknown): string {\n if (typeof value !== \"string\") return \"\";\n return value.replace(/\\s+/g, \" \").trim();\n}\n\nfunction normalizeOptionalText(value: unknown): string | null {\n const text = normalizeText(value);\n return text.length ? text : null;\n}\n\nfunction absoluteUrl(raw: string | null | undefined): string | null {\n if (!raw) return null;\n const value = raw.trim();\n if (!value) return null;\n if (value.startsWith(\"data:\")) return null;\n if (value.startsWith(\"//\")) return `https:${value}`;\n try {\n return new URL(value, SOURCE_URL).toString();\n } catch {\n return null;\n }\n}\n\nfunction parseSrcset(srcset: string | null | undefined): string[] {\n if (!srcset) return [];\n return srcset\n .split(\",\")\n .map((part) => part.trim().split(/\\s+/)[0] ?? \"\")\n .map((part) => part.trim())\n .filter(Boolean)\n .map((part) => absoluteUrl(part))\n .filter((part): part is string => Boolean(part));\n}\n\nfunction uniqueStrings(values: Array): string[] {\n const seen = new Set();\n const out: string[] = [];\n\n for (const value of values) {\n const normalized = normalizeText(value);\n if (!normalized || seen.has(normalized.toLowerCase())) continue;\n seen.add(normalized.toLowerCase());\n out.push(normalized);\n }\n\n return out;\n}\n\nfunction commaJoin(values: Array): string | null {\n const items = uniqueStrings(values);\n return items.length ? items.join(\", \") : null;\n}\n\nfunction extractLinkId(href: string | null | undefined): string | null {\n if (!href) return null;\n const match = href.match(/\\/series\\/([A-Za-z0-9_-]+)\\//);\n return match?.[1] ?? null;\n}\n\nfunction firstTextByLabels(article: Element, labels: string[]): string | null {\n const strongs = Array.from(article.querySelectorAll(\"strong\"));\n for (const label of labels) {\n const strong = strongs.find((node) => normalizeText(node.textContent).toLowerCase().includes(label.toLowerCase()));\n if (!strong) continue;\n const parent = strong.parentElement;\n if (!parent) continue;\n const spans = Array.from(parent.querySelectorAll(\"span\"));\n const text = spans[0]?.textContent ?? null;\n const normalized = normalizeOptionalText(text);\n if (normalized) return normalized;\n }\n return null;\n}\n\nfunction extractMultiTextByLabel(article: Element, labels: string[]): string[] {\n const strongs = Array.from(article.querySelectorAll(\"strong\"));\n for (const label of labels) {\n const strong = strongs.find((node) => normalizeText(node.textContent).toLowerCase().includes(label.toLowerCase()));\n if (!strong) continue;\n const parent = strong.parentElement;\n if (!parent) continue;\n const spans = Array.from(parent.querySelectorAll(\"span\"));\n return spans\n .map((span) => normalizeOptionalText(span.textContent))\n .filter((value): value is string => Boolean(value));\n }\n return [];\n}\n\nfunction extractKey(article: Element): string | null {\n const firstAnchor = article.querySelector(\"a[href]\");\n const href = firstAnchor?.getAttribute(\"href\") ?? \"\";\n const key = extractLinkId(href);\n if (key) return key;\n\n const altAnchor = Array.from(article.querySelectorAll(\"a[href]\"))\n .map((a) => extractLinkId(a.getAttribute(\"href\")))\n .find((value): value is string => Boolean(value));\n\n return altAnchor ?? null;\n}\n\nfunction extractTitle(article: Element): string | null {\n const candidates = [\n \".text-ellipsis.truncate\",\n \"a.line-clamp-1\",\n \"[data-title]\",\n \"h1\",\n \"h2\",\n \"h3\",\n ];\n\n for (const selector of candidates) {\n const node = article.querySelector(selector);\n const text = normalizeOptionalText(node?.textContent);\n if (text) return text;\n }\n\n const fallback = normalizeOptionalText(article.querySelector(\"a[href]\")?.textContent);\n return fallback;\n}\n\nfunction extractBannerUrls(article: Element): { banner1: string; banner2: string | null } {\n const sourceUrls = Array.from(article.querySelectorAll(\"source\"))\n .flatMap((source) => parseSrcset(source.getAttribute(\"srcset\")));\n const imgUrls = Array.from(article.querySelectorAll(\"img\"))\n .map((img) => absoluteUrl(img.getAttribute(\"src\")))\n .filter((value): value is string => Boolean(value));\n\n const all = [...sourceUrls, ...imgUrls];\n const deduped = uniqueStrings(all);\n\n const banner1 = deduped[0] ?? PLACEHOLDER_BANNER;\n const banner2 = deduped.find((value) => value !== banner1) ?? null;\n\n return { banner1, banner2 };\n}\n\nfunction extractRow(article: Element): MangaRow | null {\n const key = extractKey(article);\n const title = extractTitle(article);\n if (!key || !title) return null;\n\n const { banner1, banner2 } = extractBannerUrls(article);\n\n const type = firstTextByLabels(article, [\"Type\"]);\n const status = firstTextByLabels(article, [\"Status\"]);\n const yearText = firstTextByLabels(article, [\"Year\"]);\n const year = yearText ? Number.parseInt(yearText, 10) : null;\n\n const author = commaJoin(extractMultiTextByLabel(article, [\"Author\"]));\n const tags = commaJoin(extractMultiTextByLabel(article, [\"Tag\"]));\n\n return {\n key,\n title,\n type,\n year: Number.isFinite(year) ? year : null,\n banner_1: banner1,\n banner_2: banner2,\n author,\n tags,\n status,\n };\n}\n\nasync function fetchHtml(url: string): Promise {\n const controller = new AbortController();\n const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);\n\n try {\n const response = await fetch(url, {\n method: \"GET\",\n headers: {\n accept: \"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\",\n \"user-agent\": USER_AGENT,\n \"cache-control\": \"no-cache\",\n pragma: \"no-cache\",\n },\n signal: controller.signal,\n });\n\n if (!response.ok) {\n throw new Error(`Upstream HTTP ${response.status} ${response.statusText}`);\n }\n\n return await response.text();\n } finally {\n clearTimeout(timer);\n }\n}\n\nfunction buildSourceUrl(offset: number): string {\n const params = new URLSearchParams({\n limit: String(PAGE_SIZE),\n offset: String(offset),\n sort: \"Alphabet\",\n order: \"Ascending\",\n official: \"Any\",\n anime: \"Any\",\n adult: \"Any\",\n display_mode: \"Full Display\",\n });\n\n return `${SOURCE_URL}?${params.toString()}`;\n}\n\nasync function getStoredOffset(): Promise {\n const raw = await redis.get(OFFSET_KEY);\n if (typeof raw === \"number\" && Number.isFinite(raw)) return Math.max(0, Math.trunc(raw));\n if (typeof raw === \"string\") {\n const parsed = Number.parseInt(raw, 10);\n if (Number.isFinite(parsed) && parsed >= 0) return parsed;\n }\n return 0;\n}\n\nasync function setStoredOffset(offset: number): Promise {\n await redis.set(OFFSET_KEY, String(Math.max(0, Math.trunc(offset))));\n}\n\nasync function storeRows(rows: MangaRow[]): Promise {\n let stored = 0;\n\n for (const row of rows) {\n await sql.prepare(UPSERT_SQL).bind(\n row.key,\n row.title,\n row.type,\n row.year,\n row.banner_1,\n row.banner_2,\n row.author,\n row.tags,\n row.status,\n ).run();\n stored++;\n }\n\n return stored;\n}\n\nasync function runBatch(offset: number): Promise {\n const url = buildSourceUrl(offset);\n const html = await fetchHtml(url);\n const document = new DOMParser().parseFromString(html, \"text/html\");\n const articles = document?.querySelectorAll(\"article.bg-base-300\") ?? [];\n\n const rows: MangaRow[] = [];\n for (const article of Array.from(articles)) {\n const row = extractRow(article);\n if (row) rows.push(row);\n }\n\n const stored = rows.length ? await storeRows(rows) : 0;\n\n return {\n offset,\n ok: true,\n fetched: rows.length,\n stored,\n };\n}\n\nasync function handleFetchAndUpload(): Promise {\n const startOffset = await getStoredOffset();\n let currentOffset = startOffset;\n\n const runs: RunItem[] = [];\n\n for (let index = 0; index < BATCHES_PER_INVOCATION; index++) {\n try {\n const result = await runBatch(currentOffset);\n runs.push(result);\n } catch (error) {\n runs.push({\n offset: currentOffset,\n ok: false,\n fetched: 0,\n stored: 0,\n error: error instanceof Error ? error.message : String(error),\n });\n } finally {\n currentOffset += PAGE_SIZE;\n await setStoredOffset(currentOffset);\n }\n }\n\n return jsonResponse({\n ok: true,\n endpoint: \"GET /fetch-and-upload\",\n source: SOURCE_URL,\n page_size: PAGE_SIZE,\n batches_per_invocation: BATCHES_PER_INVOCATION,\n start_offset: startOffset,\n end_offset: currentOffset,\n saved_offset_key: OFFSET_KEY,\n runs,\n });\n}\n\nexport default {\n async fetch(request: Request): Promise {\n const url = new URL(request.url);\n const path = url.pathname.replace(/\\/+$/, \"\") || \"/\";\n const method = request.method.toUpperCase();\n\n if (method === \"OPTIONS\") {\n return new Response(null, {\n status: 204,\n headers: {\n \"access-control-allow-origin\": \"*\",\n \"access-control-allow-methods\": \"GET,OPTIONS\",\n \"access-control-allow-headers\": \"content-type, authorization\",\n \"access-control-max-age\": \"86400\",\n },\n });\n }\n\n if (method !== \"GET\" || path !== \"/fetch-and-upload\") {\n return jsonResponse(\n {\n ok: false,\n error: \"not found\",\n expected: \"GET /fetch-and-upload\",\n },\n { status: 404 },\n );\n }\n\n try {\n return await handleFetchAndUpload();\n } catch (error) {\n return jsonResponse(\n {\n ok: false,\n error: error instanceof Error ? error.message : String(error),\n },\n { status: 500 },\n );\n }\n },\n};","createdAt",["D",1783929374028],"branchVersion","branchId","c3b37b47-aac6-4dd9-a90b-e4098c3a71a1","id","96c7e756-e778-4d18-b447-3242cc1845e4","CROSS-MANGA-SCRAPER",["D",1783926729718],"deletedAt","legacyEnvVars",false,"privacy","public","httpPrivacy","description","","imageUrl","019d34f5-a118-71aa-86be-a3b05ce7b06e","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","019d34f5-a118-71aa-86be-a6dc6eeabba9","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_95":94,"_102":98},"org",{"_95":94,"_96":-5,"_97":98,"_18":55,"_99":100,"_101":48},"branch",{"_13":84,"_32":85,"_42":41,"_86":43,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":44,"_93":94},"isOwned",84,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","lupinarashi","handle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoiZGVmYXVsdCIsImlpZCI6Imluc18yTm10cGhQUklwU0tqZHNiOHNnNGd2ZjJURFUiLCJyaWQiOiJ1c2VyXzNCWndldUNFUURoY3B1Uzk1d2lQOVhDTE8xaCJ9","tier","free","canRestrictVals","profileImageUrl","branchName","adoptOrgId",{"_42":43,"_32":44,"_37":153,"_46":-5,"_47":48,"_49":50,"_51":50,"_52":53,"_54":-5,"_18":55,"_56":-5,"_57":48,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":154,"_79":155,"_81":156,"_83":48},"tree",[148],"files",[148],"counts",{"_137":61,"_138":139,"_140":61,"_141":142,"_143":144},"contents",[124],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_42":125,"_32":126,"_30":127,"_128":-5,"_40":41,"_39":129,"_130":48,"_37":131,"_13":132,"_33":133,"_35":134,"_24":48,"_135":-5,"_27":136},"19c70e1b-0905-4a04-9d30-cd82bd0459bb","customize.js","519e67de-7e8a-11f1-a7fd-1607ee4eb77e","parentId",8,"latest",["D",1783926834582],2,"script","export const SOURCE_URL = \"https://weebcentral.com/search/data\";\nexport const TURSO_DATABASE_URL = \"https://neon-manga-db-lupinarashi.aws-ap-south-1.turso.io\";\nexport const TURSO_AUTH_TOKEN = \"eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJhIjoicnciLCJpYXQiOjE3ODM5MjY0NDQsImlkIjoiMDE5ZjVhNGMtMWEwMS03NmM1LWEzZGItNmVjOTZhZTFkOWNjIiwia2lkIjoibTRxTWl4ZVRaM0xGelBSc3AyVnFYMTRQLVRKSllNOWVkUmxNQWllQ3c2TSIsInJpZCI6ImM1N2Q0NThjLWU3MmMtNGM4OC1iYmIxLTE2YjkyOWYzNzk3ZSJ9.tfdqmDSy2ndG9jGp3l4CLuJ_JRXkeZAEfoUKBRUPIgvEijXtfuzytZFGeXMjnS6fmNFN3ZQDaTLs7DRhZGwUCg\";\nexport const UPSTASH_REDIS_REST_URL = \"https://magical-giraffe-89191.upstash.io\";\nexport const UPSTASH_REDIS_REST_TOKEN = \"gQAAAAAAAVxnAAIncDJmY2MwNTdiZDAyNDQ0OTY3ODhkMjJjMWFkNDlkZGExMnAyODkxOTE\";\nexport const OFFSET_KEY = \"neon_manga_library:offset\";\nexport const PLACEHOLDER_BANNER = \"https://via.placeholder.com/480x720?text=No+Banner\";","label",[126],"pulls","branches",1,"forks","envVars",{"_145":61,"_147":61},"connections",{"_145":61,"_146":61},"count","unfilledCount","emptyCount",{"_42":125,"_32":126,"_30":127,"_128":-5,"_40":41,"_39":129,"_130":48,"_37":149,"_13":132,"_33":133,"_35":134,"_24":48,"_135":-5,"_150":151,"_27":152},["D",1783926834582],"children",[],[126],["D",1783926729718],{"_95":94,"_102":98},{"_95":94,"_96":-5,"_97":98,"_18":55,"_99":100,"_101":48},{"_13":84,"_32":85,"_42":41,"_86":43,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":44,"_93":94},"ctx",{"_77":-5,"_121":190,"_191":192,"_193":194,"_195":48,"_196":48,"_197":48,"_198":25,"_199":-5,"_200":-7,"_201":-7,"_202":61,"_203":48,"_204":48,"_205":-5,"_206":48,"_207":48,"_208":48,"_209":-5},"ENV",{"_161":162,"_163":164,"_165":85,"_166":167,"_168":169,"_170":171,"_172":173,"_174":175,"_176":177,"_178":179,"_180":181,"_182":183,"_184":185,"_186":187,"_188":189},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v33