[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_167":168,"_169":170},"routes/_app.x.$handle.$name",{"_96":95,"_33":46,"_104":86,"_28":54,"_13":14,"_105":-5,"_11":106,"_107":108,"_109":110,"_111":112,"_113":114,"_20":-5,"_115":-5,"_116":-5,"_117":27,"_118":119,"_120":25,"_121":-5,"_122":-5,"_123":124},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_44":45,"_33":46,"_39":47,"_48":-5,"_49":27,"_50":51,"_52":51,"_53":54,"_55":-5,"_18":56,"_57":-5,"_58":27,"_59":-5,"_60":-5,"_61":62,"_63":64,"_65":-5,"_66":-5,"_67":-5,"_68":-5,"_69":70,"_71":70,"_72":73,"_74":73,"_75":73,"_76":70,"_77":70,"_78":79,"_80":81,"_82":83,"_84":27},"version",38,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":27,"_28":29,"_30":29},"file",{"_31":32,"_33":29,"_34":35,"_36":38,"_39":40,"_41":14,"_42":43,"_24":27},"previous",{"_31":32,"_33":29,"_34":35,"_36":37},"deleted",true,"deletedHere",false,"path","Worker.ts","latestPath","fileId","54b3ab4a-7e90-11f1-9ea0-1607ee4eb77e","name","fileType","http","content","// Learn more: https://docs.val.town/vals/http/\nexport default async function (req: Request): Promise {\n return Response.json({ ok: true })\n}","import { createClient } from \"npm:@libsql/client/web\";\nimport { Redis } from \"npm:@upstash/redis\";\nimport { DOMParser } from \"npm:linkedom\"; // Added for Val Town/Deno DOM parsing compatibility\n\nimport {\n OFFSET_KEY,\n PLACEHOLDER_BANNER,\n SOURCE_URL,\n TURSO_AUTH_TOKEN,\n TURSO_DATABASE_URL,\n UPSTASH_REDIS_REST_TOKEN,\n UPSTASH_REDIS_REST_URL,\n} from \"./customize.js\";\n\nconst PAGE_SIZE = 32;\nconst BATCHES_PER_INVOCATION = 5;\nconst REQUEST_TIMEOUT_MS = 20_000;\nconst USER_AGENT =\n \"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0\";\n\ntype MangaRow = {\n key: string;\n title: string;\n type: string | null;\n year: number | null;\n banner_1: string;\n banner_2: string | null;\n author: string | null;\n tags: string | null;\n status: string | null;\n};\n\ntype RunItem = {\n offset: number;\n ok: boolean;\n fetched: number;\n stored: number;\n key?: string;\n error?: string;\n};\n\nconst sql = createClient({\n url: TURSO_DATABASE_URL,\n authToken: TURSO_AUTH_TOKEN,\n});\n\nconst redis = new Redis({\n url: UPSTASH_REDIS_REST_URL,\n token: UPSTASH_REDIS_REST_TOKEN,\n});\n\nconst UPSERT_SQL = `\nINSERT INTO neon_manga_library (\n key,\n title,\n type,\n year,\n banner_1,\n banner_2,\n author,\n tags,\n status\n) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)\nON CONFLICT(key) DO UPDATE SET\n title = excluded.title,\n type = excluded.type,\n year = excluded.year,\n banner_1 = excluded.banner_1,\n banner_2 = excluded.banner_2,\n author = excluded.author,\n tags = excluded.tags,\n status = excluded.status\n`;\n\nfunction jsonResponse(body: unknown, init: ResponseInit = {}): Response {\n const headers = new Headers(init.headers);\n headers.set(\"content-type\", \"application/json; charset=utf-8\");\n headers.set(\"access-control-allow-origin\", \"*\");\n headers.set(\"access-control-allow-methods\", \"GET,OPTIONS\");\n headers.set(\"access-control-allow-headers\", \"content-type, authorization\");\n return new Response(JSON.stringify(body, null, 2), { ...init, headers });\n}\n\nfunction normalizeText(value: unknown): string {\n if (typeof value !== \"string\") return \"\";\n return value.replace(/\\s+/g, \" \").trim();\n}\n\nfunction normalizeOptionalText(value: unknown): string | null {\n const text = normalizeText(value);\n return text.length ? text : null;\n}\n\nfunction absoluteUrl(raw: string | null | undefined): string | null {\n if (!raw) return null;\n const value = raw.trim();\n if (!value) return null;\n if (value.startsWith(\"data:\")) return null;\n if (value.startsWith(\"//\")) return `https:${value}`;\n try {\n return new URL(value, SOURCE_URL).toString();\n } catch {\n return null;\n }\n}\n\nfunction parseSrcset(srcset: string | null | undefined): string[] {\n if (!srcset) return [];\n return srcset\n .split(\",\")\n .map((part) => part.trim().split(/\\s+/)[0] ?? \"\")\n .map((part) => part.trim())\n .filter(Boolean)\n .map((part) => absoluteUrl(part))\n .filter((part): part is string => Boolean(part));\n}\n\nfunction uniqueStrings(values: Array): string[] {\n const seen = new Set();\n const out: string[] = [];\n\n for (const value of values) {\n const normalized = normalizeText(value);\n if (!normalized || seen.has(normalized.toLowerCase())) continue;\n seen.add(normalized.toLowerCase());\n out.push(normalized);\n }\n\n return out;\n}\n\nfunction commaJoin(values: Array): string | null {\n const items = uniqueStrings(values);\n return items.length ? items.join(\", \") : null;\n}\n\nfunction extractLinkId(href: string | null | undefined): string | null {\n if (!href) return null;\n const match = href.match(/\\/series\\/([A-Za-z0-9_-]+)\\//);\n return match?.[1] ?? null;\n}\n\nfunction firstTextByLabels(article: any, labels: string[]): string | null {\n const strongs = Array.from(article.querySelectorAll(\"strong\")) as any[];\n for (const label of labels) {\n const strong = strongs.find((node) => normalizeText(node.textContent).toLowerCase().includes(label.toLowerCase()));\n if (!strong) continue;\n const parent = strong.parentElement;\n if (!parent) continue;\n const spans = Array.from(parent.querySelectorAll(\"span\")) as any[];\n const text = spans[0]?.textContent ?? null;\n const normalized = normalizeOptionalText(text);\n if (normalized) return normalized;\n }\n return null;\n}\n\nfunction extractMultiTextByLabel(article: any, labels: string[]): string[] {\n const strongs = Array.from(article.querySelectorAll(\"strong\")) as any[];\n for (const label of labels) {\n const strong = strongs.find((node) => normalizeText(node.textContent).toLowerCase().includes(label.toLowerCase()));\n if (!strong) continue;\n const parent = strong.parentElement;\n if (!parent) continue;\n const spans = Array.from(parent.querySelectorAll(\"span\")) as any[];\n return spans\n .map((span) => normalizeOptionalText(span.textContent))\n .filter((value): value is string => Boolean(value));\n }\n return [];\n}\n\nfunction extractKey(article: any): string | null {\n const firstAnchor = article.querySelector(\"a[href]\");\n const href = firstAnchor?.getAttribute(\"href\") ?? \"\";\n const key = extractLinkId(href);\n if (key) return key;\n\n const altAnchor = (Array.from(article.querySelectorAll(\"a[href]\")) as any[])\n .map((a) => extractLinkId(a.getAttribute(\"href\")))\n .find((value): value is string => Boolean(value));\n\n return altAnchor ?? null;\n}\n\nfunction extractTitle(article: any): string | null {\n const candidates = [\n \".text-ellipsis.truncate\",\n \"a.line-clamp-1\",\n \"[data-title]\",\n \"h1\",\n \"h2\",\n \"h3\",\n ];\n\n for (const selector of candidates) {\n const node = article.querySelector(selector);\n const text = normalizeOptionalText(node?.textContent);\n if (text) return text;\n }\n\n const fallback = normalizeOptionalText(article.querySelector(\"a[href]\")?.textContent);\n return fallback;\n}\n\nfunction extractBannerUrls(article: any): { banner1: string; banner2: string | null } {\n const sourceUrls = (Array.from(article.querySelectorAll(\"source\")) as any[])\n .flatMap((source) => parseSrcset(source.getAttribute(\"srcset\")));\n const imgUrls = (Array.from(article.querySelectorAll(\"img\")) as any[])\n .map((img) => absoluteUrl(img.getAttribute(\"src\")))\n .filter((value): value is string => Boolean(value));\n\n const all = [...sourceUrls, ...imgUrls];\n const deduped = uniqueStrings(all);\n\n const banner1 = deduped[0] ?? PLACEHOLDER_BANNER;\n const banner2 = deduped.find((value) => value !== banner1) ?? null;\n\n return { banner1, banner2 };\n}\n\nfunction extractRow(article: any): MangaRow | null {\n const key = extractKey(article);\n const title = extractTitle(article);\n if (!key || !title) return null;\n\n const { banner1, banner2 } = extractBannerUrls(article);\n\n const type = firstTextByLabels(article, [\"Type\"]);\n const status = firstTextByLabels(article, [\"Status\"]);\n const yearText = firstTextByLabels(article, [\"Year\"]);\n const year = yearText ? Number.parseInt(yearText, 10) : null;\n\n const author = commaJoin(extractMultiTextByLabel(article, [\"Author\"]));\n const tags = commaJoin(extractMultiTextByLabel(article, [\"Tag\"]));\n\n return {\n key,\n title,\n type,\n year: Number.isFinite(year) ? year : null,\n banner_1: banner1,\n banner_2: banner2,\n author,\n tags,\n status,\n };\n}\n\nasync function fetchHtml(url: string): Promise {\n const controller = new AbortController();\n const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);\n\n try {\n const response = await fetch(url, {\n method: \"GET\",\n headers: {\n accept: \"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\",\n \"user-agent\": USER_AGENT,\n \"cache-control\": \"no-cache\",\n pragma: \"no-cache\",\n },\n signal: controller.signal,\n });\n\n if (!response.ok) {\n throw new Error(`Upstream HTTP ${response.status} ${response.statusText}`);\n }\n\n return await response.text();\n } finally {\n clearTimeout(timer);\n }\n}\n\nfunction buildSourceUrl(offset: number): string {\n const params = new URLSearchParams({\n limit: String(PAGE_SIZE),\n offset: String(offset),\n sort: \"Alphabet\",\n order: \"Ascending\",\n official: \"Any\",\n anime: \"Any\",\n adult: \"Any\",\n display_mode: \"Full Display\",\n });\n\n return `${SOURCE_URL}?${params.toString()}`;\n}\n\nasync function getStoredOffset(): Promise {\n const raw = await redis.get(OFFSET_KEY);\n if (typeof raw === \"number\" && Number.isFinite(raw)) return Math.max(0, Math.trunc(raw));\n if (typeof raw === \"string\") {\n const parsed = Number.parseInt(raw, 10);\n if (Number.isFinite(parsed) && parsed >= 0) return parsed;\n }\n return 0;\n}\n\nasync function setStoredOffset(offset: number): Promise {\n await redis.set(OFFSET_KEY, String(Math.max(0, Math.trunc(offset))));\n}\n\nasync function storeRows(rows: MangaRow[]): Promise {\n let stored = 0;\n\n for (const row of rows) {\n await sql.prepare(UPSERT_SQL).bind(\n row.key,\n row.title,\n row.type,\n row.year,\n row.banner_1,\n row.banner_2,\n row.author,\n row.tags,\n row.status,\n ).run();\n stored++;\n }\n\n return stored;\n}\n\nasync function runBatch(offset: number): Promise {\n const url = buildSourceUrl(offset);\n const html = await fetchHtml(url);\n \n // Using linkedom's DOMParser here instead of standard browser-only DOMParser\n const document = new DOMParser().parseFromString(html, \"text/html\");\n const articles = document?.querySelectorAll(\"article.bg-base-300\") ?? [];\n\n const rows: MangaRow[] = [];\n for (const article of Array.from(articles)) {\n const row = extractRow(article);\n if (row) rows.push(row);\n }\n\n const stored = rows.length ? await storeRows(rows) : 0;\n\n return {\n offset,\n ok: true,\n fetched: rows.length,\n stored,\n };\n}\n\nasync function handleFetchAndUpload(): Promise {\n const startOffset = await getStoredOffset();\n let currentOffset = startOffset;\n\n const runs: RunItem[] = [];\n\n for (let index = 0; index < BATCHES_PER_INVOCATION; index++) {\n try {\n const result = await runBatch(currentOffset);\n runs.push(result);\n } catch (error) {\n runs.push({\n offset: currentOffset,\n ok: false,\n fetched: 0,\n stored: 0,\n error: error instanceof Error ? error.message : String(error),\n });\n } finally {\n currentOffset += PAGE_SIZE;\n await setStoredOffset(currentOffset);\n }\n }\n\n return jsonResponse({\n ok: true,\n endpoint: \"GET /fetch-and-upload\",\n source: SOURCE_URL,\n page_size: PAGE_SIZE,\n batches_per_invocation: BATCHES_PER_INVOCATION,\n start_offset: startOffset,\n end_offset: currentOffset,\n saved_offset_key: OFFSET_KEY,\n runs,\n });\n}\n\n// Fixed Val Town entrypoint wrapper: Changed from object export to standard function export\nexport default async function (request: Request): Promise {\n const url = new URL(request.url);\n const path = url.pathname.replace(/\\/+$/, \"\") || \"/\";\n const method = request.method.toUpperCase();\n\n if (method === \"OPTIONS\") {\n return new Response(null, {\n status: 204,\n headers: {\n \"access-control-allow-origin\": \"*\",\n \"access-control-allow-methods\": \"GET,OPTIONS\",\n \"access-control-allow-headers\": \"content-type, authorization\",\n \"access-control-max-age\": \"86400\",\n },\n });\n }\n\n if (method !== \"GET\" || path !== \"/fetch-and-upload\") {\n return jsonResponse(\n {\n ok: false,\n error: \"not found\",\n expected: \"GET /fetch-and-upload\",\n },\n { status: 404 },\n );\n }\n\n try {\n return await handleFetchAndUpload();\n } catch (error) {\n return jsonResponse(\n {\n ok: false,\n error: error instanceof Error ? error.message : String(error),\n },\n { status: 500 },\n );\n }\n}","createdAt",["D",1783929396563],"branchVersion","branchId","c3b37b47-aac6-4dd9-a90b-e4098c3a71a1","id","96c7e756-e778-4d18-b447-3242cc1845e4","CROSS-MANGA-SCRAPER",["D",1783926729718],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","","imageUrl","019d34f5-a118-71aa-86be-a3b05ce7b06e","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","019d34f5-a118-71aa-86be-a6dc6eeabba9","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_96":95,"_103":99},"org",{"_96":95,"_97":-5,"_98":99,"_18":56,"_100":101,"_102":27},"branch",{"_13":85,"_33":86,"_44":43,"_87":45,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":-5,"_93":46,"_94":95},"isOwned",84,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","lupinarashi","handle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoiZGVmYXVsdCIsImlpZCI6Imluc18yTm10cGhQUklwU0tqZHNiOHNnNGd2ZjJURFUiLCJyaWQiOiJ1c2VyXzNCWndldUNFUURoY3B1Uzk1d2lQOVhDTE8xaCJ9","tier","free","canRestrictVals","profileImageUrl","branchName","adoptOrgId",{"_44":45,"_33":46,"_39":163,"_48":-5,"_49":27,"_50":51,"_52":51,"_53":54,"_55":-5,"_18":56,"_57":-5,"_58":27,"_59":-5,"_60":-5,"_61":62,"_63":64,"_65":-5,"_66":-5,"_67":-5,"_68":-5,"_69":70,"_71":70,"_72":73,"_74":73,"_75":73,"_76":70,"_77":70,"_78":164,"_80":165,"_82":166,"_84":27},"tree",[154,155],"files",[154,155],"counts",{"_143":62,"_144":145,"_146":62,"_147":148,"_149":150},"contents",[125,126],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_44":139,"_33":29,"_31":32,"_130":-5,"_42":43,"_41":14,"_132":27,"_39":140,"_13":141,"_34":35,"_36":38,"_24":27,"_137":-5,"_28":142},{"_44":127,"_33":128,"_31":129,"_130":-5,"_42":43,"_41":131,"_132":27,"_39":133,"_13":134,"_34":135,"_36":136,"_24":27,"_137":-5,"_28":138},"19c70e1b-0905-4a04-9d30-cd82bd0459bb","customize.js","519e67de-7e8a-11f1-a7fd-1607ee4eb77e","parentId",8,"latest",["D",1783926834582],2,"script","export const SOURCE_URL = \"https://weebcentral.com/search/data\";\nexport const TURSO_DATABASE_URL = \"https://neon-manga-db-lupinarashi.aws-ap-south-1.turso.io\";\nexport const TURSO_AUTH_TOKEN = \"eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJhIjoicnciLCJpYXQiOjE3ODM5MjY0NDQsImlkIjoiMDE5ZjVhNGMtMWEwMS03NmM1LWEzZGItNmVjOTZhZTFkOWNjIiwia2lkIjoibTRxTWl4ZVRaM0xGelBSc3AyVnFYMTRQLVRKSllNOWVkUmxNQWllQ3c2TSIsInJpZCI6ImM1N2Q0NThjLWU3MmMtNGM4OC1iYmIxLTE2YjkyOWYzNzk3ZSJ9.tfdqmDSy2ndG9jGp3l4CLuJ_JRXkeZAEfoUKBRUPIgvEijXtfuzytZFGeXMjnS6fmNFN3ZQDaTLs7DRhZGwUCg\";\nexport const UPSTASH_REDIS_REST_URL = \"https://magical-giraffe-89191.upstash.io\";\nexport const UPSTASH_REDIS_REST_TOKEN = \"gQAAAAAAAVxnAAIncDJmY2MwNTdiZDAyNDQ0OTY3ODhkMjJjMWFkNDlkZGExMnAyODkxOTE\";\nexport const OFFSET_KEY = \"neon_manga_library:offset\";\nexport const PLACEHOLDER_BANNER = \"https://via.placeholder.com/480x720?text=No+Banner\";","label",[128],"cec22f1e-de72-4d5e-a2d0-e2517b83b656",["D",1783929396563],4,[29],"pulls","branches",1,"forks","envVars",{"_151":62,"_153":62},"connections",{"_151":62,"_152":62},"count","unfilledCount","emptyCount",{"_44":139,"_33":29,"_31":32,"_130":-5,"_42":43,"_41":14,"_132":27,"_39":160,"_13":141,"_34":35,"_36":38,"_24":27,"_137":-5,"_157":161,"_28":162},{"_44":127,"_33":128,"_31":129,"_130":-5,"_42":43,"_41":131,"_132":27,"_39":156,"_13":134,"_34":135,"_36":136,"_24":27,"_137":-5,"_157":158,"_28":159},["D",1783926834582],"children",[],[128],["D",1783929396563],[],[29],["D",1783926729718],{"_96":95,"_103":99},{"_96":95,"_97":-5,"_98":99,"_18":56,"_100":101,"_102":27},{"_13":85,"_33":86,"_44":43,"_87":45,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":-5,"_93":46,"_94":95},"ctx",{"_78":-5,"_122":200,"_201":202,"_203":204,"_205":27,"_206":27,"_207":27,"_208":25,"_209":-5,"_210":-7,"_211":-7,"_212":62,"_213":27,"_214":27,"_215":-5,"_216":27,"_217":27,"_218":27,"_219":-5},"ENV",{"_171":172,"_173":174,"_175":86,"_176":177,"_178":179,"_180":181,"_182":183,"_184":185,"_186":187,"_188":189,"_190":191,"_192":193,"_194":195,"_196":197,"_198":199},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v38