[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_167":168,"_169":170},"routes/_app.x.$handle.$name",{"_96":95,"_33":46,"_104":86,"_28":54,"_13":14,"_105":-5,"_11":106,"_107":108,"_109":110,"_111":112,"_113":114,"_20":-5,"_115":-5,"_116":-5,"_117":27,"_118":119,"_120":25,"_121":-5,"_122":-5,"_123":124},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_44":45,"_33":46,"_39":47,"_48":-5,"_49":27,"_50":51,"_52":51,"_53":54,"_55":-5,"_18":56,"_57":-5,"_58":27,"_59":-5,"_60":-5,"_61":62,"_63":64,"_65":-5,"_66":-5,"_67":-5,"_68":-5,"_69":70,"_71":70,"_72":73,"_74":73,"_75":73,"_76":70,"_77":70,"_78":79,"_80":81,"_82":83,"_84":27},"version",44,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":27,"_28":29,"_30":29},"file",{"_31":32,"_33":29,"_34":35,"_36":38,"_39":40,"_41":14,"_42":43,"_24":27},"previous",{"_31":32,"_33":29,"_34":35,"_36":37},"deleted",true,"deletedHere",false,"path","neon-api(manga).ts","latestPath","fileId","cfca5d18-7eb5-11f1-bd87-1607ee4eb77e","name","fileType","http","content","// Learn more: https://docs.val.town/vals/http/\nexport default async function (req: Request): Promise {\n return Response.json({ ok: true })\n}","import { load } from \"npm:cheerio\";\nimport { createClient } from \"npm:@libsql/client/web\";\nimport { Redis } from \"npm:@upstash/redis\";\nimport { CONFIG } from \"./customize.js\";\n\ntype MangaRow = {\n key: string;\n title: string;\n type: string | null;\n year: number | null;\n banner_1: string;\n banner_2: string | null;\n author: string | null;\n tags: string | null;\n status: string | null;\n};\n\nconst FALLBACK_BANNER =\n `data:image/svg+xml;charset=UTF-8,${encodeURIComponent(`\n\n`)}`;\n\nconst db = createClient({\n url: CONFIG.TURSO_URL,\n authToken: CONFIG.TURSO_AUTH_TOKEN,\n});\n\nconst redis = new Redis({\n url: CONFIG.UPSTASH_REDIS_REST_URL,\n token: CONFIG.UPSTASH_REDIS_REST_TOKEN,\n});\n\nconst CREATE_TABLE_SQL = `\nCREATE TABLE IF NOT EXISTS neon_manga_library (\n key TEXT UNIQUE,\n title TEXT NOT NULL,\n type TEXT,\n year INTEGER,\n banner_1 TEXT NOT NULL,\n banner_2 TEXT,\n author TEXT,\n tags TEXT,\n status TEXT,\n created_at TEXT DEFAULT CURRENT_TIMESTAMP\n);\n`;\n\nconst UPSERT_SQL = `\nINSERT INTO neon_manga_library (\n key, title, type, year, banner_1, banner_2, author, tags, status\n) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)\nON CONFLICT(key) DO UPDATE SET\n title = excluded.title,\n type = excluded.type,\n year = excluded.year,\n banner_1 = excluded.banner_1,\n banner_2 = excluded.banner_2,\n author = excluded.author,\n tags = excluded.tags,\n status = excluded.status\n`;\n\nlet schemaReady: Promise | null = null;\n\nfunction ensureSchema(): Promise {\n schemaReady ??= (async () => {\n await db.execute(CREATE_TABLE_SQL);\n })();\n\n return schemaReady;\n}\n\nfunction normalizeText(value: string | undefined | null): string {\n return String(value ?? \"\")\n .replace(/\\s+/g, \" \")\n .trim();\n}\n\nfunction slugify(value: string): string {\n const base = normalizeText(value)\n .toLowerCase()\n .replace(/[^a-z0-9]+/g, \"-\")\n .replace(/^-+|-+$/g, \"\");\n\n return base || \"manga\";\n}\n\nfunction firstHttpUrl(list: string[]): string | null {\n for (const item of list) {\n const trimmed = normalizeText(item);\n if (!trimmed) continue;\n if (/^https?:\\/\\//i.test(trimmed)) return trimmed;\n if (/^\\/\\//.test(trimmed)) return `https:${trimmed}`;\n }\n return null;\n}\n\nfunction parseSrcSet(srcset: string | undefined | null): string[] {\n if (!srcset) return [];\n return srcset\n .split(\",\")\n .map((part) => normalizeText(part.split(/\\s+/)[0]))\n .filter(Boolean);\n}\n\nfunction extractKeyFromLink(href: string | undefined | null): string | null {\n const link = normalizeText(href);\n if (!link) return null;\n\n const match = link.match(/\\/series\\/([A-Z0-9]+)\\//i);\n return match?.[1] ?? null;\n}\n\nfunction getLabelValues($card: ReturnType, label: string): string[] {\n const strongs = $card(\"strong\").toArray();\n\n const target = strongs.find((el) =>\n normalizeText($card(el).text()).toLowerCase().includes(label.toLowerCase())\n );\n\n if (!target) return [];\n\n const parent = $card(target).parent();\n const values = parent.find(\"span\").toArray().map((el) => normalizeText($card(el).text()));\n\n return values.filter(Boolean);\n}\n\nfunction extractBanners($card: ReturnType): { banner_1: string; banner_2: string | null } {\n const candidates: string[] = [];\n\n const sourceDesktop = $card('source[media=\"(min-width: 768px)\"]').attr(\"srcset\");\n const sourceAny = $card(\"source[srcset]\").first().attr(\"srcset\");\n const img = $card('img[alt*=\"cover\" i]').first();\n const imgSrc = img.attr(\"src\");\n const imgDataSrc = img.attr(\"data-src\");\n const imgDataLazy = img.attr(\"data-lazy-src\");\n\n candidates.push(...parseSrcSet(sourceDesktop));\n candidates.push(...parseSrcSet(sourceAny));\n\n if (imgSrc) candidates.push(imgSrc);\n if (imgDataSrc) candidates.push(imgDataSrc);\n if (imgDataLazy) candidates.push(imgDataLazy);\n\n const unique = Array.from(new Set(candidates.map(normalizeText).filter(Boolean)));\n const banner_1 = firstHttpUrl(unique) ?? FALLBACK_BANNER;\n const banner_2 = unique.find((u) => normalizeText(u) !== banner_1) ?? null;\n\n return {\n banner_1,\n banner_2: banner_2 && /^https?:\\/\\//i.test(banner_2) ? banner_2 : null,\n };\n}\n\nfunction buildFallbackKey(title: string, pageOffset: number, index: number): string {\n return `${slugify(title)}-${pageOffset}-${index}`;\n}\n\nfunction parsePage(html: string, pageOffset: number): MangaRow[] {\n const $ = load(html);\n const cards = $(\"article.bg-base-300\");\n\n const rows: MangaRow[] = [];\n\n cards.each((index, el) => {\n const card = $(el);\n\n const title =\n normalizeText(\n card.find(\".text-ellipsis.truncate\").first().text() ||\n card.find(\"a.line-clamp-1\").first().text() ||\n card.find(\"h1, h2, h3\").first().text()\n ) || \"Untitled\";\n\n const firstAnchor = card.find(\"a\").first();\n const href = firstAnchor.attr(\"href\");\n const key = extractKeyFromLink(href) ?? buildFallbackKey(title, pageOffset, index);\n\n const { banner_1, banner_2 } = extractBanners(card);\n\n const yearRaw = getLabelValues(card, \"Year\")[0] ?? \"\";\n const type = getLabelValues(card, \"Type\")[0] ?? null;\n const status = getLabelValues(card, \"Status\")[0] ?? null;\n const authors = getLabelValues(card, \"Author\");\n const tags = getLabelValues(card, \"Tag\");\n\n const year = (() => {\n const n = Number.parseInt(yearRaw, 10);\n return Number.isFinite(n) ? n : null;\n })();\n\n rows.push({\n key,\n title,\n type: type ? normalizeText(type) : null,\n year,\n banner_1,\n banner_2,\n author: authors.length ? authors.map(normalizeText).filter(Boolean).join(\", \") : null,\n tags: tags.length ? tags.map(normalizeText).filter(Boolean).join(\", \") : null,\n status: status ? normalizeText(status) : null,\n });\n });\n\n return rows;\n}\n\nasync function fetchHtml(url: string): Promise {\n const controller = new AbortController();\n const timer = setTimeout(() => controller.abort(), CONFIG.REQUEST_TIMEOUT_MS);\n\n try {\n const res = await fetch(url, {\n method: \"GET\",\n signal: controller.signal,\n headers: {\n accept: \"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\",\n \"user-agent\":\n \"Mozilla/5.0 (compatible; ValTownIngestBot/1.0; +https://val.town/)\",\n },\n });\n\n if (!res.ok) {\n throw new Error(`Upstream HTTP ${res.status}`);\n }\n\n return await res.text();\n } finally {\n clearTimeout(timer);\n }\n}\n\nasync function upsertRows(rows: MangaRow[]): Promise<{ inserted: number; updatedLike: number }> {\n if (!rows.length) return { inserted: 0, updatedLike: 0 };\n\n await db.execute(\"BEGIN\");\n try {\n for (const row of rows) {\n await db.execute({\n sql: UPSERT_SQL,\n args: [\n row.key,\n row.title,\n row.type,\n row.year,\n row.banner_1,\n row.banner_2,\n row.author,\n row.tags,\n row.status,\n ],\n });\n }\n\n await db.execute(\"COMMIT\");\n return { inserted: rows.length, updatedLike: rows.length };\n } catch (error) {\n await db.execute(\"ROLLBACK\").catch(() => {});\n throw error;\n }\n}\n\nasync function readOffset(): Promise {\n const raw = await redis.get(CONFIG.OFFSET_KEY);\n const parsed = Number.parseInt(String(raw ?? \"0\"), 10);\n return Number.isFinite(parsed) && parsed >= 0 ? parsed : 0;\n}\n\nasync function saveOffset(offset: number): Promise {\n await redis.set(CONFIG.OFFSET_KEY, String(offset));\n}\n\nfunction buildSourceUrl(offset: number): string {\n const params = new URLSearchParams({\n limit: String(CONFIG.PAGE_SIZE),\n offset: String(offset),\n sort: \"Alphabet\",\n order: \"Ascending\",\n official: \"Any\",\n anime: \"Any\",\n adult: \"Any\",\n display_mode: \"Full Display\",\n });\n\n return `${CONFIG.SOURCE_BASE_URL}?${params.toString()}`;\n}\n\nexport default async function handler(req: Request): Promise {\n const url = new URL(req.url);\n\n if (url.pathname !== \"/fetch-and-upload\") {\n return new Response(\"Not Found\", { status: 404 });\n }\n\n if (req.method !== \"GET\") {\n return new Response(\"Method Not Allowed\", {\n status: 405,\n headers: { Allow: \"GET\" },\n });\n }\n\n const startedAt = new Date().toISOString();\n const runId = crypto.randomUUID();\n\n await ensureSchema();\n\n let offset = await readOffset();\n const initialOffset = offset;\n const pages: Array<{\n page: number;\n offset: number;\n url: string;\n count: number;\n status: \"ok\" | \"empty\" | \"error\";\n error?: string;\n }> = [];\n\n console.log(`[${runId}] run started`, {\n startedAt,\n initialOffset,\n });\n\n for (let pageIndex = 0; pageIndex < CONFIG.PAGES_PER_RUN; pageIndex++) {\n const pageOffset = offset;\n const sourceUrl = buildSourceUrl(pageOffset);\n\n try {\n const html = await fetchHtml(sourceUrl);\n const rows = parsePage(html, pageOffset);\n\n if (rows.length > 0) {\n const result = await upsertRows(rows);\n pages.push({\n page: pageIndex + 1,\n offset: pageOffset,\n url: sourceUrl,\n count: result.inserted,\n status: \"ok\",\n });\n } else {\n pages.push({\n page: pageIndex + 1,\n offset: pageOffset,\n url: sourceUrl,\n count: 0,\n status: \"empty\",\n });\n }\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error);\n pages.push({\n page: pageIndex + 1,\n offset: pageOffset,\n url: sourceUrl,\n count: 0,\n status: \"error\",\n error: message,\n });\n console.error(`[${runId}] page ${pageIndex + 1} failed`, message);\n } finally {\n offset += CONFIG.PAGE_SIZE;\n await saveOffset(offset);\n }\n }\n\n const finishedAt = new Date().toISOString();\n\n const summary = {\n ok: true,\n runId,\n startedAt,\n finishedAt,\n initialOffset,\n finalOffset: offset,\n pagesProcessed: CONFIG.PAGES_PER_RUN,\n pageSize: CONFIG.PAGE_SIZE,\n totalAttemptedOffsetAdvance: CONFIG.PAGES_PER_RUN * CONFIG.PAGE_SIZE,\n pages,\n };\n\n console.log(`[${runId}] run complete`, summary);\n\n return Response.json(summary, {\n headers: {\n \"cache-control\": \"no-store\",\n },\n });\n}","createdAt",["D",1783945532880],"branchVersion","branchId","c3b37b47-aac6-4dd9-a90b-e4098c3a71a1","id","96c7e756-e778-4d18-b447-3242cc1845e4","CROSS-MANGA-SCRAPER",["D",1783926729718],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","","imageUrl","019d34f5-a118-71aa-86be-a3b05ce7b06e","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","019d34f5-a118-71aa-86be-a6dc6eeabba9","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_96":95,"_103":99},"org",{"_96":95,"_97":-5,"_98":99,"_18":56,"_100":101,"_102":27},"branch",{"_13":85,"_33":86,"_44":43,"_87":45,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":-5,"_93":46,"_94":95},"isOwned",84,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","lupinarashi","handle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoiZGVmYXVsdCIsImlpZCI6Imluc18yTm10cGhQUklwU0tqZHNiOHNnNGd2ZjJURFUiLCJyaWQiOiJ1c2VyXzNCWndldUNFUURoY3B1Uzk1d2lQOVhDTE8xaCJ9","tier","free","canRestrictVals","profileImageUrl","branchName","adoptOrgId",{"_44":45,"_33":46,"_39":163,"_48":-5,"_49":27,"_50":51,"_52":51,"_53":54,"_55":-5,"_18":56,"_57":-5,"_58":27,"_59":-5,"_60":-5,"_61":62,"_63":64,"_65":-5,"_66":-5,"_67":-5,"_68":-5,"_69":70,"_71":70,"_72":73,"_74":73,"_75":73,"_76":70,"_77":70,"_78":164,"_80":165,"_82":166,"_84":27},"tree",[154,155],"files",[154,155],"counts",{"_143":62,"_144":145,"_146":62,"_147":148,"_149":150},"contents",[125,126],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_44":134,"_33":135,"_31":136,"_128":-5,"_42":43,"_41":137,"_129":27,"_39":138,"_13":139,"_34":140,"_36":141,"_24":27,"_132":-5,"_28":142},{"_44":127,"_33":29,"_31":32,"_128":-5,"_42":43,"_41":14,"_129":27,"_39":130,"_13":131,"_34":35,"_36":38,"_24":27,"_132":-5,"_28":133},"724d8689-2331-47a1-b238-e900a19f8e1a","parentId","latest",["D",1783945532880],4,"label",[29],"19c70e1b-0905-4a04-9d30-cd82bd0459bb","customize.js","519e67de-7e8a-11f1-a7fd-1607ee4eb77e",8,["D",1783926834582],2,"script","export const SOURCE_URL = \"https://weebcentral.com/search/data\";\nexport const TURSO_DATABASE_URL = \"https://neon-manga-db-lupinarashi.aws-ap-south-1.turso.io\";\nexport const TURSO_AUTH_TOKEN = \"eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJhIjoicnciLCJpYXQiOjE3ODM5MjY0NDQsImlkIjoiMDE5ZjVhNGMtMWEwMS03NmM1LWEzZGItNmVjOTZhZTFkOWNjIiwia2lkIjoibTRxTWl4ZVRaM0xGelBSc3AyVnFYMTRQLVRKSllNOWVkUmxNQWllQ3c2TSIsInJpZCI6ImM1N2Q0NThjLWU3MmMtNGM4OC1iYmIxLTE2YjkyOWYzNzk3ZSJ9.tfdqmDSy2ndG9jGp3l4CLuJ_JRXkeZAEfoUKBRUPIgvEijXtfuzytZFGeXMjnS6fmNFN3ZQDaTLs7DRhZGwUCg\";\nexport const UPSTASH_REDIS_REST_URL = \"https://magical-giraffe-89191.upstash.io\";\nexport const UPSTASH_REDIS_REST_TOKEN = \"gQAAAAAAAVxnAAIncDJmY2MwNTdiZDAyNDQ0OTY3ODhkMjJjMWFkNDlkZGExMnAyODkxOTE\";\nexport const OFFSET_KEY = \"neon_manga_library:offset\";\nexport const PLACEHOLDER_BANNER = \"https://via.placeholder.com/480x720?text=No+Banner\";",[135],"pulls","branches",1,"forks","envVars",{"_151":62,"_153":62},"connections",{"_151":62,"_152":62},"count","unfilledCount","emptyCount",{"_44":134,"_33":135,"_31":136,"_128":-5,"_42":43,"_41":137,"_129":27,"_39":160,"_13":139,"_34":140,"_36":141,"_24":27,"_132":-5,"_157":161,"_28":162},{"_44":127,"_33":29,"_31":32,"_128":-5,"_42":43,"_41":14,"_129":27,"_39":156,"_13":131,"_34":35,"_36":38,"_24":27,"_132":-5,"_157":158,"_28":159},["D",1783945532880],"children",[],[29],["D",1783926834582],[],[135],["D",1783926729718],{"_96":95,"_103":99},{"_96":95,"_97":-5,"_98":99,"_18":56,"_100":101,"_102":27},{"_13":85,"_33":86,"_44":43,"_87":45,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":-5,"_93":46,"_94":95},"ctx",{"_78":-5,"_122":200,"_201":202,"_203":204,"_205":27,"_206":27,"_207":27,"_208":25,"_209":-5,"_210":-7,"_211":-7,"_212":62,"_213":27,"_214":27,"_215":-5,"_216":27,"_217":27,"_218":27,"_219":-5},"ENV",{"_171":172,"_173":174,"_175":86,"_176":177,"_178":179,"_180":181,"_182":183,"_184":185,"_186":187,"_188":189,"_190":191,"_192":193,"_194":195,"_196":197,"_198":199},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v44