[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_166":167,"_168":169},"routes/_app.x.$handle.$name",{"_95":94,"_33":46,"_103":85,"_28":38,"_13":14,"_104":-5,"_11":105,"_106":107,"_108":109,"_110":111,"_112":113,"_20":-5,"_114":-5,"_115":-5,"_116":27,"_117":118,"_119":25,"_120":-5,"_121":-5,"_122":123},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_44":45,"_33":46,"_39":47,"_48":-5,"_49":27,"_50":51,"_52":51,"_53":38,"_54":-5,"_18":55,"_56":-5,"_57":27,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":78,"_79":80,"_81":82,"_83":27},"version",11,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":27,"_28":29,"_30":29},"file",{"_31":32,"_33":29,"_34":35,"_36":38,"_39":40,"_41":14,"_42":43,"_24":27},"previous",{"_31":32,"_33":29,"_34":35,"_36":37},"deleted",true,"deletedHere",false,"path","NEON-API(manga).ts","latestPath","fileId","29d54cae-7e8a-11f1-9d66-1607ee4eb77e","name","fileType","http","content","import { createClient } from \"npm:@libsql/client/web\";\nimport * as customize from \"./customize.js\";\n\ntype MangaRow = {\nkey: string;\ntitle: string;\ntype: string | null;\nyear: number | null;\nbanner_1: string;\nbanner_2: string | null;\nauthor: string | null;\ntags: string | null;\nstatus: string | null;\n};\n\ntype ScrapeResult = {\nrows: MangaRow[];\nfound: number;\n};\n\nconst cfg = customize as Record;\n\nconst SOURCE_URL = cfg.SOURCE_URL ?? \"https://weebcentral.com/search/data\";\nconst LIMIT = 32;\nconst REQUESTS_PER_RUN = 5;\n\nconst OFFSET_KEY = cfg.OFFSET_KEY ?? \"neon_manga_library:offset\";\nconst PLACEHOLDER_BANNER =\ncfg.PLACEHOLDER_BANNER ??\n\"https://via.placeholder.com/480x720?text=No+Banner\";\n\nconst TURSO_DATABASE_URL =\ncfg.TURSO_DATABASE_URL ?? cfg.TURSO_URL ?? cfg.DATABASE_URL;\nconst TURSO_AUTH_TOKEN =\ncfg.TURSO_AUTH_TOKEN ?? cfg.TURSO_TOKEN ?? cfg.DATABASE_TOKEN;\n\nconst UPSTASH_REDIS_REST_URL =\ncfg.UPSTASH_REDIS_REST_URL ?? cfg.UPSTASH_URL;\nconst UPSTASH_REDIS_REST_TOKEN =\ncfg.UPSTASH_REDIS_REST_TOKEN ?? cfg.UPSTASH_TOKEN;\n\nif (!TURSO_DATABASE_URL || !TURSO_AUTH_TOKEN) {\nthrow new Error(\n\"Missing Turso config in customize.js. Expected TURSO_DATABASE_URL and TURSO_AUTH_TOKEN (or TURSO_URL / TURSO_TOKEN / DATABASE_URL / DATABASE_TOKEN aliases).\",\n);\n}\n\nif (!UPSTASH_REDIS_REST_URL || !UPSTASH_REDIS_REST_TOKEN) {\nthrow new Error(\n\"Missing Upstash config in customize.js. Expected UPSTASH_REDIS_REST_URL and UPSTASH_REDIS_REST_TOKEN (or UPSTASH_URL / UPSTASH_TOKEN aliases).\",\n);\n}\n\nconst db = createClient({\nurl: TURSO_DATABASE_URL,\nauthToken: TURSO_AUTH_TOKEN,\n});\n\nconst SQL_SCHEMA = \"CREATE TABLE IF NOT EXISTS neon_manga_library ( key TEXT UNIQUE, title TEXT NOT NULL, type TEXT, year INTEGER, banner_1 TEXT NOT NULL, banner_2 TEXT, author TEXT, tags TEXT, status TEXT, created_at TEXT DEFAULT CURRENT_TIMESTAMP );\";\n\nfunction cleanText(value: string | null | undefined): string {\nreturn (value ?? \"\")\n.replace(/\\s+/g, \" \")\n.replace(/\\u00A0/g, \" \")\n.trim();\n}\n\nfunction firstNonEmpty(...values: Array): string {\nfor (const value of values) {\nconst cleaned = cleanText(value);\nif (cleaned) return cleaned;\n}\nreturn \"\";\n}\n\nfunction toIntOrNull(value: string | null | undefined): number | null {\nconst cleaned = cleanText(value);\nif (!cleaned) return null;\nconst match = cleaned.match(/\\d{4}/);\nreturn match ? Number(match[0]) : null;\n}\n\nfunction normalizeImageUrl(value: string | null | undefined): string {\nconst cleaned = cleanText(value);\nif (!cleaned) return PLACEHOLDER_BANNER;\n\nconst firstToken = cleaned.split(\",\")[0]?.trim() ?? cleaned;\nconst firstUrl = firstToken.split(/\\s+/)[0]?.trim() ?? firstToken;\n\nreturn firstUrl || PLACEHOLDER_BANNER;\n}\n\nfunction safeJoin(values: string[]): string | null {\nconst filtered = values.map(cleanText).filter(Boolean);\nreturn filtered.length ? filtered.join(\", \") : null;\n}\n\nasync function sha256Hex(input: string): Promise {\nconst bytes = new TextEncoder().encode(input);\nconst hash = await crypto.subtle.digest(\"SHA-256\", bytes);\nreturn Array.from(new Uint8Array(hash))\n.map((b) => b.toString(16).padStart(2, \"0\"))\n.join(\"\");\n}\n\nasync function upstashGet(key: string): Promise {\nconst url =\n\"${UPSTASH_REDIS_REST_URL.replace(/\\/$/, \"\")}/get/${encodeURIComponent(key)}\";\n\nconst res = await fetch(url, {\nheaders: {\nAuthorization: \"Bearer ${UPSTASH_REDIS_REST_TOKEN}\",\n},\n});\n\nif (!res.ok) {\nthrow new Error(\"Upstash GET failed (${res.status})\");\n}\n\nconst data = await res.json() as { result?: string | null };\nreturn data.result ?? null;\n}\n\nasync function upstashSet(key: string, value: string): Promise {\nconst url =\n\"${UPSTASH_REDIS_REST_URL.replace(/\\/$/, \"\")}/set/${encodeURIComponent(key)}/${encodeURIComponent(value)}\";\n\nconst res = await fetch(url, {\nheaders: {\nAuthorization: \"Bearer ${UPSTASH_REDIS_REST_TOKEN}\",\n},\n});\n\nif (!res.ok) {\nthrow new Error(\"Upstash SET failed (${res.status})\");\n}\n}\n\nasync function ensureSchema(): Promise {\nawait db.execute(SQL_SCHEMA);\n}\n\nfunction buildSourceUrl(offset: number): string {\nconst params = new URLSearchParams({\nlimit: String(LIMIT),\noffset: String(offset),\nsort: \"Alphabet\",\norder: \"Ascending\",\nofficial: \"Any\",\nanime: \"Any\",\nadult: \"Any\",\ndisplay_mode: \"Full Display\",\n});\n\nreturn \"${SOURCE_URL}?${params.toString()}\";\n}\n\nfunction extractList(article: Element, label: string): string[] {\nconst strong = Array.from(article.querySelectorAll(\"strong\")).find((el) =>\ncleanText(el.textContent).toLowerCase().includes(label.toLowerCase())\n);\n\nif (!strong) return [];\n\nconst parent = strong.parentElement;\nif (!parent) return [];\n\nreturn Array.from(parent.querySelectorAll(\"span\"))\n.map((span) => cleanText(span.textContent))\n.filter(Boolean);\n}\n\nasync function articleToRow(article: Element): Promise {\nconst titleEl =\narticle.querySelector(\".text-ellipsis.truncate\") ??\narticle.querySelector(\"a.line-clamp-1\") ??\narticle.querySelector(\"a[href]\");\n\nconst title = firstNonEmpty(titleEl?.textContent);\nif (!title) return null;\n\nconst anchor = article.querySelector(\"a[href]\");\nconst hrefRaw = anchor?.getAttribute(\"href\") ?? \"\";\nconst href = hrefRaw\n? new URL(hrefRaw, SOURCE_URL).toString()\n: \"\";\n\nconst keyMatch = href.match(//series/([A-Z0-9]+)//i);\nconst derivedKey = keyMatch?.[1] ?? \"\";\nconst key = derivedKey || (await sha256Hex(\"${title}|${href}\")).slice(0, 24);\n\nconst normalSource = article.querySelector('source[media=\"(min-width: 768px)\"]');\nconst fallbackImgEl = article.querySelector('img[alt*=\"cover\" i]');\n\nconst banner_1 = normalizeImageUrl(\nnormalSource?.getAttribute(\"srcset\") ||\nnormalSource?.getAttribute(\"src\") ||\nfallbackImgEl?.getAttribute(\"src\") ||\nfallbackImgEl?.getAttribute(\"data-src\") ||\n\"\",\n);\n\nconst type = firstNonEmpty(\n...extractList(article, \"Type\"),\n...extractList(article, \"type\"),\n) || null;\n\nconst status = firstNonEmpty(\n...extractList(article, \"Status\"),\n...extractList(article, \"status\"),\n) || null;\n\nconst year =\ntoIntOrNull(\nfirstNonEmpty(\n...extractList(article, \"Year\"),\n...extractList(article, \"year\"),\n),\n) ?? null;\n\nconst author = safeJoin(extractList(article, \"Author\"));\nconst tags = safeJoin(extractList(article, \"Tag\"));\n\nreturn {\nkey,\ntitle,\ntype,\nyear,\nbanner_1: banner_1 || PLACEHOLDER_BANNER,\nbanner_2: null,\nauthor,\ntags,\nstatus,\n};\n}\n\nasync function fetchAndParse(offset: number): Promise {\nconst sourceUrl = buildSourceUrl(offset);\n\nconst response = await fetch(sourceUrl, {\nmethod: \"GET\",\nheaders: {\nAccept: \"text/html,application/xhtml+xml,application/xml;q=0.9,/;q=0.8\",\n\"User-Agent\": \"Mozilla/5.0 (ValTown Manga Sync Bot)\",\n},\n});\n\nif (!response.ok) {\nthrow new Error(\"Source fetch failed (${response.status}) at offset ${offset}\");\n}\n\nconst html = await response.text();\nconst doc = new DOMParser().parseFromString(html, \"text/html\");\n\nif (!doc) {\nthrow new Error(\"Unable to parse HTML at offset ${offset}\");\n}\n\nconst articles = Array.from(doc.querySelectorAll(\"article.bg-base-300\"));\nconst rows: MangaRow[] = [];\n\nfor (const article of articles) {\nconst row = await articleToRow(article);\nif (!row) continue;\nrows.push(row);\n}\n\nreturn { rows, found: articles.length };\n}\n\nasync function upsertRows(rows: MangaRow[]): Promise {\nif (!rows.length) return 0;\n\nconst batch = rows.map((row) => ({\nsql: \"INSERT INTO neon_manga_library ( key, title, type, year, banner_1, banner_2, author, tags, status ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(key) DO UPDATE SET title = excluded.title, type = excluded.type, year = excluded.year, banner_1 = excluded.banner_1, banner_2 = excluded.banner_2, author = excluded.author, tags = excluded.tags, status = excluded.status\",\nargs: [\nrow.key,\nrow.title,\nrow.type,\nrow.year,\nrow.banner_1,\nrow.banner_2,\nrow.author,\nrow.tags,\nrow.status,\n],\n}));\n\nawait db.batch(batch);\nreturn rows.length;\n}\n\nasync function readOffset(): Promise {\nconst stored = await upstashGet(OFFSET_KEY);\nconst parsed = Number.parseInt(stored ?? \"0\", 10);\nreturn Number.isFinite(parsed) && parsed >= 0 ? parsed : 0;\n}\n\nasync function writeOffset(offset: number): Promise {\nawait upstashSet(OFFSET_KEY, String(offset));\n}\n\nasync function runSync(): Promise {\nawait ensureSchema();\n\nlet offset = await readOffset();\nconst startOffset = offset;\n\nconst runLog: Array<{\nrequestOffset: number;\nfound: number;\ninsertedOrUpdated: number;\nok: boolean;\nerror?: string;\n}> = [];\n\nlet totalInserted = 0;\n\nfor (let i = 0; i < REQUESTS_PER_RUN; i++) {\nconst requestOffset = offset;\n\ntry {\n const { rows, found } = await fetchAndParse(requestOffset);\n const insertedOrUpdated = await upsertRows(rows);\n\n totalInserted += insertedOrUpdated;\n runLog.push({\n requestOffset,\n found,\n insertedOrUpdated,\n ok: true,\n });\n} catch (error) {\n runLog.push({\n requestOffset,\n found: 0,\n insertedOrUpdated: 0,\n ok: false,\n error: error instanceof Error ? error.message : String(error),\n });\n} finally {\n offset += LIMIT;\n}\n\n}\n\nawait writeOffset(offset);\n\nreturn Response.json({\nok: true,\nendpoint: \"/fetch-and-upload\",\nsource: SOURCE_URL,\nstartOffset,\nnextOffset: offset,\npagesProcessed: REQUESTS_PER_RUN,\nrowsWritten: totalInserted,\nlog: runLog,\n});\n}\n\nexport default async function handler(req: Request): Promise {\nconst url = new URL(req.url);\n\nif (req.method !== \"GET\") {\nreturn new Response(\"Method Not Allowed\", { status: 405 });\n}\n\nif (url.pathname !== \"/fetch-and-upload\") {\nreturn new Response(\"Not Found\", { status: 404 });\n}\n\ntry {\nreturn await runSync();\n} catch (error) {\nreturn Response.json(\n{\nok: false,\nerror: error instanceof Error ? error.message : String(error),\n},\n{ status: 500 },\n);\n}\n}","","createdAt",["D",1783927185547],"branchVersion","branchId","c3b37b47-aac6-4dd9-a90b-e4098c3a71a1","id","96c7e756-e778-4d18-b447-3242cc1845e4","CROSS-MANGA-SCRAPER",["D",1783926729718],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","imageUrl","019d34f5-a118-71aa-86be-a3b05ce7b06e","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","019d34f5-a118-71aa-86be-a6dc6eeabba9","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_95":94,"_102":98},"org",{"_95":94,"_96":-5,"_97":98,"_18":55,"_99":100,"_101":27},"branch",{"_13":84,"_33":85,"_44":43,"_86":45,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":46,"_93":94},"isOwned",84,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","lupinarashi","handle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoiZGVmYXVsdCIsImlpZCI6Imluc18yTm10cGhQUklwU0tqZHNiOHNnNGd2ZjJURFUiLCJyaWQiOiJ1c2VyXzNCWndldUNFUURoY3B1Uzk1d2lQOVhDTE8xaCJ9","tier","free","canRestrictVals","profileImageUrl","branchName","adoptOrgId",{"_44":45,"_33":46,"_39":162,"_48":-5,"_49":27,"_50":51,"_52":51,"_53":38,"_54":-5,"_18":55,"_56":-5,"_57":27,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":163,"_79":164,"_81":165,"_83":27},"tree",[153,154],"files",[153,154],"counts",{"_142":61,"_143":144,"_145":61,"_146":147,"_148":149},"contents",[124,125],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_44":138,"_33":29,"_31":32,"_129":-5,"_42":43,"_41":14,"_131":27,"_39":139,"_13":140,"_34":35,"_36":38,"_24":27,"_136":-5,"_28":141},{"_44":126,"_33":127,"_31":128,"_129":-5,"_42":43,"_41":130,"_131":27,"_39":132,"_13":133,"_34":134,"_36":135,"_24":27,"_136":-5,"_28":137},"19c70e1b-0905-4a04-9d30-cd82bd0459bb","customize.js","519e67de-7e8a-11f1-a7fd-1607ee4eb77e","parentId",8,"latest",["D",1783926834582],2,"script","export const SOURCE_URL = \"https://weebcentral.com/search/data\";\nexport const TURSO_DATABASE_URL = \"https://neon-manga-db-lupinarashi.aws-ap-south-1.turso.io\";\nexport const TURSO_AUTH_TOKEN = \"eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJhIjoicnciLCJpYXQiOjE3ODM5MjY0NDQsImlkIjoiMDE5ZjVhNGMtMWEwMS03NmM1LWEzZGItNmVjOTZhZTFkOWNjIiwia2lkIjoibTRxTWl4ZVRaM0xGelBSc3AyVnFYMTRQLVRKSllNOWVkUmxNQWllQ3c2TSIsInJpZCI6ImM1N2Q0NThjLWU3MmMtNGM4OC1iYmIxLTE2YjkyOWYzNzk3ZSJ9.tfdqmDSy2ndG9jGp3l4CLuJ_JRXkeZAEfoUKBRUPIgvEijXtfuzytZFGeXMjnS6fmNFN3ZQDaTLs7DRhZGwUCg\";\nexport const UPSTASH_REDIS_REST_URL = \"https://magical-giraffe-89191.upstash.io\";\nexport const UPSTASH_REDIS_REST_TOKEN = \"gQAAAAAAAVxnAAIncDJmY2MwNTdiZDAyNDQ0OTY3ODhkMjJjMWFkNDlkZGExMnAyODkxOTE\";\nexport const OFFSET_KEY = \"neon_manga_library:offset\";\nexport const PLACEHOLDER_BANNER = \"https://via.placeholder.com/480x720?text=No+Banner\";","label",[127],"eb449989-1acf-4d44-8893-a564f6f9fde1",["D",1783927185547],7,[29],"pulls","branches",1,"forks","envVars",{"_150":61,"_152":61},"connections",{"_150":61,"_151":61},"count","unfilledCount","emptyCount",{"_44":138,"_33":29,"_31":32,"_129":-5,"_42":43,"_41":14,"_131":27,"_39":159,"_13":140,"_34":35,"_36":38,"_24":27,"_136":-5,"_156":160,"_28":161},{"_44":126,"_33":127,"_31":128,"_129":-5,"_42":43,"_41":130,"_131":27,"_39":155,"_13":133,"_34":134,"_36":135,"_24":27,"_136":-5,"_156":157,"_28":158},["D",1783926834582],"children",[],[127],["D",1783927185547],[],[29],["D",1783926729718],{"_95":94,"_102":98},{"_95":94,"_96":-5,"_97":98,"_18":55,"_99":100,"_101":27},{"_13":84,"_33":85,"_44":43,"_86":45,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":46,"_93":94},"ctx",{"_77":-5,"_121":199,"_200":201,"_202":203,"_204":27,"_205":27,"_206":27,"_207":25,"_208":-5,"_209":-7,"_210":-7,"_211":61,"_212":27,"_213":27,"_214":-5,"_215":27,"_216":27,"_217":27,"_218":-5},"ENV",{"_170":171,"_172":173,"_174":85,"_175":176,"_177":178,"_179":180,"_181":182,"_183":184,"_185":186,"_187":188,"_189":190,"_191":192,"_193":194,"_195":196,"_197":198},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v11