[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_165":166,"_167":168},"routes/_app.x.$handle.$name",{"_94":93,"_32":45,"_102":84,"_27":37,"_13":14,"_103":-5,"_11":104,"_105":106,"_107":108,"_109":110,"_111":112,"_20":-5,"_113":-5,"_114":-5,"_115":25,"_116":117,"_118":119,"_120":-5,"_121":-5,"_122":123},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_43":44,"_32":45,"_38":46,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":37,"_53":-5,"_18":54,"_55":-5,"_56":25,"_57":-5,"_58":-5,"_59":60,"_61":62,"_63":-5,"_64":-5,"_65":-5,"_66":-5,"_67":68,"_69":68,"_70":71,"_72":71,"_73":71,"_74":68,"_75":68,"_76":77,"_78":79,"_80":81,"_82":25},"version",7,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_34":35,"_36":37,"_38":39,"_40":14,"_41":42,"_24":25},"previous",{"_30":31,"_32":33,"_34":35,"_36":37},"deleted",false,"deletedHere","path","customize.js","latestPath","fileId","519e67de-7e8a-11f1-a7fd-1607ee4eb77e","name","new-file-3405.ts","fileType","script","content","","createdAt",["D",1783926823711],"branchVersion","branchId","c3b37b47-aac6-4dd9-a90b-e4098c3a71a1","id","96c7e756-e778-4d18-b447-3242cc1845e4","CROSS-MANGA-SCRAPER",["D",1783926729718],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","imageUrl","019d34f5-a118-71aa-86be-a3b05ce7b06e","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","019d34f5-a118-71aa-86be-a6dc6eeabba9","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_94":93,"_101":97},"org",{"_94":93,"_95":-5,"_96":97,"_18":54,"_98":99,"_100":25},"branch",{"_13":83,"_32":84,"_43":42,"_85":44,"_86":-5,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":45,"_92":93},"isOwned",84,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","lupinarashi","handle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoiZGVmYXVsdCIsImlpZCI6Imluc18yTm10cGhQUklwU0tqZHNiOHNnNGd2ZjJURFUiLCJyaWQiOiJ1c2VyXzNCWndldUNFUURoY3B1Uzk1d2lQOVhDTE8xaCJ9","tier","free","canRestrictVals","profileImageUrl","branchName","adoptOrgId",{"_43":44,"_32":45,"_38":161,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":37,"_53":-5,"_18":54,"_55":-5,"_56":25,"_57":-5,"_58":-5,"_59":60,"_61":62,"_63":-5,"_64":-5,"_65":-5,"_66":-5,"_67":68,"_69":68,"_70":71,"_72":71,"_73":71,"_74":68,"_75":68,"_76":162,"_78":163,"_80":164,"_82":25},"tree",[152,153],"files",[152,153],"counts",{"_142":60,"_143":130,"_144":60,"_145":146,"_147":148},"contents",[124,125],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles",true,"grants","orgs","customDomains",{},{"_43":133,"_32":134,"_30":135,"_127":-5,"_41":42,"_40":136,"_128":25,"_38":137,"_13":138,"_34":139,"_36":140,"_24":25,"_131":-5,"_27":141},{"_43":126,"_32":28,"_30":31,"_127":-5,"_41":42,"_40":14,"_128":25,"_38":129,"_13":130,"_34":35,"_36":37,"_24":25,"_131":-5,"_27":132},"7cd9a9e6-f061-4533-9929-eda4cad8fc13","parentId","latest",["D",1783926823711],1,"label",[28],"334b7bbb-8c1f-4c32-a0c9-e1df71c60ffe","NEON-API(manga).ts","29d54cae-7e8a-11f1-9d66-1607ee4eb77e",5,["D",1783926790400],4,"http","import { createClient } from \"npm:@libsql/client\";\nimport * as customize from \"./customize.js\";\n\ntype MangaRow = {\n key: string;\n title: string;\n type: string | null;\n year: number | null;\n banner_1: string;\n banner_2: string | null;\n author: string | null;\n tags: string | null;\n status: string | null;\n};\n\ntype ScrapeResult = {\n rows: MangaRow[];\n found: number;\n};\n\nconst SOURCE_URL =\n (customize as Record).SOURCE_URL ??\n \"https://weebcentral.com/search/data\";\n\nconst LIMIT = 32;\nconst REQUESTS_PER_RUN = 5;\nconst OFFSET_KEY =\n (customize as Record).OFFSET_KEY ??\n \"neon_manga_library:offset\";\nconst PLACEHOLDER_BANNER =\n (customize as Record).PLACEHOLDER_BANNER ??\n \"https://via.placeholder.com/480x720?text=No+Banner\";\n\nconst TURSO_DATABASE_URL =\n (customize as Record).TURSO_DATABASE_URL ??\n (customize as Record).TURSO_URL ??\n (customize as Record).DATABASE_URL;\nconst TURSO_AUTH_TOKEN =\n (customize as Record).TURSO_AUTH_TOKEN ??\n (customize as Record).TURSO_TOKEN ??\n (customize as Record).DATABASE_TOKEN;\nconst UPSTASH_REDIS_REST_URL =\n (customize as Record).UPSTASH_REDIS_REST_URL ??\n (customize as Record).UPSTASH_URL;\nconst UPSTASH_REDIS_REST_TOKEN =\n (customize as Record).UPSTASH_REDIS_REST_TOKEN ??\n (customize as Record).UPSTASH_TOKEN;\n\nif (!TURSO_DATABASE_URL || !TURSO_AUTH_TOKEN) {\n throw new Error(\n \"Missing Turso config in customize.js. Expected TURSO_DATABASE_URL and TURSO_AUTH_TOKEN (or TURSO_URL / TURSO_TOKEN aliases).\",\n );\n}\n\nif (!UPSTASH_REDIS_REST_URL || !UPSTASH_REDIS_REST_TOKEN) {\n throw new Error(\n \"Missing Upstash config in customize.js. Expected UPSTASH_REDIS_REST_URL and UPSTASH_REDIS_REST_TOKEN (or UPSTASH_URL / UPSTASH_TOKEN aliases).\",\n );\n}\n\nconst db = createClient({\n url: TURSO_DATABASE_URL,\n authToken: TURSO_AUTH_TOKEN,\n});\n\nconst SQL_SCHEMA = `\nCREATE TABLE IF NOT EXISTS neon_manga_library (\n key TEXT UNIQUE,\n title TEXT NOT NULL,\n type TEXT,\n year INTEGER,\n banner_1 TEXT NOT NULL,\n banner_2 TEXT,\n author TEXT,\n tags TEXT,\n status TEXT,\n created_at TEXT DEFAULT CURRENT_TIMESTAMP\n);\n`;\n\nfunction cleanText(value: string | null | undefined): string {\n return (value ?? \"\")\n .replace(/\\s+/g, \" \")\n .replace(/\\u00A0/g, \" \")\n .trim();\n}\n\nfunction firstNonEmpty(...values: Array): string {\n for (const value of values) {\n const cleaned = cleanText(value);\n if (cleaned) return cleaned;\n }\n return \"\";\n}\n\nfunction toIntOrNull(value: string | null | undefined): number | null {\n const cleaned = cleanText(value);\n if (!cleaned) return null;\n const match = cleaned.match(/\\d{4}/);\n return match ? Number(match[0]) : null;\n}\n\nfunction normalizeImageUrl(value: string | null | undefined): string {\n const cleaned = cleanText(value);\n if (!cleaned) return PLACEHOLDER_BANNER;\n const firstToken = cleaned.split(\",\")[0]?.trim() ?? cleaned;\n const firstUrl = firstToken.split(/\\s+/)[0]?.trim() ?? firstToken;\n return firstUrl || PLACEHOLDER_BANNER;\n}\n\nfunction safeJoin(values: string[]): string | null {\n const filtered = values.map(cleanText).filter(Boolean);\n return filtered.length ? filtered.join(\", \") : null;\n}\n\nasync function sha256Hex(input: string): Promise {\n const bytes = new TextEncoder().encode(input);\n const hash = await crypto.subtle.digest(\"SHA-256\", bytes);\n return Array.from(new Uint8Array(hash))\n .map((b) => b.toString(16).padStart(2, \"0\"))\n .join(\"\");\n}\n\nasync function upstashGet(key: string): Promise {\n const url = `${UPSTASH_REDIS_REST_URL.replace(/\\/$/, \"\")}/get/${encodeURIComponent(key)}`;\n const res = await fetch(url, {\n headers: { Authorization: `Bearer ${UPSTASH_REDIS_REST_TOKEN}` },\n });\n if (!res.ok) {\n throw new Error(`Upstash GET failed (${res.status})`);\n }\n const data = await res.json() as { result?: string | null };\n return data.result ?? null;\n}\n\nasync function upstashSet(key: string, value: string): Promise {\n const url = `${UPSTASH_REDIS_REST_URL.replace(/\\/$/, \"\")}/set/${encodeURIComponent(key)}/${encodeURIComponent(value)}`;\n const res = await fetch(url, {\n headers: { Authorization: `Bearer ${UPSTASH_REDIS_REST_TOKEN}` },\n });\n if (!res.ok) {\n throw new Error(`Upstash SET failed (${res.status})`);\n }\n}\n\nasync function ensureSchema(): Promise {\n await db.execute(SQL_SCHEMA);\n}\n\nfunction buildSourceUrl(offset: number): string {\n const params = new URLSearchParams({\n limit: String(LIMIT),\n offset: String(offset),\n sort: \"Alphabet\",\n order: \"Ascending\",\n official: \"Any\",\n anime: \"Any\",\n adult: \"Any\",\n display_mode: \"Full Display\",\n });\n return `${SOURCE_URL}?${params.toString()}`;\n}\n\nfunction extractList(article: Element, label: string): string[] {\n const strong = Array.from(article.querySelectorAll(\"strong\")).find((el) =>\n cleanText(el.textContent).includes(label)\n );\n if (!strong) return [];\n const parent = strong.parentElement;\n if (!parent) return [];\n return Array.from(parent.querySelectorAll(\"span\"))\n .map((span) => cleanText(span.textContent))\n .filter(Boolean);\n}\n\nasync function articleToRow(article: Element): Promise {\n const titleEl =\n article.querySelector(\".text-ellipsis.truncate\") ??\n article.querySelector(\"a.line-clamp-1\") ??\n article.querySelector(\"a[href]\");\n const title = firstNonEmpty(titleEl?.textContent);\n if (!title) return null;\n\n const anchor = article.querySelector(\"a[href]\") as HTMLAnchorElement | null;\n const href = anchor?.href ?? \"\";\n const keyMatch = href.match(/\\/series\\/([A-Z0-9]+)\\//i);\n const derivedKey = keyMatch?.[1] ?? \"\";\n const key = derivedKey || (await sha256Hex(`${title}|${href}`)).slice(0, 24);\n\n const normalSource = article.querySelector('source[media=\"(min-width: 768px)\"]');\n const fallbackImgEl = article.querySelector('img[alt*=\"cover\" i]');\n const banner1 = normalizeImageUrl(\n normalSource?.getAttribute(\"srcset\") ||\n normalSource?.getAttribute(\"src\") ||\n fallbackImgEl?.getAttribute(\"src\") ||\n fallbackImgEl?.getAttribute(\"data-src\") ||\n \"\",\n );\n\n const type = firstNonEmpty(\n ...extractList(article, \"Type\"),\n ...extractList(article, \"type\"),\n ) || null;\n const status = firstNonEmpty(\n ...extractList(article, \"Status\"),\n ...extractList(article, \"status\"),\n ) || null;\n const year =\n toIntOrNull(firstNonEmpty(...extractList(article, \"Year\"), ...extractList(article, \"year\"))) ??\n null;\n const authors = safeJoin(extractList(article, \"Author\"));\n const tags = safeJoin(extractList(article, \"Tag\"));\n\n return {\n key,\n title,\n type,\n year,\n banner_1: banner1,\n banner_2: null,\n author: authors,\n tags,\n status,\n };\n}\n\nasync function fetchAndParse(offset: number): Promise {\n const sourceUrl = buildSourceUrl(offset);\n const response = await fetch(sourceUrl, {\n method: \"GET\",\n headers: {\n Accept: \"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\",\n \"User-Agent\": \"Mozilla/5.0 (ValTown Manga Sync Bot)\",\n },\n });\n\n if (!response.ok) {\n throw new Error(`Source fetch failed (${response.status}) at offset ${offset}`);\n }\n\n const html = await response.text();\n const doc = new DOMParser().parseFromString(html, \"text/html\");\n if (!doc) {\n throw new Error(`Unable to parse HTML at offset ${offset}`);\n }\n\n const articles = Array.from(doc.querySelectorAll(\"article.bg-base-300\"));\n const rows: MangaRow[] = [];\n\n for (const article of articles) {\n const row = await articleToRow(article);\n if (!row) continue;\n if (!row.banner_1) row.banner_1 = PLACEHOLDER_BANNER;\n rows.push(row);\n }\n\n return { rows, found: articles.length };\n}\n\nasync function upsertRows(rows: MangaRow[]): Promise {\n if (!rows.length) return 0;\n\n const batch = rows.map((row) => ({\n sql: `\n INSERT INTO neon_manga_library (\n key, title, type, year, banner_1, banner_2, author, tags, status\n ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)\n ON CONFLICT(key) DO UPDATE SET\n title = excluded.title,\n type = excluded.type,\n year = excluded.year,\n banner_1 = excluded.banner_1,\n banner_2 = excluded.banner_2,\n author = excluded.author,\n tags = excluded.tags,\n status = excluded.status\n `,\n args: [\n row.key,\n row.title,\n row.type,\n row.year,\n row.banner_1,\n row.banner_2,\n row.author,\n row.tags,\n row.status,\n ],\n }));\n\n await db.batch(batch);\n return rows.length;\n}\n\nasync function readOffset(): Promise {\n const stored = await upstashGet(OFFSET_KEY);\n const parsed = Number.parseInt(stored ?? \"0\", 10);\n return Number.isFinite(parsed) && parsed >= 0 ? parsed : 0;\n}\n\nasync function writeOffset(offset: number): Promise {\n await upstashSet(OFFSET_KEY, String(offset));\n}\n\nasync function runSync(): Promise {\n await ensureSchema();\n\n let offset = await readOffset();\n const startOffset = offset;\n const runLog: Array<{\n requestOffset: number;\n found: number;\n insertedOrUpdated: number;\n ok: boolean;\n error?: string;\n }> = [];\n\n let totalInserted = 0;\n\n for (let i = 0; i < REQUESTS_PER_RUN; i++) {\n const requestOffset = offset;\n try {\n const { rows, found } = await fetchAndParse(requestOffset);\n const insertedOrUpdated = await upsertRows(rows);\n totalInserted += insertedOrUpdated;\n runLog.push({ requestOffset, found, insertedOrUpdated, ok: true });\n } catch (error) {\n runLog.push({\n requestOffset,\n found: 0,\n insertedOrUpdated: 0,\n ok: false,\n error: error instanceof Error ? error.message : String(error),\n });\n } finally {\n offset += LIMIT;\n }\n }\n\n await writeOffset(offset);\n\n return Response.json({\n ok: true,\n endpoint: \"/fetch-and-upload\",\n source: SOURCE_URL,\n startOffset,\n nextOffset: offset,\n pagesProcessed: REQUESTS_PER_RUN,\n rowsWritten: totalInserted,\n log: runLog,\n });\n}\n\nDeno.serve(async (req) => {\n const url = new URL(req.url);\n\n if (req.method !== \"GET\") {\n return new Response(\"Method Not Allowed\", { status: 405 });\n }\n\n if (url.pathname !== \"/fetch-and-upload\") {\n return new Response(\"Not Found\", { status: 404 });\n }\n\n try {\n return await runSync();\n } catch (error) {\n return Response.json(\n {\n ok: false,\n error: error instanceof Error ? error.message : String(error),\n },\n { status: 500 },\n );\n }\n});",[134],"pulls","branches","forks","envVars",{"_149":60,"_151":60},"connections",{"_149":60,"_150":60},"count","unfilledCount","emptyCount",{"_43":133,"_32":134,"_30":135,"_127":-5,"_41":42,"_40":136,"_128":25,"_38":158,"_13":138,"_34":139,"_36":140,"_24":25,"_131":-5,"_155":159,"_27":160},{"_43":126,"_32":28,"_30":31,"_127":-5,"_41":42,"_40":14,"_128":25,"_38":154,"_13":130,"_34":35,"_36":37,"_24":25,"_131":-5,"_155":156,"_27":157},["D",1783926823711],"children",[],[28],["D",1783926790400],[],[134],["D",1783926729718],{"_94":93,"_101":97},{"_94":93,"_95":-5,"_96":97,"_18":54,"_98":99,"_100":25},{"_13":83,"_32":84,"_43":42,"_85":44,"_86":-5,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":45,"_92":93},"ctx",{"_76":-5,"_121":198,"_199":200,"_201":202,"_203":25,"_204":25,"_205":25,"_206":119,"_207":-5,"_208":-7,"_209":-7,"_210":60,"_211":25,"_212":25,"_213":-5,"_214":25,"_215":25,"_216":25,"_217":-5},"ENV",{"_169":170,"_171":172,"_173":84,"_174":175,"_176":177,"_178":179,"_180":181,"_182":183,"_184":185,"_186":187,"_188":189,"_190":191,"_192":193,"_194":195,"_196":197},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v7