[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_164":165,"_166":167},"routes/_app.x.$handle.$name",{"_93":92,"_32":44,"_101":83,"_26":36,"_13":14,"_102":-5,"_11":103,"_104":105,"_106":107,"_108":109,"_110":111,"_20":-5,"_112":-5,"_113":-5,"_114":24,"_115":116,"_117":118,"_119":-5,"_120":-5,"_121":122},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_42":43,"_32":44,"_37":45,"_46":-5,"_47":24,"_48":49,"_50":49,"_51":36,"_52":-5,"_18":53,"_54":-5,"_55":24,"_56":-5,"_57":-5,"_58":59,"_60":61,"_62":-5,"_63":-5,"_64":-5,"_65":-5,"_66":67,"_68":67,"_69":70,"_71":70,"_72":70,"_73":67,"_74":67,"_75":76,"_77":78,"_79":80,"_81":24},"version",6,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":-7,"_23":24,"_25":24,"_26":27,"_28":29},"file",{"_30":31,"_32":27,"_33":34,"_35":36,"_37":38,"_39":14,"_40":41,"_23":24},"previous","deleted",false,"deletedHere","path","new-file-3405.ts","latestPath","customize.js","fileId","519e67de-7e8a-11f1-a7fd-1607ee4eb77e","name","fileType","script","content","","createdAt",["D",1783926796497],"branchVersion","branchId","c3b37b47-aac6-4dd9-a90b-e4098c3a71a1","id","96c7e756-e778-4d18-b447-3242cc1845e4","CROSS-MANGA-SCRAPER",["D",1783926729718],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","imageUrl","019d34f5-a118-71aa-86be-a3b05ce7b06e","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","019d34f5-a118-71aa-86be-a6dc6eeabba9","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_93":92,"_100":96},"org",{"_93":92,"_94":-5,"_95":96,"_18":53,"_97":98,"_99":24},"branch",{"_13":82,"_32":83,"_42":41,"_84":43,"_85":-5,"_86":-5,"_87":-5,"_88":-5,"_89":-5,"_90":44,"_91":92},"isOwned",84,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","lupinarashi","handle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoiZGVmYXVsdCIsImlpZCI6Imluc18yTm10cGhQUklwU0tqZHNiOHNnNGd2ZjJURFUiLCJyaWQiOiJ1c2VyXzNCWndldUNFUURoY3B1Uzk1d2lQOVhDTE8xaCJ9","tier","free","canRestrictVals","profileImageUrl","branchName","adoptOrgId",{"_42":43,"_32":44,"_37":160,"_46":-5,"_47":24,"_48":49,"_50":49,"_51":36,"_52":-5,"_18":53,"_54":-5,"_55":24,"_56":-5,"_57":-5,"_58":59,"_60":61,"_62":-5,"_63":-5,"_64":-5,"_65":-5,"_66":67,"_68":67,"_69":70,"_71":70,"_72":70,"_73":67,"_74":67,"_75":161,"_77":162,"_79":163,"_81":24},"tree",[151,152],"files",[151,152],"counts",{"_140":59,"_141":142,"_143":59,"_144":145,"_146":147},"contents",[123,124],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles",true,"grants","orgs","customDomains",{},{"_42":131,"_32":132,"_30":133,"_126":-5,"_40":41,"_39":134,"_127":24,"_37":135,"_13":136,"_33":137,"_35":138,"_23":24,"_129":-5,"_26":139},{"_42":125,"_32":27,"_30":31,"_126":-5,"_40":41,"_39":14,"_127":24,"_37":128,"_13":59,"_33":34,"_35":36,"_23":24,"_129":-5,"_26":130},"325ace47-a412-4988-aca8-54a13f9d4ee1","parentId","latest",["D",1783926796497],"label",[27],"334b7bbb-8c1f-4c32-a0c9-e1df71c60ffe","NEON-API(manga).ts","29d54cae-7e8a-11f1-9d66-1607ee4eb77e",5,["D",1783926790400],4,"http","import { createClient } from \"npm:@libsql/client\";\nimport * as customize from \"./customize.js\";\n\ntype MangaRow = {\n key: string;\n title: string;\n type: string | null;\n year: number | null;\n banner_1: string;\n banner_2: string | null;\n author: string | null;\n tags: string | null;\n status: string | null;\n};\n\ntype ScrapeResult = {\n rows: MangaRow[];\n found: number;\n};\n\nconst SOURCE_URL =\n (customize as Record).SOURCE_URL ??\n \"https://weebcentral.com/search/data\";\n\nconst LIMIT = 32;\nconst REQUESTS_PER_RUN = 5;\nconst OFFSET_KEY =\n (customize as Record).OFFSET_KEY ??\n \"neon_manga_library:offset\";\nconst PLACEHOLDER_BANNER =\n (customize as Record).PLACEHOLDER_BANNER ??\n \"https://via.placeholder.com/480x720?text=No+Banner\";\n\nconst TURSO_DATABASE_URL =\n (customize as Record).TURSO_DATABASE_URL ??\n (customize as Record).TURSO_URL ??\n (customize as Record).DATABASE_URL;\nconst TURSO_AUTH_TOKEN =\n (customize as Record).TURSO_AUTH_TOKEN ??\n (customize as Record).TURSO_TOKEN ??\n (customize as Record).DATABASE_TOKEN;\nconst UPSTASH_REDIS_REST_URL =\n (customize as Record).UPSTASH_REDIS_REST_URL ??\n (customize as Record).UPSTASH_URL;\nconst UPSTASH_REDIS_REST_TOKEN =\n (customize as Record).UPSTASH_REDIS_REST_TOKEN ??\n (customize as Record).UPSTASH_TOKEN;\n\nif (!TURSO_DATABASE_URL || !TURSO_AUTH_TOKEN) {\n throw new Error(\n \"Missing Turso config in customize.js. Expected TURSO_DATABASE_URL and TURSO_AUTH_TOKEN (or TURSO_URL / TURSO_TOKEN aliases).\",\n );\n}\n\nif (!UPSTASH_REDIS_REST_URL || !UPSTASH_REDIS_REST_TOKEN) {\n throw new Error(\n \"Missing Upstash config in customize.js. Expected UPSTASH_REDIS_REST_URL and UPSTASH_REDIS_REST_TOKEN (or UPSTASH_URL / UPSTASH_TOKEN aliases).\",\n );\n}\n\nconst db = createClient({\n url: TURSO_DATABASE_URL,\n authToken: TURSO_AUTH_TOKEN,\n});\n\nconst SQL_SCHEMA = `\nCREATE TABLE IF NOT EXISTS neon_manga_library (\n key TEXT UNIQUE,\n title TEXT NOT NULL,\n type TEXT,\n year INTEGER,\n banner_1 TEXT NOT NULL,\n banner_2 TEXT,\n author TEXT,\n tags TEXT,\n status TEXT,\n created_at TEXT DEFAULT CURRENT_TIMESTAMP\n);\n`;\n\nfunction cleanText(value: string | null | undefined): string {\n return (value ?? \"\")\n .replace(/\\s+/g, \" \")\n .replace(/\\u00A0/g, \" \")\n .trim();\n}\n\nfunction firstNonEmpty(...values: Array): string {\n for (const value of values) {\n const cleaned = cleanText(value);\n if (cleaned) return cleaned;\n }\n return \"\";\n}\n\nfunction toIntOrNull(value: string | null | undefined): number | null {\n const cleaned = cleanText(value);\n if (!cleaned) return null;\n const match = cleaned.match(/\\d{4}/);\n return match ? Number(match[0]) : null;\n}\n\nfunction normalizeImageUrl(value: string | null | undefined): string {\n const cleaned = cleanText(value);\n if (!cleaned) return PLACEHOLDER_BANNER;\n const firstToken = cleaned.split(\",\")[0]?.trim() ?? cleaned;\n const firstUrl = firstToken.split(/\\s+/)[0]?.trim() ?? firstToken;\n return firstUrl || PLACEHOLDER_BANNER;\n}\n\nfunction safeJoin(values: string[]): string | null {\n const filtered = values.map(cleanText).filter(Boolean);\n return filtered.length ? filtered.join(\", \") : null;\n}\n\nasync function sha256Hex(input: string): Promise {\n const bytes = new TextEncoder().encode(input);\n const hash = await crypto.subtle.digest(\"SHA-256\", bytes);\n return Array.from(new Uint8Array(hash))\n .map((b) => b.toString(16).padStart(2, \"0\"))\n .join(\"\");\n}\n\nasync function upstashGet(key: string): Promise {\n const url = `${UPSTASH_REDIS_REST_URL.replace(/\\/$/, \"\")}/get/${encodeURIComponent(key)}`;\n const res = await fetch(url, {\n headers: { Authorization: `Bearer ${UPSTASH_REDIS_REST_TOKEN}` },\n });\n if (!res.ok) {\n throw new Error(`Upstash GET failed (${res.status})`);\n }\n const data = await res.json() as { result?: string | null };\n return data.result ?? null;\n}\n\nasync function upstashSet(key: string, value: string): Promise {\n const url = `${UPSTASH_REDIS_REST_URL.replace(/\\/$/, \"\")}/set/${encodeURIComponent(key)}/${encodeURIComponent(value)}`;\n const res = await fetch(url, {\n headers: { Authorization: `Bearer ${UPSTASH_REDIS_REST_TOKEN}` },\n });\n if (!res.ok) {\n throw new Error(`Upstash SET failed (${res.status})`);\n }\n}\n\nasync function ensureSchema(): Promise {\n await db.execute(SQL_SCHEMA);\n}\n\nfunction buildSourceUrl(offset: number): string {\n const params = new URLSearchParams({\n limit: String(LIMIT),\n offset: String(offset),\n sort: \"Alphabet\",\n order: \"Ascending\",\n official: \"Any\",\n anime: \"Any\",\n adult: \"Any\",\n display_mode: \"Full Display\",\n });\n return `${SOURCE_URL}?${params.toString()}`;\n}\n\nfunction extractList(article: Element, label: string): string[] {\n const strong = Array.from(article.querySelectorAll(\"strong\")).find((el) =>\n cleanText(el.textContent).includes(label)\n );\n if (!strong) return [];\n const parent = strong.parentElement;\n if (!parent) return [];\n return Array.from(parent.querySelectorAll(\"span\"))\n .map((span) => cleanText(span.textContent))\n .filter(Boolean);\n}\n\nasync function articleToRow(article: Element): Promise {\n const titleEl =\n article.querySelector(\".text-ellipsis.truncate\") ??\n article.querySelector(\"a.line-clamp-1\") ??\n article.querySelector(\"a[href]\");\n const title = firstNonEmpty(titleEl?.textContent);\n if (!title) return null;\n\n const anchor = article.querySelector(\"a[href]\") as HTMLAnchorElement | null;\n const href = anchor?.href ?? \"\";\n const keyMatch = href.match(/\\/series\\/([A-Z0-9]+)\\//i);\n const derivedKey = keyMatch?.[1] ?? \"\";\n const key = derivedKey || (await sha256Hex(`${title}|${href}`)).slice(0, 24);\n\n const normalSource = article.querySelector('source[media=\"(min-width: 768px)\"]');\n const fallbackImgEl = article.querySelector('img[alt*=\"cover\" i]');\n const banner1 = normalizeImageUrl(\n normalSource?.getAttribute(\"srcset\") ||\n normalSource?.getAttribute(\"src\") ||\n fallbackImgEl?.getAttribute(\"src\") ||\n fallbackImgEl?.getAttribute(\"data-src\") ||\n \"\",\n );\n\n const type = firstNonEmpty(\n ...extractList(article, \"Type\"),\n ...extractList(article, \"type\"),\n ) || null;\n const status = firstNonEmpty(\n ...extractList(article, \"Status\"),\n ...extractList(article, \"status\"),\n ) || null;\n const year =\n toIntOrNull(firstNonEmpty(...extractList(article, \"Year\"), ...extractList(article, \"year\"))) ??\n null;\n const authors = safeJoin(extractList(article, \"Author\"));\n const tags = safeJoin(extractList(article, \"Tag\"));\n\n return {\n key,\n title,\n type,\n year,\n banner_1: banner1,\n banner_2: null,\n author: authors,\n tags,\n status,\n };\n}\n\nasync function fetchAndParse(offset: number): Promise {\n const sourceUrl = buildSourceUrl(offset);\n const response = await fetch(sourceUrl, {\n method: \"GET\",\n headers: {\n Accept: \"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\",\n \"User-Agent\": \"Mozilla/5.0 (ValTown Manga Sync Bot)\",\n },\n });\n\n if (!response.ok) {\n throw new Error(`Source fetch failed (${response.status}) at offset ${offset}`);\n }\n\n const html = await response.text();\n const doc = new DOMParser().parseFromString(html, \"text/html\");\n if (!doc) {\n throw new Error(`Unable to parse HTML at offset ${offset}`);\n }\n\n const articles = Array.from(doc.querySelectorAll(\"article.bg-base-300\"));\n const rows: MangaRow[] = [];\n\n for (const article of articles) {\n const row = await articleToRow(article);\n if (!row) continue;\n if (!row.banner_1) row.banner_1 = PLACEHOLDER_BANNER;\n rows.push(row);\n }\n\n return { rows, found: articles.length };\n}\n\nasync function upsertRows(rows: MangaRow[]): Promise {\n if (!rows.length) return 0;\n\n const batch = rows.map((row) => ({\n sql: `\n INSERT INTO neon_manga_library (\n key, title, type, year, banner_1, banner_2, author, tags, status\n ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)\n ON CONFLICT(key) DO UPDATE SET\n title = excluded.title,\n type = excluded.type,\n year = excluded.year,\n banner_1 = excluded.banner_1,\n banner_2 = excluded.banner_2,\n author = excluded.author,\n tags = excluded.tags,\n status = excluded.status\n `,\n args: [\n row.key,\n row.title,\n row.type,\n row.year,\n row.banner_1,\n row.banner_2,\n row.author,\n row.tags,\n row.status,\n ],\n }));\n\n await db.batch(batch);\n return rows.length;\n}\n\nasync function readOffset(): Promise {\n const stored = await upstashGet(OFFSET_KEY);\n const parsed = Number.parseInt(stored ?? \"0\", 10);\n return Number.isFinite(parsed) && parsed >= 0 ? parsed : 0;\n}\n\nasync function writeOffset(offset: number): Promise {\n await upstashSet(OFFSET_KEY, String(offset));\n}\n\nasync function runSync(): Promise {\n await ensureSchema();\n\n let offset = await readOffset();\n const startOffset = offset;\n const runLog: Array<{\n requestOffset: number;\n found: number;\n insertedOrUpdated: number;\n ok: boolean;\n error?: string;\n }> = [];\n\n let totalInserted = 0;\n\n for (let i = 0; i < REQUESTS_PER_RUN; i++) {\n const requestOffset = offset;\n try {\n const { rows, found } = await fetchAndParse(requestOffset);\n const insertedOrUpdated = await upsertRows(rows);\n totalInserted += insertedOrUpdated;\n runLog.push({ requestOffset, found, insertedOrUpdated, ok: true });\n } catch (error) {\n runLog.push({\n requestOffset,\n found: 0,\n insertedOrUpdated: 0,\n ok: false,\n error: error instanceof Error ? error.message : String(error),\n });\n } finally {\n offset += LIMIT;\n }\n }\n\n await writeOffset(offset);\n\n return Response.json({\n ok: true,\n endpoint: \"/fetch-and-upload\",\n source: SOURCE_URL,\n startOffset,\n nextOffset: offset,\n pagesProcessed: REQUESTS_PER_RUN,\n rowsWritten: totalInserted,\n log: runLog,\n });\n}\n\nDeno.serve(async (req) => {\n const url = new URL(req.url);\n\n if (req.method !== \"GET\") {\n return new Response(\"Method Not Allowed\", { status: 405 });\n }\n\n if (url.pathname !== \"/fetch-and-upload\") {\n return new Response(\"Not Found\", { status: 404 });\n }\n\n try {\n return await runSync();\n } catch (error) {\n return Response.json(\n {\n ok: false,\n error: error instanceof Error ? error.message : String(error),\n },\n { status: 500 },\n );\n }\n});",[132],"pulls","branches",1,"forks","envVars",{"_148":59,"_150":59},"connections",{"_148":59,"_149":59},"count","unfilledCount","emptyCount",{"_42":131,"_32":132,"_30":133,"_126":-5,"_40":41,"_39":134,"_127":24,"_37":157,"_13":136,"_33":137,"_35":138,"_23":24,"_129":-5,"_154":158,"_26":159},{"_42":125,"_32":27,"_30":31,"_126":-5,"_40":41,"_39":14,"_127":24,"_37":153,"_13":59,"_33":34,"_35":36,"_23":24,"_129":-5,"_154":155,"_26":156},["D",1783926796497],"children",[],[27],["D",1783926790400],[],[132],["D",1783926729718],{"_93":92,"_100":96},{"_93":92,"_94":-5,"_95":96,"_18":53,"_97":98,"_99":24},{"_13":82,"_32":83,"_42":41,"_84":43,"_85":-5,"_86":-5,"_87":-5,"_88":-5,"_89":-5,"_90":44,"_91":92},"ctx",{"_75":-5,"_120":197,"_198":199,"_200":201,"_202":24,"_203":24,"_204":24,"_205":118,"_206":-5,"_207":-7,"_208":-7,"_209":59,"_210":24,"_211":24,"_212":-5,"_213":24,"_214":24,"_215":24,"_216":-5},"ENV",{"_168":169,"_170":171,"_172":83,"_173":174,"_175":176,"_177":178,"_179":180,"_181":182,"_183":184,"_185":186,"_187":188,"_189":190,"_191":192,"_193":194,"_195":196},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v6