[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_175":176,"_177":178},"routes/_app.x.$handle.$name",{"_102":101,"_38":51,"_111":92,"_33":112,"_13":14,"_113":-5,"_11":114,"_115":116,"_117":118,"_119":120,"_121":122,"_26":-5,"_123":-5,"_124":-5,"_125":31,"_126":127,"_128":64,"_129":-5,"_130":-5,"_131":132},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":18,"_19":-5},"project",{"_49":50,"_38":51,"_44":52,"_53":-5,"_54":31,"_55":56,"_57":56,"_58":59,"_60":-5,"_19":61,"_62":37,"_63":64,"_65":-5,"_66":-5,"_67":68,"_69":70,"_71":-5,"_72":-5,"_73":-5,"_74":-5,"_75":76,"_77":76,"_78":79,"_80":79,"_81":79,"_82":76,"_83":76,"_84":85,"_86":87,"_88":89,"_90":31},"version",8,"changes",[25],"aiChatInfo",{"_20":21,"_22":-5,"_23":-5,"_24":-5},"userId","aiChatMessageId","4c37dd80-85b4-43e2-a063-09bcb32b83d8","aiChatThreadId","aiChatThreadUserId","aiChatThreadTitle",{"_26":27,"_28":29,"_30":31,"_32":31,"_33":34,"_35":34},"file",{"_36":37,"_38":34,"_39":40,"_41":43,"_44":45,"_46":14,"_47":48,"_30":31},"previous",{"_36":37,"_38":34,"_39":40,"_41":42},"deleted",false,"deletedHere","path","main.ts","latestPath","fileId","32c89862-a878-11f1-946c-1607ee4eb77e","name","fileType","http","content","/**\n * URL Archiver\n *\n * Archive one or more URLs into readable article content plus link markdown.\n *\n * Any HTTP method (GET/POST/PUT/PATCH/...) is accepted. URLs may be passed\n * via query params (?url=https://a.com&site=https://b.com&url=https://c.com)\n * or in the request body (JSON or form-encoded). Accepted aliases:\n * url, site, website, link (+ their plural forms)\n * Each value may be a string or an array, and multiple params are flattened\n * into a single list of URLs.\n *\n * POST / { \"url\": [\"https://a.com\", \"https://b.com\"] }\n * GET /?link=https://a.com&website=https://b.com\n */\nimport { extract, extractFromHtml } from \"npm:@extractus/article-extractor@9.0.1\";\nimport { Cairn } from \"npm:@wabarc/cairn@3.0.0\";\n\nconst userAgent = `heidi/url-archiver`;\n\nconst ALIASES = new Set([\n \"url\", \"urls\", \"site\", \"sites\", \"website\", \"websites\",\n \"link\", \"links\",\n]);\n\nexport default async function handler(req: Request): Promise {\n // Parse raw values from body (JSON or form) and query string.\n const collected: Record = {};\n\n function collect(source: unknown) {\n if (!source || typeof source !== \"object\") return;\n for (const [key, value] of Object.entries(source as Record)) {\n if (ALIASES.has(key)) {\n if (Array.isArray(collected[key])) {\n (collected[key] as unknown[]).push(...flatten(value));\n } else {\n collected[key] = flatten(value);\n }\n }\n }\n }\n\n // Body: try JSON first, fall back to form-encoded.\n try {\n const text = await req.text();\n if (text) {\n const type = req.headers.get(\"content-type\") || \"\";\n if (type.includes(\"application/json\")) {\n collect(JSON.parse(text));\n } else {\n collected[\"url\"] = Array.from(new URLSearchParams(text).entries())\n .filter(([k]) => ALIASES.has(k))\n .map(([, v]) => v);\n }\n }\n } catch {\n /* ignore unparseable body */\n }\n\n // Query string.\n const qs = new URL(req.url).searchParams;\n for (const key of Array.from(qs.keys())) {\n if (ALIASES.has(key)) {\n const vals = qs.getAll(key);\n if (Array.isArray(collected[key])) {\n (collected[key] as string[]).push(...vals);\n } else {\n collected[key] = vals;\n }\n }\n }\n\n // Flatten every alias into one unique list of URLs.\n const urls = Array.from(\n new Set(Object.values(collected).flatMap((v) => v as string[]) ?? []),\n ).filter((u): u is string => typeof u === \"string\" && u.length > 0);\n\n if (urls.length === 0) {\n return json(\n { error: \"No URL provided. Use url/site/website/link (query or body).\" },\n 400,\n );\n }\n\n // Archive each URL; one bad URL doesn't sink the rest.\n const results = await Promise.all(urls.map((u) => archive(u)));\n const ok = results.filter((r) => r.ok).map((r) => r.record);\n const failed = results.filter((r) => !r.ok).map((r) => r.error);\n\n const body: Record = {\n archived_at: new Date().toISOString(),\n count: ok.length + failed.length,\n results: ok,\n };\n if (failed.length) body.failed = failed;\n\n return json(body, failed.length && !ok.length ? 500 : 200);\n}\n\nasync function archive(url: string): Promise<\n | { ok: true; record: Record }\n | { ok: false; error: { url: string; error: string } }\n> {\n try {\n const cairn = new Cairn();\n const archived = await cairn.request({ url }).options({ userAgent }).archive();\n const webpage = {\n url: archived.url,\n webpage: archived.webpage.html(),\n status: archived.status,\n contentType: archived.contentType,\n };\n\n const article = webpage?.webpage\n ? extractFromHtml(webpage.webpage, webpage.url)\n : await extract(url);\n\n return {\n ok: true,\n record: {\n originalUrl: url,\n archived_at: new Date().toISOString(),\n ...webpage,\n ...article,\n link_markdown: archived.toc ?? [],\n },\n };\n } catch (err) {\n return { ok: false, error: { url, error: (err as Error).message } };\n }\n}\n\nfunction flatten(value: unknown): Array {\n return Array.isArray(value) ? value.flatMap(flatten) : [value];\n}\n\nfunction json(body: unknown, status: number): Response {\n return new Response(JSON.stringify(body, null, 2), {\n status,\n headers: { \"content-type\": \"application/json\" },\n });\n}\n","/**\n * URL Archiver\n *\n * Archive one or more URLs into a readable article record (clean content,\n * metadata) plus a snapshot of the fetched HTML.\n *\n * Any HTTP method (GET/POST/PUT/PATCH/...) is accepted. URLs may be passed\n * via query params (?url=https://a.com&site=https://b.com&url=https://c.com)\n * or in the request body (JSON or form-encoded). Accepted aliases:\n * url, site, website, link (+ their plural forms)\n * Each value may be a string or an array, and multiple params are flattened\n * into a single list of URLs.\n *\n * POST / { \"url\": [\"https://a.com\", \"https://b.com\"] }\n * GET /?link=https://a.com&website=https://b.com\n *\n * Note: cairn's own fetch path doesn't load under Deno, so this uses plain\n * `fetch` to snapshot the page (cairn's job) and article-extractor to derive\n * the readable record — same combined intent, works in the sandbox.\n */\nimport { extractFromHtml } from \"npm:@extractus/article-extractor@9.0.1\";\n\nconst ALIASES = new Set([\n \"url\", \"urls\", \"site\", \"sites\", \"website\", \"websites\",\n \"link\", \"links\",\n]);\n\nconst USER_AGENT = \"Mozilla/5.0 (compatible; heidi/url-archiver)\";\n\nexport default async function handler(req: Request): Promise {\n // Parse raw values from body (JSON or form) and query string.\n const collected: Record = {};\n\n function collect(source: unknown) {\n if (!source || typeof source !== \"object\") return;\n for (const [key, value] of Object.entries(source as Record)) {\n if (ALIASES.has(key)) {\n if (Array.isArray(collected[key])) {\n (collected[key] as unknown[]).push(...flatten(value));\n } else {\n collected[key] = flatten(value);\n }\n }\n }\n }\n\n // Body: try JSON first, fall back to form-encoded.\n try {\n const text = await req.text();\n if (text) {\n const type = req.headers.get(\"content-type\") || \"\";\n if (type.includes(\"application/json\")) {\n collect(JSON.parse(text));\n } else {\n collected[\"url\"] = Array.from(new URLSearchParams(text).entries())\n .filter(([k]) => ALIASES.has(k))\n .map(([, v]) => v);\n }\n }\n } catch {\n /* ignore unparseable body */\n }\n\n // Query string.\n const qs = new URL(req.url).searchParams;\n for (const key of Array.from(qs.keys())) {\n if (ALIASES.has(key)) {\n const vals = qs.getAll(key);\n if (Array.isArray(collected[key])) {\n (collected[key] as string[]).push(...vals);\n } else {\n collected[key] = vals;\n }\n }\n }\n\n // Flatten every alias into one unique list of URLs.\n const urls = Array.from(\n new Set(Object.values(collected).flatMap((v) => v as string[]) ?? []),\n ).filter((u): u is string => typeof u === \"string\" && u.length > 0);\n\n if (urls.length === 0) {\n return json(\n { error: \"No URL provided. Use url/site/website/link (query or body).\" },\n 400,\n );\n }\n\n // Archive each URL; one bad URL doesn't sink the rest.\n const results = await Promise.all(urls.map((u) => archive(u)));\n const ok = results.filter((r) => r.ok).map((r) => r.record);\n const failed = results.filter((r) => !r.ok).map((r) => r.error);\n\n const body: Record = {\n archived_at: new Date().toISOString(),\n count: ok.length + failed.length,\n results: ok,\n };\n if (failed.length) body.failed = failed;\n\n return json(body, failed.length && !ok.length ? 500 : 200);\n}\n\nasync function archive(url: string): Promise<\n | { ok: true; record: Record }\n | { ok: false; error: { url: string; error: string } }\n> {\n try {\n // Snapshot the page (this is cairn's job, done with plain fetch).\n const res = await fetch(url, {\n headers: { \"user-agent\": USER_AGENT },\n redirect: \"follow\",\n });\n const finalUrl = res.url || url;\n const contentType = res.headers.get(\"content-type\") || \"\";\n const status = res.status;\n if (!res.ok) throw new Error(`HTTP ${status}`);\n const html = await res.text();\n\n // Derive the readable article from the fetched HTML (extractor's job).\n const article = await extractFromHtml(html, finalUrl).catch(() => null);\n\n return {\n ok: true,\n record: {\n originalUrl: url,\n url: finalUrl,\n archived_at: new Date().toISOString(),\n status,\n contentType,\n title: article?.title ?? null,\n author: article?.author ?? null,\n description: article?.description ?? null,\n image: article?.image ?? null,\n publisher: article?.publisher ?? null,\n published: article?.published ?? null,\n content: article?.content ?? null,\n // Snapshot retained as fetched HTML.\n html_snapshot: html.slice(0, 2_000_000),\n },\n };\n } catch (err) {\n return { ok: false, error: { url, error: (err as Error).message } };\n }\n}\n\nfunction flatten(value: unknown): Array {\n return Array.isArray(value) ? value.flatMap(flatten) : [value];\n}\n\nfunction json(body: unknown, status: number): Response {\n return new Response(JSON.stringify(body, null, 2), {\n status,\n headers: { \"content-type\": \"application/json\" },\n });\n}\n","createdAt",["D",1788537791776],"branchVersion","branchId","3f798dd9-8407-4149-b148-45d4da89ccbd","id","ce845deb-58e1-4474-aea4-5bbf1cd9e916","url-archiver",["D",1788536954683],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Archive any URL: git repos, music, video, papers, Wikipedia","imageUrl","124f6344-3784-4835-926c-5b9b29b366cc","pinnedFileId","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","01a06d17-30e7-7396-bb69-029ba771a7d7","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_102":101,"_110":106},"org",{"_102":101,"_103":104,"_105":106,"_19":61,"_107":108,"_109":31},"branch",{"_13":91,"_38":92,"_49":48,"_93":50,"_94":-5,"_95":-5,"_96":-5,"_97":-5,"_98":-5,"_99":51,"_100":101},"isOwned",21,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","heidi","handle","displayName","Heidi Hedh","avatar","https://lh3.googleusercontent.com/a/ACg8ocK_QSHhMpajSrYgGIPk8bOKlftGgK0BEPGcExr97zjm8MHbQA=s96-c","tier","free","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_49":50,"_38":51,"_44":171,"_53":-5,"_54":31,"_55":56,"_57":56,"_58":59,"_60":-5,"_19":61,"_62":37,"_63":64,"_65":-5,"_66":-5,"_67":68,"_69":70,"_71":-5,"_72":-5,"_73":-5,"_74":-5,"_75":76,"_77":76,"_78":79,"_80":79,"_81":79,"_82":76,"_83":76,"_84":172,"_86":173,"_88":174,"_90":31},"tree",[162,163],"files",[162,163],"counts",{"_151":68,"_152":153,"_154":68,"_155":156,"_157":158},"contents",[133,134],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_49":142,"_38":143,"_36":144,"_136":-5,"_47":48,"_46":145,"_137":31,"_44":146,"_13":147,"_39":148,"_41":149,"_30":31,"_140":-5,"_33":150},{"_49":135,"_38":34,"_36":37,"_136":-5,"_47":48,"_46":14,"_137":31,"_44":138,"_13":139,"_39":40,"_41":43,"_30":31,"_140":51,"_33":141},"a95cd808-08e7-4af6-ba4f-24da7983d34a","parentId","latest",["D",1788537791776],4,"label",[34],"f102bb57-2ec4-4687-b94d-b0de8d44c59a","cairn-test.ts","f85dabc0-a879-11f1-bf8d-1607ee4eb77e",7,["D",1788537744646],2,"script","import { Cairn } from \"npm:@wabarc/cairn@2\";\n\nconst cairn = new Cairn();\ntry {\n const archived = await cairn.request({ url: \"https://example.com\" })\n .options({ userAgent: \"heidi-cairn-test\", timeout: 30 })\n .archive();\n console.log(\"archive OK\");\n console.log(\"url:\", archived.url, \"status:\", archived.status);\n console.log(\"keys:\", Object.keys(archived));\n console.log(\"has webpage:\", !!archived.webpage, \"type:\", typeof archived.webpage);\n} catch (e) {\n console.error(\"CAIRN 3 FAIL:\", (e as Error).message);\n}\n",[143],"pulls","branches",1,"forks","envVars",{"_159":68,"_161":68},"connections",{"_159":68,"_160":68},"count","unfilledCount","emptyCount",{"_49":142,"_38":143,"_36":144,"_136":-5,"_47":48,"_46":145,"_137":31,"_44":168,"_13":147,"_39":148,"_41":149,"_30":31,"_140":-5,"_165":169,"_33":170},{"_49":135,"_38":34,"_36":37,"_136":-5,"_47":48,"_46":14,"_137":31,"_44":164,"_13":139,"_39":40,"_41":43,"_30":31,"_140":51,"_165":166,"_33":167},["D",1788537791776],"children",[],[34],["D",1788537744646],[],[143],["D",1788536954683],{"_102":101,"_110":106},{"_102":101,"_103":104,"_105":106,"_19":61,"_107":108,"_109":31},{"_13":91,"_38":92,"_49":48,"_93":50,"_94":-5,"_95":-5,"_96":-5,"_97":-5,"_98":-5,"_99":51,"_100":101},"ctx",{"_84":-5,"_130":208,"_209":210,"_211":212,"_213":31,"_214":31,"_215":31,"_216":64,"_217":-5,"_218":-7,"_219":-7,"_220":68,"_221":31,"_222":31,"_223":-5,"_224":31,"_225":31,"_226":31,"_227":-5},"ENV",{"_179":180,"_181":182,"_183":92,"_184":185,"_186":187,"_188":189,"_190":191,"_192":193,"_194":195,"_196":197,"_198":199,"_200":201,"_202":203,"_204":205,"_206":207},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v8