[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8},"actionData","errors","root",{"_103":104,"_105":106},"routes/embed.x.$handle.$name.$",{"_9":10,"_11":12,"_13":14,"_15":16,"_17":18,"_19":20,"_21":-5},"handle","heidi","name","url-archiver","path","main.ts","project",{"_22":45,"_11":12,"_33":46,"_47":-5,"_48":42,"_49":50,"_51":50,"_52":53,"_54":-5,"_55":56,"_57":25,"_58":32,"_59":-5,"_60":-5,"_61":62,"_63":64,"_65":-5,"_66":-5,"_67":-5,"_68":-5,"_69":70,"_71":70,"_72":73,"_74":73,"_75":73,"_76":70,"_77":70,"_78":79,"_80":81,"_82":83,"_84":42},"file",{"_22":23,"_11":14,"_24":25,"_26":-5,"_27":28,"_29":30,"_31":32,"_33":34,"_35":36,"_37":38,"_39":40,"_41":42,"_43":12,"_13":44},"permalink","https://www.val.town/x/heidi/url-archiver/code/main.ts","extension","id","c549e592-a93e-4116-922d-7b61704d6245","fileId","32c89862-a878-11f1-946c-1607ee4eb77e","parentId","branchId","3f798dd9-8407-4149-b148-45d4da89ccbd","branchVersion",20,"latest",true,"createdAt",["D",1788768338783],"version",7,"fileType","http","content","/**\n * URL Archiver\n *\n * Archive one or more URLs into a rich record. Any HTTP method is accepted.\n * URLs may come via query params or body (JSON or form), under any of the\n * aliases: url, urls, site, sites, website, websites, link, links. Each may\n * be a string or an array; all are flattened & deduped.\n *\n * Extractor options may be passed as a nested `options` object or flat keys:\n * wordsPerMinute, descriptionTruncateLen, descriptionLengthThreshold,\n * contentLengthThreshold, allowedTags, allowedAttributes, allowedIframeDomains\n *\n * A single URL returns the record directly; multiple URLs return an array.\n * The page type is recognized smartly and routed to a dedicated provider:\n * git GitHub / GitLab / Bitbucket / Codeberg / Gitea repos\n * music Spotify / Deezer / SoundCloud / Bandcamp / YouTube Music,\n * plus MusicBrainz / ListenBrainz / Last.fm\n * video YouTube / Vimeo / Dailymotion\n * wikipedia Wikipedia articles (+ REST summary)\n * paper DOIs (Crossref) / arXiv / PubMed\n * file direct asset downloads\n * article generic article extraction (default fallback)\n *\n * POST / { \"url\": [\"https://a.com\", \"https://b.com\"], \"options\": { \"wordsPerMinute\": 250 } }\n * GET /?link=https://a.com&wordsPerMinute=250\n */\nimport { extractFromHtml } from \"npm:@extractus/article-extractor@9.0.1\";\nimport { classify } from \"./src/classify.ts\";\nimport { archiveGit } from \"./src/git.ts\";\nimport { archiveMusic } from \"./src/music.ts\";\nimport { archiveVideo } from \"./src/video.ts\";\nimport { enrichWikipedia } from \"./src/wikipedia.ts\";\nimport { archivePaper } from \"./src/paper.ts\";\nimport { FILE_RE, USER_AGENT } from \"./src/util.ts\";\n\ntype ParserOptions = {\n wordsPerMinute?: number;\n descriptionTruncateLen?: number;\n descriptionLengthThreshold?: number;\n contentLengthThreshold?: number;\n allowedTags?: string[];\n allowedAttributes?: Record;\n allowedIframeDomains?: string[];\n};\n\nconst ALIASES = new Set([\n \"url\", \"urls\", \"site\", \"sites\", \"website\", \"websites\",\n \"link\", \"links\",\n]);\n\nconst OPTION_KEYS = new Set([\n \"wordsPerMinute\", \"descriptionTruncateLen\", \"descriptionLengthThreshold\",\n \"contentLengthThreshold\", \"allowedTags\", \"allowedAttributes\",\n \"allowedIframeDomains\",\n]);\n\nexport default async function handler(req: Request): Promise {\n const collected: Record = {};\n const rawOptions: Record = {};\n\n function collect(source: unknown) {\n if (!source || typeof source !== \"object\") return;\n for (const [key, value] of Object.entries(source as Record)) {\n if (ALIASES.has(key)) {\n if (Array.isArray(collected[key])) {\n (collected[key] as unknown[]).push(...flatten(value));\n } else {\n collected[key] = flatten(value);\n }\n } else if (key === \"options\" && value && typeof value === \"object\") {\n Object.assign(rawOptions, value);\n } else if (OPTION_KEYS.has(key)) {\n rawOptions[key] = value;\n }\n }\n }\n\n // Body\n try {\n const text = await req.text();\n if (text) {\n const type = req.headers.get(\"content-type\") || \"\";\n let parsed = false;\n if (!type.includes(\"application/x-www-form-urlencoded\")) {\n try {\n collect(JSON.parse(text));\n parsed = true;\n } catch {\n /* not JSON */\n }\n }\n if (!parsed) {\n const sp = new URLSearchParams(text);\n for (const [k, v] of sp.entries()) {\n if (ALIASES.has(k)) collect({ [k]: v });\n else if (OPTION_KEYS.has(k)) rawOptions[k] = v;\n else if (k === \"options\") {\n try { Object.assign(rawOptions, JSON.parse(v)); } catch { /* skip */ }\n }\n }\n }\n }\n } catch {\n /* ignore unparseable body */\n }\n\n // Query string\n const qs = new URL(req.url).searchParams;\n for (const key of Array.from(qs.keys())) {\n if (ALIASES.has(key)) {\n const vals = qs.getAll(key);\n if (Array.isArray(collected[key])) (collected[key] as string[]).push(...vals);\n else collected[key] = vals;\n } else if (OPTION_KEYS.has(key)) {\n rawOptions[key] = qs.getAll(key).length > 1 ? qs.getAll(key) : qs.get(key);\n } else if (key === \"options\") {\n try { Object.assign(rawOptions, JSON.parse(qs.get(key) || \"{}\")); } catch { /* skip */ }\n }\n }\n\n const urls = Array.from(\n new Set(Object.values(collected).flatMap((v) => v as string[]) ?? []),\n ).filter((u): u is string => typeof u === \"string\" && u.length > 0);\n\n if (urls.length === 0) {\n return json({ error: \"No URL provided. Use url/site/website/link (query or body).\" }, 400);\n }\n\n const options = parseOptions(rawOptions);\n\n const results = await Promise.all(urls.map((u) => archive(u, options)));\n const ok = results.filter((r) => r.ok).map((r) => r.record);\n const failed = results.filter((r) => !r.ok).map((r) => r.error);\n\n // Single URL -> return the record directly; multiple -> array wrapper.\n if (urls.length === 1) {\n if (ok.length === 1) return json(ok[0], 200);\n return json({ error: failed[0]?.error }, 500);\n }\n\n const body: Record = {\n archived_at: new Date().toISOString(),\n count: ok.length + failed.length,\n results: ok,\n };\n if (failed.length) body.failed = failed;\n return json(body, failed.length && !ok.length ? 500 : 200);\n}\n\n// ---------------------------------------------------------------------------\n// Archive: classify, route to a specialized provider, else extract article.\n// ---------------------------------------------------------------------------\nasync function archive(\n url: string,\n options: ParserOptions,\n): Promise<{ ok: true; record: Record } | { ok: false; error: { url: string; error: string } }> {\n try {\n const kind = classify(url);\n\n // Dedicated providers know the page shape best.\n if (kind === \"git\") return { ok: true, record: await archiveGit(url) };\n if (kind === \"music\") return { ok: true, record: await archiveMusic(url) };\n if (kind === \"video\") return { ok: true, record: await archiveVideo(url) };\n if (kind === \"paper\") return { ok: true, record: await archivePaper(url) };\n\n // Generic: fetch the page, then extract the article.\n const res = await fetch(url, { headers: { \"user-agent\": USER_AGENT }, redirect: \"follow\" });\n const finalUrl = res.url || url;\n const contentType = res.headers.get(\"content-type\") || \"\";\n const status = res.status;\n if (!res.ok) throw new Error(`HTTP ${status}`);\n const html = await res.text();\n const htmlBytes = new TextEncoder().encode(html).length;\n\n const article = await extractFromHtml(html, finalUrl, options).catch(() => null);\n const content = article?.content ?? null;\n const contentBytes = content ? new TextEncoder().encode(content).length : 0;\n\n const record: Record = {\n kind,\n originalUrl: url,\n url: finalUrl,\n archived_at: new Date().toISOString(),\n status,\n contentType,\n title: article?.title ?? null,\n author: article?.author ?? null,\n description: article?.description ?? null,\n image: article?.image ?? null,\n publisher: article?.publisher ?? null,\n published: article?.published ?? null,\n content,\n content_size: contentBytes,\n html_size: htmlBytes,\n };\n\n // Wikipedia: add canonical title + REST summary.\n if (kind === \"wikipedia\") {\n return { ok: true, record: await enrichWikipedia(record, finalUrl) };\n }\n\n return { ok: true, record };\n } catch (err) {\n return { ok: false, error: { url, error: (err as Error).message } };\n }\n}\n\n// ---------------------------------------------------------------------------\n// Options parsing\n// ---------------------------------------------------------------------------\nfunction parseOptions(raw: Record): ParserOptions {\n const o: ParserOptions = {};\n if (raw.wordsPerMinute != null) o.wordsPerMinute = Number(raw.wordsPerMinute);\n if (raw.descriptionTruncateLen != null) o.descriptionTruncateLen = Number(raw.descriptionTruncateLen);\n if (raw.descriptionLengthThreshold != null) o.descriptionLengthThreshold = Number(raw.descriptionLengthThreshold);\n if (raw.contentLengthThreshold != null) o.contentLengthThreshold = Number(raw.contentLengthThreshold);\n if (raw.allowedTags != null) o.allowedTags = toArray(raw.allowedTags);\n if (raw.allowedIframeDomains != null) o.allowedIframeDomains = toArray(raw.allowedIframeDomains);\n if (raw.allowedAttributes != null) {\n if (typeof raw.allowedAttributes === \"string\") {\n try { o.allowedAttributes = JSON.parse(raw.allowedAttributes); } catch { /* skip */ }\n } else if (raw.allowedAttributes && typeof raw.allowedAttributes === \"object\") {\n o.allowedAttributes = raw.allowedAttributes as Record;\n }\n }\n return o;\n}\n\nfunction toArray(v: unknown): string[] {\n if (Array.isArray(v)) return v.map(String);\n if (typeof v === \"string\") return v.split(\",\").map((s) => s.trim()).filter(Boolean);\n return [];\n}\n\nfunction flatten(value: unknown): Array {\n return Array.isArray(value) ? value.flatMap(flatten) : [value];\n}\n\nfunction json(body: unknown, status: number): Response {\n const payload = JSON.stringify(body, null, 2);\n const bytes = new TextEncoder().encode(payload).length;\n return new Response(payload, {\n status,\n headers: {\n \"content-type\": \"application/json; charset=utf-8\",\n \"content-length\": String(bytes),\n \"x-archiver-count\": String(Array.isArray(body) ? body.length : 1),\n },\n });\n}\n","deleted",false,"label",[14],"ce845deb-58e1-4474-aea4-5bbf1cd9e916",["D",1788536954683],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Archive any URL: git repos, music, video, papers, Wikipedia","imageUrl","userId","124f6344-3784-4835-926c-5b9b29b366cc","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","01a06d17-30e7-7396-bb69-029ba771a7d7","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_9":10,"_102":98},"org",{"_9":10,"_95":96,"_97":98,"_55":56,"_99":100,"_101":42},"branch",{"_35":85,"_11":86,"_22":28,"_87":45,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":-5,"_93":12,"_94":10},"isOwned",21,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","displayName","Heidi Hedh","avatar","https://lh3.googleusercontent.com/a/ACg8ocK_QSHhMpajSrYgGIPk8bOKlftGgK0BEPGcExr97zjm8MHbQA=s96-c","tier","free","canRestrictVals","profileImageUrl","ctx",{"_78":-5,"_136":137,"_138":139,"_140":141,"_142":42,"_143":42,"_144":42,"_145":32,"_146":-5,"_147":-7,"_148":-7,"_149":62,"_150":42,"_151":42,"_152":-5,"_153":42,"_154":42,"_155":42,"_156":-5},"ENV",{"_107":108,"_109":110,"_111":86,"_112":113,"_114":115,"_116":117,"_118":119,"_120":121,"_122":123,"_124":125,"_126":127,"_128":129,"_130":131,"_132":133,"_134":135},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town","orgs",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]