[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_236":237,"_238":239},"routes/_app.x.$handle.$name",{"_102":101,"_38":51,"_111":92,"_33":112,"_13":14,"_113":-5,"_11":114,"_115":116,"_117":118,"_119":120,"_121":122,"_26":-5,"_123":-5,"_124":-5,"_125":31,"_126":127,"_128":64,"_129":-5,"_130":-5,"_131":132},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":18,"_19":-5},"project",{"_49":50,"_38":51,"_44":52,"_53":-5,"_54":31,"_55":56,"_57":56,"_58":59,"_60":-5,"_19":61,"_62":37,"_63":64,"_65":-5,"_66":-5,"_67":68,"_69":70,"_71":-5,"_72":-5,"_73":-5,"_74":-5,"_75":76,"_77":76,"_78":79,"_80":79,"_81":79,"_82":76,"_83":76,"_84":85,"_86":87,"_88":89,"_90":31},"version",20,"changes",[25],"aiChatInfo",{"_20":21,"_22":-5,"_23":-5,"_24":-5},"userId","aiChatMessageId","02a7b804-ff16-4769-b212-998c9b8f5b48","aiChatThreadId","aiChatThreadUserId","aiChatThreadTitle",{"_26":27,"_28":29,"_30":31,"_32":31,"_33":34,"_35":34},"file",{"_36":37,"_38":34,"_39":40,"_41":43,"_44":45,"_46":14,"_47":48,"_30":31},"previous",{"_36":37,"_38":34,"_39":40,"_41":42},"deleted",false,"deletedHere","path","main.ts","latestPath","fileId","32c89862-a878-11f1-946c-1607ee4eb77e","name","fileType","http","content","/**\n * URL Archiver\n *\n * Archive one or more URLs into a rich record. Any HTTP method is accepted.\n * URLs may come via query params or body (JSON or form), under any of the\n * aliases: url, urls, site, sites, website, websites, link, links. Each may\n * be a string or an array; all are flattened & deduped.\n *\n * Extractor options may be passed as a nested `options` object or flat keys:\n * wordsPerMinute, descriptionTruncateLen, descriptionLengthThreshold,\n * contentLengthThreshold, allowedTags, allowedAttributes, allowedIframeDomains\n *\n * A single URL returns the record directly; multiple URLs return an array.\n * The page type is recognized smartly (git repo, video, wikipedia, song,\n * file, article, ...) and the record is shaped to fit.\n *\n * POST / { \"url\": [\"https://a.com\", \"https://b.com\"], \"options\": { \"wordsPerMinute\": 250 } }\n * GET /?link=https://a.com&wordsPerMinute=250\n */\nimport { extractFromHtml } from \"npm:@extractus/article-extractor@9.0.1\";\n\ntype ParserOptions = {\n wordsPerMinute?: number;\n descriptionTruncateLen?: number;\n descriptionLengthThreshold?: number;\n contentLengthThreshold?: number;\n allowedTags?: string[];\n allowedAttributes?: Record;\n allowedIframeDomains?: string[];\n};\n\nconst ALIASES = new Set([\n \"url\", \"urls\", \"site\", \"sites\", \"website\", \"websites\",\n \"link\", \"links\",\n]);\n\nconst OPTION_KEYS = new Set([\n \"wordsPerMinute\", \"descriptionTruncateLen\", \"descriptionLengthThreshold\",\n \"contentLengthThreshold\", \"allowedTags\", \"allowedAttributes\",\n \"allowedIframeDomains\",\n]);\n\nconst USER_AGENT = \"Mozilla/5.0 (compatible; heidi/url-archiver)\";\n\nexport default async function handler(req: Request): Promise {\n const collected: Record = {};\n const rawOptions: Record = {};\n\n function collect(source: unknown) {\n if (!source || typeof source !== \"object\") return;\n for (const [key, value] of Object.entries(source as Record)) {\n if (ALIASES.has(key)) {\n if (Array.isArray(collected[key])) {\n (collected[key] as unknown[]).push(...flatten(value));\n } else {\n collected[key] = flatten(value);\n }\n } else if (key === \"options\" && value && typeof value === \"object\") {\n Object.assign(rawOptions, value);\n } else if (OPTION_KEYS.has(key)) {\n rawOptions[key] = value;\n }\n }\n }\n\n // Body\n try {\n const text = await req.text();\n if (text) {\n const type = req.headers.get(\"content-type\") || \"\";\n let parsed = false;\n if (!type.includes(\"application/x-www-form-urlencoded\")) {\n try {\n collect(JSON.parse(text));\n parsed = true;\n } catch {\n /* not JSON */\n }\n }\n if (!parsed) {\n const sp = new URLSearchParams(text);\n for (const [k, v] of sp.entries()) {\n if (ALIASES.has(k)) collect({ [k]: v });\n else if (OPTION_KEYS.has(k)) rawOptions[k] = v;\n else if (k === \"options\") {\n try { Object.assign(rawOptions, JSON.parse(v)); } catch { /* skip */ }\n }\n }\n }\n }\n } catch {\n /* ignore unparseable body */\n }\n\n // Query string\n const qs = new URL(req.url).searchParams;\n for (const key of Array.from(qs.keys())) {\n if (ALIASES.has(key)) {\n const vals = qs.getAll(key);\n if (Array.isArray(collected[key])) (collected[key] as string[]).push(...vals);\n else collected[key] = vals;\n } else if (OPTION_KEYS.has(key)) {\n rawOptions[key] = qs.getAll(key).length > 1 ? qs.getAll(key) : qs.get(key);\n } else if (key === \"options\") {\n try { Object.assign(rawOptions, JSON.parse(qs.get(key) || \"{}\")); } catch { /* skip */ }\n }\n }\n\n const urls = Array.from(\n new Set(Object.values(collected).flatMap((v) => v as string[]) ?? []),\n ).filter((u): u is string => typeof u === \"string\" && u.length > 0);\n\n if (urls.length === 0) {\n return json({ error: \"No URL provided. Use url/site/website/link (query or body).\" }, 400);\n }\n\n const options = parseOptions(rawOptions);\n\n const results = await Promise.all(urls.map((u) => archive(u, options)));\n const ok = results.filter((r) => r.ok).map((r) => r.record);\n const failed = results.filter((r) => !r.ok).map((r) => r.error);\n\n // Single URL -> return the record directly; multiple -> array wrapper.\n if (urls.length === 1) {\n if (ok.length === 1) return json(ok[0], 200);\n return json({ error: failed[0]?.error }, 500);\n }\n\n const body: Record = {\n archived_at: new Date().toISOString(),\n count: ok.length + failed.length,\n results: ok,\n };\n if (failed.length) body.failed = failed;\n return json(body, failed.length && !ok.length ? 500 : 200);\n}\n\n// ---------------------------------------------------------------------------\n// Archive: fetch the page, classify it, build a fitting record.\n// ---------------------------------------------------------------------------\nasync function archive(\n url: string,\n options: ParserOptions,\n): Promise<{ ok: true; record: Record } | { ok: false; error: { url: string; error: string } }> {\n try {\n const kind = classify(url);\n\n // Specialized handlers first (they know the page shape best).\n if (kind === \"git\") return { ok: true, record: await archiveGit(url) };\n if (kind === \"video\") return { ok: true, record: await archiveOembed(url, \"video\") };\n if (kind === \"song\") return { ok: true, record: await archiveOembed(url, \"song\") };\n\n // Generic: fetch the page, then extract the article.\n const res = await fetch(url, { headers: { \"user-agent\": USER_AGENT }, redirect: \"follow\" });\n const finalUrl = res.url || url;\n const contentType = res.headers.get(\"content-type\") || \"\";\n const status = res.status;\n if (!res.ok) throw new Error(`HTTP ${status}`);\n const html = await res.text();\n const htmlBytes = new TextEncoder().encode(html).length;\n\n const article = await extractFromHtml(html, finalUrl, options).catch(() => null);\n const content = article?.content ?? null;\n const contentBytes = content ? new TextEncoder().encode(content).length : 0;\n\n const record: Record = {\n kind,\n originalUrl: url,\n url: finalUrl,\n archived_at: new Date().toISOString(),\n status,\n contentType,\n title: article?.title ?? null,\n author: article?.author ?? null,\n description: article?.description ?? null,\n image: article?.image ?? null,\n publisher: article?.publisher ?? null,\n published: article?.published ?? null,\n content,\n content_size: contentBytes,\n html_size: htmlBytes,\n };\n\n // Wikipedia: add the canonical page title.\n if (kind === \"wikipedia\") {\n const m = finalUrl.match(/\\/wiki\\/([^?#]+)/);\n if (m) record.wiki_title = decodeURIComponent(m[1].replace(/_/g, \" \"));\n }\n\n return { ok: true, record };\n } catch (err) {\n return { ok: false, error: { url, error: (err as Error).message } };\n }\n}\n\n// ---------------------------------------------------------------------------\n// Classification\n// ---------------------------------------------------------------------------\nfunction classify(url: string): string {\n let host = \"\";\n let path = \"\";\n try {\n const u = new URL(url);\n host = u.hostname.toLowerCase().replace(/^www\\./, \"\");\n path = u.pathname;\n } catch {\n return \"unknown\";\n }\n\n if (/(^|\\.)(github|gitlab|bitbucket|codeberg|gitea)\\.(com|org|io)$/.test(host) ||\n host === \"dev.azure.com\") return \"git\";\n\n if (/(^|\\.)(youtube|youtu|youtube-nocookie)\\./.test(host) ||\n /(^|\\.)vimeo\\./.test(host) ||\n /(^|\\.)dailymotion\\./.test(host)) return \"video\";\n\n if (/(^|\\.)wikipedia\\.org$/.test(host) && path.startsWith(\"/wiki/\")) return \"wikipedia\";\n\n if (/(^|\\.)(spotify|soundcloud|musicbrainz|deezer|bandcamp)\\./.test(host) ||\n /music\\.youtube\\.com/.test(host)) return \"song\";\n\n if (isFileUrl(path, url)) return \"file\";\n\n return \"article\";\n}\n\nconst FILE_EXT = /\\.(pdf|docx?|xlsx?|pptx?|txt|md|rtf|csv|json|xml|zip|tar|gz|7z|rar|png|jpe?g|gif|webp|svg|ico|bmp|mp3|wav|flac|ogg|m4a|mp4|webm|mov|avi|mkv|epub|mobi)$/i;\n\nfunction isFileUrl(path: string, url: string): boolean {\n return FILE_EXT.test(path) || FILE_EXT.test(url.split(\"?\")[0]);\n}\n\n// ---------------------------------------------------------------------------\n// Git repositories\n// ---------------------------------------------------------------------------\nasync function archiveGit(url: string): Promise> {\n const record: Record = {\n kind: \"git\",\n originalUrl: url,\n archived_at: new Date().toISOString(),\n };\n let host = \"\";\n let path = \"\";\n try {\n const u = new URL(url);\n host = u.hostname.toLowerCase().replace(/^www\\./, \"\");\n path = u.pathname.replace(/^\\/+/, \"\").replace(/\\.git$/, \"\");\n } catch {\n return record;\n }\n record.host = host;\n const parts = path.split(\"/\").filter(Boolean);\n const owner = parts[0];\n const repo = parts[1];\n if (owner && repo) {\n record.owner = owner;\n record.repo = repo;\n record.repo_url = `https://${host}/${owner}/${repo}`;\n }\n\n // Try the platform API for richer metadata; fall back gracefully.\n try {\n if (host === \"github.com\" && owner && repo) {\n const r = await fetch(`https://api.github.com/repos/${owner}/${repo}`, {\n headers: { \"user-agent\": USER_AGENT, accept: \"application/vnd.github+json\" },\n });\n if (r.ok) {\n const d = await r.json();\n Object.assign(record, {\n description: d.description ?? null,\n language: d.language ?? null,\n stars: d.stargazers_count ?? 0,\n forks: d.forks_count ?? 0,\n open_issues: d.open_issues_count ?? 0,\n license: d.license?.spdx_id ?? null,\n default_branch: d.default_branch ?? null,\n homepage: d.homepage ?? null,\n pushed_at: d.pushed_at ?? null,\n size_kb: d.size ?? null,\n });\n }\n } else if (host === \"gitlab.com\" && owner && repo) {\n const r = await fetch(`https://gitlab.com/api/v4/projects/${encodeURIComponent(`${owner}/${repo}`)}`);\n if (r.ok) {\n const d = await r.json();\n Object.assign(record, {\n description: d.description ?? null,\n language: d.default_branch ?? null,\n stars: d.star_count ?? 0,\n forks: d.forks_count ?? 0,\n default_branch: d.default_branch ?? null,\n homepage: d.web_url ?? null,\n last_activity: d.last_activity_at ?? null,\n });\n }\n } else if (host === \"bitbucket.org\" && owner && repo) {\n const r = await fetch(`https://api.bitbucket.org/2.0/repositories/${owner}/${repo}`);\n if (r.ok) {\n const d = await r.json();\n Object.assign(record, {\n description: d.description ?? null,\n language: d.language ?? null,\n default_branch: d.mainbranch?.name ?? null,\n homepage: d.links?.html?.href ?? null,\n created: d.created_on ?? null,\n updated: d.updated_on ?? null,\n });\n }\n }\n } catch {\n /* metadata enrichment is best-effort */\n }\n return record;\n}\n\n// ---------------------------------------------------------------------------\n// Video / song via oEmbed (public, no auth)\n// ---------------------------------------------------------------------------\nconst OEMBED: Record string> = {\n youtube: (u) => `https://www.youtube.com/oembed?url=${encodeURIComponent(u.href)}&format=json`,\n vimeo: (u) => `https://vimeo.com/api/oembed.json?url=${encodeURIComponent(u.href)}`,\n spotify: (u) => `https://open.spotify.com/oembed?url=${encodeURIComponent(u.href)}&format=json`,\n soundcloud: (u) => `https://soundcloud.com/oembed?url=${encodeURIComponent(u.href)}&format=json`,\n};\n\nasync function archiveOembed(url: string, kind: \"video\" | \"song\"): Promise> {\n const record: Record = {\n kind,\n originalUrl: url,\n archived_at: new Date().toISOString(),\n };\n let host = \"\";\n try {\n const u = new URL(url);\n host = u.hostname.toLowerCase().replace(/^www\\./, \"\");\n record.host = host;\n record.url = url;\n\n // Pick an oEmbed provider.\n let provider: string | null = null;\n if (/(^|\\.)(youtube|youtu|youtube-nocookie)\\./.test(host)) provider = \"youtube\";\n else if (/(^|\\.)vimeo\\./.test(host)) provider = \"vimeo\";\n else if (/(^|\\.)spotify\\./.test(host)) provider = \"spotify\";\n else if (/(^|\\.)soundcloud\\./.test(host)) provider = \"soundcloud\";\n\n if (provider && OEMBED[provider]) {\n const r = await fetch(OEMBED[provider](u));\n if (r.ok) {\n const d = await r.json();\n Object.assign(record, {\n title: d.title ?? null,\n author: d.author_name ?? null,\n author_url: d.author_url ?? null,\n thumbnail: d.thumbnail_url ?? null,\n thumbnail_width: d.thumbnail_width ?? null,\n thumbnail_height: d.thumbnail_height ?? null,\n provider: d.provider_name ?? null,\n embed_html: d.html ?? null,\n width: d.width ?? null,\n height: d.height ?? null,\n });\n return record;\n }\n }\n\n // Fallback: extract an id for known hosts and build useful fields.\n if (provider === \"youtube\") {\n const id = youtubeId(url);\n if (id) {\n record.video_id = id;\n record.thumbnail = `https://img.youtube.com/vi/${id}/hqdefault.jpg`;\n record.embed_url = `https://www.youtube.com/embed/${id}`;\n }\n } else if (provider === \"vimeo\") {\n const m = url.match(/vimeo\\.com\\/(?:video\\/)?(\\d+)/);\n if (m) {\n record.video_id = m[1];\n record.embed_url = `https://player.vimeo.com/video/${m[1]}`;\n }\n } else if (provider === \"spotify\") {\n const m = url.match(/spotify\\.com\\/(track|album|playlist|artist)\\/([A-Za-z0-9]+)/);\n if (m) {\n record.spotify_type = m[1];\n record.spotify_id = m[2];\n record.embed_url = `https://open.spotify.com/embed/${m[1]}/${m[2]}`;\n }\n }\n } catch {\n /* best-effort */\n }\n return record;\n}\n\nfunction youtubeId(url: string): string | null {\n const m = url.match(/(?:v=|youtu\\.be\\/|embed\\/|shorts\\/|live\\/)([A-Za-z0-9_-]{11})/);\n return m ? m[1] : null;\n}\n\n// ---------------------------------------------------------------------------\n// Options parsing\n// ---------------------------------------------------------------------------\nfunction parseOptions(raw: Record): ParserOptions {\n const o: ParserOptions = {};\n if (raw.wordsPerMinute != null) o.wordsPerMinute = Number(raw.wordsPerMinute);\n if (raw.descriptionTruncateLen != null) o.descriptionTruncateLen = Number(raw.descriptionTruncateLen);\n if (raw.descriptionLengthThreshold != null) o.descriptionLengthThreshold = Number(raw.descriptionLengthThreshold);\n if (raw.contentLengthThreshold != null) o.contentLengthThreshold = Number(raw.contentLengthThreshold);\n if (raw.allowedTags != null) o.allowedTags = toArray(raw.allowedTags);\n if (raw.allowedIframeDomains != null) o.allowedIframeDomains = toArray(raw.allowedIframeDomains);\n if (raw.allowedAttributes != null) {\n if (typeof raw.allowedAttributes === \"string\") {\n try { o.allowedAttributes = JSON.parse(raw.allowedAttributes); } catch { /* skip */ }\n } else if (raw.allowedAttributes && typeof raw.allowedAttributes === \"object\") {\n o.allowedAttributes = raw.allowedAttributes as Record;\n }\n }\n return o;\n}\n\nfunction toArray(v: unknown): string[] {\n if (Array.isArray(v)) return v.map(String);\n if (typeof v === \"string\") return v.split(\",\").map((s) => s.trim()).filter(Boolean);\n return [];\n}\n\nfunction flatten(value: unknown): Array {\n return Array.isArray(value) ? value.flatMap(flatten) : [value];\n}\n\nfunction json(body: unknown, status: number): Response {\n const payload = JSON.stringify(body, null, 2);\n const bytes = new TextEncoder().encode(payload).length;\n return new Response(payload, {\n status,\n headers: {\n \"content-type\": \"application/json; charset=utf-8\",\n \"content-length\": String(bytes),\n \"x-archiver-count\": String(Array.isArray(body) ? body.length : 1),\n },\n });\n}\n","/**\n * URL Archiver\n *\n * Archive one or more URLs into a rich record. Any HTTP method is accepted.\n * URLs may come via query params or body (JSON or form), under any of the\n * aliases: url, urls, site, sites, website, websites, link, links. Each may\n * be a string or an array; all are flattened & deduped.\n *\n * Extractor options may be passed as a nested `options` object or flat keys:\n * wordsPerMinute, descriptionTruncateLen, descriptionLengthThreshold,\n * contentLengthThreshold, allowedTags, allowedAttributes, allowedIframeDomains\n *\n * A single URL returns the record directly; multiple URLs return an array.\n * The page type is recognized smartly and routed to a dedicated provider:\n * git GitHub / GitLab / Bitbucket / Codeberg / Gitea repos\n * music Spotify / Deezer / SoundCloud / Bandcamp / YouTube Music,\n * plus MusicBrainz / ListenBrainz / Last.fm\n * video YouTube / Vimeo / Dailymotion\n * wikipedia Wikipedia articles (+ REST summary)\n * paper DOIs (Crossref) / arXiv / PubMed\n * file direct asset downloads\n * article generic article extraction (default fallback)\n *\n * POST / { \"url\": [\"https://a.com\", \"https://b.com\"], \"options\": { \"wordsPerMinute\": 250 } }\n * GET /?link=https://a.com&wordsPerMinute=250\n */\nimport { extractFromHtml } from \"npm:@extractus/article-extractor@9.0.1\";\nimport { classify } from \"./src/classify.ts\";\nimport { archiveGit } from \"./src/git.ts\";\nimport { archiveMusic } from \"./src/music.ts\";\nimport { archiveVideo } from \"./src/video.ts\";\nimport { enrichWikipedia } from \"./src/wikipedia.ts\";\nimport { archivePaper } from \"./src/paper.ts\";\nimport { FILE_RE, USER_AGENT } from \"./src/util.ts\";\n\ntype ParserOptions = {\n wordsPerMinute?: number;\n descriptionTruncateLen?: number;\n descriptionLengthThreshold?: number;\n contentLengthThreshold?: number;\n allowedTags?: string[];\n allowedAttributes?: Record;\n allowedIframeDomains?: string[];\n};\n\nconst ALIASES = new Set([\n \"url\", \"urls\", \"site\", \"sites\", \"website\", \"websites\",\n \"link\", \"links\",\n]);\n\nconst OPTION_KEYS = new Set([\n \"wordsPerMinute\", \"descriptionTruncateLen\", \"descriptionLengthThreshold\",\n \"contentLengthThreshold\", \"allowedTags\", \"allowedAttributes\",\n \"allowedIframeDomains\",\n]);\n\nexport default async function handler(req: Request): Promise {\n const collected: Record = {};\n const rawOptions: Record = {};\n\n function collect(source: unknown) {\n if (!source || typeof source !== \"object\") return;\n for (const [key, value] of Object.entries(source as Record)) {\n if (ALIASES.has(key)) {\n if (Array.isArray(collected[key])) {\n (collected[key] as unknown[]).push(...flatten(value));\n } else {\n collected[key] = flatten(value);\n }\n } else if (key === \"options\" && value && typeof value === \"object\") {\n Object.assign(rawOptions, value);\n } else if (OPTION_KEYS.has(key)) {\n rawOptions[key] = value;\n }\n }\n }\n\n // Body\n try {\n const text = await req.text();\n if (text) {\n const type = req.headers.get(\"content-type\") || \"\";\n let parsed = false;\n if (!type.includes(\"application/x-www-form-urlencoded\")) {\n try {\n collect(JSON.parse(text));\n parsed = true;\n } catch {\n /* not JSON */\n }\n }\n if (!parsed) {\n const sp = new URLSearchParams(text);\n for (const [k, v] of sp.entries()) {\n if (ALIASES.has(k)) collect({ [k]: v });\n else if (OPTION_KEYS.has(k)) rawOptions[k] = v;\n else if (k === \"options\") {\n try { Object.assign(rawOptions, JSON.parse(v)); } catch { /* skip */ }\n }\n }\n }\n }\n } catch {\n /* ignore unparseable body */\n }\n\n // Query string\n const qs = new URL(req.url).searchParams;\n for (const key of Array.from(qs.keys())) {\n if (ALIASES.has(key)) {\n const vals = qs.getAll(key);\n if (Array.isArray(collected[key])) (collected[key] as string[]).push(...vals);\n else collected[key] = vals;\n } else if (OPTION_KEYS.has(key)) {\n rawOptions[key] = qs.getAll(key).length > 1 ? qs.getAll(key) : qs.get(key);\n } else if (key === \"options\") {\n try { Object.assign(rawOptions, JSON.parse(qs.get(key) || \"{}\")); } catch { /* skip */ }\n }\n }\n\n const urls = Array.from(\n new Set(Object.values(collected).flatMap((v) => v as string[]) ?? []),\n ).filter((u): u is string => typeof u === \"string\" && u.length > 0);\n\n if (urls.length === 0) {\n return json({ error: \"No URL provided. Use url/site/website/link (query or body).\" }, 400);\n }\n\n const options = parseOptions(rawOptions);\n\n const results = await Promise.all(urls.map((u) => archive(u, options)));\n const ok = results.filter((r) => r.ok).map((r) => r.record);\n const failed = results.filter((r) => !r.ok).map((r) => r.error);\n\n // Single URL -> return the record directly; multiple -> array wrapper.\n if (urls.length === 1) {\n if (ok.length === 1) return json(ok[0], 200);\n return json({ error: failed[0]?.error }, 500);\n }\n\n const body: Record = {\n archived_at: new Date().toISOString(),\n count: ok.length + failed.length,\n results: ok,\n };\n if (failed.length) body.failed = failed;\n return json(body, failed.length && !ok.length ? 500 : 200);\n}\n\n// ---------------------------------------------------------------------------\n// Archive: classify, route to a specialized provider, else extract article.\n// ---------------------------------------------------------------------------\nasync function archive(\n url: string,\n options: ParserOptions,\n): Promise<{ ok: true; record: Record } | { ok: false; error: { url: string; error: string } }> {\n try {\n const kind = classify(url);\n\n // Dedicated providers know the page shape best.\n if (kind === \"git\") return { ok: true, record: await archiveGit(url) };\n if (kind === \"music\") return { ok: true, record: await archiveMusic(url) };\n if (kind === \"video\") return { ok: true, record: await archiveVideo(url) };\n if (kind === \"paper\") return { ok: true, record: await archivePaper(url) };\n\n // Generic: fetch the page, then extract the article.\n const res = await fetch(url, { headers: { \"user-agent\": USER_AGENT }, redirect: \"follow\" });\n const finalUrl = res.url || url;\n const contentType = res.headers.get(\"content-type\") || \"\";\n const status = res.status;\n if (!res.ok) throw new Error(`HTTP ${status}`);\n const html = await res.text();\n const htmlBytes = new TextEncoder().encode(html).length;\n\n const article = await extractFromHtml(html, finalUrl, options).catch(() => null);\n const content = article?.content ?? null;\n const contentBytes = content ? new TextEncoder().encode(content).length : 0;\n\n const record: Record = {\n kind,\n originalUrl: url,\n url: finalUrl,\n archived_at: new Date().toISOString(),\n status,\n contentType,\n title: article?.title ?? null,\n author: article?.author ?? null,\n description: article?.description ?? null,\n image: article?.image ?? null,\n publisher: article?.publisher ?? null,\n published: article?.published ?? null,\n content,\n content_size: contentBytes,\n html_size: htmlBytes,\n };\n\n // Wikipedia: add canonical title + REST summary.\n if (kind === \"wikipedia\") {\n return { ok: true, record: await enrichWikipedia(record, finalUrl) };\n }\n\n return { ok: true, record };\n } catch (err) {\n return { ok: false, error: { url, error: (err as Error).message } };\n }\n}\n\n// ---------------------------------------------------------------------------\n// Options parsing\n// ---------------------------------------------------------------------------\nfunction parseOptions(raw: Record): ParserOptions {\n const o: ParserOptions = {};\n if (raw.wordsPerMinute != null) o.wordsPerMinute = Number(raw.wordsPerMinute);\n if (raw.descriptionTruncateLen != null) o.descriptionTruncateLen = Number(raw.descriptionTruncateLen);\n if (raw.descriptionLengthThreshold != null) o.descriptionLengthThreshold = Number(raw.descriptionLengthThreshold);\n if (raw.contentLengthThreshold != null) o.contentLengthThreshold = Number(raw.contentLengthThreshold);\n if (raw.allowedTags != null) o.allowedTags = toArray(raw.allowedTags);\n if (raw.allowedIframeDomains != null) o.allowedIframeDomains = toArray(raw.allowedIframeDomains);\n if (raw.allowedAttributes != null) {\n if (typeof raw.allowedAttributes === \"string\") {\n try { o.allowedAttributes = JSON.parse(raw.allowedAttributes); } catch { /* skip */ }\n } else if (raw.allowedAttributes && typeof raw.allowedAttributes === \"object\") {\n o.allowedAttributes = raw.allowedAttributes as Record;\n }\n }\n return o;\n}\n\nfunction toArray(v: unknown): string[] {\n if (Array.isArray(v)) return v.map(String);\n if (typeof v === \"string\") return v.split(\",\").map((s) => s.trim()).filter(Boolean);\n return [];\n}\n\nfunction flatten(value: unknown): Array {\n return Array.isArray(value) ? value.flatMap(flatten) : [value];\n}\n\nfunction json(body: unknown, status: number): Response {\n const payload = JSON.stringify(body, null, 2);\n const bytes = new TextEncoder().encode(payload).length;\n return new Response(payload, {\n status,\n headers: {\n \"content-type\": \"application/json; charset=utf-8\",\n \"content-length\": String(bytes),\n \"x-archiver-count\": String(Array.isArray(body) ? body.length : 1),\n },\n });\n}\n","createdAt",["D",1788768338783],"branchVersion","branchId","3f798dd9-8407-4149-b148-45d4da89ccbd","id","ce845deb-58e1-4474-aea4-5bbf1cd9e916","url-archiver",["D",1788536954683],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Archive any URL: git repos, music, video, papers, Wikipedia","imageUrl","124f6344-3784-4835-926c-5b9b29b366cc","pinnedFileId","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","01a06d17-30e7-7396-bb69-029ba771a7d7","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_102":101,"_110":106},"org",{"_102":101,"_103":104,"_105":106,"_19":61,"_107":108,"_109":31},"branch",{"_13":91,"_38":92,"_49":48,"_93":50,"_94":-5,"_95":-5,"_96":-5,"_97":-5,"_98":-5,"_99":51,"_100":101},"isOwned",21,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","heidi","handle","displayName","Heidi Hedh","avatar","https://lh3.googleusercontent.com/a/ACg8ocK_QSHhMpajSrYgGIPk8bOKlftGgK0BEPGcExr97zjm8MHbQA=s96-c","tier","free","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_49":50,"_38":51,"_44":232,"_53":-5,"_54":31,"_55":56,"_57":56,"_58":59,"_60":-5,"_19":61,"_62":37,"_63":64,"_65":-5,"_66":-5,"_67":68,"_69":70,"_71":-5,"_72":-5,"_73":-5,"_74":-5,"_75":76,"_77":76,"_78":79,"_80":79,"_81":79,"_82":76,"_83":76,"_84":233,"_86":234,"_88":235,"_90":31},"tree",[160,168],"files",[160,161,162,163,164,165,166,167,168],"counts",{"_149":68,"_150":151,"_152":68,"_153":154,"_155":156},"contents",[133,134],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_49":142,"_38":143,"_36":144,"_136":-5,"_47":48,"_46":145,"_137":64,"_44":146,"_13":68,"_39":147,"_41":-5,"_30":31,"_140":-5,"_33":148},{"_49":135,"_38":34,"_36":37,"_136":-5,"_47":48,"_46":14,"_137":64,"_44":138,"_13":139,"_39":40,"_41":43,"_30":31,"_140":51,"_33":141},"c549e592-a93e-4116-922d-7b61704d6245","parentId","latest",["D",1788768338783],7,"label",[34],"750c7c9a-1fc4-4b17-a501-2260cf1e9def","src","2ad11f9a-53a3-433f-af2e-7dc42ce19589",12,["D",1788768264288],"directory",[143],"pulls","branches",1,"forks","envVars",{"_157":68,"_159":68},"connections",{"_157":68,"_158":68},"count","unfilledCount","emptyCount",{"_49":142,"_38":143,"_36":144,"_136":-5,"_47":48,"_46":145,"_137":64,"_44":229,"_13":68,"_39":147,"_41":-5,"_30":31,"_140":-5,"_170":230,"_33":231},{"_49":221,"_38":222,"_36":223,"_136":144,"_47":48,"_46":224,"_137":64,"_44":225,"_13":68,"_39":178,"_41":226,"_30":31,"_140":-5,"_170":227,"_33":228},{"_49":213,"_38":214,"_36":215,"_136":144,"_47":48,"_46":216,"_137":64,"_44":217,"_13":68,"_39":178,"_41":218,"_30":31,"_140":-5,"_170":219,"_33":220},{"_49":205,"_38":206,"_36":207,"_136":144,"_47":48,"_46":208,"_137":64,"_44":209,"_13":68,"_39":178,"_41":210,"_30":31,"_140":-5,"_170":211,"_33":212},{"_49":197,"_38":198,"_36":199,"_136":144,"_47":48,"_46":200,"_137":64,"_44":201,"_13":151,"_39":178,"_41":202,"_30":31,"_140":-5,"_170":203,"_33":204},{"_49":190,"_38":191,"_36":192,"_136":144,"_47":48,"_46":145,"_137":64,"_44":193,"_13":68,"_39":178,"_41":194,"_30":31,"_140":-5,"_170":195,"_33":196},{"_49":182,"_38":183,"_36":184,"_136":144,"_47":48,"_46":185,"_137":64,"_44":186,"_13":68,"_39":178,"_41":187,"_30":31,"_140":-5,"_170":188,"_33":189},{"_49":173,"_38":174,"_36":175,"_136":144,"_47":48,"_46":176,"_137":64,"_44":177,"_13":68,"_39":178,"_41":179,"_30":31,"_140":-5,"_170":180,"_33":181},{"_49":135,"_38":34,"_36":37,"_136":-5,"_47":48,"_46":14,"_137":64,"_44":169,"_13":139,"_39":40,"_41":43,"_30":31,"_140":51,"_170":171,"_33":172},["D",1788768338783],"children",[],[34],"4a723b41-1ad2-45cf-ba6b-f685f69b4422","wikipedia.ts","d74960f8-aa92-11f1-aa53-1607ee4eb77e",17,["D",1788768307894],"script","import { hostOf, USER_AGENT } from \"./util.ts\";\n\n/**\n * Enrich a Wikipedia page record. The `record` is built by the generic article\n * extractor; here we add the canonical title, language and a REST `/summary`\n * pull (title, extract, description, thumbnail, Wikidata id) — free, no key.\n */\nexport async function enrichWikipedia(\n record: Record,\n finalUrl: string,\n): Promise> {\n const host = hostOf(finalUrl);\n const titlePath = pathOf(finalUrl);\n\n if (!(/\\.wikipedia\\.org$/.test(host)) || !titlePath) {\n record.wiki_title = record.title ?? null;\n return record;\n }\n\n const lang = host.startsWith(\"www.\") ? \"en\" : host.split(\".\")[0];\n const title = decodeURIComponent(titlePath.replace(/_/g, \" \"));\n record.wiki_title = title;\n record.wiki_lang = lang;\n\n try {\n const r = await fetch(\n `https://${lang}.wikipedia.org/api/rest_v1/page/summary/${encodeURIComponent(title)}`,\n { headers: { \"user-agent\": USER_AGENT, accept: \"application/json\" } },\n );\n if (!r.ok) return record;\n const d = (await r.json()) as Record;\n record.summary = record.summary ?? d.extract ?? null;\n record.wiki_description = d.description ?? null;\n record.thumbnail = record.thumbnail ?? (d.thumbnail as Record)?.source ?? null;\n record.wiki_image = (d.originalimage as Record)?.source ?? null;\n record.wikidata_id = d.wikibase_item ?? null;\n record.wiki_intro_html = d.extract_html ?? null;\n record.wiki_type = d.type ?? null;\n } catch {\n /* best-effort */\n }\n return record;\n}\n\nfunction pathOf(url: string): string | null {\n try {\n const p = new URL(url).pathname;\n const m = p.match(/\\/wiki\\/(.+)/);\n return m ? m[1] : null;\n } catch {\n return null;\n }\n}\n",[],[143,174],"5881f64d-5f03-41c6-83f9-ad42743d595d","video.ts","d4020378-aa92-11f1-aa53-1607ee4eb77e",16,["D",1788768302390],"import { fetchOembed, hostOf, youtubeId } from \"./util.ts\";\n\n// Video providers with public oEmbed (no auth).\nconst OEMBED: Record string> = {\n youtube: (u) =>\n `https://www.youtube.com/oembed?url=${encodeURIComponent(u.href)}&format=json`,\n vimeo: (u) => `https://vimeo.com/api/oembed.json?url=${encodeURIComponent(u.href)}`,\n dailymotion: (u) =>\n `https://www.dailymotion.com/services/oembed?url=${encodeURIComponent(u.href)}`,\n};\n\n/** Archive a video URL (YouTube, Vimeo, Dailymotion) via oEmbed + ID enrichment. */\nexport async function archiveVideo(url: string): Promise> {\n const record: Record = {\n kind: \"video\",\n originalUrl: url,\n archived_at: new Date().toISOString(),\n };\n\n let u: URL | null = null;\n try {\n u = new URL(url);\n } catch {\n return record;\n }\n record.host = hostOf(url);\n record.url = url;\n\n const host = record.host as string;\n let provider: string | null = null;\n if (/(^|\\.)(youtube|youtu|youtube-nocookie)\\./.test(host)) provider = \"youtube\";\n else if (/(^|\\.)vimeo\\./.test(host)) provider = \"vimeo\";\n else if (/(^|\\.)dailymotion\\./.test(host)) provider = \"dailymotion\";\n\n if (provider && OEMBED[provider]) {\n const d = await fetchOembed(OEMBED[provider](u));\n if (d) {\n record.title = d.title ?? null;\n record.author = d.author_name ?? null;\n record.author_url = d.author_url ?? null;\n record.thumbnail = d.thumbnail_url ?? null;\n record.provider_name = d.provider_name ?? null;\n record.embed_html = d.html ?? null;\n record.width = d.width ?? null;\n record.height = d.height ?? null;\n }\n }\n\n if (provider === \"youtube\") {\n const id = youtubeId(url);\n if (id) {\n record.video_id = id;\n record.thumbnail = record.thumbnail ??\n `https://img.youtube.com/vi/${id}/hqdefault.jpg`;\n record.embed_url = `https://www.youtube.com/embed/${id}`;\n }\n } else if (provider === \"vimeo\") {\n const m = url.match(/vimeo\\.com\\/(?:video\\/)?(\\d+)/);\n if (m) {\n record.video_id = m[1];\n record.embed_url = record.embed_url ?? `https://player.vimeo.com/video/${m[1]}`;\n }\n } else if (provider === \"dailymotion\") {\n const m = url.match(/dailymotion\\.com\\/video\\/([A-Za-z0-9]+)/);\n if (m) record.video_id = m[1];\n }\n\n return record;\n}\n",[],[143,183],"1c6ffaaf-1ff2-4eca-8fa6-b84b9267c9fc","util.ts","bd4c5dd6-aa92-11f1-907d-1607ee4eb77e",["D",1788768264288],"/** Shared helpers for the URL archiver providers. */\n\nexport const USER_AGENT =\n \"Mozilla/5.0 (compatible; heidi/url-archiver/1.0; +https://www.val.town/x/heidi/url-archiver)\";\n\n/** File extensions that indicate a download/asset rather than a page. */\nexport const FILE_RE =\n /\\.(pdf|docx?|xlsx?|pptx?|txt|md|rtf|csv|json|xml|zip|tar|gz|7z|rar|png|jpe?g|gif|webp|svg|ico|bmp|mp3|wav|flac|ogg|m4a|mp4|webm|mov|avi|mkv|epub|mobi)$/i;\n\n/** Fetch a provider's oEmbed JSON; returns null on any failure. */\nexport async function fetchOembed(\n oembedUrl: string,\n): Promise | null> {\n try {\n const r = await fetch(oembedUrl, {\n headers: { \"user-agent\": USER_AGENT, accept: \"application/json\" },\n });\n if (!r.ok) return null;\n const d: unknown = await r.json();\n return (typeof d === \"object\" && d !== null)\n ? (d as Record)\n : null;\n } catch {\n return null;\n }\n}\n\n/** Fetch a page and read its tags (returns {} on failure). */\nexport async function fetchMeta(\n url: string,\n): Promise> {\n try {\n const r = await fetch(url, {\n headers: { \"user-agent\": USER_AGENT },\n redirect: \"follow\",\n });\n if (!r.ok) return {};\n return metaTags(await r.text());\n } catch {\n return {};\n }\n}\n\n/** Parse tags out of an HTML string. */\nexport function metaTags(html: string): Record {\n const out: Record = {};\n const tagRe = /]*>/gi;\n const attrRe = /([a-zA-Z-]+)\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)')/g;\n let tag: RegExpExecArray | null;\n while ((tag = tagRe.exec(html)) !== null) {\n const attrs: Record = {};\n let a: RegExpExecArray | null;\n attrRe.lastIndex = 0;\n while ((a = attrRe.exec(tag[0])) !== null) {\n attrs[a[1].toLowerCase()] = a[2] ?? a[3] ?? \"\";\n }\n const name = attrs.property || attrs.name;\n if (name && attrs.content != null && !(name in out)) {\n out[name] = decodeEntities(attrs.content);\n }\n }\n return out;\n}\n\n/** Decode common HTML entities in a string. */\nexport function decodeEntities(s: string): string {\n return s\n .replace(/&#x([0-9a-fA-F]+);/g, (_, h) => String.fromCodePoint(parseInt(h, 16)))\n .replace(/&#(\\d+);/g, (_, d) => String.fromCodePoint(parseInt(d, 10)))\n .replace(/&/g, \"&\")\n .replace(/</g, \"<\")\n .replace(/>/g, \">\")\n .replace(/"/g, '\"')\n .replace(/'/g, \"'\")\n .replace(/ /g, \" \")\n .trim();\n}\n\n/** Strip JATS/HTML tags, collapse whitespace. */\nexport function stripTags(s: string): string {\n return s.replace(/<[^>]+>/g, \" \").replace(/\\s+/g, \" \").trim();\n}\n\n/** Extract a YouTube/Music video ID from a URL. */\nexport function youtubeId(url: string): string | null {\n const m = url.match(/[?&]v=([A-Za-z0-9_-]{11})/) ||\n url.match(/(?:youtu\\.be\\/|embed\\/|shorts\\/|live\\/)([A-Za-z0-9_-]{11})/);\n return m ? m[1] : null;\n}\n\n/** Normalize a hostname: lowercase, strip www. */\nexport function hostOf(url: string): string {\n try {\n return new URL(url).hostname.toLowerCase().replace(/^www\\./, \"\");\n } catch {\n return \"\";\n }\n}\n",[],[143,191],"7428e5a7-dc59-4108-a7c6-baf0d088fd73","paper.ts","df07b88a-aa92-11f1-8552-1607ee4eb77e",19,["D",1788768324395],"import { hostOf, stripTags, USER_AGENT } from \"./util.ts\";\n\n/**\n * Archive a scientific paper / DOI URL. Routes by host:\n * doi.org -> Crossref works API\n * arxiv -> export.arxiv.org Atom API\n * pubmed -> NCBI eutils esummary\n * All are public and keyless.\n */\nexport async function archivePaper(url: string): Promise> {\n const record: Record = {\n kind: \"paper\",\n originalUrl: url,\n archived_at: new Date().toISOString(),\n };\n const host = hostOf(url);\n record.host = host;\n record.url = url;\n if (!host) return record;\n\n if (host === \"doi.org\") {\n const doi = pathOf(url);\n if (doi) {\n record.doi = doi;\n await enrichCrossref(record, doi);\n }\n return record;\n }\n if (host === \"arxiv.org\" || host.endsWith(\".arxiv.org\")) {\n return await enrichArxiv(record, url);\n }\n if (host === \"pubmed.ncbi.nlm.nih.gov\") {\n return await enrichPubmed(record, url);\n }\n return record;\n}\n\n// ---------------------------------------------------------------------------\n// Crossref (DOI registration agency — free REST API)\n// ---------------------------------------------------------------------------\nasync function enrichCrossref(\n record: Record,\n doi: string,\n): Promise {\n try {\n const r = await fetch(`https://api.crossref.org/works/${encodeURIComponent(doi)}`, {\n headers: { \"user-agent\": USER_AGENT, accept: \"application/json\" },\n });\n if (!r.ok) return;\n const res = (await r.json()) as Record;\n const m = (res.message ?? {}) as Record;\n record.title = firstArray(m.title);\n record.authors = (m.author as Array> | undefined)\n ?.map((a) => [a.given, a.family].filter(Boolean).join(\" \") || String(a.name ?? \"\"))\n .filter(Boolean);\n record.journal = firstArray(m[\"container-title\"]);\n record.published = pickDate(m.published) ?? pickDate(m[\"published-online\"]) ??\n pickDate(m[\"published-print\"]) ?? pickDate(m.issued);\n record.publisher = m.publisher ?? null;\n record.volume = m.volume ?? null;\n record.issue = m.issue ?? null;\n record.pages = m.page ?? null;\n record.type = m.type ?? null;\n record.abstract = typeof m.abstract === \"string\" ? stripTags(m.abstract) : null;\n record.subjects = Array.isArray(m.subject) ? m.subject : undefined;\n record.license = (m.license as Array> | undefined)?.[0]?.URL ?? null;\n record.url = (m.URL as string) ?? record.url;\n record.paper_urls = (m.link as Array> | undefined)\n ?.map((l) => l.URL)\n .filter((v): v is string => typeof v === \"string\");\n } catch {\n /* best-effort */\n }\n}\n\n// ---------------------------------------------------------------------------\n// arXiv (export API returns Atom XML)\n// ---------------------------------------------------------------------------\nasync function enrichArxiv(\n record: Record,\n url: string,\n): Promise> {\n const m = url.match(/arxiv\\.org\\/(?:abs|pdf)\\/([^?#]+)/) ||\n url.match(/arxiv:([\\d.]+)/);\n const id = m ? m[1].replace(/\\.pdf$/, \"\").trim() : null;\n if (!id) return record;\n record.arxiv_id = id;\n record.url = `https://arxiv.org/abs/${id}`;\n\n try {\n const r = await fetch(`https://export.arxiv.org/api/query?id_list=${id}`, {\n headers: { \"user-agent\": USER_AGENT, accept: \"application/atom+xml\" },\n });\n if (!r.ok) return record;\n const xml = await r.text();\n const entry = firstEntry(xml) ?? \"\";\n record.title = stripTags(field(entry, \"title\"));\n const abstract = stripTags(field(entry, \"summary\"));\n if (abstract) record.abstract = abstract;\n const published = field(entry, \"published\");\n if (published) record.published = published.slice(0, 10);\n record.authors = Array.from(entry.matchAll(/([\\s\\S]*?)<\\/name>/g))\n .map((g) => stripTags(g[1]))\n .filter(Boolean);\n const doi = field(entry, \"arxiv:doi\");\n if (doi) record.doi = doi;\n record.journal = field(entry, \"arxiv:journal_ref\") || record.journal || null;\n } catch {\n /* best-effort */\n }\n return record;\n}\n\n// ---------------------------------------------------------------------------\n// PubMed (NCBI eutils — free for moderate use)\n// ---------------------------------------------------------------------------\nasync function enrichPubmed(\n record: Record,\n url: string,\n): Promise> {\n const pmid = url.match(/pubmed\\.ncbi\\.nlm\\.nih\\.gov\\/(\\d+)/)?.[1];\n if (!pmid) return record;\n record.pubmed_id = pmid;\n try {\n const r = await fetch(\n `https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id=${pmid}&retmode=json`,\n { headers: { \"user-agent\": USER_AGENT, accept: \"application/json\" } },\n );\n if (!r.ok) return record;\n const d = (await r.json()) as Record;\n const it = (d.result as Record)?.[pmid] as Record | undefined;\n if (!it) return record;\n record.title = it.title ?? null;\n record.published = it.pubdate ?? null;\n record.journal = it.fulljournalname ?? it.source ?? null;\n record.authors = (it.authors as Array> | undefined)\n ?.map((a) => a.name ?? null);\n record.type = \"journal-article\";\n const ids = (it.articleids as Array> | undefined) ?? [];\n record.doi = ids.find((x) => x.idtype === \"doi\")?.value ?? null;\n record.pmc_id = ids.find((x) => x.idtype === \"pmc\")?.value ?? null;\n } catch {\n /* best-effort */\n }\n return record;\n}\n\n// ---------------------------------------------------------------------------\n// Helpers\n// ---------------------------------------------------------------------------\nfunction pathOf(url: string): string | null {\n try {\n return new URL(url).pathname.replace(/^\\/+/, \"\") || null;\n } catch {\n return null;\n }\n}\n\nfunction firstArray(v: unknown): string | null {\n return Array.isArray(v) && v.length ? String(v[0]) : null;\n}\n\nfunction pickDate(o: unknown): string | null {\n if (!o || typeof o !== \"object\") return null;\n const parts = (o as Record)[\"date-parts\"] as Array> | undefined;\n if (!Array.isArray(parts) || !parts[0] || !parts[0][0]) return null;\n const [y, mo, d] = parts[0];\n return [y, mo, d].filter((x) => x != null).join(\"-\");\n}\n\nfunction field(xml: string, tag: string): string {\n const m = xml.match(new RegExp(`<${tag}>([\\\\s\\\\S]*?)`));\n return m ? m[1] : \"\";\n}\n\nfunction firstEntry(xml: string): string | null {\n const m = xml.match(/([\\s\\S]*?)<\\/entry>/);\n return m ? m[1] : null;\n}\n",[],[143,198],"dfc6f840-fc1e-4eda-befe-594dc563fac0","music.ts","d0e8d11c-aa92-11f1-bb4e-1607ee4eb77e",15,["D",1788768297194],"import {\n fetchMeta,\n fetchOembed,\n hostOf,\n USER_AGENT,\n youtubeId,\n} from \"./util.ts\";\n\n// Streaming providers (all support public oEmbed, no auth).\nconst OEMBED: Record string> = {\n spotify: (u) =>\n `https://open.spotify.com/oembed?url=${encodeURIComponent(u.href)}&format=json`,\n soundcloud: (u) =>\n `https://soundcloud.com/oembed?url=${encodeURIComponent(u.href)}&format=json`,\n bandcamp: (u) => `https://bandcamp.com/oembed?url=${encodeURIComponent(u.href)}`,\n deezer: (u) =>\n `https://www.deezer.com/oembed?url=${encodeURIComponent(u.href)}&format=json`,\n};\n\n/**\n * Archive a music URL: streaming providers via oEmbed, music metadata\n * databases via their public APIs. Best-effort; never throws.\n */\nexport async function archiveMusic(url: string): Promise> {\n const record: Record = {\n kind: \"music\",\n originalUrl: url,\n archived_at: new Date().toISOString(),\n };\n\n const host = hostOf(url);\n let u: URL | null = null;\n try {\n u = new URL(url);\n } catch {\n return record;\n }\n record.host = host;\n record.url = url;\n\n // --- Metadata databases ---\n if (/(^|\\.)musicbrainz\\./.test(host)) return archiveMusicBrainz(record, u);\n if (/(^|\\.)listenbrainz\\./.test(host)) return archiveListenBrainz(record, u);\n if (host.endsWith(\".last.fm\")) return archiveLastFm(record, u);\n\n // --- Streaming providers ---\n let provider: string | null = null;\n if (/(^|\\.)spotify\\./.test(host)) provider = \"spotify\";\n else if (/(^|\\.)soundcloud\\./.test(host)) provider = \"soundcloud\";\n else if (/(^|\\.)bandcamp\\./.test(host)) provider = \"bandcamp\";\n else if (/(^|\\.)deezer\\./.test(host)) provider = \"deezer\";\n else if (host === \"music.youtube.com\") provider = \"youtube-music\";\n\n if (provider && OEMBED[provider]) {\n const d = await fetchOembed(OEMBED[provider](u));\n if (d) {\n record.title = d.title ?? null;\n record.author = d.author_name ?? null;\n record.author_url = d.author_url ?? null;\n record.thumbnail = d.thumbnail_url ?? null;\n record.provider_name = d.provider_name ?? null;\n record.embed_html = d.html ?? null;\n }\n }\n\n if (provider === \"spotify\") {\n const m = url.match(\n /spotify\\.com\\/(track|album|playlist|artist|episode|show)\\/([A-Za-z0-9]+)/,\n );\n if (m) {\n record.spotify_type = m[1];\n record.spotify_id = m[2];\n record.spotify_uri = `spotify:${m[1]}:${m[2]}`;\n record.embed_url = `https://open.spotify.com/embed/${m[1]}/${m[2]}`;\n }\n } else if (provider === \"youtube-music\") {\n const id = youtubeId(url);\n if (id) {\n record.video_id = id;\n record.thumbnail = record.thumbnail ??\n `https://img.youtube.com/vi/${id}/hqdefault.jpg`;\n record.embed_url = `https://music.youtube.com/watch?v=${id}`;\n }\n } else if (provider === \"deezer\") {\n const m = url.match(\n /deezer\\.com\\/(?:[a-z]{2}\\/)?(track|album|playlist|artist|show|episode)\\/(\\d+)/,\n );\n if (m) {\n record.deezer_type = m[1];\n record.deezer_id = m[2];\n }\n } else if (provider === \"soundcloud\") {\n const m = url.match(/soundcloud\\.com\\/([^/?#]+)\\/([^/?#]+)/);\n if (m) {\n record.soundcloud_user = m[1];\n record.soundcloud_track = m[2];\n }\n } else if (provider === \"bandcamp\") {\n const m = url.match(/bandcamp\\.com\\/track\\/([^/?#]+)\\/?$/);\n if (m) record.bandcamp_slug = m[1];\n }\n\n return record;\n}\n\n// ---------------------------------------------------------------------------\n// MusicBrainz (public ws/2 API, no key, requires a polite User-Agent)\n// ---------------------------------------------------------------------------\nconst MB_ENTITIES = new Set([\n \"artist\", \"release-group\", \"release\", \"recording\", \"work\",\n \"label\", \"event\", \"place\", \"series\", \"area\", \"url\",\n]);\n\nasync function archiveMusicBrainz(\n record: Record,\n u: URL,\n): Promise> {\n const m = u.pathname.match(\n /\\/(artist|release-group|release|recording|work|label|event|place|series|area|url)\\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/,\n );\n if (!m || !MB_ENTITIES.has(m[1])) {\n return fromPageMeta(record, u.href);\n }\n record.musicbrainz_entity = m[1];\n record.musicbrainz_id = m[2];\n record.url = `https://musicbrainz.org/${m[1]}/${m[2]}`;\n\n const inc = m[1] === \"release\"\n ? \"release-groups+recordings+artist-credits\"\n : m[1] === \"recording\" ? \"artist-credits+releases\" : undefined;\n const qs = inc ? `?inc=${inc}&fmt=json` : \"?fmt=json\";\n try {\n const r = await fetch(`https://musicbrainz.org/ws/2/${m[1]}/${m[2]}${qs}`, {\n headers: { \"user-agent\": USER_AGENT, accept: \"application/json\" },\n });\n if (!r.ok) return fromPageMeta(record, u.href);\n const d = (await r.json()) as Record;\n\n record.title = d.title ?? d.name ?? null;\n record.musicbrainz_type = d.type ?? null;\n if (m[1] === \"release-group\") record.musicbrainz_primary_type = d[\"primary-type\"] ?? null;\n record.disambiguation = d.disambiguation ?? null;\n\n if (d.artist) {\n const artist = d.artist as Record;\n record.author = artist.sort_name ?? artist.name ?? null;\n record.author_mbid = artist.id ?? null;\n }\n if (d[\"artist-credit\"]) {\n record.artist_credit = (d[\"artist-credit\"] as Array>)\n .map((a) => String(a.name ?? \"\")).join(\" / \");\n }\n if (d.country) record.country = d.country;\n if (d.area) record.area = (d.area as Record)?.name ?? null;\n if (d.date) record.release_date = d.date;\n if (d[\"first-release-date\"]) record.first_release_date = d[\"first-release-date\"];\n if (Array.isArray(d[\"release-events\"]) && d[\"release-events\"][0]) {\n const ev = d[\"release-events\"][0] as Record;\n if (ev.date) record.release_date = ev.date;\n }\n if (d[\"release-group\"]) {\n const rg = d[\"release-group\"] as Record;\n record.release_group = rg.title ?? null;\n record.musicbrainz_primary_type = rg[\"primary-type\"] ?? null;\n }\n // Rating/metadata stats where present\n if (d.rating) record.musicbrainz_rating = d.rating;\n if (d.score != null) record.musicbrainz_score = d.score;\n if (m[1] === \"release\" && d.media) {\n record.release_format = (d.media as Array>)[0]?.format ?? null;\n record.track_count = (d.media as Array>)[0]?.track_count ?? null;\n }\n if (m[1] === \"recording\" && typeof d.length === \"number\") {\n record.duration_ms = d.length;\n }\n } catch {\n return fromPageMeta(record, u.href);\n }\n return record;\n}\n\n// ---------------------------------------------------------------------------\n// ListenBrainz (public API; enrich via page meta / MBID when linkable)\n// ---------------------------------------------------------------------------\nasync function archiveListenBrainz(\n record: Record,\n u: URL,\n): Promise> {\n // ListenBrainz artist/release/track pages often embed a MusicBrainz ID.\n const mb = u.pathname.match(/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/);\n if (mb) record.musicbrainz_id = mb[1];\n return fromPageMeta(record, u.href);\n}\n\n// ---------------------------------------------------------------------------\n// Last.fm (page meta; optional API enrichment when LASTFM_API_KEY is set)\n// ---------------------------------------------------------------------------\nasync function archiveLastFm(\n record: Record,\n u: URL,\n): Promise> {\n const seg = u.pathname.split(\"/\").filter(Boolean);\n if (seg[0] === \"music\" && seg.length >= 2) {\n record.artist = decodeURIComponent(seg[1]);\n if (seg.length >= 4) {\n record.lastfm_type = \"track\";\n record.track = decodeURIComponent(seg[3]);\n } else if (seg.length === 3) {\n record.lastfm_type = \"album\";\n record.album = decodeURIComponent(seg[2]);\n } else {\n record.lastfm_type = \"artist\";\n }\n }\n\n const key = Deno.env.get(\"LASTFM_API_KEY\");\n if (key && record.artist) {\n const p = new URLSearchParams({ api_key: key, format: \"json\", artist: String(record.artist) });\n let method = \"artist.getInfo\";\n if (record.lastfm_type === \"album\") { method = \"album.getInfo\"; p.set(\"album\", String(record.album)); }\n if (record.lastfm_type === \"track\") { method = \"track.getInfo\"; p.set(\"track\", String(record.track)); }\n try {\n const r = await fetch(`https://ws.audioscrobbler.com/2.0/?method=${method}&${p.toString()}`, {\n headers: { \"user-agent\": USER_AGENT },\n });\n if (r.ok) {\n const d = await r.json();\n const info = d?.artist ?? d?.album ?? d?.track;\n if (info) {\n record.title = info.name ?? record.title ?? null;\n if (record.lastfm_type === \"artist\") record.summary = innerText(info.bio?.summary);\n if (record.lastfm_type === \"album\" && info.artist) record.author = info.artist ?? null;\n const img = (Array.isArray(info.image) ? info.image : []).find(\n (i) => (i as Record)?.[\"#text\"],\n );\n record.thumbnail = record.thumbnail ??\n ((img as Record)?.[\"#text\"] ?? null);\n }\n }\n } catch {\n /* best-effort */\n }\n }\n return record;\n}\n\n// ---------------------------------------------------------------------------\n// Fallback: pull title/description/image from the page's tags.\n// ---------------------------------------------------------------------------\nasync function fromPageMeta(\n record: Record,\n url: string,\n): Promise> {\n const meta = await fetchMeta(url);\n record.title ??= meta[\"og:title\"] ?? meta.title ?? null;\n record.description ??= meta[\"og:description\"] ?? meta.description ?? null;\n record.thumbnail ??= meta[\"og:image\"] ?? meta[\"twitter:image\"] ?? null;\n return record;\n}\n\nfunction innerText(v: unknown): string | null {\n if (typeof v !== \"string\") return null;\n return v.replace(/<[^>]+>/g, \" \").replace(/\\s+/g, \" \").trim() || null;\n}\n",[],[143,206],"fbfbca7a-7cad-40ed-ab8d-e3950fdb57b9","git.ts","c60442cc-aa92-11f1-8348-1607ee4eb77e",14,["D",1788768278908],"import { hostOf, USER_AGENT } from \"./util.ts\";\n\n/** Indefinite GitHub API JSON fetch; returns null on any failure. */\nasync function apiJson(url: string): Promise | null> {\n try {\n const r = await fetch(url, {\n headers: { \"user-agent\": USER_AGENT, accept: \"application/json\" },\n redirect: \"follow\",\n });\n if (!r.ok) return null;\n const d: unknown = await r.json();\n return (typeof d === \"object\" && d !== null)\n ? (d as Record)\n : null;\n } catch {\n return null;\n }\n}\n\n/**\n * Archive a source-control repository URL. Parses owner/repo out of the path\n * (works for repo, subdir, file and tag pages) and enriches via the provider's\n * public repository API where one exists.\n */\nexport async function archiveGit(url: string): Promise> {\n const record: Record = {\n kind: \"git\",\n originalUrl: url,\n archived_at: new Date().toISOString(),\n };\n\n const host = hostOf(url);\n let path = \"\";\n try {\n path = new URL(url).pathname.replace(/^\\/+/, \"\").replace(/\\.git$/, \"\");\n } catch {\n return record;\n }\n record.host = host;\n record.url = host && path ? `https://${host}/${path}` : url;\n\n const parts = path.split(\"/\").filter(Boolean);\n const owner = parts[0];\n const repo = parts[1];\n if (owner && repo) {\n record.owner = owner;\n record.repo = repo;\n record.repo_url = `https://${host}/${owner}/${repo}`;\n }\n if (!owner || !repo) return record;\n\n try {\n if (host === \"github.com\") {\n const d = await apiJson(`https://api.github.com/repos/${owner}/${repo}`);\n if (d) {\n record.default_branch = d.default_branch ?? null;\n record.description = d.description ?? null;\n record.language = d.language ?? null;\n record.stars = d.stargazers_count ?? 0;\n record.forks = d.forks_count ?? 0;\n record.watchers = d.subscribers_count ?? null;\n record.open_issues = d.open_issues_count ?? 0;\n record.license = licenseName(d.license as Record | null);\n record.homepage = d.homepage ?? null;\n record.parent = d.fork ? (d.parent as Record)?.full_name ?? null : null;\n record.created = d.created_at ?? null;\n record.pushed = d.pushed_at ?? null;\n record.size_kb = d.size ?? null;\n record.topics = Array.isArray(d.topics) ? d.topics : [];\n record.archived = !!d.archived;\n }\n } else if (host === \"gitlab.com\") {\n const d = await apiJson(\n `https://gitlab.com/api/v4/projects/${encodeURIComponent(`${owner}/${repo}`)}`,\n );\n if (d) {\n record.description = d.description ?? null;\n record.default_branch = d.default_branch ?? null;\n record.host_type = d.visibility ?? null;\n record.stars = d.star_count ?? 0;\n record.forks = d.forks_count ?? 0;\n record.open_issues = d.open_issues_count ?? null;\n record.homepage = d.web_url ?? null;\n record.created = d.created_at ?? null;\n record.last_activity = d.last_activity_at ?? null;\n record.license = licenseName(d.license as Record | null);\n record.topics = Array.isArray(d.topics) ? d.topics : [];\n }\n } else if (host === \"bitbucket.org\") {\n const d = await apiJson(\n `https://api.bitbucket.org/2.0/repositories/${owner}/${repo}`,\n );\n if (d) {\n record.description = d.description ?? null;\n record.language = d.language ?? null;\n record.default_branch = (d.mainbranch as Record)?.name ?? null;\n record.homepage = (d.links as Record)?.html\n ? ((d.links as Record).html as Record)?.href\n : null;\n record.created = d.created_on ?? null;\n record.updated = d.updated_on ?? null;\n record.parent = (d.parent as Record)?.full_name ?? null;\n }\n } else if (host === \"codeberg.org\" || /gitea/.test(host)) {\n // Codeberg and Gitea expose a Gitea-compatible API.\n const d = await apiJson(`https://${host}/api/v1/repos/${owner}/${repo}`);\n if (d) {\n record.description = d.description ?? null;\n record.language = d.language ?? null;\n record.default_branch = d.default_branch ?? null;\n record.stars = d.stars_count ?? 0;\n record.forks = d.forks_count ?? 0;\n record.open_issues = d.open_issues_count ?? null;\n record.license = licenseName(d.license as Record | null);\n record.homepage = d.website ?? null;\n record.created = d.created_at ?? null;\n record.updated = d.updated_at ?? null;\n record.private = !!d.private;\n record.fork = !!d.fork;\n }\n }\n } catch {\n /* metadata enrichment is best-effort */\n }\n return record;\n}\n\nfunction licenseName(license: Record | null | undefined): unknown {\n if (!license) return null;\n return license.spdx_id ?? license.name ?? null;\n}\n",[],[143,214],"31061663-3127-496a-9652-6c22a485375a","classify.ts","bfa724ee-aa92-11f1-907d-1607ee4eb77e",13,["D",1788768268241],"import { FILE_RE, hostOf } from \"./util.ts\";\n\nexport type UrlKind =\n | \"git\"\n | \"music\"\n | \"video\"\n | \"wikipedia\"\n | \"paper\"\n | \"file\"\n | \"article\"\n | \"unknown\";\n\n/**\n * Classify a URL into the archiver's handler kinds.\n * Order matters: YouTube Music must win over the generic YouTube video check.\n */\nexport function classify(url: string): UrlKind {\n const host = hostOf(url);\n let path = \"\";\n try {\n path = new URL(url).pathname;\n } catch {\n return \"unknown\";\n }\n if (!host) return \"unknown\";\n\n // Repositories\n if (/(^|\\.)(github|gitlab|bitbucket|codeberg|gitea)\\.(com|net|org|io|dev)$/.test(host) ||\n host === \"dev.azure.com\") {\n return \"git\";\n }\n\n // Music streaming + metadata databases\n if (host === \"music.youtube.com\" ||\n /(^|\\.)(spotify|deezer|soundcloud|bandcamp|musicbrainz|listenbrainz)\\./.test(host) ||\n host.endsWith(\".last.fm\")) {\n return \"music\";\n }\n\n // Video (Youtube, Vimeo, Dailymotion)\n if (/(^|\\.)(youtube|youtu|youtube-nocookie)\\./.test(host) ||\n /(^|\\.)vimeo\\./.test(host) ||\n /(^|\\.)dailymotion\\./.test(host)) {\n return \"video\";\n }\n\n // Wikipedia\n if (/(^|\\.)wikipedia\\.org$/.test(host) && path.startsWith(\"/wiki/\")) {\n return \"wikipedia\";\n }\n\n // Scientific papers / DOIs\n if (host === \"doi.org\" ||\n /(^|\\.)arxiv\\.org$/.test(host) ||\n /(^|\\.)pubmed\\.ncbi\\.nlm\\.nih\\.gov/.test(host)) {\n return \"paper\";\n }\n\n // Direct file/asset download\n if (FILE_RE.test(url.split(/[?#]/)[0])) return \"file\";\n\n return \"article\";\n}\n",[],[143,222],["D",1788768264288],[161,162,163,164,165,166,167],[143],["D",1788536954683],{"_102":101,"_110":106},{"_102":101,"_103":104,"_105":106,"_19":61,"_107":108,"_109":31},{"_13":91,"_38":92,"_49":48,"_93":50,"_94":-5,"_95":-5,"_96":-5,"_97":-5,"_98":-5,"_99":51,"_100":101},"ctx",{"_84":-5,"_130":269,"_270":271,"_272":273,"_274":31,"_275":31,"_276":31,"_277":64,"_278":-5,"_279":-7,"_280":-7,"_281":68,"_282":31,"_283":31,"_284":-5,"_285":31,"_286":31,"_287":31,"_288":-5},"ENV",{"_240":241,"_242":243,"_244":92,"_245":246,"_247":248,"_249":250,"_251":252,"_253":254,"_255":256,"_257":258,"_259":260,"_261":262,"_263":264,"_265":266,"_267":268},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v20