[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_253":254,"_255":256},"routes/_app.x.$handle.$name",{"_11":12,"_13":14,"_15":16,"_17":18,"_38":-7,"_124":-5,"_19":125,"_126":127,"_128":129,"_130":131,"_132":133,"_21":134,"_135":-5,"_136":-5,"_137":24,"_138":139,"_140":86,"_141":-5,"_142":-5,"_143":144},"routes/_app.x.$handle.$name.branch.$branch.history_.$",{"_11":12,"_13":14,"_15":16,"_17":18,"_19":20,"_21":22,"_23":24,"_25":26,"_27":28,"_29":24,"_30":-5},"handle","heidi","name","url-archiver","branchName","main","path","src/util.ts","project",{"_32":73,"_13":14,"_36":74,"_75":-5,"_76":24,"_77":78,"_79":78,"_80":81,"_82":-5,"_30":69,"_83":84,"_85":86,"_87":-5,"_88":-5,"_89":26,"_40":41,"_90":-5,"_91":-5,"_92":-5,"_93":-5,"_94":95,"_96":95,"_97":98,"_99":98,"_100":98,"_101":95,"_102":95,"_53":103,"_104":105,"_106":107,"_108":24},"file",{"_63":64,"_13":65,"_66":67,"_68":24,"_72":-5},"isDirectory",false,"page",0,"history",[31],"hasNext","userId",{"_32":33,"_34":35,"_36":37,"_38":39,"_40":41,"_42":43,"_44":-5,"_45":46,"_47":-5,"_48":-5,"_49":-5,"_50":51,"_52":-5,"_53":54,"_21":55,"_56":-5,"_57":-5,"_58":-5,"_59":-5,"_60":61},"id","d5ae7274-637d-4da2-b0bc-ccb012c22561","branchId","3f798dd9-8407-4149-b148-45d4da89ccbd","createdAt",["D",1788768264288],"version",12,"orgId","01a06d17-30e7-7396-bb69-029ba771a7d7","parentId","c968ec7c-419e-4cea-9d54-9231947c003e","incomingCommitId","aiChatMessageId","02a7b804-ff16-4769-b212-998c9b8f5b48","aiChatThreadId","aiChatThreadUserId","aiChatThreadTitle","source","townie","clientLabel","user",{"_32":69,"_40":41,"_11":12,"_70":71},{"_63":64,"_13":65,"_66":67,"_68":24},"pull","merge","revert","multiple","commits",[62],{"_32":33,"_34":35,"_36":37,"_38":39,"_40":41,"_42":43,"_44":-5,"_45":46,"_47":-5,"_48":-5,"_49":-5,"_50":51,"_52":-5,"_53":54,"_21":55,"_56":-5,"_57":-5,"_58":-5},"fileId","bd4c5dd6-aa92-11f1-907d-1607ee4eb77e","util.ts","fileType","script","deleted","124f6344-3784-4835-926c-5b9b29b366cc","avatar","https://lh3.googleusercontent.com/a/ACg8ocK_QSHhMpajSrYgGIPk8bOKlftGgK0BEPGcExr97zjm8MHbQA=s96-c","deletedAtVersion","ce845deb-58e1-4474-aea4-5bbf1cd9e916",["D",1788536954683],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Archive any URL: git repos, music, video, papers, Wikipedia","imageUrl","pinnedFileId","32c89862-a878-11f1-946c-1607ee4eb77e","pinPreview",true,"pinMigrated","screenshotVersion","likeCount","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm",{"_11":12,"_123":71},"org",{"_11":12,"_118":119,"_70":71,"_30":69,"_120":121,"_122":24},"branch",{"_38":109,"_13":16,"_32":35,"_110":73,"_111":-5,"_112":-5,"_113":-5,"_114":-5,"_115":-5,"_116":14,"_117":12},"isOwned",21,"projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","displayName","Heidi Hedh","tier","free","canRestrictVals","profileImageUrl","adoptOrgId",{"_32":73,"_13":14,"_36":249,"_75":-5,"_76":24,"_77":78,"_79":78,"_80":81,"_82":-5,"_30":69,"_83":84,"_85":86,"_87":-5,"_88":-5,"_89":26,"_40":41,"_90":-5,"_91":-5,"_92":-5,"_93":-5,"_94":95,"_96":95,"_97":98,"_99":98,"_100":98,"_101":95,"_102":95,"_53":250,"_104":251,"_106":252,"_108":24},"tree",[165,173,174],"files",[165,166,167,168,169,170,171,172,173,174],"counts",{"_154":26,"_155":156,"_157":26,"_158":159,"_160":161},"contents",[],{"_32":145,"_34":35,"_63":64,"_13":65,"_146":147,"_66":67,"_38":26,"_42":148,"_36":149,"_150":39,"_151":-5,"_17":152},"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},"1c6ffaaf-1ff2-4eca-8fa6-b84b9267c9fc","content","/** Shared helpers for the URL archiver providers. */\n\nexport const USER_AGENT =\n \"Mozilla/5.0 (compatible; heidi/url-archiver/1.0; +https://www.val.town/x/heidi/url-archiver)\";\n\n/** File extensions that indicate a download/asset rather than a page. */\nexport const FILE_RE =\n /\\.(pdf|docx?|xlsx?|pptx?|txt|md|rtf|csv|json|xml|zip|tar|gz|7z|rar|png|jpe?g|gif|webp|svg|ico|bmp|mp3|wav|flac|ogg|m4a|mp4|webm|mov|avi|mkv|epub|mobi)$/i;\n\n/** Fetch a provider's oEmbed JSON; returns null on any failure. */\nexport async function fetchOembed(\n oembedUrl: string,\n): Promise | null> {\n try {\n const r = await fetch(oembedUrl, {\n headers: { \"user-agent\": USER_AGENT, accept: \"application/json\" },\n });\n if (!r.ok) return null;\n const d: unknown = await r.json();\n return (typeof d === \"object\" && d !== null)\n ? (d as Record)\n : null;\n } catch {\n return null;\n }\n}\n\n/** Fetch a page and read its tags (returns {} on failure). */\nexport async function fetchMeta(\n url: string,\n): Promise> {\n try {\n const r = await fetch(url, {\n headers: { \"user-agent\": USER_AGENT },\n redirect: \"follow\",\n });\n if (!r.ok) return {};\n return metaTags(await r.text());\n } catch {\n return {};\n }\n}\n\n/** Parse tags out of an HTML string. */\nexport function metaTags(html: string): Record {\n const out: Record = {};\n const tagRe = /]*>/gi;\n const attrRe = /([a-zA-Z-]+)\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)')/g;\n let tag: RegExpExecArray | null;\n while ((tag = tagRe.exec(html)) !== null) {\n const attrs: Record = {};\n let a: RegExpExecArray | null;\n attrRe.lastIndex = 0;\n while ((a = attrRe.exec(tag[0])) !== null) {\n attrs[a[1].toLowerCase()] = a[2] ?? a[3] ?? \"\";\n }\n const name = attrs.property || attrs.name;\n if (name && attrs.content != null && !(name in out)) {\n out[name] = decodeEntities(attrs.content);\n }\n }\n return out;\n}\n\n/** Decode common HTML entities in a string. */\nexport function decodeEntities(s: string): string {\n return s\n .replace(/&#x([0-9a-fA-F]+);/g, (_, h) => String.fromCodePoint(parseInt(h, 16)))\n .replace(/&#(\\d+);/g, (_, d) => String.fromCodePoint(parseInt(d, 10)))\n .replace(/&/g, \"&\")\n .replace(/</g, \"<\")\n .replace(/>/g, \">\")\n .replace(/"/g, '\"')\n .replace(/'/g, \"'\")\n .replace(/ /g, \" \")\n .trim();\n}\n\n/** Strip JATS/HTML tags, collapse whitespace. */\nexport function stripTags(s: string): string {\n return s.replace(/<[^>]+>/g, \" \").replace(/\\s+/g, \" \").trim();\n}\n\n/** Extract a YouTube/Music video ID from a URL. */\nexport function youtubeId(url: string): string | null {\n const m = url.match(/[?&]v=([A-Za-z0-9_-]{11})/) ||\n url.match(/(?:youtu\\.be\\/|embed\\/|shorts\\/|live\\/)([A-Za-z0-9_-]{11})/);\n return m ? m[1] : null;\n}\n\n/** Normalize a hostname: lowercase, strip www. */\nexport function hostOf(url: string): string {\n try {\n return new URL(url).hostname.toLowerCase().replace(/^www\\./, \"\");\n } catch {\n return \"\";\n }\n}\n","2ad11f9a-53a3-433f-af2e-7dc42ce19589",["D",1788768264288],"branchVersion","label",[153,65],"src","pulls","branches",1,"forks","envVars",{"_162":26,"_164":26},"connections",{"_162":26,"_163":26},"count","unfilledCount","emptyCount",{"_32":244,"_13":153,"_63":148,"_42":-5,"_34":35,"_150":39,"_178":86,"_36":245,"_38":26,"_66":246,"_146":-5,"_68":24,"_151":-5,"_183":247,"_17":248},{"_32":236,"_13":237,"_63":238,"_42":148,"_34":35,"_150":239,"_178":86,"_36":240,"_38":26,"_66":67,"_146":241,"_68":24,"_151":-5,"_183":242,"_17":243},{"_32":228,"_13":229,"_63":230,"_42":148,"_34":35,"_150":231,"_178":86,"_36":232,"_38":26,"_66":67,"_146":233,"_68":24,"_151":-5,"_183":234,"_17":235},{"_32":220,"_13":221,"_63":222,"_42":148,"_34":35,"_150":223,"_178":86,"_36":224,"_38":26,"_66":67,"_146":225,"_68":24,"_151":-5,"_183":226,"_17":227},{"_32":212,"_13":213,"_63":214,"_42":148,"_34":35,"_150":215,"_178":86,"_36":216,"_38":156,"_66":67,"_146":217,"_68":24,"_151":-5,"_183":218,"_17":219},{"_32":145,"_13":65,"_63":64,"_42":148,"_34":35,"_150":39,"_178":86,"_36":209,"_38":26,"_66":67,"_146":147,"_68":24,"_151":-5,"_183":210,"_17":211},{"_32":201,"_13":202,"_63":203,"_42":148,"_34":35,"_150":204,"_178":86,"_36":205,"_38":26,"_66":67,"_146":206,"_68":24,"_151":-5,"_183":207,"_17":208},{"_32":193,"_13":194,"_63":195,"_42":148,"_34":35,"_150":196,"_178":86,"_36":197,"_38":26,"_66":67,"_146":198,"_68":24,"_151":-5,"_183":199,"_17":200},{"_32":186,"_13":187,"_63":188,"_42":-5,"_34":35,"_150":109,"_178":86,"_36":189,"_38":26,"_66":21,"_146":190,"_68":24,"_151":-5,"_183":191,"_17":192},{"_32":175,"_13":176,"_63":84,"_42":-5,"_34":35,"_150":177,"_178":86,"_36":179,"_38":180,"_66":181,"_146":182,"_68":24,"_151":14,"_183":184,"_17":185},"c549e592-a93e-4116-922d-7b61704d6245","main.ts",20,"latest",["D",1788768338783],7,"http","/**\n * URL Archiver\n *\n * Archive one or more URLs into a rich record. Any HTTP method is accepted.\n * URLs may come via query params or body (JSON or form), under any of the\n * aliases: url, urls, site, sites, website, websites, link, links. Each may\n * be a string or an array; all are flattened & deduped.\n *\n * Extractor options may be passed as a nested `options` object or flat keys:\n * wordsPerMinute, descriptionTruncateLen, descriptionLengthThreshold,\n * contentLengthThreshold, allowedTags, allowedAttributes, allowedIframeDomains\n *\n * A single URL returns the record directly; multiple URLs return an array.\n * The page type is recognized smartly and routed to a dedicated provider:\n * git GitHub / GitLab / Bitbucket / Codeberg / Gitea repos\n * music Spotify / Deezer / SoundCloud / Bandcamp / YouTube Music,\n * plus MusicBrainz / ListenBrainz / Last.fm\n * video YouTube / Vimeo / Dailymotion\n * wikipedia Wikipedia articles (+ REST summary)\n * paper DOIs (Crossref) / arXiv / PubMed\n * file direct asset downloads\n * article generic article extraction (default fallback)\n *\n * POST / { \"url\": [\"https://a.com\", \"https://b.com\"], \"options\": { \"wordsPerMinute\": 250 } }\n * GET /?link=https://a.com&wordsPerMinute=250\n */\nimport { extractFromHtml } from \"npm:@extractus/article-extractor@9.0.1\";\nimport { classify } from \"./src/classify.ts\";\nimport { archiveGit } from \"./src/git.ts\";\nimport { archiveMusic } from \"./src/music.ts\";\nimport { archiveVideo } from \"./src/video.ts\";\nimport { enrichWikipedia } from \"./src/wikipedia.ts\";\nimport { archivePaper } from \"./src/paper.ts\";\nimport { FILE_RE, USER_AGENT } from \"./src/util.ts\";\n\ntype ParserOptions = {\n wordsPerMinute?: number;\n descriptionTruncateLen?: number;\n descriptionLengthThreshold?: number;\n contentLengthThreshold?: number;\n allowedTags?: string[];\n allowedAttributes?: Record;\n allowedIframeDomains?: string[];\n};\n\nconst ALIASES = new Set([\n \"url\", \"urls\", \"site\", \"sites\", \"website\", \"websites\",\n \"link\", \"links\",\n]);\n\nconst OPTION_KEYS = new Set([\n \"wordsPerMinute\", \"descriptionTruncateLen\", \"descriptionLengthThreshold\",\n \"contentLengthThreshold\", \"allowedTags\", \"allowedAttributes\",\n \"allowedIframeDomains\",\n]);\n\nexport default async function handler(req: Request): Promise {\n const collected: Record = {};\n const rawOptions: Record = {};\n\n function collect(source: unknown) {\n if (!source || typeof source !== \"object\") return;\n for (const [key, value] of Object.entries(source as Record)) {\n if (ALIASES.has(key)) {\n if (Array.isArray(collected[key])) {\n (collected[key] as unknown[]).push(...flatten(value));\n } else {\n collected[key] = flatten(value);\n }\n } else if (key === \"options\" && value && typeof value === \"object\") {\n Object.assign(rawOptions, value);\n } else if (OPTION_KEYS.has(key)) {\n rawOptions[key] = value;\n }\n }\n }\n\n // Body\n try {\n const text = await req.text();\n if (text) {\n const type = req.headers.get(\"content-type\") || \"\";\n let parsed = false;\n if (!type.includes(\"application/x-www-form-urlencoded\")) {\n try {\n collect(JSON.parse(text));\n parsed = true;\n } catch {\n /* not JSON */\n }\n }\n if (!parsed) {\n const sp = new URLSearchParams(text);\n for (const [k, v] of sp.entries()) {\n if (ALIASES.has(k)) collect({ [k]: v });\n else if (OPTION_KEYS.has(k)) rawOptions[k] = v;\n else if (k === \"options\") {\n try { Object.assign(rawOptions, JSON.parse(v)); } catch { /* skip */ }\n }\n }\n }\n }\n } catch {\n /* ignore unparseable body */\n }\n\n // Query string\n const qs = new URL(req.url).searchParams;\n for (const key of Array.from(qs.keys())) {\n if (ALIASES.has(key)) {\n const vals = qs.getAll(key);\n if (Array.isArray(collected[key])) (collected[key] as string[]).push(...vals);\n else collected[key] = vals;\n } else if (OPTION_KEYS.has(key)) {\n rawOptions[key] = qs.getAll(key).length > 1 ? qs.getAll(key) : qs.get(key);\n } else if (key === \"options\") {\n try { Object.assign(rawOptions, JSON.parse(qs.get(key) || \"{}\")); } catch { /* skip */ }\n }\n }\n\n const urls = Array.from(\n new Set(Object.values(collected).flatMap((v) => v as string[]) ?? []),\n ).filter((u): u is string => typeof u === \"string\" && u.length > 0);\n\n if (urls.length === 0) {\n return json({ error: \"No URL provided. Use url/site/website/link (query or body).\" }, 400);\n }\n\n const options = parseOptions(rawOptions);\n\n const results = await Promise.all(urls.map((u) => archive(u, options)));\n const ok = results.filter((r) => r.ok).map((r) => r.record);\n const failed = results.filter((r) => !r.ok).map((r) => r.error);\n\n // Single URL -> return the record directly; multiple -> array wrapper.\n if (urls.length === 1) {\n if (ok.length === 1) return json(ok[0], 200);\n return json({ error: failed[0]?.error }, 500);\n }\n\n const body: Record = {\n archived_at: new Date().toISOString(),\n count: ok.length + failed.length,\n results: ok,\n };\n if (failed.length) body.failed = failed;\n return json(body, failed.length && !ok.length ? 500 : 200);\n}\n\n// ---------------------------------------------------------------------------\n// Archive: classify, route to a specialized provider, else extract article.\n// ---------------------------------------------------------------------------\nasync function archive(\n url: string,\n options: ParserOptions,\n): Promise<{ ok: true; record: Record } | { ok: false; error: { url: string; error: string } }> {\n try {\n const kind = classify(url);\n\n // Dedicated providers know the page shape best.\n if (kind === \"git\") return { ok: true, record: await archiveGit(url) };\n if (kind === \"music\") return { ok: true, record: await archiveMusic(url) };\n if (kind === \"video\") return { ok: true, record: await archiveVideo(url) };\n if (kind === \"paper\") return { ok: true, record: await archivePaper(url) };\n\n // Generic: fetch the page, then extract the article.\n const res = await fetch(url, { headers: { \"user-agent\": USER_AGENT }, redirect: \"follow\" });\n const finalUrl = res.url || url;\n const contentType = res.headers.get(\"content-type\") || \"\";\n const status = res.status;\n if (!res.ok) throw new Error(`HTTP ${status}`);\n const html = await res.text();\n const htmlBytes = new TextEncoder().encode(html).length;\n\n const article = await extractFromHtml(html, finalUrl, options).catch(() => null);\n const content = article?.content ?? null;\n const contentBytes = content ? new TextEncoder().encode(content).length : 0;\n\n const record: Record = {\n kind,\n originalUrl: url,\n url: finalUrl,\n archived_at: new Date().toISOString(),\n status,\n contentType,\n title: article?.title ?? null,\n author: article?.author ?? null,\n description: article?.description ?? null,\n image: article?.image ?? null,\n publisher: article?.publisher ?? null,\n published: article?.published ?? null,\n content,\n content_size: contentBytes,\n html_size: htmlBytes,\n };\n\n // Wikipedia: add canonical title + REST summary.\n if (kind === \"wikipedia\") {\n return { ok: true, record: await enrichWikipedia(record, finalUrl) };\n }\n\n return { ok: true, record };\n } catch (err) {\n return { ok: false, error: { url, error: (err as Error).message } };\n }\n}\n\n// ---------------------------------------------------------------------------\n// Options parsing\n// ---------------------------------------------------------------------------\nfunction parseOptions(raw: Record): ParserOptions {\n const o: ParserOptions = {};\n if (raw.wordsPerMinute != null) o.wordsPerMinute = Number(raw.wordsPerMinute);\n if (raw.descriptionTruncateLen != null) o.descriptionTruncateLen = Number(raw.descriptionTruncateLen);\n if (raw.descriptionLengthThreshold != null) o.descriptionLengthThreshold = Number(raw.descriptionLengthThreshold);\n if (raw.contentLengthThreshold != null) o.contentLengthThreshold = Number(raw.contentLengthThreshold);\n if (raw.allowedTags != null) o.allowedTags = toArray(raw.allowedTags);\n if (raw.allowedIframeDomains != null) o.allowedIframeDomains = toArray(raw.allowedIframeDomains);\n if (raw.allowedAttributes != null) {\n if (typeof raw.allowedAttributes === \"string\") {\n try { o.allowedAttributes = JSON.parse(raw.allowedAttributes); } catch { /* skip */ }\n } else if (raw.allowedAttributes && typeof raw.allowedAttributes === \"object\") {\n o.allowedAttributes = raw.allowedAttributes as Record;\n }\n }\n return o;\n}\n\nfunction toArray(v: unknown): string[] {\n if (Array.isArray(v)) return v.map(String);\n if (typeof v === \"string\") return v.split(\",\").map((s) => s.trim()).filter(Boolean);\n return [];\n}\n\nfunction flatten(value: unknown): Array {\n return Array.isArray(value) ? value.flatMap(flatten) : [value];\n}\n\nfunction json(body: unknown, status: number): Response {\n const payload = JSON.stringify(body, null, 2);\n const bytes = new TextEncoder().encode(payload).length;\n return new Response(payload, {\n status,\n headers: {\n \"content-type\": \"application/json; charset=utf-8\",\n \"content-length\": String(bytes),\n \"x-archiver-count\": String(Array.isArray(body) ? body.length : 1),\n },\n });\n}\n","children",[],[176],"b78d0f4a-39ed-4eb6-a62e-e7fee1abe631","README.md","1cc31818-aa93-11f1-848a-1607ee4eb77e",["D",1788768424451],"# URL Archiver\n\nArchive one or more URLs into a rich, per-provider record. The page type is\ndetected automatically and routed to a matching extraction provider. Any HTTP\nmethod is accepted; URLs come via query params or body (JSON or form).\n\n**Try it:** [Try the app](url:main.ts)\n\nEndpoint: `https://url-archiver.val.run`\n\n```bash\n# Single URL (GET)\ncurl \"https://url-archiver.val.run/?url=https://github.com/denoland/deno\"\n\n# Many URLs (GET, repeated params)\ncurl \"https://url-archiver.val.run/?url=https://a.com&url=https://b.com\"\n\n# Many URLs (POST, JSON)\ncurl -X POST https://url-archiver.val.run \\\n -H 'content-type: application/json' \\\n -d '{\"url\":[\"https://a.com\",\"https://b.com\"],\"options\":{\"wordsPerMinute\":250}}'\n```\n\nURL aliases (any may be a string or array): `url`, `urls`, `site`, `sites`,\n`website`, `website`, `link`, `links`.\n\n## Recognized types & providers\n\n| kind | Handled URLs | Source of metadata |\n|-------------|-------------------------------------------------------|----------------------------------------------------|\n| `git` | GitHub, GitLab, Bitbucket, Codeberg/Gitea | provider repo APIs (stars, forks, topics, …) |\n| `music` | Spotify, Deezer, SoundCloud, Bandcamp, YouTube Music | oEmbed + ID extraction |\n| `music` | MusicBrainz | MusicBrainz `ws/2` API (keyless) |\n| `music` | ListenBrainz, Last.fm | page meta (+ Last.fm API if key set) |\n| `video` | YouTube, Vimeo, Dailymotion | oEmbed + ID extraction |\n| `wikipedia` | `*.wikipedia.org/wiki/…` | article extract + REST `summary` (keyless) |\n| `paper` | `doi.org/…`, arXiv, PubMed | Crossref / arXiv Atom / NCBI eutils (all keyless) |\n| `file` | direct asset downloads (`*.pdf`, images, …) | — |\n| `article` | everything else | `@extractus/article-extractor` |\n\n### Example outputs\n\n```jsonc\n// https://github.com/denoland/deno\n{ \"kind\": \"git\", \"owner\": \"denoland\", \"repo\": \"deno\", \"language\": \"Rust\",\n \"stars\": 108390, \"forks\": 6373, \"license\": \"MIT\", \"topics\": [\"deno\", \"typescript\", \"rust\"], … }\n\n// https://doi.org/10.1038/s41586-020-2649-2\n{ \"kind\": \"paper\", \"doi\": \"10.1038/s41586-020-2649-2\", \"title\": \"Array programming with NumPy\",\n \"authors\": [\"Charles R. Harris\", \"…\"], \"journal\": \"Nature\", \"published\": \"2020-9-16\", \"abstract\": \"…\", … }\n\n// https://musicbrainz.org/recording/a4fd9f68-…\n{ \"kind\": \"music\", \"musicbrainz_entity\": \"recording\", \"title\": \"Never Gonna Give You Up\",\n \"artist_credit\": \"Rick Astley\", \"duration_ms\": 91000, … }\n```\n\n## Optional env vars\n\nNo credentials are required for any provider — everything public/keyless works\nout of the box. The only optional key enables richer Last.fm enrichment:\n\n- `LASTFM_API_KEY` — enrich Last.fm artist/album/track pages via the Last.fm API.\n\n👉 Add LASTFM_API_KEY here: https://www.val.town/x/heidi/url-archiver/environment-variables?key=LASTFM_API_KEY\n\n## Layout\n\n- `main.ts` — HTTP handler, URL collection, classification & routing\n- `src/classify.ts` — URL → kind detection\n- `src/git.ts`, `src/music.ts`, `src/video.ts`, `src/paper.ts`, `src/wikipedia.ts` — providers\n- `src/util.ts` — shared helpers (oEmbed, meta-tag parsing, entity decoding)\n\n```mermaid\nflowchart LR\n A[Request] --> B{classify} --> G[git]\n B --> M[music]\n B --> V[video]\n B --> W[wikipedia]\n B --> P[paper]\n B --> F[file / article]\n G --> GI[Platform API]\n M --> MO[oEmbed]\n M --> MB[MusicBrainz]\n V --> VO[oEmbed]\n W --> WE[Article Extract]\n W --> WR[REST summary]\n P --> PC[Crossref / arXiv / PubMed]\n```\n\nEverything is best-effort: if a provider API is down or the page doesn't match,\nthe archiver still returns a useful base record (IDs, host, original URL, title)\nrather than failing.\n",[],[187],"4a723b41-1ad2-45cf-ba6b-f685f69b4422","wikipedia.ts","d74960f8-aa92-11f1-aa53-1607ee4eb77e",17,["D",1788768307894],"import { hostOf, USER_AGENT } from \"./util.ts\";\n\n/**\n * Enrich a Wikipedia page record. The `record` is built by the generic article\n * extractor; here we add the canonical title, language and a REST `/summary`\n * pull (title, extract, description, thumbnail, Wikidata id) — free, no key.\n */\nexport async function enrichWikipedia(\n record: Record,\n finalUrl: string,\n): Promise> {\n const host = hostOf(finalUrl);\n const titlePath = pathOf(finalUrl);\n\n if (!(/\\.wikipedia\\.org$/.test(host)) || !titlePath) {\n record.wiki_title = record.title ?? null;\n return record;\n }\n\n const lang = host.startsWith(\"www.\") ? \"en\" : host.split(\".\")[0];\n const title = decodeURIComponent(titlePath.replace(/_/g, \" \"));\n record.wiki_title = title;\n record.wiki_lang = lang;\n\n try {\n const r = await fetch(\n `https://${lang}.wikipedia.org/api/rest_v1/page/summary/${encodeURIComponent(title)}`,\n { headers: { \"user-agent\": USER_AGENT, accept: \"application/json\" } },\n );\n if (!r.ok) return record;\n const d = (await r.json()) as Record;\n record.summary = record.summary ?? d.extract ?? null;\n record.wiki_description = d.description ?? null;\n record.thumbnail = record.thumbnail ?? (d.thumbnail as Record)?.source ?? null;\n record.wiki_image = (d.originalimage as Record)?.source ?? null;\n record.wikidata_id = d.wikibase_item ?? null;\n record.wiki_intro_html = d.extract_html ?? null;\n record.wiki_type = d.type ?? null;\n } catch {\n /* best-effort */\n }\n return record;\n}\n\nfunction pathOf(url: string): string | null {\n try {\n const p = new URL(url).pathname;\n const m = p.match(/\\/wiki\\/(.+)/);\n return m ? m[1] : null;\n } catch {\n return null;\n }\n}\n",[],[153,194],"5881f64d-5f03-41c6-83f9-ad42743d595d","video.ts","d4020378-aa92-11f1-aa53-1607ee4eb77e",16,["D",1788768302390],"import { fetchOembed, hostOf, youtubeId } from \"./util.ts\";\n\n// Video providers with public oEmbed (no auth).\nconst OEMBED: Record string> = {\n youtube: (u) =>\n `https://www.youtube.com/oembed?url=${encodeURIComponent(u.href)}&format=json`,\n vimeo: (u) => `https://vimeo.com/api/oembed.json?url=${encodeURIComponent(u.href)}`,\n dailymotion: (u) =>\n `https://www.dailymotion.com/services/oembed?url=${encodeURIComponent(u.href)}`,\n};\n\n/** Archive a video URL (YouTube, Vimeo, Dailymotion) via oEmbed + ID enrichment. */\nexport async function archiveVideo(url: string): Promise> {\n const record: Record = {\n kind: \"video\",\n originalUrl: url,\n archived_at: new Date().toISOString(),\n };\n\n let u: URL | null = null;\n try {\n u = new URL(url);\n } catch {\n return record;\n }\n record.host = hostOf(url);\n record.url = url;\n\n const host = record.host as string;\n let provider: string | null = null;\n if (/(^|\\.)(youtube|youtu|youtube-nocookie)\\./.test(host)) provider = \"youtube\";\n else if (/(^|\\.)vimeo\\./.test(host)) provider = \"vimeo\";\n else if (/(^|\\.)dailymotion\\./.test(host)) provider = \"dailymotion\";\n\n if (provider && OEMBED[provider]) {\n const d = await fetchOembed(OEMBED[provider](u));\n if (d) {\n record.title = d.title ?? null;\n record.author = d.author_name ?? null;\n record.author_url = d.author_url ?? null;\n record.thumbnail = d.thumbnail_url ?? null;\n record.provider_name = d.provider_name ?? null;\n record.embed_html = d.html ?? null;\n record.width = d.width ?? null;\n record.height = d.height ?? null;\n }\n }\n\n if (provider === \"youtube\") {\n const id = youtubeId(url);\n if (id) {\n record.video_id = id;\n record.thumbnail = record.thumbnail ??\n `https://img.youtube.com/vi/${id}/hqdefault.jpg`;\n record.embed_url = `https://www.youtube.com/embed/${id}`;\n }\n } else if (provider === \"vimeo\") {\n const m = url.match(/vimeo\\.com\\/(?:video\\/)?(\\d+)/);\n if (m) {\n record.video_id = m[1];\n record.embed_url = record.embed_url ?? `https://player.vimeo.com/video/${m[1]}`;\n }\n } else if (provider === \"dailymotion\") {\n const m = url.match(/dailymotion\\.com\\/video\\/([A-Za-z0-9]+)/);\n if (m) record.video_id = m[1];\n }\n\n return record;\n}\n",[],[153,202],["D",1788768264288],[],[153,65],"7428e5a7-dc59-4108-a7c6-baf0d088fd73","paper.ts","df07b88a-aa92-11f1-8552-1607ee4eb77e",19,["D",1788768324395],"import { hostOf, stripTags, USER_AGENT } from \"./util.ts\";\n\n/**\n * Archive a scientific paper / DOI URL. Routes by host:\n * doi.org -> Crossref works API\n * arxiv -> export.arxiv.org Atom API\n * pubmed -> NCBI eutils esummary\n * All are public and keyless.\n */\nexport async function archivePaper(url: string): Promise> {\n const record: Record = {\n kind: \"paper\",\n originalUrl: url,\n archived_at: new Date().toISOString(),\n };\n const host = hostOf(url);\n record.host = host;\n record.url = url;\n if (!host) return record;\n\n if (host === \"doi.org\") {\n const doi = pathOf(url);\n if (doi) {\n record.doi = doi;\n await enrichCrossref(record, doi);\n }\n return record;\n }\n if (host === \"arxiv.org\" || host.endsWith(\".arxiv.org\")) {\n return await enrichArxiv(record, url);\n }\n if (host === \"pubmed.ncbi.nlm.nih.gov\") {\n return await enrichPubmed(record, url);\n }\n return record;\n}\n\n// ---------------------------------------------------------------------------\n// Crossref (DOI registration agency — free REST API)\n// ---------------------------------------------------------------------------\nasync function enrichCrossref(\n record: Record,\n doi: string,\n): Promise {\n try {\n const r = await fetch(`https://api.crossref.org/works/${encodeURIComponent(doi)}`, {\n headers: { \"user-agent\": USER_AGENT, accept: \"application/json\" },\n });\n if (!r.ok) return;\n const res = (await r.json()) as Record;\n const m = (res.message ?? {}) as Record;\n record.title = firstArray(m.title);\n record.authors = (m.author as Array> | undefined)\n ?.map((a) => [a.given, a.family].filter(Boolean).join(\" \") || String(a.name ?? \"\"))\n .filter(Boolean);\n record.journal = firstArray(m[\"container-title\"]);\n record.published = pickDate(m.published) ?? pickDate(m[\"published-online\"]) ??\n pickDate(m[\"published-print\"]) ?? pickDate(m.issued);\n record.publisher = m.publisher ?? null;\n record.volume = m.volume ?? null;\n record.issue = m.issue ?? null;\n record.pages = m.page ?? null;\n record.type = m.type ?? null;\n record.abstract = typeof m.abstract === \"string\" ? stripTags(m.abstract) : null;\n record.subjects = Array.isArray(m.subject) ? m.subject : undefined;\n record.license = (m.license as Array> | undefined)?.[0]?.URL ?? null;\n record.url = (m.URL as string) ?? record.url;\n record.paper_urls = (m.link as Array> | undefined)\n ?.map((l) => l.URL)\n .filter((v): v is string => typeof v === \"string\");\n } catch {\n /* best-effort */\n }\n}\n\n// ---------------------------------------------------------------------------\n// arXiv (export API returns Atom XML)\n// ---------------------------------------------------------------------------\nasync function enrichArxiv(\n record: Record,\n url: string,\n): Promise> {\n const m = url.match(/arxiv\\.org\\/(?:abs|pdf)\\/([^?#]+)/) ||\n url.match(/arxiv:([\\d.]+)/);\n const id = m ? m[1].replace(/\\.pdf$/, \"\").trim() : null;\n if (!id) return record;\n record.arxiv_id = id;\n record.url = `https://arxiv.org/abs/${id}`;\n\n try {\n const r = await fetch(`https://export.arxiv.org/api/query?id_list=${id}`, {\n headers: { \"user-agent\": USER_AGENT, accept: \"application/atom+xml\" },\n });\n if (!r.ok) return record;\n const xml = await r.text();\n const entry = firstEntry(xml) ?? \"\";\n record.title = stripTags(field(entry, \"title\"));\n const abstract = stripTags(field(entry, \"summary\"));\n if (abstract) record.abstract = abstract;\n const published = field(entry, \"published\");\n if (published) record.published = published.slice(0, 10);\n record.authors = Array.from(entry.matchAll(/([\\s\\S]*?)<\\/name>/g))\n .map((g) => stripTags(g[1]))\n .filter(Boolean);\n const doi = field(entry, \"arxiv:doi\");\n if (doi) record.doi = doi;\n record.journal = field(entry, \"arxiv:journal_ref\") || record.journal || null;\n } catch {\n /* best-effort */\n }\n return record;\n}\n\n// ---------------------------------------------------------------------------\n// PubMed (NCBI eutils — free for moderate use)\n// ---------------------------------------------------------------------------\nasync function enrichPubmed(\n record: Record,\n url: string,\n): Promise> {\n const pmid = url.match(/pubmed\\.ncbi\\.nlm\\.nih\\.gov\\/(\\d+)/)?.[1];\n if (!pmid) return record;\n record.pubmed_id = pmid;\n try {\n const r = await fetch(\n `https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id=${pmid}&retmode=json`,\n { headers: { \"user-agent\": USER_AGENT, accept: \"application/json\" } },\n );\n if (!r.ok) return record;\n const d = (await r.json()) as Record;\n const it = (d.result as Record)?.[pmid] as Record | undefined;\n if (!it) return record;\n record.title = it.title ?? null;\n record.published = it.pubdate ?? null;\n record.journal = it.fulljournalname ?? it.source ?? null;\n record.authors = (it.authors as Array> | undefined)\n ?.map((a) => a.name ?? null);\n record.type = \"journal-article\";\n const ids = (it.articleids as Array> | undefined) ?? [];\n record.doi = ids.find((x) => x.idtype === \"doi\")?.value ?? null;\n record.pmc_id = ids.find((x) => x.idtype === \"pmc\")?.value ?? null;\n } catch {\n /* best-effort */\n }\n return record;\n}\n\n// ---------------------------------------------------------------------------\n// Helpers\n// ---------------------------------------------------------------------------\nfunction pathOf(url: string): string | null {\n try {\n return new URL(url).pathname.replace(/^\\/+/, \"\") || null;\n } catch {\n return null;\n }\n}\n\nfunction firstArray(v: unknown): string | null {\n return Array.isArray(v) && v.length ? String(v[0]) : null;\n}\n\nfunction pickDate(o: unknown): string | null {\n if (!o || typeof o !== \"object\") return null;\n const parts = (o as Record)[\"date-parts\"] as Array> | undefined;\n if (!Array.isArray(parts) || !parts[0] || !parts[0][0]) return null;\n const [y, mo, d] = parts[0];\n return [y, mo, d].filter((x) => x != null).join(\"-\");\n}\n\nfunction field(xml: string, tag: string): string {\n const m = xml.match(new RegExp(`<${tag}>([\\\\s\\\\S]*?)`));\n return m ? m[1] : \"\";\n}\n\nfunction firstEntry(xml: string): string | null {\n const m = xml.match(/([\\s\\S]*?)<\\/entry>/);\n return m ? m[1] : null;\n}\n",[],[153,213],"dfc6f840-fc1e-4eda-befe-594dc563fac0","music.ts","d0e8d11c-aa92-11f1-bb4e-1607ee4eb77e",15,["D",1788768297194],"import {\n fetchMeta,\n fetchOembed,\n hostOf,\n USER_AGENT,\n youtubeId,\n} from \"./util.ts\";\n\n// Streaming providers (all support public oEmbed, no auth).\nconst OEMBED: Record string> = {\n spotify: (u) =>\n `https://open.spotify.com/oembed?url=${encodeURIComponent(u.href)}&format=json`,\n soundcloud: (u) =>\n `https://soundcloud.com/oembed?url=${encodeURIComponent(u.href)}&format=json`,\n bandcamp: (u) => `https://bandcamp.com/oembed?url=${encodeURIComponent(u.href)}`,\n deezer: (u) =>\n `https://www.deezer.com/oembed?url=${encodeURIComponent(u.href)}&format=json`,\n};\n\n/**\n * Archive a music URL: streaming providers via oEmbed, music metadata\n * databases via their public APIs. Best-effort; never throws.\n */\nexport async function archiveMusic(url: string): Promise> {\n const record: Record = {\n kind: \"music\",\n originalUrl: url,\n archived_at: new Date().toISOString(),\n };\n\n const host = hostOf(url);\n let u: URL | null = null;\n try {\n u = new URL(url);\n } catch {\n return record;\n }\n record.host = host;\n record.url = url;\n\n // --- Metadata databases ---\n if (/(^|\\.)musicbrainz\\./.test(host)) return archiveMusicBrainz(record, u);\n if (/(^|\\.)listenbrainz\\./.test(host)) return archiveListenBrainz(record, u);\n if (host.endsWith(\".last.fm\")) return archiveLastFm(record, u);\n\n // --- Streaming providers ---\n let provider: string | null = null;\n if (/(^|\\.)spotify\\./.test(host)) provider = \"spotify\";\n else if (/(^|\\.)soundcloud\\./.test(host)) provider = \"soundcloud\";\n else if (/(^|\\.)bandcamp\\./.test(host)) provider = \"bandcamp\";\n else if (/(^|\\.)deezer\\./.test(host)) provider = \"deezer\";\n else if (host === \"music.youtube.com\") provider = \"youtube-music\";\n\n if (provider && OEMBED[provider]) {\n const d = await fetchOembed(OEMBED[provider](u));\n if (d) {\n record.title = d.title ?? null;\n record.author = d.author_name ?? null;\n record.author_url = d.author_url ?? null;\n record.thumbnail = d.thumbnail_url ?? null;\n record.provider_name = d.provider_name ?? null;\n record.embed_html = d.html ?? null;\n }\n }\n\n if (provider === \"spotify\") {\n const m = url.match(\n /spotify\\.com\\/(track|album|playlist|artist|episode|show)\\/([A-Za-z0-9]+)/,\n );\n if (m) {\n record.spotify_type = m[1];\n record.spotify_id = m[2];\n record.spotify_uri = `spotify:${m[1]}:${m[2]}`;\n record.embed_url = `https://open.spotify.com/embed/${m[1]}/${m[2]}`;\n }\n } else if (provider === \"youtube-music\") {\n const id = youtubeId(url);\n if (id) {\n record.video_id = id;\n record.thumbnail = record.thumbnail ??\n `https://img.youtube.com/vi/${id}/hqdefault.jpg`;\n record.embed_url = `https://music.youtube.com/watch?v=${id}`;\n }\n } else if (provider === \"deezer\") {\n const m = url.match(\n /deezer\\.com\\/(?:[a-z]{2}\\/)?(track|album|playlist|artist|show|episode)\\/(\\d+)/,\n );\n if (m) {\n record.deezer_type = m[1];\n record.deezer_id = m[2];\n }\n } else if (provider === \"soundcloud\") {\n const m = url.match(/soundcloud\\.com\\/([^/?#]+)\\/([^/?#]+)/);\n if (m) {\n record.soundcloud_user = m[1];\n record.soundcloud_track = m[2];\n }\n } else if (provider === \"bandcamp\") {\n const m = url.match(/bandcamp\\.com\\/track\\/([^/?#]+)\\/?$/);\n if (m) record.bandcamp_slug = m[1];\n }\n\n return record;\n}\n\n// ---------------------------------------------------------------------------\n// MusicBrainz (public ws/2 API, no key, requires a polite User-Agent)\n// ---------------------------------------------------------------------------\nconst MB_ENTITIES = new Set([\n \"artist\", \"release-group\", \"release\", \"recording\", \"work\",\n \"label\", \"event\", \"place\", \"series\", \"area\", \"url\",\n]);\n\nasync function archiveMusicBrainz(\n record: Record,\n u: URL,\n): Promise> {\n const m = u.pathname.match(\n /\\/(artist|release-group|release|recording|work|label|event|place|series|area|url)\\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/,\n );\n if (!m || !MB_ENTITIES.has(m[1])) {\n return fromPageMeta(record, u.href);\n }\n record.musicbrainz_entity = m[1];\n record.musicbrainz_id = m[2];\n record.url = `https://musicbrainz.org/${m[1]}/${m[2]}`;\n\n const inc = m[1] === \"release\"\n ? \"release-groups+recordings+artist-credits\"\n : m[1] === \"recording\" ? \"artist-credits+releases\" : undefined;\n const qs = inc ? `?inc=${inc}&fmt=json` : \"?fmt=json\";\n try {\n const r = await fetch(`https://musicbrainz.org/ws/2/${m[1]}/${m[2]}${qs}`, {\n headers: { \"user-agent\": USER_AGENT, accept: \"application/json\" },\n });\n if (!r.ok) return fromPageMeta(record, u.href);\n const d = (await r.json()) as Record;\n\n record.title = d.title ?? d.name ?? null;\n record.musicbrainz_type = d.type ?? null;\n if (m[1] === \"release-group\") record.musicbrainz_primary_type = d[\"primary-type\"] ?? null;\n record.disambiguation = d.disambiguation ?? null;\n\n if (d.artist) {\n const artist = d.artist as Record;\n record.author = artist.sort_name ?? artist.name ?? null;\n record.author_mbid = artist.id ?? null;\n }\n if (d[\"artist-credit\"]) {\n record.artist_credit = (d[\"artist-credit\"] as Array>)\n .map((a) => String(a.name ?? \"\")).join(\" / \");\n }\n if (d.country) record.country = d.country;\n if (d.area) record.area = (d.area as Record)?.name ?? null;\n if (d.date) record.release_date = d.date;\n if (d[\"first-release-date\"]) record.first_release_date = d[\"first-release-date\"];\n if (Array.isArray(d[\"release-events\"]) && d[\"release-events\"][0]) {\n const ev = d[\"release-events\"][0] as Record;\n if (ev.date) record.release_date = ev.date;\n }\n if (d[\"release-group\"]) {\n const rg = d[\"release-group\"] as Record;\n record.release_group = rg.title ?? null;\n record.musicbrainz_primary_type = rg[\"primary-type\"] ?? null;\n }\n // Rating/metadata stats where present\n if (d.rating) record.musicbrainz_rating = d.rating;\n if (d.score != null) record.musicbrainz_score = d.score;\n if (m[1] === \"release\" && d.media) {\n record.release_format = (d.media as Array>)[0]?.format ?? null;\n record.track_count = (d.media as Array>)[0]?.track_count ?? null;\n }\n if (m[1] === \"recording\" && typeof d.length === \"number\") {\n record.duration_ms = d.length;\n }\n } catch {\n return fromPageMeta(record, u.href);\n }\n return record;\n}\n\n// ---------------------------------------------------------------------------\n// ListenBrainz (public API; enrich via page meta / MBID when linkable)\n// ---------------------------------------------------------------------------\nasync function archiveListenBrainz(\n record: Record,\n u: URL,\n): Promise> {\n // ListenBrainz artist/release/track pages often embed a MusicBrainz ID.\n const mb = u.pathname.match(/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/);\n if (mb) record.musicbrainz_id = mb[1];\n return fromPageMeta(record, u.href);\n}\n\n// ---------------------------------------------------------------------------\n// Last.fm (page meta; optional API enrichment when LASTFM_API_KEY is set)\n// ---------------------------------------------------------------------------\nasync function archiveLastFm(\n record: Record,\n u: URL,\n): Promise> {\n const seg = u.pathname.split(\"/\").filter(Boolean);\n if (seg[0] === \"music\" && seg.length >= 2) {\n record.artist = decodeURIComponent(seg[1]);\n if (seg.length >= 4) {\n record.lastfm_type = \"track\";\n record.track = decodeURIComponent(seg[3]);\n } else if (seg.length === 3) {\n record.lastfm_type = \"album\";\n record.album = decodeURIComponent(seg[2]);\n } else {\n record.lastfm_type = \"artist\";\n }\n }\n\n const key = Deno.env.get(\"LASTFM_API_KEY\");\n if (key && record.artist) {\n const p = new URLSearchParams({ api_key: key, format: \"json\", artist: String(record.artist) });\n let method = \"artist.getInfo\";\n if (record.lastfm_type === \"album\") { method = \"album.getInfo\"; p.set(\"album\", String(record.album)); }\n if (record.lastfm_type === \"track\") { method = \"track.getInfo\"; p.set(\"track\", String(record.track)); }\n try {\n const r = await fetch(`https://ws.audioscrobbler.com/2.0/?method=${method}&${p.toString()}`, {\n headers: { \"user-agent\": USER_AGENT },\n });\n if (r.ok) {\n const d = await r.json();\n const info = d?.artist ?? d?.album ?? d?.track;\n if (info) {\n record.title = info.name ?? record.title ?? null;\n if (record.lastfm_type === \"artist\") record.summary = innerText(info.bio?.summary);\n if (record.lastfm_type === \"album\" && info.artist) record.author = info.artist ?? null;\n const img = (Array.isArray(info.image) ? info.image : []).find(\n (i) => (i as Record)?.[\"#text\"],\n );\n record.thumbnail = record.thumbnail ??\n ((img as Record)?.[\"#text\"] ?? null);\n }\n }\n } catch {\n /* best-effort */\n }\n }\n return record;\n}\n\n// ---------------------------------------------------------------------------\n// Fallback: pull title/description/image from the page's tags.\n// ---------------------------------------------------------------------------\nasync function fromPageMeta(\n record: Record,\n url: string,\n): Promise> {\n const meta = await fetchMeta(url);\n record.title ??= meta[\"og:title\"] ?? meta.title ?? null;\n record.description ??= meta[\"og:description\"] ?? meta.description ?? null;\n record.thumbnail ??= meta[\"og:image\"] ?? meta[\"twitter:image\"] ?? null;\n return record;\n}\n\nfunction innerText(v: unknown): string | null {\n if (typeof v !== \"string\") return null;\n return v.replace(/<[^>]+>/g, \" \").replace(/\\s+/g, \" \").trim() || null;\n}\n",[],[153,221],"fbfbca7a-7cad-40ed-ab8d-e3950fdb57b9","git.ts","c60442cc-aa92-11f1-8348-1607ee4eb77e",14,["D",1788768278908],"import { hostOf, USER_AGENT } from \"./util.ts\";\n\n/** Indefinite GitHub API JSON fetch; returns null on any failure. */\nasync function apiJson(url: string): Promise | null> {\n try {\n const r = await fetch(url, {\n headers: { \"user-agent\": USER_AGENT, accept: \"application/json\" },\n redirect: \"follow\",\n });\n if (!r.ok) return null;\n const d: unknown = await r.json();\n return (typeof d === \"object\" && d !== null)\n ? (d as Record)\n : null;\n } catch {\n return null;\n }\n}\n\n/**\n * Archive a source-control repository URL. Parses owner/repo out of the path\n * (works for repo, subdir, file and tag pages) and enriches via the provider's\n * public repository API where one exists.\n */\nexport async function archiveGit(url: string): Promise> {\n const record: Record = {\n kind: \"git\",\n originalUrl: url,\n archived_at: new Date().toISOString(),\n };\n\n const host = hostOf(url);\n let path = \"\";\n try {\n path = new URL(url).pathname.replace(/^\\/+/, \"\").replace(/\\.git$/, \"\");\n } catch {\n return record;\n }\n record.host = host;\n record.url = host && path ? `https://${host}/${path}` : url;\n\n const parts = path.split(\"/\").filter(Boolean);\n const owner = parts[0];\n const repo = parts[1];\n if (owner && repo) {\n record.owner = owner;\n record.repo = repo;\n record.repo_url = `https://${host}/${owner}/${repo}`;\n }\n if (!owner || !repo) return record;\n\n try {\n if (host === \"github.com\") {\n const d = await apiJson(`https://api.github.com/repos/${owner}/${repo}`);\n if (d) {\n record.default_branch = d.default_branch ?? null;\n record.description = d.description ?? null;\n record.language = d.language ?? null;\n record.stars = d.stargazers_count ?? 0;\n record.forks = d.forks_count ?? 0;\n record.watchers = d.subscribers_count ?? null;\n record.open_issues = d.open_issues_count ?? 0;\n record.license = licenseName(d.license as Record | null);\n record.homepage = d.homepage ?? null;\n record.parent = d.fork ? (d.parent as Record)?.full_name ?? null : null;\n record.created = d.created_at ?? null;\n record.pushed = d.pushed_at ?? null;\n record.size_kb = d.size ?? null;\n record.topics = Array.isArray(d.topics) ? d.topics : [];\n record.archived = !!d.archived;\n }\n } else if (host === \"gitlab.com\") {\n const d = await apiJson(\n `https://gitlab.com/api/v4/projects/${encodeURIComponent(`${owner}/${repo}`)}`,\n );\n if (d) {\n record.description = d.description ?? null;\n record.default_branch = d.default_branch ?? null;\n record.host_type = d.visibility ?? null;\n record.stars = d.star_count ?? 0;\n record.forks = d.forks_count ?? 0;\n record.open_issues = d.open_issues_count ?? null;\n record.homepage = d.web_url ?? null;\n record.created = d.created_at ?? null;\n record.last_activity = d.last_activity_at ?? null;\n record.license = licenseName(d.license as Record | null);\n record.topics = Array.isArray(d.topics) ? d.topics : [];\n }\n } else if (host === \"bitbucket.org\") {\n const d = await apiJson(\n `https://api.bitbucket.org/2.0/repositories/${owner}/${repo}`,\n );\n if (d) {\n record.description = d.description ?? null;\n record.language = d.language ?? null;\n record.default_branch = (d.mainbranch as Record)?.name ?? null;\n record.homepage = (d.links as Record)?.html\n ? ((d.links as Record).html as Record)?.href\n : null;\n record.created = d.created_on ?? null;\n record.updated = d.updated_on ?? null;\n record.parent = (d.parent as Record)?.full_name ?? null;\n }\n } else if (host === \"codeberg.org\" || /gitea/.test(host)) {\n // Codeberg and Gitea expose a Gitea-compatible API.\n const d = await apiJson(`https://${host}/api/v1/repos/${owner}/${repo}`);\n if (d) {\n record.description = d.description ?? null;\n record.language = d.language ?? null;\n record.default_branch = d.default_branch ?? null;\n record.stars = d.stars_count ?? 0;\n record.forks = d.forks_count ?? 0;\n record.open_issues = d.open_issues_count ?? null;\n record.license = licenseName(d.license as Record | null);\n record.homepage = d.website ?? null;\n record.created = d.created_at ?? null;\n record.updated = d.updated_at ?? null;\n record.private = !!d.private;\n record.fork = !!d.fork;\n }\n }\n } catch {\n /* metadata enrichment is best-effort */\n }\n return record;\n}\n\nfunction licenseName(license: Record | null | undefined): unknown {\n if (!license) return null;\n return license.spdx_id ?? license.name ?? null;\n}\n",[],[153,229],"31061663-3127-496a-9652-6c22a485375a","classify.ts","bfa724ee-aa92-11f1-907d-1607ee4eb77e",13,["D",1788768268241],"import { FILE_RE, hostOf } from \"./util.ts\";\n\nexport type UrlKind =\n | \"git\"\n | \"music\"\n | \"video\"\n | \"wikipedia\"\n | \"paper\"\n | \"file\"\n | \"article\"\n | \"unknown\";\n\n/**\n * Classify a URL into the archiver's handler kinds.\n * Order matters: YouTube Music must win over the generic YouTube video check.\n */\nexport function classify(url: string): UrlKind {\n const host = hostOf(url);\n let path = \"\";\n try {\n path = new URL(url).pathname;\n } catch {\n return \"unknown\";\n }\n if (!host) return \"unknown\";\n\n // Repositories\n if (/(^|\\.)(github|gitlab|bitbucket|codeberg|gitea)\\.(com|net|org|io|dev)$/.test(host) ||\n host === \"dev.azure.com\") {\n return \"git\";\n }\n\n // Music streaming + metadata databases\n if (host === \"music.youtube.com\" ||\n /(^|\\.)(spotify|deezer|soundcloud|bandcamp|musicbrainz|listenbrainz)\\./.test(host) ||\n host.endsWith(\".last.fm\")) {\n return \"music\";\n }\n\n // Video (Youtube, Vimeo, Dailymotion)\n if (/(^|\\.)(youtube|youtu|youtube-nocookie)\\./.test(host) ||\n /(^|\\.)vimeo\\./.test(host) ||\n /(^|\\.)dailymotion\\./.test(host)) {\n return \"video\";\n }\n\n // Wikipedia\n if (/(^|\\.)wikipedia\\.org$/.test(host) && path.startsWith(\"/wiki/\")) {\n return \"wikipedia\";\n }\n\n // Scientific papers / DOIs\n if (host === \"doi.org\" ||\n /(^|\\.)arxiv\\.org$/.test(host) ||\n /(^|\\.)pubmed\\.ncbi\\.nlm\\.nih\\.gov/.test(host)) {\n return \"paper\";\n }\n\n // Direct file/asset download\n if (FILE_RE.test(url.split(/[?#]/)[0])) return \"file\";\n\n return \"article\";\n}\n",[],[153,237],"750c7c9a-1fc4-4b17-a501-2260cf1e9def",["D",1788768264288],"directory",[166,167,168,169,170,171,172],[153],["D",1788536954683],{"_11":12,"_123":71},{"_11":12,"_118":119,"_70":71,"_30":69,"_120":121,"_122":24},{"_38":109,"_13":16,"_32":35,"_110":73,"_111":-5,"_112":-5,"_113":-5,"_114":-5,"_115":-5,"_116":14,"_117":12},"ctx",{"_53":-5,"_142":286,"_287":288,"_289":290,"_291":24,"_292":24,"_293":24,"_294":86,"_295":-5,"_296":-7,"_297":-7,"_298":26,"_299":24,"_300":24,"_301":-5,"_302":24,"_303":24,"_304":24,"_305":-5},"ENV",{"_257":258,"_259":260,"_261":16,"_262":263,"_264":265,"_266":267,"_268":269,"_270":271,"_272":273,"_274":275,"_276":277,"_278":279,"_280":281,"_282":283,"_284":285},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

History · util.ts