${esc(it.title)}

可调参数（跟在地址后面，第一个用 ? 之后用 &）：

` +\n `` +\n `` +\n `` +\n `` +\n `

参数	作用
`?days=N`	只保留最近 N 天（默认不限）
`?pages=N`	翻几页，每页 10 条（默认 1，最多 3）
`?sponsored=1`	保留付费软文（默认丢弃）
`?diag=0`	不要这条诊断

`;\n\nexport default async function (req: Request): Promise {\n const url = new URL(req.url);\n const q = url.searchParams;\n const path = url.pathname.replace(/^\\/|\\/$/g, \"\");\n const plain = (b: string, s = 200) =>\n new Response(b, {\n status: s,\n headers: { \"Content-Type\": \"text/plain; charset=utf-8\" },\n });\n\n const src = TOPICS[path];\n if (!src) {\n return plain(\n \"可用地址：\\n\" + Object.values(TOPICS).map((t) =>\n ` /${t.key}${\" \".repeat(11 - t.key.length)}${t.title}`\n ).join(\"\\n\") +\n \"\\n\\n参数：?days= ?pages= ?sponsored=1 ?diag=0\\n\",\n 404,\n );\n }\n\n const pages = Math.min(3, Math.max(1, Number(q.get(\"pages\")) || 1));\n const daysRaw = Number(q.get(\"days\"));\n const cutoff = daysRaw > 0\n ? new Date(Date.now() - Math.min(365, daysRaw) * 864e5)\n : null;\n const keepSpon = q.get(\"sponsored\") === \"1\";\n\n const all: Item[] = [];\n for (let p = 1; p <= pages; p++) {\n const r = await fetch(src.url + (p > 1 ? `?paged=${p}` : \"\"), {\n headers: { \"User-Agent\": UA },\n });\n if (!r.ok) {\n return plain(`${src.key} 第 ${p} 页取不到，HTTP ${r.status}`, 502);\n }\n const got = parse(await r.text());\n if (!got.length) break;\n all.push(...got);\n }\n if (!all.length) {\n return plain(`${src.key} 一条都没抓到。把这句话告诉 Claude。`, 502);\n }\n\n const seen = new Set();\n const kept: Item[] = [];\n const ads: Item[] = [];\n for (const it of all.sort((a, b) => +b.date - +a.date)) {\n if (cutoff && it.date < cutoff) continue;\n const k = key(it.link);\n if (seen.has(k)) continue;\n seen.add(k);\n if (it.sponsored && !keepSpon) {\n ads.push(it);\n continue;\n }\n kept.push(it);\n }\n\n const body = kept.map((it) => {\n const d = it.creator ? `${it.creator} —— ${it.desc}` : it.desc;\n return `${esc(it.title)}${esc(it.link)}` +\n `${esc(key(it.link))}` +\n `${esc(d)}` +\n `${it.date.toUTCString()}`;\n });\n\n if (q.get(\"diag\") !== \"0\") {\n const rows = ads.map((a) =>\n `${esc(a.date.toISOString().slice(0, 10))}${\n esc(a.title)\n }
${esc(a.creator)}`\n ).join(\"\");\n const html =\n `

抓到 ${all.length} 条 → 保留 ${kept.length} 条，丢弃软文 ${ads.length} 条。

` +\n (ads.length\n ? `${rows}

日期	丢弃的软文

`\n : \"

这一轮没有软文。

\") +\n PARAMS;\n const pin = new Date((kept[0]?.date.getTime() ?? Date.now()) + 60_000);\n body.unshift(\n `${\n esc(\n `软文过滤 · ${day()}（保留 ${kept.length} 条 / 丢弃 ${ads.length} 条）`,\n )\n }` +\n `${esc(url.origin + url.pathname)}` +\n `diag-${esc(src.key)}` +\n `${esc(html)}` +\n `${pin.toUTCString()}`,\n );\n }\n\n return new Response(\n `\n\n${esc(src.title)}\n${esc(B + src.key + \"/\")}\nMIT Technology Review ${\n esc(src.key)\n } 板块，已剔除带 sponsored 标记的付费软文。\nen-US\n${new Date().toUTCString()}\n${body.join(\"\")}\n`,\n {\n headers: {\n \"Content-Type\": \"application/rss+xml; charset=utf-8\",\n \"Cache-Control\": \"public, max-age=300\",\n },\n },\n );\n}",3,["D",1788576841172],"branchVersion","label",[18],"pulls","branches",1,"forks","envVars",{"_184":26,"_186":26},"connections",{"_184":26,"_185":26},"count","unfilledCount","emptyCount",{"_35":203,"_13":204,"_64":205,"_45":-5,"_37":38,"_173":132,"_190":109,"_39":206,"_41":199,"_67":21,"_169":207,"_69":24,"_174":-5,"_192":208,"_17":209},{"_35":195,"_13":196,"_64":107,"_45":-5,"_37":38,"_173":197,"_190":109,"_39":198,"_41":199,"_67":94,"_169":200,"_69":24,"_174":-5,"_192":201,"_17":202},{"_35":168,"_13":18,"_64":65,"_45":-5,"_37":38,"_173":89,"_190":109,"_39":191,"_41":171,"_67":94,"_169":170,"_69":24,"_174":-5,"_192":193,"_17":194},"latest",["D",1788576841172],"children",[],[18],"536c2dd7-6517-4be1-b7d1-34e54d8b21ce","main.ts",9,["D",1788575212783],4,"/**\n * MIT Technology Review · AI —— 三个 feed 合并去重\n *\n * 源（优先级从具体到宽泛，重复时保留更具体的那个标签）：\n * 算法 https://www.technologyreview.com/tag/the-algorithm/feed/\n * WDH https://www.technologyreview.com/author/will-douglas-heaven/feed/\n * AI https://www.technologyreview.com/topic/artificial-intelligence/feed/\n *\n * 去重只按文章链接（同一家媒体，同一篇就是同一个 URL），不做相似度匹配 ——\n * 相似度只会带来误杀，这里一点用都没有。\n *\n * 默认过滤掉付费软文：这三个 feed 里的广告稿都带 sponsored，\n * 2026-09-05 实测 AI 板块 40 条里有 9 条（22.5%）是软文。?sponsored=1 可以留下。\n *\n * 参数：\n * ?days=N 只保留最近 N 天（默认不限，三个 feed 一共就几十条）\n * ?pages=N 每个源翻几页，每页 10 条（默认 1，最多 3）\n * ?sponsored=1 保留付费软文（默认丢弃）\n * ?tag=0 标题前不加来源前缀\n * ?diag=0 不要「合并诊断」那条\n */\ntype Src = { tag: string; pri: number; url: string };\ntype Item = {\n tag: string;\n pri: number;\n title: string;\n link: string;\n date: Date;\n desc: string;\n creator: string;\n sponsored: boolean;\n};\ntype Dropped = { kept: Item; dropped: Item; why: string };\n\nconst SRC: Src[] = [\n {\n tag: \"算法\",\n pri: 0,\n url: \"https://www.technologyreview.com/tag/the-algorithm/feed/\",\n },\n {\n tag: \"WDH\",\n pri: 1,\n url: \"https://www.technologyreview.com/author/will-douglas-heaven/feed/\",\n },\n {\n tag: \"AI\",\n pri: 2,\n url: \"https://www.technologyreview.com/topic/artificial-intelligence/feed/\",\n },\n];\n\nconst UA =\n \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36\";\n\nconst unwrap = (s: string) => s.replace(//g, \"\").trim();\nconst strip = (s: string) =>\n unwrap(s).replace(/<[^>]*>/g, \" \").replace(/\\s+/g, \" \").trim();\nconst esc = (s: string) =>\n s.replace(/&/g, \"&\").replace(//g, \">\")\n .replace(/\"/g, \""\").replace(/'/g, \"'\");\n\n/** 去掉查询串和末尾斜杠，作为唯一键 */\nconst key = (u: string) =>\n u.replace(/[?#].*$/, \"\").replace(/\\/+$/, \"\").toLowerCase();\n\nconst one = (block: string, tag: string) =>\n unwrap(\n (block.match(new RegExp(`<${tag}>([\\\\s\\\\S]*?)`)) || [])[1] || \"\",\n );\n\nfunction parse(xml: string, src: Src): Item[] {\n return xml.split(\"\").slice(1).map((s) => s.split(\"\")[0]).map(\n (b) => {\n const cats = [...b.matchAll(/([\\s\\S]*?)<\\/category>/g)].map((\n m,\n ) => unwrap(m[1]));\n return {\n tag: src.tag,\n pri: src.pri,\n title: one(b, \"title\"),\n link: one(b, \"link\"),\n date: new Date(one(b, \"pubDate\")),\n desc: strip(\n (b.match(/([\\s\\S]*?)<\\/description>/) || [])[1] || \"\",\n ),\n creator: unwrap(\n (b.match(/([\\s\\S]*?)<\\/dc:creator>/) || [])[1] || \"\",\n ),\n sponsored: cats.some((c) => /^sponsored$/i.test(c)),\n };\n },\n ).filter((it) => it.title && it.link && !isNaN(+it.date));\n}\n\nasync function collect(pages: number) {\n const items: Item[] = [];\n const counts: Record = {};\n const errors: string[] = [];\n await Promise.all(SRC.map(async (src) => {\n let n = 0;\n for (let p = 1; p <= pages; p++) {\n const u = src.url + (p > 1 ? `?paged=${p}` : \"\");\n try {\n const r = await fetch(u, { headers: { \"User-Agent\": UA } });\n if (!r.ok) {\n errors.push(`${src.tag} 第 ${p} 页 HTTP ${r.status}`);\n break;\n }\n const got = parse(await r.text(), src);\n if (!got.length) break;\n items.push(...got);\n n += got.length;\n } catch (e) {\n errors.push(`${src.tag} 第 ${p} 页 ${String(e).slice(0, 80)}`);\n break;\n }\n }\n counts[src.tag] = n;\n }));\n return { items, counts, errors };\n}\n\nfunction reduce(items: Item[], keepSponsored: boolean, cutoff: Date | null) {\n const ordered = [...items].sort((a, b) => a.pri - b.pri || +b.date - +a.date);\n const kept: Item[] = [];\n const dropped: Dropped[] = [];\n const seen = new Map();\n for (const it of ordered) {\n if (cutoff && it.date < cutoff) continue;\n if (!keepSponsored && it.sponsored) {\n dropped.push({ kept: it, dropped: it, why: \"付费软文（sponsored）\" });\n continue;\n }\n const k = key(it.link);\n const twin = seen.get(k);\n if (twin) {\n dropped.push({\n kept: twin,\n dropped: it,\n why: `与「${twin.tag}」里同一篇重复`,\n });\n continue;\n }\n seen.set(k, it);\n kept.push(it);\n }\n kept.sort((a, b) => +b.date - +a.date);\n return { kept, dropped };\n}\n\nfunction diagEntry(\n counts: Record,\n kept: Item[],\n dropped: Dropped[],\n base: URL,\n) {\n const day = (() => {\n const iso = () => new Date().toISOString().slice(0, 10);\n try {\n const s = new Date().toLocaleDateString(\"en-CA\", {\n timeZone: \"America/New_York\",\n });\n return /^\\d{4}-\\d{2}-\\d{2}$/.test(s) ? s : iso();\n } catch {\n return iso();\n }\n })();\n const spon = dropped.filter((d) => d.why.startsWith(\"付费\"));\n const dup = dropped.filter((d) => !d.why.startsWith(\"付费\"));\n const rows = (list: Dropped[]) =>\n list.map((d) =>\n `${esc(d.dropped.tag)}丢弃 ${esc(d.dropped.title)}
${\n d.kept === d.dropped\n ? esc(d.why)\n : `因为已有 ${esc(d.kept.tag)} ${esc(d.kept.title)}`\n }`\n ).join(\"\");\n const body =\n `

抓到 ${\n SRC.map((s) => `${s.tag} ${counts[s.tag] ?? 0} 条`).join(\"、\")\n }` +\n ` → 保留 ${kept.length} 条，丢弃 ${dropped.length} 条` +\n `（软文 ${spon.length}、重复 ${dup.length}）。

` +\n (dropped.length\n ? `${\n rows(spon)\n }${rows(dup)}

来源	丢弃了什么 / 为什么

`\n : \"

这一轮没有丢弃任何条目。

\") +\n `

可调参数（跟在地址后面，第一个用 ? 之后用 &）：

参数	作用
`?days=N`	只保留最近 N 天（默认不限）
`?pages=N`	每个源翻几页，每页 10 条（默认 1，最多 3）
`?sponsored=1`	保留付费软文（默认丢弃）
`?tag=0`	标题前不加来源前缀
`?diag=0`	不要这条诊断

`;\n const pin = new Date((kept[0]?.date.getTime() ?? Date.now()) + 60_000);\n return {\n title:\n `合并诊断 · ${day}（保留 ${kept.length} 条 / 丢弃 ${dropped.length} 条）`,\n link: base.origin + base.pathname,\n guid: \"diag-mit-tr\",\n date: pin,\n html: body,\n };\n}\n\nfunction buildRss(\n kept: Item[],\n showTag: boolean,\n diag: ReturnType | null,\n) {\n const items = kept.map((it) => {\n const t = (showTag ? `${it.tag} · ` : \"\") + it.title;\n const d = it.creator ? `${it.creator} —— ${it.desc}` : it.desc;\n return `${esc(t)}${esc(it.link)}` +\n `${esc(key(it.link))}` +\n `${esc(d)}` +\n `${it.date.toUTCString()}`;\n });\n if (diag) {\n items.unshift(\n `${esc(diag.title)}${esc(diag.link)}` +\n `${esc(diag.guid)}` +\n `${esc(diag.html)}` +\n `${diag.date.toUTCString()}`,\n );\n }\n return `\n\nMIT Tech Review · AI（三源合并去重）\nhttps://www.technologyreview.com/topic/artificial-intelligence/\nMIT Technology Review 的 AI 板块、The Algorithm、Will Douglas Heaven 三个 feed 合并，同一篇只留一份，默认剔除付费软文。\nen-US\n${new Date().toUTCString()}\n${items.join(\"\")}\n`;\n}\n\nexport default async function (req: Request): Promise {\n const url = new URL(req.url);\n const q = url.searchParams;\n const pages = Math.min(3, Math.max(1, Number(q.get(\"pages\")) || 1));\n const daysRaw = Number(q.get(\"days\"));\n const cutoff = daysRaw > 0\n ? new Date(Date.now() - Math.min(365, daysRaw) * 864e5)\n : null;\n const keepSponsored = q.get(\"sponsored\") === \"1\";\n\n const { items, counts, errors } = await collect(pages);\n if (!items.length) {\n return new Response(\n \"三个 feed 一条都没抓到。把这句话告诉 Claude。\\n\" + errors.join(\"\\n\"),\n {\n status: 502,\n headers: { \"Content-Type\": \"text/plain; charset=utf-8\" },\n },\n );\n }\n const { kept, dropped } = reduce(items, keepSponsored, cutoff);\n\n const diag = q.get(\"diag\") === \"0\"\n ? null\n : diagEntry(counts, kept, dropped, url);\n return new Response(buildRss(kept, q.get(\"tag\") !== \"0\", diag), {\n headers: {\n \"Content-Type\": \"application/rss+xml; charset=utf-8\",\n \"Cache-Control\": \"public, max-age=300\",\n },\n });\n}",[],[196],"607b88cc-bcdf-460d-9774-17cb916c43fa","README.md","86dbb7a8-a8ce-11f1-82c6-1607ee4eb77e",["D",1788576997009],"# mit-tr-ai\n\nMIT Technology Review 的三个 AI 相关 feed 合并去重。 Base:\nhttps://anmeimei--6e26e1e8a8cd11f1b08b1607ee4eb77e.web.val.run/\n\n源（优先级从具体到宽泛，重复时保留更具体的那个标签）：\n\n| 标签 | feed |\n| ---- | -------------------------------------------------------------------- |\n| 算法 | https://www.technologyreview.com/tag/the-algorithm/feed/ |\n| WDH | https://www.technologyreview.com/author/will-douglas-heaven/feed/ |\n| AI | https://www.technologyreview.com/topic/artificial-intelligence/feed/ |\n\n三个 feed 都不需要登录，匿名请求拿到的内容完全一样。每页固定 10 条，`?paged=N`\n翻页。\n\n## 去重\n\n**只按文章链接（去掉查询串和末尾斜杠），不做相似度匹配。**\n同一家媒体、同一篇文章就是同一个 URL，相似度在这里只会带来误杀，没有任何收益。\n这一点和 us-topnews-merge 不同——那边跨 CNBC/CBS 两家，必须靠近似匹配。\n\n## 默认剔除付费软文\n\nMIT TR 的广告稿（\"MIT Technology Review Insights\" 及署名的合作稿）都带\n`sponsored`。2026-09-05 实测：AI 板块 40 条里 9\n条是软文（22.5%）。注意别用作者名判断——实测有 2 条 sponsored\n稿件署的是普通人名，不是 Insights。 `?sponsored=1` 可以把软文放回来。\n\n## 参数\n\n ?days=N 只保留最近 N 天（默认不限）\n ?pages=N 每个源翻几页，每页 10 条（默认 1，最多 3）\n ?sponsored=1 保留付费软文（默认丢弃）\n ?tag=0 标题前不加来源前缀\n ?diag=0 不要「合并诊断」那条\n\nfeed 第一条是「合并诊断」，guid 固定为 `diag-mit-tr`，永远只有一条，就地更新。\n日期按美东算。\n\n## 实测 2026-09-04（美东）\n\n| 参数 | 保留 | 丢弃 |\n| ------------ | ---- | ------------------- |\n| 默认 | 26 | 4（软文 2、重复 2） |\n| ?sponsored=1 | 28 | 2 |\n| ?pages=3 | 76 | 14 |\n| ?days=14 | 8 | 3 |\n\n默认约 130ms，?pages=3 约 1.1 秒。\n\n三个 feed 的时间跨度差很多：AI 板块 10 条只回溯约 11 天，The Algorithm 约 2.5\n个月， WDH 约 3.5\n个月。所以首次订阅会一次进来二十几条、最早到五月，之后就只有新的了。\n\n## topics.ts —— 商业 / 计算 / 太空（只去软广，不合并）\n\nhttps://anmeimei--a800ede4a8d411f1bbee1607ee4eb77e.web.val.run/business\nhttps://anmeimei--a800ede4a8d411f1bbee1607ee4eb77e.web.val.run/computing\nhttps://anmeimei--a800ede4a8d411f1bbee1607ee4eb77e.web.val.run/space\n\n每条路由对应 MIT TR 的一个 topic feed，只做一件事：剔除带 sponsored 的付费软文。\n参数 ?days=N、?pages=N（默认 1，最多 3）、?sponsored=1、?diag=0。 feed\n第一条是「软文过滤」诊断，列出这一轮丢了哪几条，guid 固定为 diag-<板块>。\n\n### 为什么不做跨板块去重\n\n2026-09-05 实测，四个板块各取 40 条（AI / 商业 / 计算 / 太空，共 160 条），\n两两之间的 URL 重叠：\n\n AI ∩ 商业 = 0 AI ∩ 计算 = 0 AI ∩ 太空 = 0\n 商业 ∩ 计算 = 0 商业 ∩ 太空 = 0 计算 ∩ 太空 = 0\n\n**一条都不重叠。** MIT TR 每篇文章只归一个 topic，跨板块重复根本不存在，\n所以去重是个不存在的问题，不用做。\n\n### 软文比例差别很大（2026-09-05，各 40 条）\n\n| 板块 | 软文 | 占比 |\n| ---- | ------- | --------- |\n| 计算 | 24 / 40 | **60.0%** |\n| AI | 9 / 40 | 22.5% |\n| 商业 | 3 / 40 | 7.5% |\n| 太空 | 0 / 40 | 0.0% |\n\n计算板块超过一半是广告稿。另外注意这几个 topic 的发稿量很低：过去 90\n天里非软文的条数，AI 31 条、太空 7 条、计算 2 条、商业 0 条（商业 feed\n最新一条停在 2026-06-30）。所以商业和计算这两个 feed 平时基本不动，\n偶尔来一条真报道——这对低密度阅读反而是好事，但别指望它们每天有东西。\n\n## 相关\n\n- CNBC + CBS 合并去重：val `anmeimei/us-topnews-merge`\n- 财新滚动要闻 / 晨读荐闻 / T早报：val `anmeimei/caixin-briefing-rss`",[],[204],["D",1788573569461],{"_11":12,"_146":72},{"_11":12,"_141":142,"_71":72,"_30":70,"_143":144,"_145":24},{"_41":132,"_13":16,"_35":38,"_133":96,"_134":-5,"_135":-5,"_136":-5,"_137":-5,"_138":-5,"_139":14,"_140":12},"ctx",{"_54":-5,"_165":247,"_248":249,"_250":251,"_252":24,"_253":24,"_254":24,"_255":109,"_256":-5,"_257":-7,"_258":-7,"_259":26,"_260":24,"_261":24,"_262":-5,"_263":24,"_264":24,"_265":24,"_266":-5},"ENV",{"_218":219,"_220":221,"_222":16,"_223":224,"_225":226,"_227":228,"_229":230,"_231":232,"_233":234,"_235":236,"_237":238,"_239":240,"_241":242,"_243":244,"_245":246},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

History · topics.ts