${esc(title)}

[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_153":154,"_155":156},"routes/_app.x.$handle.$name",{"_96":95,"_32":45,"_105":86,"_27":36,"_13":14,"_106":-5,"_11":107,"_108":109,"_110":111,"_112":113,"_114":115,"_20":-5,"_116":-5,"_117":-5,"_118":25,"_119":120,"_121":25,"_122":-5,"_123":-5,"_124":125},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_43":44,"_32":45,"_38":46,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":53,"_54":-5,"_18":55,"_56":31,"_57":58,"_59":-5,"_60":-5,"_61":62,"_63":64,"_65":-5,"_66":-5,"_67":-5,"_68":-5,"_69":70,"_71":70,"_72":73,"_74":73,"_75":73,"_76":70,"_77":70,"_78":79,"_80":81,"_82":83,"_84":25},"version",2,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_33":34,"_35":37,"_38":39,"_40":14,"_41":42,"_24":25},"previous",{"_30":31,"_32":28,"_33":34,"_35":36},"deleted",false,"deletedHere","path","main.ts","latestPath","fileId","1349d12a-a7df-11f1-abc7-1607ee4eb77e","name","fileType","script","content","","/**\n * CNBC Top News + CBS Top News → 合并去重后的单个 RSS\n *\n * 为什么要去重：两家都是 \"Top News\"，选题逻辑都是「今天最大的事」，\n * 所以大新闻必然重复。但 CBS 覆盖社会/民生（Lindsay Clancy、MapQuest），\n * CNBC 不覆盖 —— 合起来去重，正好互补。\n *\n * 去重是「近似匹配」，不是精确匹配：两家对同一件事的标题写法不同，\n * 只能靠词汇重合度判断。所以有阈值，可以调，见 ?sim=\n *\n * 参数：\n * ?sim=0.36 相似度阈值，越低去重越狠（默认 0.36）\n * ?tag=0 去掉标题前的 \"CNBC · \" / \"CBS · \" 前缀\n * ?debug=1 输出人类可读的诊断页：哪些条被判为重复、分数多少\n */\n\ntype Item = {\n tag: string;\n pri: number;\n title: string;\n link: string;\n guid: string;\n date: Date;\n desc: string;\n};\n\n// pri 小的优先保留。CNBC 在前 = 重复时保留 CNBC 的版本。\nconst FEEDS = [\n {\n tag: \"CNBC\",\n pri: 0,\n url: \"https://www.cnbc.com/id/100003114/device/rss/rss.html\",\n },\n { tag: \"CBS\", pri: 1, url: \"https://www.cbsnews.com/latest/rss/main\" },\n];\n\nconst DEFAULT_SIM = 0.36;\nconst WINDOW_HOURS = 36; // 只在这个时间窗内比较，避免跨天误判\n\nconst UA =\n \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 \" +\n \"(KHTML, like Gecko) Chrome/126.0 Safari/537.36\";\n\nasync function fetchText(url: string): Promise {\n const res = await fetch(url, { headers: { \"User-Agent\": UA } });\n if (!res.ok) throw new Error(`${res.status} ${url}`);\n const buf = await res.arrayBuffer();\n const ct = res.headers.get(\"content-type\") ?? \"\";\n const m = ct.match(/charset=([\\w-]+)/i);\n let enc = (m?.[1] ?? \"utf-8\").toLowerCase();\n if (enc === \"gb2312\" || enc === \"gbk\") enc = \"gb18030\";\n try {\n return new TextDecoder(enc).decode(buf);\n } catch {\n return new TextDecoder(\"utf-8\").decode(buf);\n }\n}\n\nconst decodeEntities = (s: string) =>\n s.replace(//g, \"$1\")\n .replace(/ /g, \" \").replace(/’|’/g, \"'\")\n .replace(/‘|‘/g, \"'\").replace(/“|“/g, '\"')\n .replace(/”|”/g, '\"').replace(/—|—/g, \"—\")\n .replace(/&/g, \"&\").replace(/</g, \"<\").replace(/>/g, \">\")\n .replace(/"/g, '\"').replace(/'|'/g, \"'\");\n\nconst tagText = (xml: string, name: string) => {\n const m = xml.match(\n new RegExp(`<${name}[^>]*>([\\\\s\\\\S]*?)<\\\\/${name}>`, \"i\"),\n );\n return m ? decodeEntities(m[1]).replace(/<[^>]*>/g, \"\").trim() : \"\";\n};\n\nfunction parseFeed(xml: string, tag: string, pri: number): Item[] {\n const out: Item[] = [];\n for (const m of xml.matchAll(/][\\s\\S]*?<\\/item>/gi)) {\n const raw = m[0];\n const title = tagText(raw, \"title\");\n if (!title) continue;\n const link = tagText(raw, \"link\");\n const guid = tagText(raw, \"guid\") || link;\n const pd = tagText(raw, \"pubDate\");\n const d = new Date(pd);\n out.push({\n tag,\n pri,\n title,\n link,\n guid,\n date: isNaN(d.getTime()) ? new Date() : d,\n desc: tagText(raw, \"description\").slice(0, 400),\n });\n }\n return out;\n}\n\n// ---- 近似去重 ----\n\nconst STOP = new Set(\n (\"a an the and or but of to in on at for from by with as is are was were be \" +\n \"been being it its this that these those he she they them his her their \" +\n \"will would can could may might says said say new after over into about \" +\n \"amid up down out off more most than then there here what which who whom \" +\n \"you your we our us i me my not no nor so if because while during\")\n .split(/\\s+/),\n);\n\nfunction tokens(title: string): string[] {\n return [\n ...new Set(\n title.toLowerCase()\n .replace(/[^a-z0-9$%\\s]/g, \" \")\n .split(/\\s+/)\n .filter(Boolean)\n .filter((w) => !STOP.has(w))\n .filter((w) => w.length >= 3 || /\\d/.test(w)),\n ),\n ];\n}\n\n/**\n * 相似度 = 加权 containment。\n *\n * 为什么要加权：两家写同一件事，共享的往往是专有名词（Clancy、MapQuest、\n * Hegseth），而 \"says / new / after\" 这种词到处都是。用当天这批标题算 IDF，\n * 罕见词权重高，常见词权重低。\n *\n * 用 containment 而不是 Jaccard：两家标题长短经常差很多\n * （\"Jury is deadlocked in Lindsay Clancy's trial\" vs 一长串），\n * Jaccard 会被长的那条稀释。\n *\n * 实测（用真实标题对做的小样本）：真重复能拿到 0.41~0.77，\n * 而不同的新闻最高只到 0.30。所以默认阈值取 0.36。\n * 但这套方法抓不到「换词换到没有共同词」的重复，比如\n * \"Fed holds rates steady\" vs \"Federal Reserve leaves rates unchanged\"（0.29）。\n * 这是标题级去重的天花板，不是 bug。想更狠就调低 ?sim=，代价是误杀。\n */\nfunction makeSim(all: string[][]) {\n const n = all.length;\n const df = new Map();\n for (const doc of all) for (const w of doc) df.set(w, (df.get(w) ?? 0) + 1);\n const wt = (w: string) => Math.log((n + 1) / ((df.get(w) ?? 0) + 0.5));\n\n return (a: string[], b: string[]) => {\n const B = new Set(b);\n let inter = 0, sa = 0, sb = 0;\n for (const w of a) {\n sa += wt(w);\n if (B.has(w)) inter += wt(w);\n }\n for (const w of b) sb += wt(w);\n if (sa <= 0 || sb <= 0) return 0;\n return inter / Math.min(sa, sb);\n };\n}\n\ntype Dropped = { kept: Item; dropped: Item; score: number };\n\nfunction dedupe(items: Item[], simThreshold: number) {\n // 优先级小的排前面 → 重复时保留它（CNBC 优先），同级按时间新的在前\n const ordered = [...items].sort(\n (x, y) => x.pri - y.pri || y.date.getTime() - x.date.getTime(),\n );\n const toks = ordered.map((it) => tokens(it.title));\n const sim = makeSim(toks); // IDF 用整批标题算，所以必须先收齐再比\n\n const kept: { it: Item; tk: string[] }[] = [];\n const dropped: Dropped[] = [];\n const seenGuid = new Set();\n\n ordered.forEach((it, i) => {\n if (seenGuid.has(it.guid)) return;\n const tk = toks[i];\n let dup: { it: Item; score: number } | null = null;\n for (const k of kept) {\n const dh = Math.abs(k.it.date.getTime() - it.date.getTime()) / 36e5;\n if (dh > WINDOW_HOURS) continue;\n const s = sim(tk, k.tk);\n if (s >= simThreshold && (!dup || s > dup.score)) {\n dup = { it: k.it, score: s };\n }\n }\n if (dup) {\n dropped.push({ kept: dup.it, dropped: it, score: dup.score });\n return;\n }\n seenGuid.add(it.guid);\n kept.push({ it, tk });\n });\n\n const merged = kept.map((k) => k.it)\n .sort((x, y) => y.date.getTime() - x.date.getTime());\n return { merged, dropped };\n}\n\nconst esc = (s: string) =>\n s.replace(/&/g, \"&\").replace(//g, \">\")\n .replace(/\"/g, \""\").replace(/'/g, \"'\");\n\nfunction buildRss(items: Item[], showTag: boolean) {\n const body = items.map((it) => {\n const title = showTag ? `${it.tag} · ${it.title}` : it.title;\n return `${esc(title)}` +\n `${esc(it.link)}` +\n `${esc(it.guid)}` +\n `${it.date.toUTCString()}` +\n `${esc(it.desc)}`;\n }).join(\"\");\n\n return `\n\nUS Top News（CNBC + CBS 去重）\nhttps://www.cbsnews.com/\nCNBC Top News 与 CBS Top News 合并，同一条新闻只保留一份（优先 CNBC 的版本）。\nen-US\n${new Date().toUTCString()}\n${body}\n`;\n}\n\nfunction debugPage(\n merged: Item[],\n dropped: Dropped[],\n counts: Record,\n sim: number,\n) {\n const rows = dropped\n .sort((a, b) => b.score - a.score)\n .map((d) =>\n `${d.score.toFixed(2)}` +\n `${esc(d.kept.tag)} ${esc(d.kept.title)}` +\n `${esc(d.dropped.tag)} ${esc(d.dropped.title)}`\n ).join(\"\");\n const byTag: Record = {};\n for (const m of merged) byTag[m.tag] = (byTag[m.tag] ?? 0) + 1;\n return `\n合并诊断\n\n

合并诊断

阈值 sim=${sim}　抓到：${\n Object.entries(counts).map(([k, v]) => `${k} ${v} 条`).join(\"、\")\n }　→ 去重后 ${merged.length} 条（${\n Object.entries(byTag).map(([k, v]) => `${k} ${v}`).join(\" / \")\n }），判为重复丢弃 ${dropped.length} 条。

阈值调低去重更狠（可能误杀），调高更保守（可能漏重复）。改 ?sim= 试。

\n${rows}

相似度	保留	丢弃

`;\n}\n\nexport default async function (req: Request): Promise {\n const url = new URL(req.url);\n const sim = Math.min(\n 0.95,\n Math.max(0.2, Number(url.searchParams.get(\"sim\")) || DEFAULT_SIM),\n );\n const showTag = url.searchParams.get(\"tag\") !== \"0\";\n\n const counts: Record = {};\n const all: Item[] = [];\n const errors: string[] = [];\n await Promise.all(FEEDS.map(async (f) => {\n try {\n const items = parseFeed(await fetchText(f.url), f.tag, f.pri);\n counts[f.tag] = items.length;\n all.push(...items);\n } catch (e) {\n counts[f.tag] = 0;\n errors.push(`${f.tag}: ${e}`);\n }\n }));\n\n if (all.length === 0) {\n return new Response(\n \"两个源都没抓到内容。把这句话告诉 Claude。\\n\" + errors.join(\"\\n\"),\n { status: 500, headers: { \"Content-Type\": \"text/plain; charset=utf-8\" } },\n );\n }\n\n const { merged, dropped } = dedupe(all, sim);\n\n if (url.searchParams.get(\"debug\") === \"1\") {\n return new Response(debugPage(merged, dropped, counts, sim), {\n headers: { \"Content-Type\": \"text/html; charset=utf-8\" },\n });\n }\n\n return new Response(buildRss(merged, showTag), {\n headers: {\n \"Content-Type\": \"application/rss+xml; charset=utf-8\",\n \"Cache-Control\": \"public, max-age=300\",\n },\n });\n}","createdAt",["D",1788471485082],"branchVersion","branchId","23607ccd-58ae-46e5-8871-5f1a571d8515","id","449fec92-6e09-4aec-b796-e8acf2bfb859","us-topnews-merge",["D",1788471196775],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","CNBC + CBS Top News merged and de-duplicated into one RSS feed","imageUrl","4d930760-aac6-4857-b21b-f01a00f81422","pinnedFileId","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","01a06281-3add-7588-9875-998aa2e1101c","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_96":95,"_104":100},"org",{"_96":95,"_97":98,"_99":100,"_18":55,"_101":102,"_103":25},"branch",{"_13":85,"_32":86,"_43":42,"_87":44,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":-5,"_93":45,"_94":95},"isOwned",29,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","anmeimei","handle","displayName","Meimei An","avatar","https://lh3.googleusercontent.com/a/ACg8ocIGi6q-4ldhq-uZuUQBjSymuORY8Y80AuFVk-RYPSZTveoWJw=s96-c","tier","free","canRestrictVals","profileImageUrl","branchName","adoptOrgId",{"_43":44,"_32":45,"_38":149,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":53,"_54":-5,"_18":55,"_56":31,"_57":58,"_59":-5,"_60":-5,"_61":62,"_63":64,"_65":-5,"_66":-5,"_67":-5,"_68":-5,"_69":70,"_71":70,"_72":73,"_74":73,"_75":73,"_76":70,"_77":70,"_78":150,"_80":151,"_82":152,"_84":25},"tree",[144],"files",[144],"counts",{"_134":62,"_135":131,"_136":62,"_137":138,"_139":140},"contents",[126],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_43":127,"_32":28,"_30":31,"_128":-5,"_41":42,"_40":14,"_129":25,"_38":130,"_13":131,"_33":34,"_35":37,"_24":25,"_132":-5,"_27":133},"4f0df111-2a88-45a2-9ec2-127781450c78","parentId","latest",["D",1788471485082],1,"label",[28],"pulls","branches","forks","envVars",{"_141":62,"_143":62},"connections",{"_141":62,"_142":62},"count","unfilledCount","emptyCount",{"_43":127,"_32":28,"_30":31,"_128":-5,"_41":42,"_40":14,"_129":25,"_38":145,"_13":131,"_33":34,"_35":37,"_24":25,"_132":-5,"_146":147,"_27":148},["D",1788471485082],"children",[],[28],["D",1788471196775],{"_96":95,"_104":100},{"_96":95,"_97":98,"_99":100,"_18":55,"_101":102,"_103":25},{"_13":85,"_32":86,"_43":42,"_87":44,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":-5,"_93":45,"_94":95},"ctx",{"_78":-5,"_123":186,"_187":188,"_189":190,"_191":25,"_192":25,"_193":25,"_194":58,"_195":-5,"_196":-7,"_197":-7,"_198":62,"_199":25,"_200":25,"_201":-5,"_202":25,"_203":25,"_204":25,"_205":-5},"ENV",{"_157":158,"_159":160,"_161":86,"_162":163,"_164":165,"_166":167,"_168":169,"_170":171,"_172":173,"_174":175,"_176":177,"_178":179,"_180":181,"_182":183,"_184":185},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","518c05f2c13b243481ef29134a5672756425e459","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v2