[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8},"actionData","errors","root",{"_106":107,"_108":109},"routes/embed.x.$handle.$name.$",{"_9":10,"_11":12,"_13":14,"_15":16,"_17":18,"_19":20,"_21":-5},"handle","ClaudeAssistant","name","palantir-herald","path","news-fetch.ts","project",{"_22":45,"_11":12,"_33":46,"_47":-5,"_48":42,"_49":50,"_51":50,"_52":53,"_54":-5,"_55":56,"_57":-5,"_58":42,"_59":-5,"_60":-5,"_61":62,"_63":64,"_65":66,"_67":68,"_69":70,"_71":-5,"_72":73,"_74":73,"_75":76,"_77":76,"_78":76,"_79":73,"_80":73,"_81":82,"_83":84,"_85":86,"_87":42},"file",{"_22":23,"_11":14,"_24":25,"_26":-5,"_27":28,"_29":30,"_31":32,"_33":34,"_35":36,"_37":38,"_39":40,"_41":42,"_43":-5,"_13":44},"permalink","https://www.val.town/x/ClaudeAssistant/palantir-herald/code/news-fetch.ts","extension","id","890002d1-9851-48b4-96db-115344042c2e","fileId","99de6ef4-ae41-11f1-ab49-1607ee4eb77e","parentId","branchId","cc31caf5-d2e1-4af9-b0dd-bd82fa3e53fe","branchVersion",99,"latest",true,"createdAt",["D",1789173917809],"version",2,"fileType","interval","content","/**\n * NEWS FETCH — the val-side news fetcher, piece 2 of the cold path (task #220, 12 Sep 2026).\n *\n * Nina the News Editor runs on Alexandr's PC at 06:45 and curates what scripts/news_fetch.py fetched.\n * When the PC is off nothing is fetched, and the morning has no edition at all. Fetching is plain HTTP —\n * RSS feeds and public Telegram channel previews — and belongs in the val, which is always on; curating\n * is judgement and stays on the desktop. This file fetches on an hourly interval, stores what it finds,\n * and serves it back as rows the desktop can read exactly as if news_fetch.py had fetched them itself.\n * A morning with the PC off then costs a LATE edition instead of a MISSING one.\n *\n * It is a port of news_fetch.py's get / clean / parse_rss / parse_telegram, behaviour for behaviour,\n * because the two must produce the SAME item from the same feed: the desktop dedupes on\n * url + \"|\" + title[:80] (news_seen.json), and an item whose title differs by one character on the val\n * publishes twice. The gotchas that follow from that:\n * - Python slices strings by code point; JS .slice() by UTF-16 code unit. An emoji is one code point\n * and two code units, and 42 of the 59 sources are Telegram channels whose posts open with 🇷🇺 ⚡️ 🔥.\n * Every truncation here goes through cut(), which slices by code point, and the 40-character floor\n * on a Telegram post counts code points too (Python's len()).\n * - Python reads 200000 BYTES and then decodes; the cap here is applied to the byte stream before\n * TextDecoder, never to the decoded string.\n * - Python's html.unescape knows 2231 named entities; Deno has none. unescape() below handles every\n * numeric form (with the HTML5 remap of €–Ÿ to their cp1252 meanings) and a table of the\n * names that occur in feeds. The residual gap is a rare named entity left as written, in the title\n * and therefore in the key — see the comment on unescape().\n *\n * What it must never do: call a model; send a Telegram message; filter by age (MAX_AGE_DAYS stays in\n * news_fetch.py so both modes run ONE implementation — the `iso` column is advisory and never served);\n * touch feedback, preferences, selection or scoring; or consume anything: a pull is a pure window read,\n * nothing is marked, moved or deleted, and news_seen.json on the desktop stays the single authority on\n * what has been shown.\n *\n * The flag is settings key `flag.news_fetch_val`, \"on\" the only truthy value. It governs the DESKTOP\n * (whether news_fetch.py pulls from here instead of fetching itself); this file only reads and reports\n * it, never writes it, and fetches and stores identically either way — the off-period data is the\n * evidence for switching it on, exactly as reconciler.ts runs its full assessment while off.\n *\n * Schedule: `20 * * * *` UTC, set by hand in the Val Town UI (the API cannot read or set it). Every\n * wake does work (unlike the reconciler, no hour is skipped): a SLICE of 16 sources, least recently\n * tried first, so 59 sources rotate in under four hours; and at 05 and 06 Europe/Amsterdam a FULL\n * sweep, so fresh items are in hand just before Nina's 06:45 — 06:00 local is 04:00 UTC in summer and\n * 05:00 UTC in winter, which is why the gate is local. Two full hours so a deadline-truncated 05:20\n * sweep gets its second chance at 06:20. The file stays correct with the cron unset — deployed but\n * never firing — because the routes need no pass to have run and POST /news/fetch/run proves the code path.\n *\n * Idempotency (Law 6): one INSERT OR IGNORE on news_fetch_log's primary key, claimed before any HTTP —\n * pass:: for the interval, run: for a hand run (so it can never take the\n * hour's key from the interval), day: for the day's journal line. Items are INSERT OR IGNORE\n * on their key. A pass that throws once claimed writes a DIED receipt naming the step it died at, so it\n * is never indistinguishable from a pass that had nothing to say.\n *\n * Journal (Law 7): ONE line per local day, from the hour-6 pass — 24 lines a day would be noise; the\n * per-pass record is news_fetch_log. Source `news-fetch-val`: reconciler.ts's silence matcher looks for\n * 'nina-news-editor%' and 'news-scribe%', and this matches neither, so these lines can never be read as\n * Nina having run.\n *\n * Deployed twice from one file, like reconciler.ts: as its own interval val (the default export; its\n * import graph is this file and std/sqlite, nothing else), and imported statically by the HTTP val for\n * the routes: mountNewsFetch(app, sqlite, PULL_SECRET). Every route is ?secret=, none is flag-gated\n * (all must be exercisable while off). Stamps in responses are SQLite's 'YYYY-MM-DD HH:MM:SS' UTC\n * unless the key says otherwise (`now` is ISO).\n * GET /news/fetch/status → flag, sources, last pass, what is held, failing sources\n * GET /news/fetch/items?hours=120&limit=500&before=&debug=0\n * → rows of EXACTLY {source, topic, title, url, summary, when}, newest first: a drop-in for\n * news_raw.json[\"items\"] before the desktop's iso_of() runs. debug=1 adds _key and _first_seen.\n * POST /news/fetch/sources {sources:[…news_sources.json entries…], stamp?}\n * → upsert by name; a name absent from the push is retired (present=0, enabled=0), never deleted;\n * fewer than 5 entries is refused (400) so a truncated push can never blind the cold path.\n * POST /news/fetch/run {dry?, scope?:\"slice\"|\"full\", only?:[names], at?} → one pass, the interval's code path\n * POST /news/fetch/prune {before:\"YYYY-MM-DD\"} | {days:N}, force? → deletes held items, journals it\n * The sources are pushed by the desktop, never built in: no push means a pass tries nothing and says so.\n */\nimport { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\n\nconst TZ = \"Europe/Amsterdam\";\nconst SOURCE = \"news-fetch-val\";\nconst FLAG_KEY = \"flag.news_fetch_val\";\nexport const CRON = \"20 * * * *\"; // UTC, hourly at :20 — set by hand in the Val Town UI; reported by /news/fetch/status\n\nexport const SLICE_N = 16; // sources per hourly slice: 59 rotate in under four hours\nexport const CONCURRENCY = 6; // sources in flight at once\nexport const SOURCE_TIMEOUT_MS = 12_000; // one source's whole GET, connect to last byte\nexport const DEADLINE_MS = 45_000; // one pass's FETCH budget: a batch that cannot fetch inside it is not started.\n // The per-source store, the log row and hour 6's day line ride on top of it.\nexport const BODY_LIMIT = 200_000; // bytes read per source, before decoding — news_fetch.py's r.read(200000)\n// The desktop pulls GET /news/fetch/items?hours=120 (scripts/news_fetch.py) — five days. RETENTION_DAYS must\n// stay >= that window, or a PC that was off for a week comes back to find its window already pruned.\nexport const RETENTION_DAYS = 14;\nconst DESKTOP_WINDOW_H = 120;\nconst LOG_RETENTION_DAYS = 90; // news_fetch_log rows: ~24 a day, kept a quarter\n\n// news_fetch.py:12 — the same UA string, so a feed that answers Chrome answers both fetchers the same way.\nconst UA = { \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120 Safari/537.36\" };\n\ntype Scope = \"slice\" | \"full\";\ntype Item = { source: string; topic: string; title: string; url: string; summary: string; when: string };\ntype Src = { name: string; type: string; url: string; handle: string; topic: string; lim: number };\ntype SourceResult = { name: string; type: string; n: number; stored: number; ms: number; error: string };\n\nexport async function init(db: any = sqlite) {\n // An interval val may run before bot-webhook.ts has created anything. Byte-identical to reconciler.ts's\n // (and bot-webhook's) declaration; whichever runs first wins. The journal table is bot-webhook's and is\n // assumed to exist, as reconciler.ts assumes it.\n await db.execute(`CREATE TABLE IF NOT EXISTS settings (\n key TEXT PRIMARY KEY, value TEXT)`);\n // What was fetched. key is the desktop's dedupe key, url + \"|\" + cut(title, 80) — private to the val and\n // never served, so the desktop keeps computing its own from the six fields it is given. `when` is a\n // SQLite keyword, hence when_text. iso is a date parse of when_text, advisory and never served:\n // staleness is the desktop's call (MAX_AGE_DAYS in news_fetch.py), and there must be one implementation of it.\n await db.execute(`CREATE TABLE IF NOT EXISTS news_fetched (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n key TEXT UNIQUE NOT NULL,\n source TEXT NOT NULL, topic TEXT NOT NULL DEFAULT '',\n title TEXT NOT NULL, url TEXT NOT NULL DEFAULT '', summary TEXT NOT NULL DEFAULT '',\n when_text TEXT NOT NULL DEFAULT '',\n iso TEXT NOT NULL DEFAULT '',\n first_seen TEXT NOT NULL DEFAULT (datetime('now')),\n pass_key TEXT)`);\n await db.execute(`CREATE INDEX IF NOT EXISTS news_fetched_seen ON news_fetched(first_seen, id)`);\n // The roster as the desktop last pushed it (scripts/news_sources.json entries, verbatim). `limit` is a\n // SQL keyword, hence lim. present=0 is a name the last push no longer carried: retired, never deleted,\n // so its history and last_error survive a mistaken push. last_try is stamped on an actual attempt only —\n // the queue is ORDER BY last_try ASC NULLS FIRST, so whatever a truncated pass did not reach is tried first next hour.\n await db.execute(`CREATE TABLE IF NOT EXISTS news_fetch_sources (\n name TEXT PRIMARY KEY, type TEXT NOT NULL, url TEXT NOT NULL DEFAULT '',\n handle TEXT NOT NULL DEFAULT '', topic TEXT NOT NULL DEFAULT '',\n lim INTEGER NOT NULL DEFAULT 8,\n enabled INTEGER NOT NULL DEFAULT 1, present INTEGER NOT NULL DEFAULT 1,\n pushed_at TEXT, last_try TEXT, last_ok TEXT,\n last_error TEXT NOT NULL DEFAULT '', last_n INTEGER NOT NULL DEFAULT 0,\n stored_total INTEGER NOT NULL DEFAULT 0)`);\n // The guard and the per-pass record in one row. key: 'pass::' (the interval),\n // 'run:' (a hand run), 'day:' (the day's journal line); 'test:…' is reserved\n // for a self-test and excluded from every summary. The claim is INSERT OR IGNORE on this key and nothing else.\n await db.execute(`CREATE TABLE IF NOT EXISTS news_fetch_log (\n key TEXT PRIMARY KEY,\n ts TEXT NOT NULL DEFAULT (datetime('now')), kind TEXT NOT NULL,\n local TEXT, scope TEXT,\n tried INTEGER DEFAULT 0, ok_n INTEGER DEFAULT 0, fetched INTEGER DEFAULT 0,\n stored INTEGER DEFAULT 0, errors INTEGER DEFAULT 0, ms INTEGER DEFAULT 0,\n truncated INTEGER DEFAULT 0, detail TEXT DEFAULT '')`);\n}\n\n// ---------- time ----------\n// localParts and sqlStamp are copied from reconciler.ts, not imported: an interval val must not load that\n// file's whole module graph on every wake (reconciler.ts:66 sets the precedent). Change it there, change it here.\nconst pad = (n: number) => String(n).padStart(2, \"0\");\n\n// Local wall clock in TZ. `wd` is the weekday as Sun=0: the snapshot's schedule.days are cron numbering\n// (Sun=0) and so is JS getDay(), so the index comes from the civil date and never from Intl's weekday name.\nexport function localParts(d: Date) {\n const f = new Intl.DateTimeFormat(\"en-GB\", { timeZone: TZ, hour12: false,\n year: \"numeric\", month: \"2-digit\", day: \"2-digit\", hour: \"2-digit\", minute: \"2-digit\", weekday: \"short\" });\n const p: any = {};\n for (const x of f.formatToParts(d)) p[x.type] = x.value;\n const y = Number(p.year), m = Number(p.month), day = Number(p.day);\n return { y, m, d: day, date: `${p.year}-${p.month}-${p.day}`, hour: Number(p.hour), minute: Number(p.minute),\n weekday: String(p.weekday), wd: new Date(Date.UTC(y, m - 1, day)).getUTCDay(),\n hhmm: `${p.hour}:${p.minute}` };\n}\n\n// journal.ts is SQLite's datetime('now'): 'YYYY-MM-DD HH:MM:SS', UTC. It is compared as text, so the\n// shape must match exactly — a 'T' in the wrong place is a silent ordering bug.\nexport function sqlStamp(ms: number): string {\n return new Date(ms).toISOString().slice(0, 19).replace(\"T\", \" \");\n}\n\nconst localText = (lp: { weekday: string; date: string; hhmm: string }) => `${lp.weekday} ${lp.date} ${lp.hhmm}`;\n\n// Hours from a SQL stamp to `now`, one decimal; null when there is no stamp or it is not one.\nfunction ageH(stamp: any, now: Date): number | null {\n const s = String(stamp ?? \"\");\n if (!/^\\d{4}-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d$/.test(s)) return null;\n return Math.round((now.getTime() - Date.parse(s.replace(\" \", \"T\") + \"Z\")) / 360000) / 10;\n}\n\n// ---------- strings: the code-point rule ----------\n// Python s[:n] slices by code point; JS .slice(0, n) by UTF-16 code unit. A title that opens with an emoji\n// would be cut one character short of the desktop's, get a different key, and publish twice. Every\n// truncation in this file goes through cut(), and every length that Python takes with len() through cpLen().\nexport const cut = (s: string, n: number) => [...s].slice(0, n).join(\"\");\nconst cpLen = (s: string) => [...s].length;\n\n// ---------- html.unescape ----------\n// Deno has no html.unescape. Numeric references are handled in full: decimal and hex, the semicolon\n// optional as in Python, €–Ÿ remapped to their cp1252 meanings (HTML5's rule, Python's\n// _invalid_charrefs — sloppy feeds write – for an en dash), surrogates and > U+10FFFF → U+FFFD, and\n// the control ranges HTML5 drops. Names: the table below (~120 of the 2231 Python knows) with the\n// semicolon required, plus the handful browsers accept bare (& < > " © ®).\n// THE RESIDUAL GAP, in two parts. A named entity outside the table (Ž ∴ …) stays as\n// written, in the title and therefore in the key — that item's key would differ from the desktop's.\n// And Python's html.unescape resolves ~106 names WITHOUT the semicolon by longest-prefix match, where\n// this resolves 7; a bare é would decode there and not here. Neither occurs in the 59 sources,\n// and a bare entity is invalid XML besides. If a feed starts writing one, add the name here.\nconst CP1252: Record = {\n 0x00: \"\\uFFFD\", 0x0D: \"\\r\",\n 0x80: \"\\u20AC\", 0x81: \"\\x81\", 0x82: \"\\u201A\", 0x83: \"\\u0192\", 0x84: \"\\u201E\", 0x85: \"\\u2026\", 0x86: \"\\u2020\", 0x87: \"\\u2021\",\n 0x88: \"\\u02C6\", 0x89: \"\\u2030\", 0x8A: \"\\u0160\", 0x8B: \"\\u2039\", 0x8C: \"\\u0152\", 0x8D: \"\\x8D\", 0x8E: \"\\u017D\", 0x8F: \"\\x8F\",\n 0x90: \"\\x90\", 0x91: \"\\u2018\", 0x92: \"\\u2019\", 0x93: \"\\u201C\", 0x94: \"\\u201D\", 0x95: \"\\u2022\", 0x96: \"\\u2013\", 0x97: \"\\u2014\",\n 0x98: \"\\u02DC\", 0x99: \"\\u2122\", 0x9A: \"\\u0161\", 0x9B: \"\\u203A\", 0x9C: \"\\u0153\", 0x9D: \"\\x9D\", 0x9E: \"\\u017E\", 0x9F: \"\\u0178\",\n};\nconst NAMED: Record = {\n amp: \"&\", lt: \"<\", gt: \">\", quot: \"\\\"\", apos: \"'\", nbsp: \"\\u00A0\",\n ndash: \"\\u2013\", mdash: \"\\u2014\", hellip: \"\\u2026\", lsquo: \"\\u2018\", rsquo: \"\\u2019\", sbquo: \"\\u201A\",\n ldquo: \"\\u201C\", rdquo: \"\\u201D\", bdquo: \"\\u201E\", laquo: \"\\u00AB\", raquo: \"\\u00BB\", lsaquo: \"\\u2039\", rsaquo: \"\\u203A\",\n bull: \"\\u2022\", middot: \"\\u00B7\", times: \"\\u00D7\", divide: \"\\u00F7\", minus: \"\\u2212\", plusmn: \"\\u00B1\", deg: \"\\u00B0\",\n euro: \"\\u20AC\", pound: \"\\u00A3\", yen: \"\\u00A5\", cent: \"\\u00A2\", curren: \"\\u00A4\", copy: \"\\u00A9\", reg: \"\\u00AE\", trade: \"\\u2122\",\n sect: \"\\u00A7\", para: \"\\u00B6\", dagger: \"\\u2020\", Dagger: \"\\u2021\", permil: \"\\u2030\", prime: \"\\u2032\", Prime: \"\\u2033\",\n frac12: \"\\u00BD\", frac14: \"\\u00BC\", frac34: \"\\u00BE\", sup1: \"\\u00B9\", sup2: \"\\u00B2\", sup3: \"\\u00B3\", micro: \"\\u00B5\",\n iexcl: \"\\u00A1\", iquest: \"\\u00BF\", ordf: \"\\u00AA\", ordm: \"\\u00BA\", brvbar: \"\\u00A6\", not: \"\\u00AC\", shy: \"\\u00AD\",\n macr: \"\\u00AF\", acute: \"\\u00B4\", cedil: \"\\u00B8\", uml: \"\\u00A8\",\n ensp: \"\\u2002\", emsp: \"\\u2003\", thinsp: \"\\u2009\", zwnj: \"\\u200C\", zwj: \"\\u200D\", lrm: \"\\u200E\", rlm: \"\\u200F\",\n larr: \"\\u2190\", uarr: \"\\u2191\", rarr: \"\\u2192\", darr: \"\\u2193\", harr: \"\\u2194\", infin: \"\\u221E\", ne: \"\\u2260\", le: \"\\u2264\", ge: \"\\u2265\",\n hearts: \"\\u2665\", spades: \"\\u2660\", clubs: \"\\u2663\", diams: \"\\u2666\",\n agrave: \"\\u00E0\", aacute: \"\\u00E1\", acirc: \"\\u00E2\", atilde: \"\\u00E3\", auml: \"\\u00E4\", aring: \"\\u00E5\", aelig: \"\\u00E6\", ccedil: \"\\u00E7\",\n egrave: \"\\u00E8\", eacute: \"\\u00E9\", ecirc: \"\\u00EA\", euml: \"\\u00EB\", igrave: \"\\u00EC\", iacute: \"\\u00ED\", icirc: \"\\u00EE\", iuml: \"\\u00EF\",\n ntilde: \"\\u00F1\", ograve: \"\\u00F2\", oacute: \"\\u00F3\", ocirc: \"\\u00F4\", otilde: \"\\u00F5\", ouml: \"\\u00F6\", oslash: \"\\u00F8\",\n ugrave: \"\\u00F9\", uacute: \"\\u00FA\", ucirc: \"\\u00FB\", uuml: \"\\u00FC\", yacute: \"\\u00FD\", yuml: \"\\u00FF\", szlig: \"\\u00DF\", eth: \"\\u00F0\", thorn: \"\\u00FE\",\n Agrave: \"\\u00C0\", Aacute: \"\\u00C1\", Acirc: \"\\u00C2\", Atilde: \"\\u00C3\", Auml: \"\\u00C4\", Aring: \"\\u00C5\", AElig: \"\\u00C6\", Ccedil: \"\\u00C7\",\n Egrave: \"\\u00C8\", Eacute: \"\\u00C9\", Ecirc: \"\\u00CA\", Euml: \"\\u00CB\", Igrave: \"\\u00CC\", Iacute: \"\\u00CD\", Icirc: \"\\u00CE\", Iuml: \"\\u00CF\",\n Ntilde: \"\\u00D1\", Ograve: \"\\u00D2\", Oacute: \"\\u00D3\", Ocirc: \"\\u00D4\", Otilde: \"\\u00D5\", Ouml: \"\\u00D6\", Oslash: \"\\u00D8\",\n Ugrave: \"\\u00D9\", Uacute: \"\\u00DA\", Ucirc: \"\\u00DB\", Uuml: \"\\u00DC\", Yacute: \"\\u00DD\", ETH: \"\\u00D0\", THORN: \"\\u00DE\",\n OElig: \"\\u0152\", oelig: \"\\u0153\", Scaron: \"\\u0160\", scaron: \"\\u0161\", Yuml: \"\\u0178\", fnof: \"\\u0192\", circ: \"\\u02C6\", tilde: \"\\u02DC\",\n};\nconst BARE = new Set([\"amp\", \"lt\", \"gt\", \"quot\", \"nbsp\", \"copy\", \"reg\"]);\nconst ENTITY_RE = /&(#[0-9]+;?|#[xX][0-9a-fA-F]+;?|[A-Za-z][A-Za-z0-9]{0,31};?)/g;\n\nexport function unescape(s: string): string {\n if (!s.includes(\"&\")) return s;\n return s.replace(ENTITY_RE, (m: string, body: string) => {\n if (body[0] === \"#\") {\n const hex = body[1] === \"x\" || body[1] === \"X\";\n const num = parseInt(body.slice(hex ? 2 : 1).replace(/;$/, \"\"), hex ? 16 : 10);\n if (num in CP1252) return CP1252[num];\n if (!Number.isFinite(num) || num > 0x10FFFF || (num >= 0xD800 && num <= 0xDFFF)) return \"\\uFFFD\";\n if ((num >= 0x1 && num <= 0x8) || num === 0xB || (num >= 0xE && num <= 0x1F) || (num >= 0x7F && num <= 0x9F)\n || (num >= 0xFDD0 && num <= 0xFDEF) || (num & 0xFFFE) === 0xFFFE) return \"\";\n return String.fromCodePoint(num);\n }\n const semi = body.endsWith(\";\");\n const name = semi ? body.slice(0, -1) : body;\n if (semi && name in NAMED) return NAMED[name];\n if (!semi && BARE.has(name)) return NAMED[name];\n return m;\n });\n}\n\n// news_fetch.py:19 — the same four steps in the same order. `\\s` and trim() cover U+00A0 in both languages,\n// so an ends as one space on either side. Note the order: an entity that decodes to a tag\n// (<b>) is decoded AFTER the tag strip and so survives as text — in Python too.\nexport function clean(s: string | null | undefined): string {\n let t = s ?? \"\";\n t = t.replace(//gs, \"$1\"); // unwrap CDATA (Economist et al.)\n t = t.replace(/<[^>]+>/g, \" \");\n t = unescape(t);\n return t.replace(/\\s+/g, \" \").trim();\n}\n\n// ---------- fetching ----------\n// Deno's fetch error names the URL (\"error sending request for url (https://…): dns error\"), which would\n// fill the desktop's 80-character error slot with the URL. The clause is dropped; the source name says what failed.\nconst errText = (e: any) => String(e?.message ?? e).replace(/ for url \$[^)]*\$/, \"\").trim();\n\n// news_fetch.py:14 — one GET with the Chrome UA, at most BODY_LIMIT bytes, decoded as UTF-8. The cap is on\n// BYTES before decoding, as Python's r.read(200000).decode(\"utf-8\",\"ignore\") is; capping the decoded string\n// would let a feed of multibyte text run to a different place. Two known, harmless differences: TextDecoder\n// writes U+FFFD where Python's \"ignore\" drops a bad byte (a real feed is valid UTF-8, so this can only hit\n// the character the cap cuts in half — inside the trailing partial , which both parsers drop for lack\n// of a closing tag); and the timeout is a 12 s total, not urllib's 25 s per read. A non-2xx status is\n// thrown in urllib's words, \"HTTP Error 403: Forbidden\", so a last_error reads the same on both sides.\nexport async function get(url: string, limit = BODY_LIMIT): Promise {\n const ctl = new AbortController();\n const timer = setTimeout(() => ctl.abort(), SOURCE_TIMEOUT_MS);\n try {\n const r = await fetch(url, { headers: UA, signal: ctl.signal, redirect: \"follow\" });\n if (!r.ok) {\n await r.body?.cancel().catch(() => {});\n throw new Error(`HTTP Error ${r.status}: ${r.statusText}`.trim());\n }\n if (!r.body) return \"\";\n const reader = r.body.getReader();\n const chunks: Uint8Array[] = [];\n let n = 0, done = false;\n while (n < limit) {\n const x = await reader.read();\n if (x.done) { done = true; break; }\n chunks.push(x.value); n += x.value.byteLength;\n }\n if (!done) reader.cancel().catch(() => {}); // the rest of the body is not wanted; free the connection\n const buf = new Uint8Array(Math.min(n, limit));\n let off = 0;\n for (const c of chunks) {\n const take = Math.min(c.byteLength, buf.length - off);\n if (take <= 0) break;\n buf.set(c.subarray(0, take), off); off += take;\n }\n return new TextDecoder(\"utf-8\", { fatal: false }).decode(buf);\n } catch (e) {\n if (ctl.signal.aborted) throw new Error(`timed out after ${SOURCE_TIMEOUT_MS} ms`);\n throw e;\n } finally {\n clearTimeout(timer);\n }\n}\n\n// news_fetch.py:26 — regex parsing, not an XML parser, because the desktop does it this way and the two\n// must agree on every title. The block regex takes or ; Atom's is tried\n// before RSS's text; the first title, description and date in the block win, as in Python.\nexport function parseRss(xml: string, source: string, topic: string, limit: number): Item[] {\n const out: Item[] = [];\n const blocks = Array.from(xml.matchAll(/<(?:item|entry)\\b.*?<\\/(?:item|entry)>/gs), (m) => m[0]);\n for (const b of blocks.slice(0, limit)) {\n const t = b.match(/]*>(.*?)<\\/title>/s);\n const l = b.match(/]*href=\"([^\"]+)\"/) ?? b.match(/]*>(.*?)<\\/link>/s);\n const d = b.match(/<(?:description|summary|content)[^>]*>(.*?)<\\/(?:description|summary|content)>/s);\n const p = b.match(/<(?:pubDate|published|updated)[^>]*>(.*?)<\\/(?:pubDate|published|updated)>/s);\n if (!t) continue;\n out.push({ source, topic, title: cut(clean(t[1]), 180),\n url: l ? clean(l[1]) : \"\", summary: d ? cut(clean(d[1]), 400) : \"\",\n when: p ? cut(clean(p[1]), 31) : \"\" });\n }\n return out;\n}\n\n// news_fetch.py:40 — the public preview page t.me/s/. The last `limit` messages on the page,\n// newest first; each paired with a counted from the END of the page's time tags, as\n// Python does (times[-(i+1)]). A post under 40 code points is a sticker, a caption or a forward header,\n// not news — the floor is cpLen, Python's len(). The url is the channel, not the post: Python's choice.\nexport async function parseTelegram(hn: string, source: string, topic: string, limit: number): Promise {\n const url = `https://t.me/s/${hn}`;\n const body = await get(url);\n const msgs = Array.from(body.matchAll(/

]*>(.*?)<\\/div>/gs), (m) => m[1]);\n const times = Array.from(body.matchAll(/ m[1]);\n const out: Item[] = [];\n const tail = msgs.slice(-limit).reverse(); // Python: reversed(msgs[-limit:])\n for (let i = 0; i < tail.length; i++) {\n const txt = clean(tail[i]);\n if (cpLen(txt) < 40) continue;\n out.push({ source, topic, title: cut(txt, 150), url, summary: cut(txt, 600),\n when: i < times.length ? cut(times[times.length - 1 - i], 10) : \"\" });\n }\n return out;\n}\n\nasync function fetchSource(s: Src): Promise {\n if (s.type === \"rss\") return parseRss(await get(s.url), s.name, s.topic, s.lim);\n if (s.type === \"telegram\") return await parseTelegram(s.handle, s.name, s.topic, s.lim);\n throw new Error(`unknown source type \"${s.type}\"`);\n}\n\n// ---------- storing ----------\n// news_fetch.py:86 — the desktop's dedupe key. Private to the val: the desktop keeps computing its own\n// from the six served fields, which is the whole point of serving them unchanged.\nexport const keyOf = (it: Item) => (it.url || \"\") + \"|\" + cut(it.title, 80);\n\n// Advisory only, never served. news_fetch.py's iso_of() takes the date in the pubDate's own zone; Date's\n// is UTC, so the two can differ by a day around midnight — one more reason the desktop keeps its own.\nfunction isoOf(w: string): string {\n const s = (w ?? \"\").trim();\n if (/^\\d{4}-\\d{2}-\\d{2}/.test(s)) return s.slice(0, 10);\n const ms = Date.parse(s);\n return Number.isFinite(ms) ? new Date(ms).toISOString().slice(0, 10) : \"\";\n}\n\n// One INSERT OR IGNORE for the whole source (at most `limit` rows), not one round trip per item:\n// rowsAffected counts only the rows actually inserted, which is exactly `stored`. A key already held keeps\n// its first_seen, so re-seeing an item never moves it inside the desktop's window a second time.\nasync function store(db: any, items: Item[], passKey: string): Promise {\n if (!items.length) return 0;\n const sql = \"INSERT OR IGNORE INTO news_fetched (key, source, topic, title, url, summary, when_text, iso, pass_key) VALUES \"\n + items.map(() => \"(?,?,?,?,?,?,?,?,?)\").join(\",\");\n const args = items.flatMap((it) => [keyOf(it), it.source, it.topic, it.title, it.url, it.summary, it.when, isoOf(it.when), passKey]);\n const r = await db.execute({ sql, args });\n return Number(r.rowsAffected ?? 0);\n}\n\n// ---------- flag, journal ----------\nconst isOn = (v: any) => String(v ?? \"\").trim().toLowerCase() === \"on\";\n\n// Read and reported, never written: the flag is the desktop's switch, and a flag flipped from here would\n// be a flag flipped without the desktop's receipt.\nexport async function readFlag(db: any = sqlite): Promise<\"on\" | \"off\"> {\n const r = await db.execute({ sql: \"SELECT value FROM settings WHERE key = ?\", args: [FLAG_KEY] });\n return r.rows.length && isOn((r.rows[0] as any).value) ? \"on\" : \"off\";\n}\n\nasync function journal(db: any, summary: string, details: string) {\n await db.execute({ sql: \"INSERT INTO journal (source, summary, details) VALUES (?,?,?)\",\n args: [SOURCE, cut(summary, 400), cut(details, 3500)] });\n}\n\n// ---------- the pass ----------\n// The queue: enabled and present, least recently tried first, never-tried before all. A negative LIMIT is\n// SQLite for \"no limit\" (the full sweep). `only` names sources by hand; dry, it may reach a disabled or\n// retired source to probe it — storing from one is never allowed, so a real run keeps the enabled/present test.\nasync function selectSources(db: any, scope: Scope, only: string[] | undefined, dry: boolean): Promise {\n const limit = scope === \"full\" ? -1 : SLICE_N;\n let where = \"enabled=1 AND present=1\", args: any[] = [];\n if (only && only.length) {\n where = `name IN (${only.map(() => \"?\").join(\",\")})` + (dry ? \"\" : \" AND enabled=1 AND present=1\");\n args = only;\n }\n const r = await db.execute({ sql: `SELECT name, type, url, handle, topic, lim FROM news_fetch_sources WHERE ${where} ORDER BY last_try ASC NULLS FIRST, name LIMIT ?`,\n args: [...args, limit] });\n return (r.rows as any[]).map((x) => ({ name: String(x.name), type: String(x.type), url: String(x.url ?? \"\"),\n handle: String(x.handle ?? \"\"), topic: String(x.topic ?? \"\"), lim: Number(x.lim ?? 8) }));\n}\n\n// One source. A fetch or parse failure is the SOURCE's error: recorded on its row, returned, counted. A\n// database failure is thrown, because that is the PASS dying, not the source — the caller writes the\n// receipt. Dry: the attempt is made and reported, and no row anywhere is touched (not even last_try).\nasync function attempt(db: any, s: Src, passKey: string, dry: boolean): Promise {\n const t = Date.now();\n const out: SourceResult = { name: s.name, type: s.type, n: 0, stored: 0, ms: 0, error: \"\" };\n let items: Item[] = [];\n try { items = await fetchSource(s); } catch (e) { out.error = cut(errText(e), 300); }\n out.n = items.length;\n out.ms = Date.now() - t;\n if (dry) return out;\n const stamp = sqlStamp(Date.now());\n if (out.error) {\n await db.execute({ sql: \"UPDATE news_fetch_sources SET last_try=?, last_error=?, last_n=0 WHERE name=?\", args: [stamp, out.error, s.name] });\n } else {\n out.stored = await store(db, items, passKey);\n await db.execute({ sql: \"UPDATE news_fetch_sources SET last_try=?, last_ok=?, last_error='', last_n=?, stored_total=stored_total+? WHERE name=?\",\n args: [stamp, stamp, out.n, out.stored, s.name] });\n }\n return out;\n}\n\nexport const scopeFor = (localHour: number): Scope => localHour === 5 || localHour === 6 ? \"full\" : \"slice\";\n\n/** One pass — the single implementation the interval and POST /news/fetch/run share, so a hand run proves\n * the interval's code path. kind \"pass\" is the interval: guard key pass::, scope from the\n * local hour unless given, and at hour 6 the day's journal line. kind \"run\" is a hand run: guard key\n * run:, never the day line. dry: fetch and report, claim nothing, store nothing, journal\n * nothing. at: a test instant (the route admits it with dry only). */\nexport async function fetchOnce(o: { kind: \"pass\" | \"run\"; scope?: Scope; dry?: boolean; only?: string[]; at?: Date; db?: any }) {\n const db = o.db ?? sqlite;\n await init(db);\n const now = o.at ?? new Date();\n const lp = localParts(now);\n const local = localText(lp);\n const scope: Scope = o.scope ?? scopeFor(lp.hour);\n const dry = o.dry === true;\n const daily = o.kind === \"pass\" && lp.hour === 6;\n const passKey = o.kind === \"pass\" ? `pass:${lp.date}:${pad(lp.hour)}` : `run:${new Date().toISOString().replace(/[-:.]/g, \"\")}`;\n const t0 = Date.now();\n const marks = [dry ? \"dry\" : \"\", o.only?.length ? `only ${o.only.length}` : \"\", o.at ? `at ${now.toISOString()}` : \"\"].filter(Boolean);\n const tag = marks.length ? ` [${marks.join(\", \")}]` : \"\";\n const base: any = { ok: true, kind: o.kind, scope, local, dry, at: o.at ? now.toISOString() : null };\n\n const flag = await readFlag(db);\n const queue = await selectSources(db, scope, o.only, dry);\n\n // Law 6: the key is claimed with one statement before any HTTP. A dry pass claims nothing.\n if (!dry) {\n const claim = await db.execute({ sql: \"INSERT OR IGNORE INTO news_fetch_log (key, kind, local, scope, detail) VALUES (?,?,?,?,?)\",\n args: [passKey, o.kind, local, scope, `${queue.length} sources queued${tag}`] });\n if ((claim.rowsAffected ?? 0) === 0) return { ...base, flag, skipped: \"pass already recorded\", pass_key: passKey };\n }\n\n // From here the key is spent. Whatever dies below is journalled with the step it died at. `results` is\n // hoisted out of the try so the receipt can say how far the pass got.\n const results: SourceResult[] = [];\n let truncated = 0, step = \"fetching sources\";\n const storedSoFar = () => results.reduce((a, r) => a + r.stored, 0);\n try {\n // Batches of CONCURRENCY. A batch starts only if its worst case — one source running to the full\n // timeout — still ends inside DEADLINE_MS; otherwise `truncated` is set and the pass stops rather than\n // overrunning. What was not reached kept its old last_try and heads the next hour's queue.\n for (let i = 0; i < queue.length; i += CONCURRENCY) {\n if (Date.now() - t0 + SOURCE_TIMEOUT_MS > DEADLINE_MS) { truncated = 1; break; }\n const batch = queue.slice(i, i + CONCURRENCY);\n const settled = await Promise.allSettled(batch.map((s) => attempt(db, s, passKey, dry)));\n for (const r of settled) if (r.status === \"fulfilled\") results.push(r.value);\n const dead = settled.find((r) => r.status === \"rejected\") as PromiseRejectedResult | undefined;\n if (dead) throw dead.reason; // attempt() rejects on a database failure only — the pass's death, not a source's\n }\n const sum = { tried: results.length, ok_n: results.filter((r) => !r.error).length,\n fetched: results.reduce((a, r) => a + r.n, 0), stored: storedSoFar(), errors: results.filter((r) => r.error).length };\n const failed = results.filter((r) => r.error).map((r) => `${r.name}: ${cut(r.error, 80)}`);\n const ms = Date.now() - t0;\n let day: string | null = null;\n if (!dry) {\n step = \"recording the pass\";\n const detail = `${queue.length} queued, ${sum.tried} tried${truncated ? ` (deadline: ${queue.length - sum.tried} not reached)` : \"\"}; failed: ${failed.join(\"; \") || \"none\"}${tag}`;\n await db.execute({ sql: \"UPDATE news_fetch_log SET tried=?, ok_n=?, fetched=?, stored=?, errors=?, ms=?, truncated=?, detail=? WHERE key=?\",\n args: [sum.tried, sum.ok_n, sum.fetched, sum.stored, sum.errors, ms, truncated, cut(detail, 1500), passKey] });\n if (daily) { step = \"writing the day's journal line\"; day = await dayLine(db, now, flag); }\n }\n return { ...base, flag, ...sum, ms, truncated, queued: queue.length, not_reached: queue.length - sum.tried,\n pass_key: dry ? null : passKey, journalled: day ? 1 : 0, sources: results };\n } catch (e) {\n const error = cut(String(e), 300);\n if (dry) return { ...base, ok: false, flag, died: step, error, tried: results.length, sources: results };\n // The key is claimed and the pass died. Write the receipt that says so, with the step, and mark the\n // pass row; if even the receipt cannot be written, rethrow so Val Town's own run log shows red.\n const reached = `${results.length} of ${queue.length} sources had been tried and the ${storedSoFar()} items they stored stand`;\n try {\n await journal(db, `pass DIED while ${step}: ${cut(error, 160)}${tag}`,\n `${o.kind} ${passKey}, scope ${scope}, ${local} ${TZ}. The key was claimed, then the pass threw while ${step}; nothing after that step ran, and ${reached}. `\n + `${cut(String((e as any)?.stack ?? e), 900)} No model called, no message sent.${tag}`);\n await db.execute({ sql: \"UPDATE news_fetch_log SET tried=?, stored=?, detail = COALESCE(detail, '') || ? WHERE key = ?\",\n args: [results.length, storedSoFar(), ` — DIED while ${step}: ${cut(error, 200)}`, passKey] });\n } catch { throw e; }\n return { ...base, ok: false, flag, died: step, error, journalled: 1, pass_key: passKey, tried: results.length, stored: storedSoFar(), sources: results };\n }\n}\n\n/** The day's one journal line, from the hour-6 pass. Guard key day:, so a double-fired hour\n * writes it once. Prunes past RETENTION_DAYS first, so `held` is what remains. Returns the summary, or\n * null when the day already has its line. */\nasync function dayLine(db: any, now: Date, flag: \"on\" | \"off\"): Promise {\n const lp = localParts(now);\n const key = `day:${lp.date}`;\n const claim = await db.execute({ sql: \"INSERT OR IGNORE INTO news_fetch_log (key, kind, local, scope) VALUES (?,?,?,?)\", args: [key, \"day\", localText(lp), \"day\"] });\n if ((claim.rowsAffected ?? 0) === 0) return null;\n const pruned = Number((await db.execute({ sql: \"DELETE FROM news_fetched WHERE first_seen < datetime('now', ?)\", args: [`-${RETENTION_DAYS} days`] })).rowsAffected ?? 0);\n await db.execute({ sql: \"DELETE FROM news_fetch_log WHERE ts < datetime('now', ?) AND key NOT LIKE 'test:%'\", args: [`-${LOG_RETENTION_DAYS} days`] });\n const s = await summarize(db, now);\n const d: any = (await db.execute(\"SELECT COUNT(*) AS n, COALESCE(SUM(tried),0) AS tried, COALESCE(SUM(errors),0) AS errs, COALESCE(SUM(truncated),0) AS trunc \"\n + \"FROM news_fetch_log WHERE kind IN ('pass','run') AND key NOT LIKE 'test:%' AND ts >= datetime('now','-24 hours')\")).rows[0] ?? {};\n const failing = s.failing.map((f) => f.name);\n const summary = `day ${lp.date}: ${d.n} passes in 24 h, ${d.tried} source fetches, ${s.stored24h} new items; failing: ${failing.join(\", \") || \"none\"}; `\n + `flag.news_fetch_val ${flag}; held: ${s.storedTotal} items, oldest ${s.oldestHeld ?? \"none\"}, newest ${s.newestHeld ?? \"none\"}`;\n const details = `${localText(lp)} ${TZ}, the hour-6 pass. Sources: ${s.sources.enabled} enabled of ${s.sources.present} present (${s.sources.rss} rss, ${s.sources.telegram} telegram), `\n + `last pushed ${s.sources.pushedAt ?? \"never — POST /news/fetch/sources has not been called\"}. `\n + `Passes in 24 h: ${d.n} (${d.trunc} hit the deadline), ${d.tried} source fetches, ${d.errs} failed. Failing now: ${s.errors.join(\"; \") || \"none\"}. `\n + `Pruned ${pruned} items older than ${RETENTION_DAYS} days. Held: ${s.storedTotal} items (oldest ${s.oldestHeld ?? \"none\"}, newest ${s.newestHeld ?? \"none\"}), ${s.stored24h} new in 24 h. `\n + `flag.news_fetch_val is ${flag}: ${flag === \"on\" ? \"the desktop pulls from here\" : \"the desktop still fetches itself; this is the evidence for switching\"}. `\n + `Cron ${CRON} UTC. No model called, no message sent.`;\n await journal(db, summary, details);\n await db.execute({ sql: \"UPDATE news_fetch_log SET tried=?, stored=?, errors=?, detail=? WHERE key=?\", args: [Number(d.tried), s.stored24h, failing.length, cut(summary, 1500), key] });\n return summary;\n}\n\n// ---------- the picture ----------\n/** What status, items and the day line all report, from one place. `errors` is the desktop's exact shape,\n * news_fetch.py:66 — \"{name}: {msg[:80]}\" — for the sources whose LAST attempt failed. */\nasync function summarize(db: any, now: Date) {\n const flag = await readFlag(db);\n const src: any = (await db.execute(\"SELECT COUNT(*) AS present, COALESCE(SUM(enabled),0) AS enabled, \"\n + \"COALESCE(SUM(CASE WHEN enabled=1 AND type='rss' THEN 1 ELSE 0 END),0) AS rss, \"\n + \"COALESCE(SUM(CASE WHEN enabled=1 AND type='telegram' THEN 1 ELSE 0 END),0) AS telegram, \"\n + \"MAX(pushed_at) AS pushed_at FROM news_fetch_sources WHERE present=1\")).rows[0] ?? {};\n const last: any = (await db.execute(\"SELECT key, ts, local, scope, tried, ok_n, fetched, stored, errors, ms, truncated FROM news_fetch_log \"\n + \"WHERE kind IN ('pass','run') AND key NOT LIKE 'test:%' ORDER BY ts DESC, key DESC LIMIT 1\")).rows[0] ?? null;\n const p24: any = (await db.execute(\"SELECT COUNT(*) AS n FROM news_fetch_log WHERE kind IN ('pass','run') AND key NOT LIKE 'test:%' AND ts >= datetime('now','-24 hours')\")).rows[0] ?? {};\n const held: any = (await db.execute(\"SELECT COUNT(*) AS n, MIN(first_seen) AS oldest, MAX(first_seen) AS newest, \"\n + \"COALESCE(SUM(CASE WHEN first_seen >= datetime('now','-24 hours') THEN 1 ELSE 0 END),0) AS d1 FROM news_fetched\")).rows[0] ?? {};\n const fail = (await db.execute(\"SELECT name, type, last_error, last_ok FROM news_fetch_sources WHERE present=1 AND enabled=1 AND last_error != '' ORDER BY name\")).rows as any[];\n const failing = fail.map((f) => ({ name: String(f.name), type: String(f.type), error: String(f.last_error), lastOk: f.last_ok ?? null }));\n return {\n flag,\n sources: { present: Number(src.present ?? 0), enabled: Number(src.enabled ?? 0), rss: Number(src.rss ?? 0), telegram: Number(src.telegram ?? 0),\n pushedAt: src.pushed_at ?? null, pushAgeH: ageH(src.pushed_at, now) },\n lastPass: last ? { key: String(last.key), ts: String(last.ts), local: last.local ?? null, scope: last.scope ?? null, tried: Number(last.tried ?? 0), ok: Number(last.ok_n ?? 0),\n fetched: Number(last.fetched ?? 0), stored: Number(last.stored ?? 0), errors: Number(last.errors ?? 0), ms: Number(last.ms ?? 0), truncated: Number(last.truncated ?? 0) } : null,\n lastPassAgeH: last ? ageH(last.ts, now) : null,\n passes24h: Number(p24.n ?? 0),\n stored24h: Number(held.d1 ?? 0), storedTotal: Number(held.n ?? 0),\n oldestHeld: held.oldest ?? null, newestHeld: held.newest ?? null,\n failing,\n errors: failing.map((f) => `${f.name}: ${cut(f.error, 80)}`),\n };\n}\n\n// ---------- the interval ----------\nexport default async function (_interval: { lastRunAt?: Date }) {\n // Every wake works: a slice, or the full sweep at 05 and 06 local; the day's journal line rides the\n // hour-6 pass — after the second full sweep, before Nina's 06:45. Scope, key and the day line all\n // follow from the local hour inside fetchOnce, so the interval and a hand run cannot disagree about them.\n return await fetchOnce({ kind: \"pass\" });\n}\n\n// ---------- the routes ----------\nconst clampInt = (v: any, dflt: number, lo: number, hi: number) => {\n if (v === undefined || v === null || v === \"\") return dflt;\n const n = Math.floor(Number(v));\n return Number.isFinite(n) ? Math.min(hi, Math.max(lo, n)) : dflt;\n};\n\nexport function mountNewsFetch(app: any, db: any, PULL_SECRET: string) {\n const ready = init(db);\n const auth = (c: any) => c.req.query(\"secret\") === PULL_SECRET;\n const limits = { SLICE_N, CONCURRENCY, SOURCE_TIMEOUT_MS, DEADLINE_MS, BODY_LIMIT, RETENTION_DAYS };\n\n app.get(\"/news/fetch/status\", async (c: any) => {\n await ready;\n if (!auth(c)) return c.text(\"forbidden\", 401);\n const now = new Date();\n const s = await summarize(db, now);\n return c.json({ ok: true, flag: s.flag, now: now.toISOString(), local: localText(localParts(now)), cron: CRON,\n sources: s.sources, lastPass: s.lastPass, lastPassAgeH: s.lastPassAgeH, passes24h: s.passes24h,\n stored24h: s.stored24h, storedTotal: s.storedTotal, oldestHeld: s.oldestHeld, newestHeld: s.newestHeld,\n retentionDays: RETENTION_DAYS, failing: s.failing, errors: s.errors, limits });\n });\n\n // The pull. A row is EXACTLY {source, topic, title, url, summary, when}, in that order — an element of\n // news_raw.json[\"items\"] before iso_of() runs. No id, key, iso or first_seen: the desktop computes its own\n // key from these six and its own iso from `when`, so one implementation of each keeps running. The read\n // is pure: nothing is marked, moved or deleted by it. `next` is the last id when the page was full.\n app.get(\"/news/fetch/items\", async (c: any) => {\n await ready;\n if (!auth(c)) return c.text(\"forbidden\", 401);\n const now = new Date();\n const hours = clampInt(c.req.query(\"hours\"), 120, 1, 336);\n const limit = clampInt(c.req.query(\"limit\"), 500, 1, 1000);\n const before = Math.floor(Number(c.req.query(\"before\")));\n const paged = Number.isFinite(before) && before > 0;\n const debug = c.req.query(\"debug\") === \"1\";\n // The window is first_seen >= now − hours. The same stamp goes into the WHERE and into the response, so\n // the window reported is the window applied (first_seen is datetime('now') text, sqlStamp's shape).\n const sinceUtc = sqlStamp(now.getTime() - hours * 3600000);\n const r = await db.execute({ sql: `SELECT id, source, topic, title, url, summary, when_text, key, first_seen FROM news_fetched WHERE first_seen >= ?${paged ? \" AND id < ?\" : \"\"} ORDER BY id DESC LIMIT ?`,\n args: paged ? [sinceUtc, before, limit] : [sinceUtc, limit] });\n const raw = r.rows as any[];\n const rows = raw.map((x) => {\n const row: any = { source: String(x.source), topic: String(x.topic ?? \"\"), title: String(x.title), url: String(x.url ?? \"\"), summary: String(x.summary ?? \"\"), when: String(x.when_text ?? \"\") };\n if (debug) { row._key = x.key; row._first_seen = x.first_seen; }\n return row;\n });\n const s = await summarize(db, now);\n // lastPassAgeH and passes24h ride the pull as well as /status, so one call tells the desktop both what\n // it is holding and how fresh the val is. news_fetch.py's freshness gate and its `via val:` line both\n // read them from here; without them the provenance line prints \"? passes/24 h\".\n return c.json({ ok: true, flag: s.flag, window: { hours, sinceUtc }, n: rows.length, rows,\n next: rows.length === limit && raw.length ? Number(raw[raw.length - 1].id) : null,\n errors: s.errors, lastPass: s.lastPass, lastPassAgeH: s.lastPassAgeH, passes24h: s.passes24h,\n sources: s.sources });\n });\n\n // The roster push, from the desktop, entries of scripts/news_sources.json verbatim. Upsert by name; a\n // name the push no longer carries is retired (present=0, enabled=0), never deleted. Fewer than five\n // entries is an empty or truncated push, and one of those must never blind the cold path: refused,\n // nothing touched. A malformed entry is skipped and named, not a reason to refuse the 58 good ones.\n app.post(\"/news/fetch/sources\", async (c: any) => {\n await ready;\n if (!auth(c)) return c.text(\"forbidden\", 401);\n const b = await c.req.json().catch(() => ({}));\n if (!Array.isArray(b.sources) || b.sources.length < 5) {\n return c.json({ ok: false, error: \"sources must be an array of at least 5 entries — a shorter push is treated as truncated and refused\" }, 400);\n }\n const pushedAt = sqlStamp(Date.now());\n const have = new Map();\n for (const x of (await db.execute(\"SELECT name, present FROM news_fetch_sources\")).rows as any[]) have.set(String(x.name), Number(x.present));\n const pushed = new Set();\n const invalid: string[] = [];\n let added = 0, updated = 0;\n for (const e of b.sources as any[]) {\n const name = String(e?.name ?? \"\").trim(), type = String(e?.type ?? \"\").trim();\n const url = String(e?.url ?? \"\").trim(), handle = String(e?.handle ?? \"\").trim();\n if (!name || (type !== \"rss\" && type !== \"telegram\") || (type === \"rss\" && !url) || (type === \"telegram\" && !handle) || pushed.has(name)) {\n invalid.push(name || \"(unnamed)\"); continue;\n }\n // news_fetch.py's defaults, lines 62 and 64: limit 8 for rss and 10 for telegram; topic \"\"; enabled true.\n const rawLim = Number(e?.limit);\n const lim = e?.limit != null && e?.limit !== \"\" && Number.isFinite(rawLim) ? Math.floor(rawLim) : (type === \"telegram\" ? 10 : 8);\n const enabled = e?.enabled === false ? 0 : 1;\n const topic = String(e?.topic ?? \"\");\n pushed.add(name);\n if (have.has(name)) {\n await db.execute({ sql: \"UPDATE news_fetch_sources SET type=?, url=?, handle=?, topic=?, lim=?, enabled=?, present=1, pushed_at=? WHERE name=?\",\n args: [type, url, handle, topic, lim, enabled, pushedAt, name] });\n updated++;\n } else {\n await db.execute({ sql: \"INSERT INTO news_fetch_sources (name, type, url, handle, topic, lim, enabled, present, pushed_at) VALUES (?,?,?,?,?,?,?,1,?)\",\n args: [name, type, url, handle, topic, lim, enabled, pushedAt] });\n added++;\n }\n }\n // Retired = was present, not in this push. Computed from the names, never from a stamp comparison, so a\n // re-sent push can neither retire nor resurrect by accident.\n const gone = [...have].filter(([n, present]) => present === 1 && !pushed.has(n)).map(([n]) => n);\n let retired = 0;\n if (gone.length) {\n const r = await db.execute({ sql: `UPDATE news_fetch_sources SET present=0, enabled=0 WHERE name IN (${gone.map(() => \"?\").join(\",\")})`, args: gone });\n retired = Number(r.rowsAffected ?? 0);\n }\n // Journalled when the roster changed shape; a same-again push is not news.\n if (added || retired) {\n await journal(db, `roster push: ${pushed.size} sources (${added} new, ${retired} retired)`,\n `POST /news/fetch/sources${b.stamp !== undefined ? ` (desktop stamp ${String(b.stamp)})` : \"\"}: ${pushed.size} names, ${added} added, ${updated} updated, ${retired} retired`\n + ` (present=0, enabled=0, never deleted)${gone.length ? `: ${gone.join(\", \")}` : \"\"}. Invalid entries skipped: ${invalid.join(\", \") || \"none\"}.`);\n }\n const out: any = { ok: true, n: pushed.size, added, updated, retired, pushedAt };\n if (invalid.length) out.invalid = invalid;\n return c.json(out);\n });\n\n // The same code path the interval runs, on demand — the schedule cannot be read from any API, so the\n // proof that fetching works is a pass with a recorded result, not a deploy that returned 200. Its guard\n // key is run:, so it can never take the hour's pass: key from the interval. scope defaults\n // to the interval's rule for the instant; `at` is a test parameter and is refused unless dry.\n app.post(\"/news/fetch/run\", async (c: any) => {\n await ready;\n if (!auth(c)) return c.text(\"forbidden\", 401);\n const b = await c.req.json().catch(() => ({}));\n const dry = b.dry === true;\n if (b.scope !== undefined && b.scope !== \"slice\" && b.scope !== \"full\") return c.json({ ok: false, error: 'scope must be \"slice\" or \"full\"' }, 400);\n if (b.only !== undefined && (!Array.isArray(b.only) || !b.only.length || !b.only.every((x: any) => typeof x === \"string\"))) {\n return c.json({ ok: false, error: \"only must be a non-empty array of source names\" }, 400);\n }\n let at: Date | undefined;\n if (b.at !== undefined) {\n // A production pass may never be told what time it is.\n if (!dry) return c.json({ ok: false, error: \"at is a test parameter: pass dry:true with it\" }, 400);\n at = new Date(String(b.at));\n if (isNaN(at.getTime())) return c.json({ ok: false, error: \"at is not a date\" }, 400);\n }\n return c.json(await fetchOnce({ kind: \"run\", scope: b.scope, only: b.only, dry, at, db }));\n });\n\n // The manual prune: before:\"YYYY-MM-DD\" (a UTC day boundary) or days:N. The hour-6 pass already prunes\n // past RETENTION_DAYS; this is for a hand decision. A cut inside the desktop's 120 h pull window would\n // empty what the desktop is about to read, so that takes force:true, said deliberately. news_seen.json\n // is untouched by any of this — the desktop's memory of what was shown is its own.\n app.post(\"/news/fetch/prune\", async (c: any) => {\n await ready;\n if (!auth(c)) return c.text(\"forbidden\", 401);\n const b = await c.req.json().catch(() => ({}));\n const hasBefore = b.before !== undefined, hasDays = b.days !== undefined;\n if (hasBefore === hasDays) return c.json({ ok: false, error: 'exactly one of before:\"YYYY-MM-DD\" or days:N' }, 400);\n let cutoff: string;\n if (hasBefore) {\n if (!/^\\d{4}-\\d\\d-\\d\\d$/.test(String(b.before))) return c.json({ ok: false, error: \"before must be YYYY-MM-DD\" }, 400);\n cutoff = `${b.before} 00:00:00`;\n } else {\n const days = Number(b.days);\n if (!Number.isFinite(days) || days < 1) return c.json({ ok: false, error: \"days must be a number >= 1\" }, 400);\n cutoff = sqlStamp(Date.now() - days * 86400000);\n }\n const cutMs = Date.parse(cutoff.replace(\" \", \"T\") + \"Z\");\n if (!Number.isFinite(cutMs)) return c.json({ ok: false, error: \"before is not a date\" }, 400);\n if (cutMs > Date.now() - DESKTOP_WINDOW_H * 3600000 && b.force !== true) {\n return c.json({ ok: false, error: `the cut ${cutoff} is inside the desktop's ${DESKTOP_WINDOW_H} h pull window; pass force:true to prune it anyway` }, 400);\n }\n const r = await db.execute({ sql: \"DELETE FROM news_fetched WHERE first_seen < ?\", args: [cutoff] });\n const deleted = Number(r.rowsAffected ?? 0);\n await journal(db, `pruned ${deleted} held items first seen before ${cutoff}${b.force === true ? \" [forced, inside the pull window]\" : \"\"}`,\n `POST /news/fetch/prune ${hasBefore ? `before ${b.before}` : `days ${b.days}`}: ${deleted} rows deleted from news_fetched with first_seen < ${cutoff} UTC. Nothing else touched; news_seen.json on the desktop is unaffected.`);\n return c.json({ ok: true, deleted, before: cutoff });\n });\n}\n","deleted",false,"label",[14],"8754a5f7-f359-4e36-abb1-c12689a1658c",["D",1783928553221],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","Palantír Herald — Telegram capture + pings","imageUrl","userId","3ed96798-6351-4c0c-a152-1db6ab7640c9","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","019f5a68-d0b6-75de-80af-2b23ea5ed456","tursoName","8754a5f7f3594e36abb1c12689a1658c","tursoHostname","8754a5f7f3594e36abb1c12689a1658c-val-town.aws-us-east-2.turso.io","tursoMeasuredAt",["D",1789525245864],"pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_9":10,"_105":101},"org",{"_9":10,"_98":99,"_100":101,"_55":56,"_102":103,"_104":42},"branch",{"_35":88,"_11":89,"_22":28,"_90":45,"_91":-5,"_92":-5,"_93":-5,"_94":-5,"_95":-5,"_96":12,"_97":10},"isOwned",100,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","displayName","Alexandr Stakhov","avatar","https://lh3.googleusercontent.com/a/ACg8ocKCA_6NlmBdn2fDI25kVPZAGM3tPMEmD7U1Na63L8rtZj8kBQ=s96-c","tier","free","canRestrictVals","profileImageUrl","ctx",{"_81":-5,"_139":140,"_141":142,"_143":144,"_145":42,"_146":42,"_147":42,"_148":32,"_149":-5,"_150":-7,"_151":-7,"_152":62,"_153":42,"_154":42,"_155":-5,"_156":42,"_157":42,"_158":42,"_159":-5},"ENV",{"_110":111,"_112":113,"_114":89,"_115":116,"_117":118,"_119":120,"_121":122,"_123":124,"_125":126,"_127":128,"_129":130,"_131":132,"_133":134,"_135":136,"_137":138},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town","orgs",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]