${escapeXml(a.title)}

Month Year

followed by

with links\n const content = $(\".field--name-field-content\").first();\n\n content.children().each((_i, el) => {\n const tag = (el as cheerio.Element).tagName?.toLowerCase();\n\n if (tag === \"h2\") {\n const text = $(el).text().trim();\n if (text && text !== \"Jump to\") {\n currentMonth = text;\n }\n }\n\n if (tag === \"ul\") {\n $(el)\n .find(\"li\")\n .each((_j, li) => {\n $(li)\n .find(\"a\")\n .each((_k, a) => {\n const href = $(a).attr(\"href\") ?? \"\";\n const title = $(a).text().trim();\n if (!href || !title) return;\n\n // Only include DNR-hosted journal articles\n let fullUrl: string;\n if (href.startsWith(\"/\")) {\n fullUrl = BASE_URL + href;\n } else if (href.startsWith(\"http\")) {\n // Include external links too (St. Cloud summaries, NWS, etc.)\n fullUrl = href;\n } else {\n return;\n }\n\n entries.push({ url: fullUrl, title, month: currentMonth });\n });\n });\n }\n });\n\n return entries;\n}\n\n/** Scrape the full text content of a single article page. */\nexport async function scrapeArticle(url: string): Promise {\n // Skip external links - we can't reliably scrape those\n if (!url.includes(\"dnr.state.mn.us\")) {\n return `External article: ${url}`;\n }\n\n const res = await fetch(url, {\n headers: {\n \"User-Agent\":\n \"Mozilla/5.0 (compatible; MNClimateJournal/1.0; +https://val.town)\",\n },\n });\n if (!res.ok) {\n throw new Error(`Failed to fetch article: ${res.status} ${res.statusText}`);\n }\n\n const html = await res.text();\n const $ = cheerio.load(html);\n\n // Remove non-content elements\n $(\"script, style, nav, footer, header, aside, .sidebar, #leftmenu, .breadcrumb\").remove();\n\n // The main article content is in .field--name-field-content\n const contentEl = $(\".field--name-field-content\").first();\n if (contentEl.length === 0) {\n // Fallback to main content area\n const main = $('[role=\"main\"]').first();\n return extractText($, main);\n }\n\n return extractText($, contentEl);\n}\n\nfunction extractText($: cheerio.CheerioAPI, el: cheerio.Cheerio): string {\n const paragraphs: string[] = [];\n\n el.find(\"p, h1, h2, h3, h4, h5, h6, li, blockquote, figcaption\").each(\n (_i, elem) => {\n const text = $(elem).text().trim();\n if (text.length > 0) {\n paragraphs.push(text);\n }\n },\n );\n\n if (paragraphs.length === 0) {\n return el.text().replace(/\\s+/g, \" \").trim();\n }\n\n return paragraphs.join(\"\\n\\n\");\n}\n",[18,65],"21ed2b7c-5015-11f1-becb-ee650bb23af1",["D",1778818711685],"import { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\n\nexport async function initDb() {\n await sqlite.execute(`\n CREATE TABLE IF NOT EXISTS mn_climate_articles (\n id TEXT PRIMARY KEY,\n url TEXT NOT NULL UNIQUE,\n title TEXT NOT NULL,\n content TEXT,\n discovered_at TEXT DEFAULT (datetime('now')),\n published_at TEXT\n )\n `);\n}\n\nexport async function getKnownUrls(): Promise> {\n const result = await sqlite.execute(\n `SELECT url FROM mn_climate_articles`,\n );\n return new Set((result.rows as { url: string }[]).map((r) => r.url));\n}\n\nexport async function insertArticle(article: {\n url: string;\n title: string;\n content: string;\n publishedAt?: string;\n}): Promise {\n const id = crypto.randomUUID();\n await sqlite.execute({\n sql: `INSERT OR IGNORE INTO mn_climate_articles (id, url, title, content, published_at)\n VALUES (?, ?, ?, ?, ?)`,\n args: [id, article.url, article.title, article.content, article.publishedAt ?? null],\n });\n return id;\n}\n\nexport async function getAllArticles(): Promise {\n const result = await sqlite.execute(\n `SELECT * FROM mn_climate_articles ORDER BY discovered_at DESC`,\n );\n return result.rows as Article[];\n}\n\nexport interface Article {\n id: string;\n url: string;\n title: string;\n content: string | null;\n discovered_at: string;\n published_at: string | null;\n}\n",[18,81],"pulls","branches","forks","envVars",{"_182":26,"_184":26},"connections",{"_182":26,"_183":26},"count","unfilledCount","emptyCount",{"_34":157,"_13":18,"_63":82,"_44":-5,"_36":37,"_158":159,"_160":24,"_38":223,"_40":26,"_66":83,"_162":-5,"_68":30,"_163":-5,"_198":224,"_17":225},{"_34":171,"_13":81,"_63":80,"_44":82,"_36":37,"_158":74,"_160":24,"_38":220,"_40":26,"_66":67,"_162":173,"_68":30,"_163":-5,"_198":221,"_17":222},{"_34":167,"_13":65,"_63":64,"_44":82,"_36":37,"_158":41,"_160":24,"_38":217,"_40":26,"_66":67,"_162":169,"_68":30,"_163":-5,"_198":218,"_17":219},{"_34":209,"_13":210,"_63":211,"_44":-5,"_36":37,"_158":212,"_160":24,"_38":213,"_40":26,"_66":21,"_162":214,"_68":30,"_163":-5,"_198":215,"_17":216},{"_34":201,"_13":202,"_63":203,"_44":-5,"_36":37,"_158":122,"_160":24,"_38":204,"_40":159,"_66":205,"_162":206,"_68":30,"_163":-5,"_198":207,"_17":208},{"_34":191,"_13":192,"_63":193,"_44":-5,"_36":37,"_158":194,"_160":24,"_38":195,"_40":159,"_66":196,"_162":197,"_68":30,"_163":-5,"_198":199,"_17":200},"3111817a-5015-11f1-917a-ee650bb23af1","main.ts","21cb8012-5015-11f1-8f72-ee650bb23af1",7,["D",1778818737270],"interval","import { initDb, getKnownUrls, insertArticle } from \"./lib/db.ts\";\nimport { scrapeIndex, scrapeArticle } from \"./lib/scrape.ts\";\n\n/**\n * Cron val: checks the MN DNR Climate Journal index page for new articles.\n * Schedule: every 6 hours.\n *\n * When new articles are found, scrapes their content and stores them in SQLite.\n * The RSS feed is served by the companion feed.ts HTTP val.\n */\nexport default async function checkForNewArticles() {\n await initDb();\n\n console.log(\"[mn-climate-journal] Checking for new articles...\");\n\n const entries = await scrapeIndex();\n console.log(`[mn-climate-journal] Found ${entries.length} total entries on index page`);\n\n const knownUrls = await getKnownUrls();\n const newEntries = entries.filter((e) => !knownUrls.has(e.url));\n\n if (newEntries.length === 0) {\n console.log(\"[mn-climate-journal] No new articles found.\");\n return;\n }\n\n console.log(`[mn-climate-journal] Found ${newEntries.length} new article(s)`);\n\n for (const entry of newEntries) {\n try {\n console.log(`[mn-climate-journal] Scraping: ${entry.title} (${entry.url})`);\n const content = await scrapeArticle(entry.url);\n\n await insertArticle({\n url: entry.url,\n title: entry.title,\n content,\n publishedAt: monthToIso(entry.month),\n });\n\n console.log(`[mn-climate-journal] ✅ Stored: ${entry.title}`);\n } catch (err) {\n const msg = err instanceof Error ? err.message : String(err);\n console.error(`[mn-climate-journal] ❌ Failed to scrape ${entry.url}: ${msg}`);\n }\n }\n\n console.log(\"[mn-climate-journal] Done.\");\n}\n\n/** Convert \"May 2026\" → \"2026-05-01\" for rough publish date ordering. */\nfunction monthToIso(month: string): string | undefined {\n try {\n const d = new Date(`${month} 1`);\n if (isNaN(d.getTime())) return undefined;\n return d.toISOString().slice(0, 10);\n } catch {\n return undefined;\n }\n}\n","children",[],[192],"3e581b8c-5015-11f1-b313-ee650bb23af1","feed.ts","21c50098-5015-11f1-ad92-ee650bb23af1",["D",1778818759551],"http","import { Hono } from \"https://esm.sh/hono\";\nimport { initDb, getAllArticles } from \"./lib/db.ts\";\nimport type { Article } from \"./lib/db.ts\";\n\nfunction escapeXml(str: string): string {\n return str\n .replace(/&/g, \"&\")\n .replace(//g, \">\")\n .replace(/\"/g, \""\")\n .replace(/'/g, \"'\");\n}\n\nfunction toRfc822(dateStr: string): string {\n const d = new Date(dateStr.includes(\"T\") ? dateStr : dateStr + \"T00:00:00Z\");\n if (isNaN(d.getTime())) return new Date().toUTCString();\n return d.toUTCString();\n}\n\n/** Truncate content for the RSS description, keeping it readable. */\nfunction truncate(text: string, maxLen = 500): string {\n if (!text || text.length <= maxLen) return text ?? \"\";\n return text.slice(0, maxLen).replace(/\\s+\\S*$/, \"\") + \"…\";\n}\n\nasync function generateFeed(reqUrl: URL): Promise {\n await initDb();\n\n const articles = await getAllArticles();\n const selfUrl = new URL(\"/feed.xml\", reqUrl).toString();\n const siteUrl = \"https://www.dnr.state.mn.us/climate/journal/index.html\";\n\n const lastBuildDate = articles.length > 0\n ? toRfc822(articles[0].discovered_at)\n : new Date().toUTCString();\n\n const items = articles\n .map(\n (a: Article) => `\n \n ${escapeXml(a.title)}\n ${escapeXml(a.url)}\n ${escapeXml(a.url)}\n ${toRfc822(a.published_at ?? a.discovered_at)}\n \n `,\n )\n .join(\"\\n\");\n\n const rss = `\n\n \n \n Minnesota DNR Climate Journal\n ${escapeXml(siteUrl)}\n Articles from the Minnesota State Climatology Office's Climate Journal, tracking significant Minnesota weather events.\n en-us\n ${lastBuildDate}\n ${items}\n \n`;\n\n return new Response(rss, {\n headers: {\n \"Content-Type\": \"application/rss+xml; charset=utf-8\",\n \"Cache-Control\": \"public, max-age=900\",\n },\n });\n}\n\nconst app = new Hono();\n\napp.get(\"/feed.xml\", async (c) => {\n return generateFeed(new URL(c.req.url));\n});\n\napp.get(\"/\", (c) => c.redirect(\"/feed.xml\"));\n\nexport default app.fetch;\n",[],[202],"21e5a24e-5015-11f1-8886-ee650bb23af1","README.md","21e5a4c4-5015-11f1-8886-ee650bb23af1",5,["D",1778818711683],"# MN DNR Climate Journal → RSS\n\nTwo Val Town vals that work together:\n\n1. **`main.ts`** — Cron val (every 6 hours). Scrapes the [MN DNR Climate Journal](https://www.dnr.state.mn.us/climate/journal/index.html) index page, detects new articles, scrapes their content, and stores them in Val Town SQLite.\n\n2. **`feed.ts`** — HTTP val. Serves an RSS feed of all discovered articles at `/feed.xml`.\n\n## Setup on Val Town\n\n1. Push this folder as a val using `vt`\n2. Set `main.ts` as a **Cron** val with a 6-hour interval\n3. Set `feed.ts` as an **HTTP** val\n4. Subscribe to the feed URL in your RSS reader\n",[],[210],["D",1778818711679],[],[18,65],["D",1778818711685],[],[18,81],["D",1778818711478],[186,187],[18],["D",1778818667116],["D",1789693284357],{"_11":12,"_135":71},{"_11":12,"_131":-5,"_70":71,"_31":69,"_132":133,"_134":30},{"_40":122,"_13":16,"_34":37,"_123":85,"_124":-5,"_125":-5,"_126":-5,"_127":-5,"_128":-5,"_129":14,"_130":12},"ctx",{"_53":-5,"_154":264,"_265":266,"_267":268,"_269":30,"_270":30,"_271":30,"_272":24,"_273":-5,"_274":-7,"_275":-7,"_276":26,"_277":30,"_278":30,"_279":-5,"_280":30,"_281":30,"_282":30,"_283":-5},"ENV",{"_235":236,"_237":238,"_239":16,"_240":241,"_242":243,"_244":245,"_246":247,"_248":249,"_250":251,"_252":253,"_254":255,"_256":257,"_258":259,"_260":261,"_262":263},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","71c1b20804b3232a7e513be260cfffc1932a9e97","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

History · lib