${escapeXml(a.title)}

Month Year

followed by

with links\n const content = $(\".field--name-field-content\").first();\n\n content.children().each((_i, el) => {\n const tag = (el as cheerio.Element).tagName?.toLowerCase();\n\n if (tag === \"h2\") {\n const text = $(el).text().trim();\n if (text && text !== \"Jump to\") {\n currentMonth = text;\n }\n }\n\n if (tag === \"ul\") {\n $(el)\n .find(\"li\")\n .each((_j, li) => {\n $(li)\n .find(\"a\")\n .each((_k, a) => {\n const href = $(a).attr(\"href\") ?? \"\";\n const title = $(a).text().trim();\n if (!href || !title) return;\n\n // Only include DNR-hosted journal articles\n let fullUrl: string;\n if (href.startsWith(\"/\")) {\n fullUrl = BASE_URL + href;\n } else if (href.startsWith(\"http\")) {\n // Include external links too (St. Cloud summaries, NWS, etc.)\n fullUrl = href;\n } else {\n return;\n }\n\n entries.push({ url: fullUrl, title, month: currentMonth });\n });\n });\n }\n });\n\n return entries;\n}\n\n/** Scrape the full text content of a single article page. */\nexport async function scrapeArticle(url: string): Promise {\n // Skip external links - we can't reliably scrape those\n if (!url.includes(\"dnr.state.mn.us\")) {\n return `External article: ${url}`;\n }\n\n const res = await fetch(url, {\n headers: {\n \"User-Agent\":\n \"Mozilla/5.0 (compatible; MNClimateJournal/1.0; +https://val.town)\",\n },\n });\n if (!res.ok) {\n throw new Error(`Failed to fetch article: ${res.status} ${res.statusText}`);\n }\n\n const html = await res.text();\n const $ = cheerio.load(html);\n\n // Remove non-content elements\n $(\"script, style, nav, footer, header, aside, .sidebar, #leftmenu, .breadcrumb\").remove();\n\n // The main article content is in .field--name-field-content\n const contentEl = $(\".field--name-field-content\").first();\n if (contentEl.length === 0) {\n // Fallback to main content area\n const main = $('[role=\"main\"]').first();\n return extractText($, main);\n }\n\n return extractText($, contentEl);\n}\n\nfunction extractText($: cheerio.CheerioAPI, el: cheerio.Cheerio): string {\n const paragraphs: string[] = [];\n\n el.find(\"p, h1, h2, h3, h4, h5, h6, li, blockquote, figcaption\").each(\n (_i, elem) => {\n const text = $(elem).text().trim();\n if (text.length > 0) {\n paragraphs.push(text);\n }\n },\n );\n\n if (paragraphs.length === 0) {\n return el.text().replace(/\\s+/g, \" \").trim();\n }\n\n return paragraphs.join(\"\\n\\n\");\n}\n",[],[145,177],"21ed2b7c-5015-11f1-becb-ee650bb23af1","db.ts","21ed2c26-5015-11f1-becb-ee650bb23af1",6,["D",1778818711685],"import { sqlite } from \"https://esm.town/v/std/sqlite/main.ts\";\n\nexport async function initDb() {\n await sqlite.execute(`\n CREATE TABLE IF NOT EXISTS mn_climate_articles (\n id TEXT PRIMARY KEY,\n url TEXT NOT NULL UNIQUE,\n title TEXT NOT NULL,\n content TEXT,\n discovered_at TEXT DEFAULT (datetime('now')),\n published_at TEXT\n )\n `);\n}\n\nexport async function getKnownUrls(): Promise> {\n const result = await sqlite.execute(\n `SELECT url FROM mn_climate_articles`,\n );\n return new Set((result.rows as { url: string }[]).map((r) => r.url));\n}\n\nexport async function insertArticle(article: {\n url: string;\n title: string;\n content: string;\n publishedAt?: string;\n}): Promise {\n const id = crypto.randomUUID();\n await sqlite.execute({\n sql: `INSERT OR IGNORE INTO mn_climate_articles (id, url, title, content, published_at)\n VALUES (?, ?, ?, ?, ?)`,\n args: [id, article.url, article.title, article.content, article.publishedAt ?? null],\n });\n return id;\n}\n\nexport async function getAllArticles(): Promise {\n const result = await sqlite.execute(\n `SELECT * FROM mn_climate_articles ORDER BY discovered_at DESC`,\n );\n return result.rows as Article[];\n}\n\nexport interface Article {\n id: string;\n url: string;\n title: string;\n content: string | null;\n discovered_at: string;\n published_at: string | null;\n}\n",[],[145,186],["D",1778818711478],[161,162],[145],["D",1778818667116],["D",1789693284357],{"_16":17,"_84":80},{"_16":17,"_78":-5,"_79":80,"_33":34,"_81":82,"_83":26},{"_66":67,"_20":68,"_18":69,"_70":19,"_71":-5,"_72":-5,"_73":-5,"_74":-5,"_75":-5,"_76":21,"_77":17},"ctx",{"_59":-5,"_105":234,"_235":236,"_237":238,"_239":26,"_240":26,"_241":26,"_242":103,"_243":-5,"_244":-7,"_245":-7,"_246":40,"_247":26,"_248":26,"_249":-5,"_250":26,"_251":26,"_252":26,"_253":-5},"ENV",{"_205":206,"_207":208,"_209":68,"_210":211,"_212":213,"_214":215,"_216":217,"_218":219,"_220":221,"_222":223,"_224":225,"_226":227,"_228":229,"_230":231,"_232":233},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","71c1b20804b3232a7e513be260cfffc1932a9e97","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Search