${title}

[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_178":179,"_180":181},"routes/_app.x.$handle.$name",{"_98":97,"_32":45,"_106":85,"_27":107,"_13":14,"_108":-5,"_11":109,"_110":111,"_112":113,"_114":115,"_116":117,"_20":-5,"_118":119,"_120":121,"_122":25,"_123":124,"_125":49,"_126":-5,"_127":-5,"_128":129},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_43":44,"_32":45,"_38":46,"_47":-5,"_48":49,"_50":51,"_52":51,"_53":-5,"_54":-5,"_18":55,"_56":31,"_57":49,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":78,"_79":80,"_81":82,"_83":25},"version",2,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_33":34,"_35":37,"_38":39,"_40":14,"_41":42,"_24":25},"previous",{"_30":31,"_32":28,"_33":34,"_35":36},"deleted",false,"deletedHere","path","main.tsx","latestPath","fileId","abbbf1b0-2782-11f0-b46e-569c3dd06744","name","fileType","http","content","import { isProbablyReaderable, Readability } from \"npm:@mozilla/readability@^0.5.0\";\nimport { DOMParser } from \"npm:linkedom@0.16.10\";\nimport { marked } from \"npm:marked@12.0.1\";\nimport { getSubtitles } from \"npm:youtube-captions-scraper@^2.0.1\";\nimport { YouTube } from \"npm:youtube-sr@4.3.11\";\n\nconst isCloudflareWorker = typeof Request !== \"undefined\" && typeof Response !== \"undefined\";\n\n// init async loading of modules\nconst AgentMarkdownImport = isCloudflareWorker ? import(\"npm:agentmarkdown@6.0.0\") : null;\nconst TurndownService = isCloudflareWorker ? null : await import(\"npm:turndown@^7.1.3\");\n\n/**\n * converts HTML to markdown\n * @returns markdown in string\n */\nexport async function html2markdown(html: string): Promise {\n if (AgentMarkdownImport) {\n // TurndownService doesn't work on cf\n // Dynamically import AgentMarkdown when running in Cloudflare Worker\n const { AgentMarkdown } = await AgentMarkdownImport;\n return await AgentMarkdown.produce(html);\n } else {\n // Dynamically import TurndownService otherwise\n return new (await TurndownService)().turndown(html);\n }\n}\n\n/**\n * extracts article from html\n * then converts it to md\n * @returns markdown in string\n */\nexport async function readability2markdown(html: string): Promise<{ title: string; markdown: string }> {\n const doc = await (new DOMParser().parseFromString(html, \"text/html\"));\n\n const reader = new Readability(doc);\n const article = reader.parse();\n\n const markdown = await html2markdown(article?.content || \"\");\n return { title: doc.title.textContent, markdown };\n}\n\nfunction getYoutubeVideoID(url: URL): string | null {\n const regExp = /(?:youtube\\.com\\/(?:[^/]+\\/.+\\/|(?:v|e(?:mbed)?)\\/|.*[?&]v=)|youtu\\.be\\/)([^\"&?/\\s]{11})/i;\n const match = url.href.match(regExp);\n return match ? match[1] : null;\n}\n\nfunction response(message: string, contentType = \"text/markdown\"): Response {\n const headers = new Headers();\n headers.set(\"Access-Control-Allow-Origin\", \"*\");\n headers.set(\"Access-Control-Allow-Methods\", \"GET, POST, PUT, DELETE, OPTIONS\");\n headers.set(\"Access-Control-Allow-Headers\", \"Content-Type, Authorization\");\n headers.set(\"Access-Control-Max-Age\", \"86400\");\n headers.set(\"Content-Type\", contentType);\n return new Response(message, {\n status: 200,\n headers: headers,\n });\n}\n\nfunction err(msg: string): Response {\n const errorMessage = JSON.stringify({\n error: {\n message: msg,\n code: 400,\n },\n });\n return response(errorMessage, \"application/json\");\n}\n\nfunction fudgeURL(url: string) {\n try {\n return new URL(url);\n } catch (e) {\n // console.log(\"Url parsing failed\", e.stack);\n return new URL(\"https://\" + url);\n }\n}\n\nfunction processInput(req: Request) {\n let ret = {\n url: undefined as undefined | URL,\n response: undefined as undefined | Response,\n };\n const myurl = new URL(req.url);\n let pathname = myurl.pathname.substring(1) + myurl.search;\n if (!pathname.startsWith(\"http\")) {\n const urlAsFormParam = myurl.searchParams.get(\"url\");\n if (urlAsFormParam) {\n pathname = urlAsFormParam;\n } else if (pathname.length < 2) {\n ret.response = response(\n generate_ui(\n \"URL to convert to markdown:\",\n \"https://www.val.town/v/curtcox/markdown_download\",\n \"markdown.download\",\n ),\n \"text/html\",\n );\n return ret;\n }\n }\n ret.url = fudgeURL(pathname);\n return ret;\n}\n\nexport default async function(req: Request): Promise {\n const action = processInput(req);\n const url = action.url;\n if (!url) {\n return action.response!;\n }\n\n const html = await fetch(url.toString(), {\n method: req.method,\n headers: new Headers({\n \"User-Agent\":\n \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36\",\n \"Accept\": \"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8\",\n \"Accept-Language\": \"en-US,en;q=0.5\",\n \"Sec-Fetch-Site\": \"cross-site\",\n \"Sec-Fetch-Mode\": \"navigate\",\n \"Sec-Fetch-User\": \"?1\",\n \"Sec-Fetch-Dest\": \"document\",\n \"Referer\": \"https://www.google.com/\",\n \"sec-ch-ua\": `\"Not A Brand\";v=\"99\", \"Google Chrome\";v=\"91\", \"Chromium\";v=\"91\"`,\n \"sec-ch-ua-mobile\": \"?0\",\n \"sec-ch-ua-platform\": `\"macOS\"`,\n \"Upgrade-Insecure-Requests\": \"1\",\n // Add any other headers you need here\n }),\n }).then(r => r.text());\n\n let title = \"\";\n let markdown = \"\";\n const youtubeVideoID = getYoutubeVideoID(url);\n if (youtubeVideoID) {\n let transcript = \"## Generated Transcription\\n\\n\";\n try {\n const arr = (await getSubtitles({\n videoID: youtubeVideoID,\n })) as { text: string }[];\n transcript += arr.map(({ text }) => text).join(\"\\n\\n\");\n } catch (e) {\n transcript = `Failed to fetch transcript ${e}\\n`;\n }\n let header = \"\";\n try {\n const y = await YouTube.getVideo(url.toString());\n header = \"# \" + y.title + \"\\n\\n\" + y.embedHTML() + `\\n\\n

\\n`\n + y.description + \"\\n

\\n\\n\";\n } catch (e) {\n header = `Failed to fetch youtube metadata: ${e}\\n`;\n }\n markdown = header + \"\\n\" + transcript;\n } else {\n const r = await readability2markdown(html);\n title = r.title;\n markdown = r.markdown;\n }\n const markdown_extended = markdown + \"\\n\\n\" + url;\n if (req.headers.get(\"Accept\")?.includes(\"text/html\")) {\n const body = await marked.parse(markdown_extended);\n const html = `\n\n \n ${title}\n \n ${body}\n \n `;\n return response(html, \"text/html\");\n } else {\n return response(markdown_extended);\n }\n}\n\n/**\n * Simple UI that takes a url\n */\nexport function generate_ui(input_description: string, link: string, link_text: string): string {\n const html = `\n\n\n\n markdown.download\n \n \n\n\n

\n \n ${link_text}\n

\n\n\n`;\n return html;\n}","import { isProbablyReaderable, Readability } from \"npm:@mozilla/readability@^0.5.0\";\nimport { DOMParser } from \"npm:linkedom@0.16.10\";\nimport { marked } from \"npm:marked@12.0.1\";\nimport { getSubtitles } from \"npm:youtube-captions-scraper@^2.0.1\";\nimport { YouTube } from \"npm:youtube-sr@4.3.11\";\n\nconst isCloudflareWorker = typeof Request !== \"undefined\" && typeof Response !== \"undefined\";\n\n// init async loading of modules\nconst AgentMarkdownImport = isCloudflareWorker ? import(\"npm:agentmarkdown@6.0.0\") : null;\nconst TurndownService = isCloudflareWorker ? null : import(\"npm:turndown@^7.1.3\");\n\n/**\n * converts HTML to markdown\n * @returns markdown in string\n */\nexport async function html2markdown(html: string): Promise {\n if (AgentMarkdownImport) {\n // TurndownService doesn't work on cf\n // Dynamically import AgentMarkdown when running in Cloudflare Worker\n const { AgentMarkdown } = await AgentMarkdownImport;\n return await AgentMarkdown.produce(html);\n } else {\n // Dynamically import TurndownService otherwise\n return new (await TurndownService)().turndown(html);\n }\n}\n\n/**\n * extracts article from html\n * then converts it to md\n * @returns markdown in string\n */\nexport async function readability2markdown(html: string): Promise<{ title: string; markdown: string }> {\n const doc = await (new DOMParser().parseFromString(html, \"text/html\"));\n\n const reader = new Readability(doc);\n const article = reader.parse();\n\n const markdown = await html2markdown(article?.content || \"\");\n return { title: doc.title.textContent, markdown };\n}\n\nfunction getYoutubeVideoID(url: URL): string | null {\n const regExp = /(?:youtube\\.com\\/(?:[^/]+\\/.+\\/|(?:v|e(?:mbed)?)\\/|.*[?&]v=)|youtu\\.be\\/)([^\"&?/\\s]{11})/i;\n const match = url.href.match(regExp);\n return match ? match[1] : null;\n}\n\nfunction response(message: string, contentType = \"text/markdown\"): Response {\n const headers = new Headers();\n headers.set(\"Access-Control-Allow-Origin\", \"*\");\n headers.set(\"Access-Control-Allow-Methods\", \"GET, POST, PUT, DELETE, OPTIONS\");\n headers.set(\"Access-Control-Allow-Headers\", \"Content-Type, Authorization\");\n headers.set(\"Access-Control-Max-Age\", \"86400\");\n headers.set(\"Content-Type\", contentType);\n return new Response(message, {\n status: 200,\n headers: headers,\n });\n}\n\nfunction err(message: string): Response {\n const errorMessage = JSON.stringify({ error: { message, code: 400 } });\n return response(errorMessage, \"application/rss+xml\");\n}\n\nfunction fudgeURL(url: string) {\n try {\n return new URL(url);\n } catch (e) {\n // console.log(\"Url parsing failed\", e.stack);\n return new URL(\"https://\" + url);\n }\n}\n\nfunction processInput(req: Request) {\n let ret = {\n url: undefined as undefined | URL,\n response: undefined as undefined | Response,\n };\n const myurl = new URL(req.url);\n let pathname = myurl.pathname.substring(1) + myurl.search;\n if (!pathname.startsWith(\"http\")) {\n const urlAsFormParam = myurl.searchParams.get(\"url\");\n if (urlAsFormParam) {\n pathname = urlAsFormParam;\n } else if (pathname.length < 2) {\n ret.response = response(\n generate_ui(\n \"URL to convert to markdown:\",\n \"https://www.val.town/v/curtcox/markdown_download\",\n \"markdown.download\",\n ),\n \"text/html\",\n );\n return ret;\n }\n }\n ret.url = fudgeURL(pathname);\n return ret;\n}\n\nexport default async function(req: Request): Promise {\n const action = processInput(req);\n const url = action.url;\n if (!url) {\n return action.response!;\n }\n\n const html = await fetch(url.toString(), {\n method: req.method,\n headers: new Headers({\n \"User-Agent\":\n \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36\",\n \"Accept\": \"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8\",\n \"Accept-Language\": \"en-US,en;q=0.5\",\n \"Sec-Fetch-Site\": \"cross-site\",\n \"Sec-Fetch-Mode\": \"navigate\",\n \"Sec-Fetch-User\": \"?1\",\n \"Sec-Fetch-Dest\": \"document\",\n \"Referer\": \"https://www.google.com/\",\n \"sec-ch-ua\": `\"Not A Brand\";v=\"99\", \"Google Chrome\";v=\"91\", \"Chromium\";v=\"91\"`,\n \"sec-ch-ua-mobile\": \"?0\",\n \"sec-ch-ua-platform\": `\"macOS\"`,\n \"Upgrade-Insecure-Requests\": \"1\",\n // Add any other headers you need here\n }),\n }).then(r => r.text());\n\n let title = \"\";\n let markdown = \"\";\n const youtubeVideoID = getYoutubeVideoID(url);\n if (youtubeVideoID) {\n let transcript = \"## Generated Transcription\\n\\n\";\n try {\n const arr = (await getSubtitles({\n videoID: youtubeVideoID,\n })) as { text: string }[];\n transcript += arr.map(({ text }) => text).join(\"\\n\\n\");\n } catch (e) {\n transcript = `Failed to fetch transcript ${e}\\n`;\n }\n let header = \"\";\n try {\n const y = await YouTube.getVideo(url.toString());\n header = \"# \" + y.title + \"\\n\\n\" + y.embedHTML() + `\\n\\n

\\n`\n + y.description + \"\\n

\n \n ${link_text}\n

\n\n\n`;\n return html;\n}","createdAt",["D",1716694274253],"branchVersion","branchId","abb90e3c-2782-11f0-b46e-569c3dd06744","id","abb8c328-2782-11f0-b46e-569c3dd06744","TanStackBlogToRSS",["D",1746210262870],"deletedAt","legacyEnvVars",true,"privacy","public","httpPrivacy","description","imageUrl","90e1878f-b06e-484f-a360-bfac76c221fc","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","f6b419e6-73f8-49a1-bc2e-1301a91562fa","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_98":97,"_105":101},"org",{"_98":97,"_99":-5,"_100":101,"_18":55,"_102":103,"_104":25},"branch",{"_13":84,"_32":85,"_43":42,"_86":44,"_87":88,"_89":61,"_90":51,"_91":92,"_93":94,"_95":45,"_96":97},"isOwned",31,"main","projectId","forkedBranchId","f5c5e82a-27a9-11f0-9edc-569c3dd06744","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","f5c5a680-27a9-11f0-9edc-569c3dd06744","forkedProjectOrgId","f5b49853-c2ee-4295-85e9-f0a0c32af35f","projectName","userHandle","vogelino","handle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18yVG9JT3BsTXBxRElDdVBVbVZWdnlFWDAybUkucG5nIn0","tier","free","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_43":44,"_32":45,"_38":174,"_47":-5,"_48":49,"_50":51,"_52":51,"_53":-5,"_54":-5,"_18":55,"_56":31,"_57":49,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":175,"_79":176,"_81":177,"_83":25},"tree",[165,166],"files",[165,166],"counts",{"_155":61,"_156":146,"_157":146,"_158":159,"_160":161},"contents",[140,141],"upstreamProject",{"_43":92,"_32":137,"_53":-5,"_54":-5,"_50":51,"_77":138},"upstreamBranch",{"_43":88,"_38":130,"_131":132,"_32":85,"_86":92,"_13":14,"_87":133,"_89":61,"_134":135,"_47":-5,"_83":25,"_50":51,"_96":136,"_62":94,"_95":137},"emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},["D",1746227137479],"updatedAt",["D",1715799291658],"399f39e6-2784-11f0-b46e-569c3dd06744","lockfileKey","1746308023154-esm.town_v_curtcox_markdown_download@2-main","curtcox","markdown_download",{"_98":136,"_105":139},"https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18yYlhYcFBTUEY0bVhuenlHd1pCbmpYckRLZlEifQ",{"_43":149,"_32":150,"_30":151,"_143":-5,"_41":42,"_40":146,"_144":49,"_38":152,"_13":61,"_33":20,"_35":153,"_24":25,"_147":-5,"_27":154},{"_43":142,"_32":28,"_30":31,"_143":-5,"_41":42,"_40":14,"_144":25,"_38":145,"_13":146,"_33":34,"_35":37,"_24":25,"_147":-5,"_27":148},"abc1b4ce-2782-11f0-b46e-569c3dd06744","parentId","latest",["D",1716694274253],1,"label",[28],"abbc1262-2782-11f0-b46e-569c3dd06744","README.md","abbc12da-2782-11f0-b46e-569c3dd06744",["D",1716681815628],"# TanStack Blog to RSS\n\nHandy microservice/library to convert the TanStack Blog into an RSS Feed. Froked from curtcox/markdown_dowload. \nThe idea is to generate an RSS feed from a blog or page that doesn't provide its own. I takes the logic of the forked val, which is to convert any URL to markdown, and places it into an RSS feed for easy subscription in feed.ly.\n\n## 👇 --- ORIGINAL README -- 👇\n\nIntroductory blog post: https://taras.glek.net/post/markdown.download/\n\nPackage: https://jsr.io/@tarasglek/markdown-download\n\n### Features\n- Apply readability\n- Further convert article into markdown to simplify it\n- Allow webpages to be viewable as markdown via curl\n- Serve markdown converted to html to browsers\n- Extract youtube subtitles\n\n### Source\n\nhttps://github.com/tarasglek/markdown-download\n\nhttps://www.val.town/v/taras/markdown_download\n\nLicense: MIT\n\nUsage: \n https://markdown.download/ + URL\n\nDev:\n https://val.markdown.download/ + URL\n\n\n```",[150],"pulls","branches","forks","envVars",{"_162":61,"_164":61},"connections",{"_162":61,"_163":61},"count","unfilledCount","emptyCount",{"_43":149,"_32":150,"_30":151,"_143":-5,"_41":42,"_40":146,"_144":49,"_38":171,"_13":61,"_33":20,"_35":153,"_24":25,"_147":-5,"_168":172,"_27":173},{"_43":142,"_32":28,"_30":31,"_143":-5,"_41":42,"_40":14,"_144":25,"_38":167,"_13":146,"_33":34,"_35":37,"_24":25,"_147":-5,"_168":169,"_27":170},["D",1716694274253],"children",[],[28],["D",1716681815628],[],[150],["D",1746210262870],{"_98":97,"_105":101},{"_98":97,"_99":-5,"_100":101,"_18":55,"_102":103,"_104":25},{"_13":84,"_32":85,"_43":42,"_86":44,"_87":88,"_89":61,"_90":51,"_91":92,"_93":94,"_95":45,"_96":97},"ctx",{"_77":-5,"_127":211,"_212":213,"_214":215,"_216":25,"_217":25,"_218":25,"_219":49,"_220":-5,"_221":-7,"_222":-7,"_223":61,"_224":25,"_225":25,"_226":-5,"_227":25,"_228":25,"_229":25,"_230":-5},"ENV",{"_182":183,"_184":185,"_186":85,"_187":188,"_189":190,"_191":192,"_193":194,"_195":196,"_197":198,"_199":200,"_201":202,"_203":204,"_205":206,"_207":208,"_209":210},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","5a94df3abe1fc7a20cd77b19bfb04bf16e60a929","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v2