markdown.download

import { isProbablyReaderable, Readability } from "npm:@mozilla/readability@^0.5.0"; import { DOMParser } from "npm:linkedom@0.16.10"; import { marked } from "npm:marked@12.0.1"; import { getSubtitles } from "npm:youtube-captions-scraper@^2.0.1"; import { YouTube } from "npm:youtube-sr@4.3.11"; const isCloudflareWorker = typeof Request !== "undefined" && typeof Response !== "undefined"; // import { convertHtmlToMarkdown } from ; // init async loading of modules const AgentMarkdownImport = isCloudflareWorker ? import("npm:agentmarkdown@6.0.0") : null; const dom2semanticMarkdown = isCloudflareWorker ? null : await import("npm:dom-to-semantic-markdown@1.2.4") /** * converts HTML to markdown * @returns markdown in string */ export async function html2markdown(html: string): Promise { if (AgentMarkdownImport) { // TurndownService doesn't work on cf // Dynamically import AgentMarkdown when running in Cloudflare Worker const { AgentMarkdown } = await AgentMarkdownImport; return await AgentMarkdown.produce(html); } else { const ret = (await dom2semanticMarkdown).convertHtmlToMarkdown(html, { overrideDOMParser: new DOMParser() as any }); console.log(ret); return ret; } } /** * extracts article from html * then converts it to md * @returns markdown in string */ export async function readability2markdown(html: string): Promise<{ title: string; markdown: string }> { const doc = await (new DOMParser().parseFromString(html, "text/html")); const reader = new Readability(doc); const article = reader.parse(); const markdown = await html2markdown(article?.content || ""); return { title: doc.title.textContent || "", markdown }; } function response(message: string, contentType = "text/markdown"): Response { const headers = new Headers(); headers.set("Access-Control-Allow-Origin", "*"); headers.set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS"); headers.set("Access-Control-Allow-Headers", "Content-Type, Authorization"); headers.set("Access-Control-Max-Age", "86400"); // Check if charset is already specified in the contentType if (!contentType.toLowerCase().includes("charset=")) { contentType = `${contentType}; charset=utf-8`; } headers.set("Content-Type", contentType); return new Response(message, { status: 200, headers: headers, }); } function err(msg: string): Response { const errorMessage = JSON.stringify({ error: { message: msg, code: 400, }, }); return response(errorMessage, "application/json"); } function fudgeURL(url: string) { try { return new URL(url); } catch (e) { // console.log("Url parsing failed", e.stack); return new URL("https://" + url); } } async function processInput(req: Request) { let ret = { url: undefined as undefined | URL, response: undefined as undefined | Response, htmlContent: undefined as undefined | string, }; const myurl = new URL(req.url); let pathname = myurl.pathname.substring(1) + myurl.search; if (req.method === "POST") { const formData = new URLSearchParams(await req.text()); const url = formData.get("url"); const htmlContent = formData.get("htmlContent"); if (htmlContent) { ret.htmlContent = htmlContent; return ret; } if (url) { pathname = url; } } if (!pathname.startsWith("http")) { const urlAsFormParam = myurl.searchParams.get("url"); if (urlAsFormParam) { pathname = urlAsFormParam; } else if (pathname.length < 2) { ret.response = response( generate_ui( "URL to convert to markdown:", "https://www.val.town/v/taras/markdown_download", "markdown.download", ), "text/html", ); return ret; } } ret.url = fudgeURL(pathname); return ret; } export async function youtube2markdown(url: URL): Promise { function getYoutubeVideoID(url: URL): string | null { const regExp = /(?:youtube\.com\/(?:[^/]+\/.+\/|(?:v|e(?:mbed)?)\/|.*[?&]v=)|youtu\.be\/)([^"&?/\s]{11})/i; const match = url.href.match(regExp); return match ? match[1] : null; } const youtubeVideoID = getYoutubeVideoID(url); if (!youtubeVideoID) return undefined; let transcript = "## Generated Transcription\n\n"; try { const arr = (await getSubtitles({ videoID: youtubeVideoID, })) as { text: string }[]; transcript += arr.map(({ text }) => text).join("\n\n"); } catch (e) { transcript = `Failed to fetch transcript ${e}\n`; } let header = ""; try { const y = await YouTube.getVideo(url.toString()); header = "# " + y.title + "\n\n" + y.embedHTML() + `\n\n

\n`
      + y.description + "\n

\n\n"; } catch (e) { header = `Failed to fetch youtube metadata: ${e}\n`; } return header + "\n" + transcript; } /** * Simple UI that takes a url */ export function generate_ui(input_description: string, link: string, link_text: string): string { const html = ` markdown.download

${link_text}

`; return html; } export default async function(req: Request): Promise { const action = await processInput(req); const url = action.url; if (action.response) { return action.response!; } let html = action.htmlContent; let fetch_response; if (!html) { fetch_response = await fetch(url.toString(), { method: req.method, headers: new Headers({ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", "Sec-Fetch-Site": "cross-site", "Sec-Fetch-Mode": "navigate", "Sec-Fetch-User": "?1", "Sec-Fetch-Dest": "document", "Referer": "https://www.google.com/", "sec-ch-ua": `"Not A Brand";v="99", "Google Chrome";v="91", "Chromium";v="91"`, "sec-ch-ua-mobile": "?0", "sec-ch-ua-platform": `"macOS"`, "Upgrade-Insecure-Requests": "1", // Add any other headers you need here }), }); console.log(fetch_response); html = await fetch_response.text(); } let title = ""; let markdown = ""; const youtube_markdown = url ? await youtube2markdown(url) : undefined; if (youtube_markdown) { markdown = youtube_markdown; } else { const r = await html2markdown(html); title = ""; markdown = r; } let markdown_extended = markdown + "\n\n" + url; if (fetch_response && Math.floor(fetch_response.status / 200) * 200 != 200 || markdown === "") { const jina_url = `https://r.jina.ai/${url.toString()}`; console.log(`jina fallback:${jina_url}`); const fallback_fetch_response = await fetch(jina_url); markdown_extended = await fallback_fetch_response.text(); } if (req.headers.get("Accept")?.includes("text/html")) { const body = await marked.parse(markdown_extended); const html = ` ${title} ${body} `; return response(html, "text/html"); } else { return response(markdown_extended); } }

scrape2md