scrape2md ui

import { isProbablyReaderable, Readability } from "npm:@mozilla/readability@^0.5.0"; import { DOMParser } from "npm:linkedom@0.16.10"; import { marked } from "npm:marked@12.0.1"; import TurndownService from "npm:turndown@^7.1.3"; import { getSubtitles } from "npm:youtube-captions-scraper@^2.0.1"; function getYoutubeVideoID(url: URL): string | null { const regExp = /(?:youtube\.com\/(?:[^/]+\/.+\/|(?:v|e(?:mbed)?)\/|.*[?&]v=)|youtu\.be\/)([^"&?/\s]{11})/i; const match = url.href.match(regExp); return match ? match[1] : null; } function response(message: string, contentType = "text/markdown"): Response { const headers = new Headers(); headers.set("Access-Control-Allow-Origin", "*"); headers.set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS"); headers.set("Access-Control-Allow-Headers", "Content-Type, Authorization"); headers.set("Access-Control-Max-Age", "86400"); headers.set("Content-Type", contentType); return new Response(message, { status: 200, headers: headers, }); } function err(msg: string): Response { const errorMessage = JSON.stringify({ error: { message: msg, code: 400, }, }); return response(errorMessage, "application/json"); } export default async function(req: Request): Promise { const myurl = new URL(req.url); let pathname = myurl.pathname.substring(1) + myurl.search; if (!pathname.startsWith("http")) { const urlAsFormParam = myurl.searchParams.get("url"); if (urlAsFormParam) { pathname = urlAsFormParam; } else { return response(html, "text/html"); } } const url = new URL(pathname); const youtubeVideoID = getYoutubeVideoID(url); if (youtubeVideoID) { const arr = (await getSubtitles({ videoID: youtubeVideoID, })) as { text: string }[]; const description = "## Generated Transcription\n\n" + arr.map(({ text }) => text).join("\n"); return response(description); } const fetch_response = await fetch(url.toString(), { method: req.method, headers: new Headers({ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", "Sec-Fetch-Site": "cross-site", "Sec-Fetch-Mode": "navigate", "Sec-Fetch-User": "?1", "Sec-Fetch-Dest": "document", "Referer": "https://www.google.com/", "sec-ch-ua": `"Not A Brand";v="99", "Google Chrome";v="91", "Chromium";v="91"`, "sec-ch-ua-mobile": "?0", "sec-ch-ua-platform": `"macOS"`, "Upgrade-Insecure-Requests": "1", // Add any other headers you need here }), }); const contentType = fetch_response.headers.get("content-type"); const response_text = await fetch_response.text(); // We only handle html, pass content through otherwise if (!contentType?.includes("text/html")) { return response(response_text, contentType); } const doc = new DOMParser().parseFromString(response_text, "text/html"); const reader = new Readability(doc); const article = reader.parse(); console.log("content", typeof article?.content, article?.content); const markdown = new TurndownService().turndown(article?.content || "") + "\n\n" + url; if (req.headers.get("Accept")?.includes("text/html")) { return response(await marked.parse(markdown), "text/html"); } else { return response(markdown); } } const html = ` scrape2md ui

scrape2md