scrape2md ui

[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_155":156,"_157":158},"routes/_app.x.$handle.$name",{"_11":12,"_13":14,"_15":-7,"_18":19,"_17":-7,"_20":120,"_22":121,"_122":123,"_124":125,"_26":126,"_24":127,"_128":-5,"_129":-5,"_130":52,"_131":132,"_133":52},"routes/_app.x.$handle.$name.code.$",{"_11":12,"_13":14,"_15":16,"_17":-7,"_18":19,"_20":21,"_22":23,"_24":25,"_26":27,"_28":-5,"_29":-5,"_30":-5,"_31":-5,"_32":-5},"handle","taras","name","scrape2md","branchName","main","version","path","main.tsx","project",{"_33":70,"_13":14,"_44":71,"_72":-5,"_73":43,"_74":75,"_76":-5,"_77":-5,"_78":79,"_80":36,"_81":43,"_82":-5,"_83":-5,"_84":64,"_85":86,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":92,"_93":92,"_94":95,"_96":95,"_97":95,"_98":92,"_99":92,"_100":101,"_102":103,"_104":105,"_106":52},"tree",[55,56],"file",{"_33":34,"_13":19,"_35":36,"_37":-5,"_38":39,"_40":41,"_42":43,"_44":45,"_17":46,"_47":48,"_49":50,"_51":52,"_53":-5,"_18":54},"contents",[],"upstream","pull","ahead","behind","envVars","id","2258ca68-2784-11f0-b46e-569c3dd06744","fileId","201736ae-2784-11f0-b46e-569c3dd06744","parentId","branchId","201461c2-2784-11f0-b46e-569c3dd06744","branchVersion",102,"latest",true,"createdAt",["D",1725873826897],101,"fileType","http","content","import { isProbablyReaderable, Readability } from \"npm:@mozilla/readability@^0.5.0\";\nimport { DOMParser } from \"npm:linkedom@0.16.10\";\nimport { marked } from \"npm:marked@12.0.1\";\nimport TurndownService from \"npm:turndown@^7.1.3\";\nimport { getSubtitles } from \"npm:youtube-captions-scraper@^2.0.1\";\n\nfunction getYoutubeVideoID(url: URL): string | null {\n const regExp = /(?:youtube\\.com\\/(?:[^/]+\\/.+\\/|(?:v|e(?:mbed)?)\\/|.*[?&]v=)|youtu\\.be\\/)([^\"&?/\\s]{11})/i;\n const match = url.href.match(regExp);\n return match ? match[1] : null;\n}\n\nfunction response(message: string, contentType = \"text/markdown\"): Response {\n const headers = new Headers();\n headers.set(\"Access-Control-Allow-Origin\", \"*\");\n headers.set(\"Access-Control-Allow-Methods\", \"GET, POST, PUT, DELETE, OPTIONS\");\n headers.set(\"Access-Control-Allow-Headers\", \"Content-Type, Authorization\");\n headers.set(\"Access-Control-Max-Age\", \"86400\");\n headers.set(\"Content-Type\", contentType);\n return new Response(message, {\n status: 200,\n headers: headers,\n });\n}\n\nfunction err(msg: string): Response {\n const errorMessage = JSON.stringify({\n error: {\n message: msg,\n code: 400,\n },\n });\n return response(errorMessage, \"application/json\");\n}\n\nexport default async function(req: Request): Promise {\n const myurl = new URL(req.url);\n let pathname = myurl.pathname.substring(1) + myurl.search;\n if (!pathname.startsWith(\"http\")) {\n const urlAsFormParam = myurl.searchParams.get(\"url\");\n if (urlAsFormParam) {\n pathname = urlAsFormParam;\n } else {\n return response(html, \"text/html\");\n }\n }\n const url = new URL(pathname);\n\n const youtubeVideoID = getYoutubeVideoID(url);\n if (youtubeVideoID) {\n const arr = (await getSubtitles({\n videoID: youtubeVideoID,\n })) as { text: string }[];\n const description = \"## Generated Transcription\\n\\n\"\n + arr.map(({ text }) => text).join(\"\\n\");\n return response(description);\n }\n\n const fetch_response = await fetch(url.toString(), {\n method: req.method,\n headers: new Headers({\n \"User-Agent\":\n \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36\",\n \"Accept\": \"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8\",\n \"Accept-Language\": \"en-US,en;q=0.5\",\n \"Sec-Fetch-Site\": \"cross-site\",\n \"Sec-Fetch-Mode\": \"navigate\",\n \"Sec-Fetch-User\": \"?1\",\n \"Sec-Fetch-Dest\": \"document\",\n \"Referer\": \"https://www.google.com/\",\n \"sec-ch-ua\": `\"Not A Brand\";v=\"99\", \"Google Chrome\";v=\"91\", \"Chromium\";v=\"91\"`,\n \"sec-ch-ua-mobile\": \"?0\",\n \"sec-ch-ua-platform\": `\"macOS\"`,\n \"Upgrade-Insecure-Requests\": \"1\",\n // Add any other headers you need here\n }),\n });\n const contentType = fetch_response.headers.get(\"content-type\");\n const response_text = await fetch_response.text();\n // We only handle html, pass content through otherwise\n if (!contentType?.includes(\"text/html\")) {\n return response(response_text, contentType);\n }\n const doc = new DOMParser().parseFromString(response_text, \"text/html\");\n\n const reader = new Readability(doc);\n const article = reader.parse();\n\n console.log(\"content\", typeof article?.content, article?.content);\n const markdown = new TurndownService().turndown(article?.content || \"\") + \"\\n\\n\" + url;\n\n if (req.headers.get(\"Accept\")?.includes(\"text/html\")) {\n return response(await marked.parse(markdown), \"text/html\");\n } else {\n return response(markdown);\n }\n}\n\nconst html = `\n\n\n\n scrape2md ui\n \n \n\n\n

\n \n scrape2md\n

\n \n\n\n`;","deleted",false,"label",[19],{"_33":61,"_13":62,"_35":63,"_37":-5,"_38":39,"_40":64,"_42":43,"_44":65,"_17":66,"_47":24,"_49":67,"_51":52,"_53":-5,"_58":68,"_18":69},{"_33":34,"_13":19,"_35":36,"_37":-5,"_38":39,"_40":41,"_42":43,"_44":57,"_17":46,"_47":48,"_49":50,"_51":52,"_53":-5,"_58":59,"_18":60},["D",1725873826897],"children",[],[19],"201753be-2784-11f0-b46e-569c3dd06744","README.md","20175454-2784-11f0-b46e-569c3dd06744",1,["D",1708160544547],0,"This is a deno/valtown port in progress of https://github.com/tarasglek/scrape2md\n\nLicense: MIT\n\nHandy script to scrape various data sources into markdown. Intended to feed llms in https://chatcraft.org\n\nUsage: \n https://taras-scrape2md.web.val.run/ + URL_TO_SCRAPE\n\nOr just visit in browser and paste your url\n\n### TODO\nhttps://chatcraft.org/api/share/tarasglek/IDYChVAilfePgVZb_T5pH POST from browser\nhttps://www.val.town/v/nbbaier/valToGH sync to github\n\nMetadata for use with https://github.com/tarasglek/valtown2js:\n```json\n{\n \"typeCheck\": false,\n \"mappings\": {\n \"https://esm.sh/linkedom\": {\n \"name\": \"linkedom\",\n \"version\": \"^0.16.8\"\n }\n },\n \"package\": {\n \"name\": \"scrape2md\",\n \"version\": \"1.0.0\",\n \"devDependencies\": {\n \"@types/turndown\": \"^5.0.4\"\n }\n }\n}\n```",[],[62],"201411a4-2784-11f0-b46e-569c3dd06744",["D",1746210887581],"deletedAt","legacyEnvVars","privacy","public","description","imageUrl","userId","9416cc25-0df9-471f-98f1-36648a8b3220","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount","orgId","76c8eb7e-9c0e-4fc4-8e76-59b97fa16865","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_11":12,"_119":116},"org",{"_11":12,"_115":116,"_78":79,"_117":118},"branch",{"_17":41,"_13":16,"_33":39,"_107":70,"_108":-5,"_109":-5,"_110":-5,"_111":-5,"_112":-5,"_113":14,"_114":12},"isOwned","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","avatar","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18yYmpLdHhWY2t6b1lwdXlxZUpTc09WUDVZQngifQ","tier","free","profileImageUrl",{"_33":70,"_13":14,"_44":151,"_72":-5,"_73":43,"_74":75,"_76":-5,"_77":-5,"_78":79,"_80":36,"_81":43,"_82":-5,"_83":-5,"_84":64,"_85":86,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":92,"_93":92,"_94":95,"_96":95,"_97":95,"_98":92,"_99":92,"_100":152,"_102":153,"_104":154,"_106":52},[145,146],"files",[145,146],"counts",{"_136":66,"_137":64,"_138":64,"_32":139,"_140":141},[],{"_33":34,"_38":39,"_35":36,"_13":19,"_49":50,"_47":48,"_17":46,"_37":-5,"_44":134,"_40":41,"_53":-5,"_18":135},"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"navCollapsed",["D",1725873826897],[19],"pulls","branches","forks",{"_142":66,"_144":66},"connections",{"_142":66,"_143":66},"count","unfilledCount","emptyCount",{"_33":61,"_13":62,"_35":63,"_37":-5,"_38":39,"_40":64,"_42":43,"_44":65,"_17":66,"_47":24,"_49":67,"_51":52,"_53":-5,"_58":149,"_18":150},{"_33":34,"_13":19,"_35":36,"_37":-5,"_38":39,"_40":41,"_42":43,"_44":57,"_17":46,"_47":48,"_49":50,"_51":52,"_53":-5,"_58":147,"_18":148},[],[19],[],[62],["D",1746210887581],{"_11":12,"_119":116},{"_11":12,"_115":116,"_78":79,"_117":118},{"_17":41,"_13":16,"_33":39,"_107":70,"_108":-5,"_109":-5,"_110":-5,"_111":-5,"_112":-5,"_113":14,"_114":12},"ctx",{"_100":-5,"_188":189,"_190":191,"_192":52,"_193":52,"_194":52,"_195":-5,"_196":-7,"_197":-7,"_198":66,"_199":52,"_200":52,"_201":-5,"_202":52,"_203":52},"ENV",{"_159":160,"_161":162,"_163":164,"_165":16,"_166":167,"_168":169,"_170":171,"_172":173,"_174":175,"_176":177,"_178":179,"_180":181,"_182":183,"_184":185,"_186":187},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","VAL_RUNTIME_NAME","prod","RENDER_GIT_COMMIT","f1316bb1ce3ad41ce227a17bcad7725ad044419f","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","API_URL","https://api.val.town","preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier"]