scrape2md ui

[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_163":164,"_165":166},"routes/_app.x.$handle.$name",{"_94":93,"_32":44,"_102":84,"_27":103,"_13":14,"_104":-5,"_11":105,"_106":107,"_108":109,"_110":111,"_112":113,"_20":-5,"_114":-5,"_115":-5,"_116":25,"_117":118,"_119":48,"_120":-5,"_121":-5,"_122":123},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_42":43,"_32":44,"_37":45,"_46":-5,"_47":48,"_49":50,"_51":50,"_52":-5,"_53":-5,"_18":54,"_55":31,"_56":48,"_57":-5,"_58":-5,"_59":60,"_61":62,"_63":-5,"_64":-5,"_65":-5,"_66":-5,"_67":68,"_69":68,"_70":71,"_72":71,"_73":71,"_74":68,"_75":68,"_76":77,"_78":79,"_80":81,"_82":25},"version",66,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_33":34,"_35":36,"_37":38,"_39":14,"_40":41,"_24":25},"previous",{"_30":31,"_32":28,"_33":34,"_35":36},"deleted",false,"deletedHere","path","main.tsx","latestPath","fileId","201736ae-2784-11f0-b46e-569c3dd06744","name","fileType","http","content","import { marked } from \"https://deno.land/x/marked/mod.ts\";\nimport { DOMParser } from \"https://esm.sh/linkedom\";\nimport { isProbablyReaderable, Readability } from \"npm:@mozilla/readability@^0.5.0\";\nimport TurndownService from \"npm:turndown@^7.1.2\";\nimport { getSubtitles } from \"npm:youtube-captions-scraper@^2.0.1\";\n\nfunction getYoutubeVideoID(url: URL): string | null {\n const regExp = /(?:youtube\\.com\\/(?:[^/]+\\/.+\\/|(?:v|e(?:mbed)?)\\/|.*[?&]v=)|youtu\\.be\\/)([^\"&?/\\s]{11})/i;\n const match = url.href.match(regExp);\n return match ? match[1] : null;\n}\n\nfunction response(message: string, contentType = \"text/markdown\"): Response {\n const headers = new Headers();\n headers.set(\"Access-Control-Allow-Origin\", \"*\");\n headers.set(\"Access-Control-Allow-Methods\", \"GET, POST, PUT, DELETE, OPTIONS\");\n headers.set(\"Access-Control-Allow-Headers\", \"Content-Type, Authorization\");\n headers.set(\"Access-Control-Max-Age\", \"86400\");\n headers.set(\"Content-Type\", contentType);\n return new Response(message, {\n status: 200,\n headers: headers,\n });\n}\n\nfunction err(msg: string): Response {\n const errorMessage = JSON.stringify({\n error: {\n message: msg,\n code: 400,\n },\n });\n return response(errorMessage, \"application/json\");\n}\n\nexport default async function(req: Request): Promise {\n const myurl = new URL(req.url);\n let pathname = myurl.pathname.substring(1) + myurl.search;\n if (!pathname.startsWith(\"http\")) {\n const urlAsFormParam = myurl.searchParams.get(\"url\");\n if (urlAsFormParam) {\n pathname = urlAsFormParam;\n } else {\n return response(html, \"text/html\");\n }\n }\n const url = new URL(pathname);\n\n const youtubeVideoID = getYoutubeVideoID(url);\n console.log(myurl, url, youtubeVideoID);\n if (youtubeVideoID) {\n const arr = (await getSubtitles({\n videoID: youtubeVideoID,\n })) as { text: string }[];\n const description = \"## Generated Transcription\\n\\n\"\n + arr.map(({ text }) => text).join(\"\\n\");\n return response(description);\n }\n\n const dom_promise = fetch(url.toString(), {\n method: req.method,\n headers: new Headers({\n \"User-Agent\":\n \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36\",\n \"Accept\": \"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8\",\n \"Accept-Language\": \"en-US,en;q=0.5\",\n \"Sec-Fetch-Site\": \"cross-site\",\n \"Sec-Fetch-Mode\": \"navigate\",\n \"Sec-Fetch-User\": \"?1\",\n \"Sec-Fetch-Dest\": \"document\",\n \"Referer\": \"https://www.google.com/\",\n \"sec-ch-ua\": `\"Not A Brand\";v=\"99\", \"Google Chrome\";v=\"91\", \"Chromium\";v=\"91\"`,\n \"sec-ch-ua-mobile\": \"?0\",\n \"sec-ch-ua-platform\": `\"macOS\"`,\n \"Upgrade-Insecure-Requests\": \"1\",\n // Add any other headers you need here\n }),\n })\n .then(r => r.text())\n .then(async html => new DOMParser().parseFromString(html, \"text/html\"));\n const doc = await dom_promise;\n\n const reader = new Readability(doc);\n const article = reader.parse();\n\n const markdown = url + \"\\n\" + new TurndownService().turndown(article?.content || \"\");\n\n if (req.headers.get(\"Accept\")?.includes(\"text/html\")) {\n return response(await marked.parse(markdown), \"text/html\");\n } else {\n return response(markdown);\n }\n}\n\nconst html = `\n\n\n\n scrape2md ui\n \n \n\n\n

\n \n scrape2md\n

\n \n\n\n`;","createdAt",["D",1710710920053],"branchVersion","branchId","201461c2-2784-11f0-b46e-569c3dd06744","id","201411a4-2784-11f0-b46e-569c3dd06744","scrape2md",["D",1746210887581],"deletedAt","legacyEnvVars",true,"privacy","public","httpPrivacy","description","imageUrl","9416cc25-0df9-471f-98f1-36648a8b3220","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",1,"orgId","76c8eb7e-9c0e-4fc4-8e76-59b97fa16865","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_94":93,"_101":97},"org",{"_94":93,"_95":-5,"_96":97,"_18":54,"_98":99,"_100":25},"branch",{"_13":83,"_32":84,"_42":41,"_85":43,"_86":-5,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":44,"_92":93},"isOwned",102,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","taras","handle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18yYmpLdHhWY2t6b1lwdXlxZUpTc09WUDVZQngifQ","tier","free","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_42":43,"_32":44,"_37":159,"_46":-5,"_47":48,"_49":50,"_51":50,"_52":-5,"_53":-5,"_18":54,"_55":31,"_56":48,"_57":-5,"_58":-5,"_59":60,"_61":62,"_63":-5,"_64":-5,"_65":-5,"_66":-5,"_67":68,"_69":68,"_70":71,"_72":71,"_73":71,"_74":68,"_75":68,"_76":160,"_78":161,"_80":162,"_82":25},"tree",[150,151],"files",[150,151],"counts",{"_140":137,"_141":60,"_142":60,"_143":144,"_145":146},"contents",[124,125],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_42":133,"_32":134,"_30":135,"_127":-5,"_40":41,"_39":60,"_128":48,"_37":136,"_13":137,"_33":20,"_35":138,"_24":25,"_131":-5,"_27":139},{"_42":126,"_32":28,"_30":31,"_127":-5,"_40":41,"_39":14,"_128":25,"_37":129,"_13":130,"_33":34,"_35":36,"_24":25,"_131":-5,"_27":132},"218a6e5c-2784-11f0-b46e-569c3dd06744","parentId","latest",["D",1710710920053],65,"label",[28],"201753be-2784-11f0-b46e-569c3dd06744","README.md","20175454-2784-11f0-b46e-569c3dd06744",["D",1708160544547],0,"This is a deno/valtown port in progress of https://github.com/tarasglek/scrape2md\n\nLicense: MIT\n\nHandy script to scrape various data sources into markdown. Intended to feed llms in https://chatcraft.org\n\nUsage: \n https://taras-scrape2md.web.val.run/ + URL_TO_SCRAPE\n\nOr just visit in browser and paste your url\n\n### TODO\nhttps://chatcraft.org/api/share/tarasglek/IDYChVAilfePgVZb_T5pH POST from browser\nhttps://www.val.town/v/nbbaier/valToGH sync to github\n\nMetadata for use with https://github.com/tarasglek/valtown2js:\n```json\n{\n \"typeCheck\": false,\n \"mappings\": {\n \"https://esm.sh/linkedom\": {\n \"name\": \"linkedom\",\n \"version\": \"^0.16.8\"\n }\n },\n \"package\": {\n \"name\": \"scrape2md\",\n \"version\": \"1.0.0\",\n \"devDependencies\": {\n \"@types/turndown\": \"^5.0.4\"\n }\n }\n}\n```",[134],"pulls","branches","forks","envVars",{"_147":137,"_149":137},"connections",{"_147":137,"_148":137},"count","unfilledCount","emptyCount",{"_42":133,"_32":134,"_30":135,"_127":-5,"_40":41,"_39":60,"_128":48,"_37":156,"_13":137,"_33":20,"_35":138,"_24":25,"_131":-5,"_153":157,"_27":158},{"_42":126,"_32":28,"_30":31,"_127":-5,"_40":41,"_39":14,"_128":25,"_37":152,"_13":130,"_33":34,"_35":36,"_24":25,"_131":-5,"_153":154,"_27":155},["D",1710710920053],"children",[],[28],["D",1708160544547],[],[134],["D",1746210887581],{"_94":93,"_101":97},{"_94":93,"_95":-5,"_96":97,"_18":54,"_98":99,"_100":25},{"_13":83,"_32":84,"_42":41,"_85":43,"_86":-5,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":44,"_92":93},"ctx",{"_76":-5,"_121":196,"_197":198,"_199":200,"_201":25,"_202":25,"_203":25,"_204":48,"_205":-5,"_206":-7,"_207":-7,"_208":137,"_209":25,"_210":25,"_211":-5,"_212":25,"_213":25,"_214":25,"_215":-5},"ENV",{"_167":168,"_169":170,"_171":84,"_172":173,"_174":175,"_176":177,"_178":179,"_180":181,"_182":183,"_184":185,"_186":187,"_188":189,"_190":191,"_192":193,"_194":195},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","a3e067d4463eea85f6334f8e583e8516ae58c4f6","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v66