scrape2md ui

\n \n scrape2md\n

\n \n\n\n`;","createdAt",["D",1725873826897],"version",101,"currentUserHasLiked","likeCount",1,"forkCount","referenceCount","dependencyCount","commentCount","prCount","mergedPr",{"_134":135,"_136":137},"handle","taras","profileImageUrl","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18yYmpLdHhWY2t6b1lwdXlxZUpTc09WUDVZQngifQ","forkedVal","fileId","interval","slug","taras/scrape2md","description","This is a deno/valtown port in progress of https://github.com/tarasglek/scrape2md License: MIT Handy script to scrape various data sources into markdown. Intended to feed llms in https://chatcraft.org Usage:\nhttps://taras-scrape2md.web.val.run/ + URL_TO_SCRAPE Or just visit in browser and paste your url TODO https://chatcraft.org/api/share/tarasglek/IDYChVAilfePgVZb_T5pH POST from browser\nhttps://www.val.town/v/nbbaier/valToGH sync to github Metadata for use with https://github.com/tarasglek/valtown2js: {\n \"typeCheck\": false,\n \"mappings\": {\n \"https://esm.sh/linkedom\": {\n \"name\": \"linkedom\",\n \"version\": \"^0.16.8\"\n }\n },\n \"package\": {\n \"name\": \"scrape2md\",\n \"version\": \"1.0.0\",\n \"devDependencies\": {\n \"@types/turndown\": \"^5.0.4\"\n }\n }\n}","versions",[147,150,154,158,162,166,170,174,178,182,186,190,194,198,202,206,210,214,218,222,226,230,234,238,242,246,250,254,258,262,266,270,274,278,282,286,290,294,298,302,306,310,314,318,322,326,330,334,338,342,346,350,354,358,362,366,370,374,378,382,386,390,394,398,402,406,410,414,418,422,426,430,434,438,442,446,450,454,458,462,466,470,474,478,482,486,490,494,498,502,506,510,514,518,522,526,530,534,538,542,545,548],{"_122":123,"_148":-5,"_120":149,"_117":115},"prId",["D",1725873826897],{"_122":151,"_148":-5,"_120":152,"_117":153},100,["D",1725873568260],"9d23ec98-6e8c-11ef-a6fb-de64eea55b61",{"_122":155,"_148":-5,"_120":156,"_117":157},99,["D",1725873387832],"3198de2a-6e8c-11ef-a96c-de64eea55b61",{"_122":159,"_148":-5,"_120":160,"_117":161},98,["D",1711295216761],"be838bbc-e9f5-11ee-8151-8aad37b30b4b",{"_122":163,"_148":-5,"_120":164,"_117":165},97,["D",1711295208816],"b9c74924-e9f5-11ee-a71c-8aad37b30b4b",{"_122":167,"_148":-5,"_120":168,"_117":169},96,["D",1711295202642],"b61927b6-e9f5-11ee-8ba8-8aad37b30b4b",{"_122":171,"_148":-5,"_120":172,"_117":173},95,["D",1711295101078],"798fc750-e9f5-11ee-825d-8aad37b30b4b",{"_122":175,"_148":-5,"_120":176,"_117":177},94,["D",1711295097459],"776784cc-e9f5-11ee-a560-8aad37b30b4b",{"_122":179,"_148":-5,"_120":180,"_117":181},93,["D",1711295097036],"77270582-e9f5-11ee-a560-8aad37b30b4b",{"_122":183,"_148":-5,"_120":184,"_117":185},92,["D",1711295035694],"5296eb42-e9f5-11ee-9456-8aad37b30b4b",{"_122":187,"_148":-5,"_120":188,"_117":189},91,["D",1711295032746],"50d50e9c-e9f5-11ee-a560-8aad37b30b4b",{"_122":191,"_148":-5,"_120":192,"_117":193},90,["D",1711294966626],"296c1878-e9f5-11ee-8ae2-8aad37b30b4b",{"_122":195,"_148":-5,"_120":196,"_117":197},89,["D",1711294963282],"276dc738-e9f5-11ee-a560-8aad37b30b4b",{"_122":199,"_148":-5,"_120":200,"_117":201},88,["D",1711294935561],"16e7ec5e-e9f5-11ee-87a1-8aad37b30b4b",{"_122":203,"_148":-5,"_120":204,"_117":205},87,["D",1711294932196],"14e662d2-e9f5-11ee-bee4-8aad37b30b4b",{"_122":207,"_148":-5,"_120":208,"_117":209},86,["D",1711294932002],"14c8c63c-e9f5-11ee-87a1-8aad37b30b4b",{"_122":211,"_148":-5,"_120":212,"_117":213},85,["D",1711294931492],"147af07e-e9f5-11ee-bee4-8aad37b30b4b",{"_122":215,"_148":-5,"_120":216,"_117":217},84,["D",1711294826974],"d62edbaa-e9f4-11ee-b338-8aad37b30b4b",{"_122":219,"_148":-5,"_120":220,"_117":221},83,["D",1711294778446],"b941fb94-e9f4-11ee-9358-8aad37b30b4b",{"_122":223,"_148":-5,"_120":224,"_117":225},82,["D",1711294773212],"b6237bfe-e9f4-11ee-aa06-8aad37b30b4b",{"_122":227,"_148":-5,"_120":228,"_117":229},81,["D",1711291534971],"2bff1eda-e9ed-11ee-9baf-8aad37b30b4b",{"_122":231,"_148":-5,"_120":232,"_117":233},80,["D",1711291490914],"11bc7c3e-e9ed-11ee-8311-8aad37b30b4b",{"_122":235,"_148":-5,"_120":236,"_117":237},79,["D",1711291473186],"072b6bfe-e9ed-11ee-9520-8aad37b30b4b",{"_122":239,"_148":-5,"_120":240,"_117":241},78,["D",1711291409689],"e152807a-e9ec-11ee-8e76-8aad37b30b4b",{"_122":243,"_148":-5,"_120":244,"_117":245},77,["D",1711291398456],"daa07e1c-e9ec-11ee-ac5f-8aad37b30b4b",{"_122":247,"_148":-5,"_120":248,"_117":249},76,["D",1711291262700],"89b5cb92-e9ec-11ee-aaf7-8aad37b30b4b",{"_122":251,"_148":-5,"_120":252,"_117":253},75,["D",1711291259654],"87e4fb80-e9ec-11ee-9152-8aad37b30b4b",{"_122":255,"_148":-5,"_120":256,"_117":257},74,["D",1711290911460],"b85ab7d8-e9eb-11ee-a4ea-8aad37b30b4b",{"_122":259,"_148":-5,"_120":260,"_117":261},73,["D",1711275630349],"241a43cc-e9c8-11ee-b8d1-8aad37b30b4b",{"_122":263,"_148":-5,"_120":264,"_117":265},72,["D",1711275627281],"22462b88-e9c8-11ee-9931-8aad37b30b4b",{"_122":267,"_148":-5,"_120":268,"_117":269},71,["D",1711275301557],"6020aeb6-e9c7-11ee-bd7e-8aad37b30b4b",{"_122":271,"_148":-5,"_120":272,"_117":273},70,["D",1711275254997],"446031ce-e9c7-11ee-b21c-8aad37b30b4b",{"_122":275,"_148":-5,"_120":276,"_117":277},69,["D",1711275239889],"3b5ecc52-e9c7-11ee-b21c-8aad37b30b4b",{"_122":279,"_148":-5,"_120":280,"_117":281},68,["D",1711275235748],"38e70f3e-e9c7-11ee-8111-8aad37b30b4b",{"_122":283,"_148":-5,"_120":284,"_117":285},67,["D",1710840463095],"f034f604-e5d2-11ee-8d63-8aad37b30b4b",{"_122":287,"_148":-5,"_120":288,"_117":289},66,["D",1710710922124],"53c55c7e-e4a5-11ee-a8d8-8aad37b30b4b",{"_122":291,"_148":-5,"_120":292,"_117":293},65,["D",1710710920053],"528961d4-e4a5-11ee-8156-8aad37b30b4b",{"_122":295,"_148":-5,"_120":296,"_117":297},64,["D",1710710917838],"5137854a-e4a5-11ee-a8d8-8aad37b30b4b",{"_122":299,"_148":-5,"_120":300,"_117":301},63,["D",1710710899638],"465e78ae-e4a5-11ee-bdf4-8aad37b30b4b",{"_122":303,"_148":-5,"_120":304,"_117":305},62,["D",1710710837911],"219387e4-e4a5-11ee-aced-8aad37b30b4b",{"_122":307,"_148":-5,"_120":308,"_117":309},61,["D",1710710834859],"1fc1cf16-e4a5-11ee-8156-8aad37b30b4b",{"_122":311,"_148":-5,"_120":312,"_117":313},60,["D",1710697158033],"47bbd874-e485-11ee-8add-8aad37b30b4b",{"_122":315,"_148":-5,"_120":316,"_117":317},59,["D",1710697153280],"44e68ad6-e485-11ee-a424-8aad37b30b4b",{"_122":319,"_148":-5,"_120":320,"_117":321},58,["D",1710241134108],"844b0aaa-e05f-11ee-bb28-8aad37b30b4b",{"_122":323,"_148":-5,"_120":324,"_117":325},57,["D",1709201430721],"c4c846e8-d6ea-11ee-997a-8aad37b30b4b",{"_122":327,"_148":-5,"_120":328,"_117":329},56,["D",1709201419994],"be6383f8-d6ea-11ee-986e-8aad37b30b4b",{"_122":331,"_148":-5,"_120":332,"_117":333},55,["D",1709201141561],"186e05fe-d6ea-11ee-b4f3-8aad37b30b4b",{"_122":335,"_148":-5,"_120":336,"_117":337},54,["D",1709201136092],"152b75e8-d6ea-11ee-9290-8aad37b30b4b",{"_122":339,"_148":-5,"_120":340,"_117":341},53,["D",1709201106244],"03612286-d6ea-11ee-b527-8aad37b30b4b",{"_122":343,"_148":-5,"_120":344,"_117":345},52,["D",1709201094019],"fc17c386-d6e9-11ee-8189-8aad37b30b4b",{"_122":347,"_148":-5,"_120":348,"_117":349},51,["D",1709201028513],"d50c3bfa-d6e9-11ee-9290-8aad37b30b4b",{"_122":351,"_148":-5,"_120":352,"_117":353},50,["D",1709032117683],"8e58ff84-d560-11ee-afdd-8aad37b30b4b",{"_122":355,"_148":-5,"_120":356,"_117":357},49,["D",1709032097375],"823e4fb0-d560-11ee-bdc5-8aad37b30b4b",{"_122":359,"_148":-5,"_120":360,"_117":361},48,["D",1709032091526],"7ec1d2a8-d560-11ee-9898-8aad37b30b4b",{"_122":363,"_148":-5,"_120":364,"_117":365},47,["D",1709032073581],"740f87ba-d560-11ee-bdc5-8aad37b30b4b",{"_122":367,"_148":-5,"_120":368,"_117":369},46,["D",1709032063051],"6dc8e130-d560-11ee-9898-8aad37b30b4b",{"_122":371,"_148":-5,"_120":372,"_117":373},45,["D",1709031896260],"0a5e8d16-d560-11ee-96da-8aad37b30b4b",{"_122":375,"_148":-5,"_120":376,"_117":377},44,["D",1708967983261],"3b40cb46-d4cb-11ee-aa7d-8aad37b30b4b",{"_122":379,"_148":-5,"_120":380,"_117":381},43,["D",1708967973391],"355eda24-d4cb-11ee-878e-8aad37b30b4b",{"_122":383,"_148":-5,"_120":384,"_117":385},42,["D",1708967954956],"2a61c474-d4cb-11ee-878e-8aad37b30b4b",{"_122":387,"_148":-5,"_120":388,"_117":389},41,["D",1708967954767],"2a451324-d4cb-11ee-878e-8aad37b30b4b",{"_122":391,"_148":-5,"_120":392,"_117":393},40,["D",1708967742545],"abc68514-d4ca-11ee-aa7d-8aad37b30b4b",{"_122":395,"_148":-5,"_120":396,"_117":397},39,["D",1708967737008],"a879a72e-d4ca-11ee-b44a-8aad37b30b4b",{"_122":399,"_148":-5,"_120":400,"_117":401},38,["D",1708171825330],"88258388-cd8d-11ee-bb9a-8aad37b30b4b",{"_122":403,"_148":-5,"_120":404,"_117":405},37,["D",1708171823775],"87385eb4-cd8d-11ee-af67-8aad37b30b4b",{"_122":407,"_148":-5,"_120":408,"_117":409},36,["D",1708171766892],"6550aeaa-cd8d-11ee-829c-8aad37b30b4b",{"_122":411,"_148":-5,"_120":412,"_117":413},35,["D",1708171695958],"3b08f918-cd8d-11ee-b1ba-8aad37b30b4b",{"_122":415,"_148":-5,"_120":416,"_117":417},34,["D",1708171693852],"39c79b4a-cd8d-11ee-b1ba-8aad37b30b4b",{"_122":419,"_148":-5,"_120":420,"_117":421},33,["D",1708171616785],"0bd80562-cd8d-11ee-b1cd-8aad37b30b4b",{"_122":423,"_148":-5,"_120":424,"_117":425},32,["D",1708171615297],"0af4fd6c-cd8d-11ee-b1cd-8aad37b30b4b",{"_122":427,"_148":-5,"_120":428,"_117":429},31,["D",1708171502729],"c7dc8dd8-cd8c-11ee-a855-8aad37b30b4b",{"_122":431,"_148":-5,"_120":432,"_117":433},30,["D",1708171501693],"c73e7b98-cd8c-11ee-83fa-8aad37b30b4b",{"_122":435,"_148":-5,"_120":436,"_117":437},29,["D",1708171464082],"b0d36ff8-cd8c-11ee-a8ff-8aad37b30b4b",{"_122":439,"_148":-5,"_120":440,"_117":441},28,["D",1708171463322],"b05f9196-cd8c-11ee-a8ff-8aad37b30b4b",{"_122":443,"_148":-5,"_120":444,"_117":445},27,["D",1708171380530],"7f0672ea-cd8c-11ee-9c20-8aad37b30b4b",{"_122":447,"_148":-5,"_120":448,"_117":449},26,["D",1708171375708],"7c26abee-cd8c-11ee-aa33-8aad37b30b4b",{"_122":451,"_148":-5,"_120":452,"_117":453},25,["D",1708171362767],"74701386-cd8c-11ee-82c1-8aad37b30b4b",{"_122":455,"_148":-5,"_120":456,"_117":457},24,["D",1708171306435],"52dc61e8-cd8c-11ee-934b-8aad37b30b4b",{"_122":459,"_148":-5,"_120":460,"_117":461},23,["D",1708171306285],"52c58144-cd8c-11ee-b1ad-8aad37b30b4b",{"_122":463,"_148":-5,"_120":464,"_117":465},22,["D",1708171288957],"48717bee-cd8c-11ee-bcf8-8aad37b30b4b",{"_122":467,"_148":-5,"_120":468,"_117":469},21,["D",1708171246931],"2f64dc18-cd8c-11ee-9d58-8aad37b30b4b",{"_122":471,"_148":-5,"_120":472,"_117":473},20,["D",1708171238569],"2a68fafa-cd8c-11ee-9d58-8aad37b30b4b",{"_122":475,"_148":-5,"_120":476,"_117":477},19,["D",1708171180252],"07a67fec-cd8c-11ee-a603-8aad37b30b4b",{"_122":479,"_148":-5,"_120":480,"_117":481},18,["D",1708171020680],"a889a8a4-cd8b-11ee-a2f2-8aad37b30b4b",{"_122":483,"_148":-5,"_120":484,"_117":485},17,["D",1708170832502],"386016da-cd8b-11ee-9a6b-8aad37b30b4b",{"_122":487,"_148":-5,"_120":488,"_117":489},16,["D",1708170820109],"30fd155a-cd8b-11ee-b1fc-8aad37b30b4b",{"_122":491,"_148":-5,"_120":492,"_117":493},15,["D",1708170800421],"2540d13e-cd8b-11ee-9a6b-8aad37b30b4b",{"_122":495,"_148":-5,"_120":496,"_117":497},14,["D",1708170797145],"234d0852-cd8b-11ee-b259-8aad37b30b4b",{"_122":499,"_148":-5,"_120":500,"_117":501},13,["D",1708170796703],"23097ba0-cd8b-11ee-b259-8aad37b30b4b",{"_122":503,"_148":-5,"_120":504,"_117":505},12,["D",1708170776763],"1726f61e-cd8b-11ee-b259-8aad37b30b4b",{"_122":507,"_148":-5,"_120":508,"_117":509},11,["D",1708170750899],"07bc6d3a-cd8b-11ee-8b31-8aad37b30b4b",{"_122":511,"_148":-5,"_120":512,"_117":513},10,["D",1708165624795],"1856f338-cd7f-11ee-bd85-8aad37b30b4b",{"_122":515,"_148":-5,"_120":516,"_117":517},9,["D",1708162712846],"50aee4b8-cd78-11ee-b135-8aad37b30b4b",{"_122":519,"_148":-5,"_120":520,"_117":521},8,["D",1708162702633],"4a988ade-cd78-11ee-b135-8aad37b30b4b",{"_122":523,"_148":-5,"_120":524,"_117":525},7,["D",1708160990399],"4e0648c2-cd74-11ee-9206-8aad37b30b4b",{"_122":527,"_148":-5,"_120":528,"_117":529},6,["D",1708160827426],"ece290b4-cd73-11ee-af28-8aad37b30b4b",{"_122":531,"_148":-5,"_120":532,"_117":533},5,["D",1708160820507],"e8c2d71e-cd73-11ee-a669-8aad37b30b4b",{"_122":535,"_148":-5,"_120":536,"_117":537},4,["D",1708160803245],"de78e4b0-cd73-11ee-a8f2-8aad37b30b4b",{"_122":539,"_148":-5,"_120":540,"_117":541},3,["D",1708160772262],"cc012e32-cd73-11ee-87ab-8aad37b30b4b",{"_122":94,"_148":-5,"_120":543,"_117":544},["D",1708160729938],"b2c70ff4-cd73-11ee-87ab-8aad37b30b4b",{"_122":126,"_148":-5,"_120":546,"_117":547},["D",1708160707564],"a5711142-cd73-11ee-87ab-8aad37b30b4b",{"_122":33,"_148":-5,"_120":549,"_117":550},["D",1708160544547],"4446b7b4-cd73-11ee-bfb1-8aad37b30b4b","recentThread","breadcrumbs","comments",[],"isOwner","routes/_app.v.$handle.$name.forks",{"_558":559,"_617":126,"_618":142,"_619":33,"_98":620},"forksList",[560],{"_104":561,"_562":563},"1ff86f92-e9f7-11ee-9325-8aad37b30b4b","initialData",{"_102":564,"_99":100,"_104":561,"_106":-5,"_107":-5,"_108":109,"_110":565,"_112":113,"_114":566,"_116":27,"_117":566,"_118":567,"_120":568,"_122":569,"_124":27,"_125":94,"_128":33,"_129":33,"_130":33,"_127":94,"_131":33,"_132":-5,"_21":570,"_138":571,"_140":-7,"_143":573,"_574":-5,"_141":575,"_145":576},"markdown_download","# markdown.download \n\nHandy microservice/library to convert various data sources into markdown. Intended to make it easier to consume the web in ereaders\n\nIntroductory blog post: https://taras.glek.net/post/markdown.download/\n\nPackage: https://jsr.io/@tarasglek/markdown-download\n\n### Features\n- Apply readability\n- Further convert article into markdown to simplify it\n- Allow webpages to be viewable as markdown via curl\n- Serve markdown converted to html to browsers\n- Extract youtube subtitles\n- Can paste htmlContent in\n- `curl 'https://val.markdown.download/' -H 'content-type: application/x-www-form-urlencoded' --data-urlencode \"htmlContent@$HOME/Downloads/oai-reference.html\"`\n\n### Source\n\nhttps://github.com/tarasglek/markdown-download\n\nhttps://www.val.town/v/taras/markdown_download\n\nLicense: MIT\n\nUsage: \n https://markdown.download/ + URL\n\nDev:\n https://val.markdown.download/ + URL\n\n\n```","20f41fca-8c60-11ef-9034-de64eea55b61","import { isProbablyReaderable, Readability } from \"npm:@mozilla/readability@^0.5.0\";\nimport { DOMParser } from \"npm:linkedom@0.16.10\";\nimport { marked } from \"npm:marked@12.0.1\";\nimport { getSubtitles } from \"npm:youtube-captions-scraper@^2.0.1\";\nimport { YouTube } from \"npm:youtube-sr@4.3.11\";\n\nconst isCloudflareWorker = typeof Request !== \"undefined\" && typeof Response !== \"undefined\";\n// import { convertHtmlToMarkdown } from ;\n\n// init async loading of modules\nconst AgentMarkdownImport = isCloudflareWorker ? import(\"npm:agentmarkdown@6.0.0\") : null;\nconst dom2semanticMarkdown = isCloudflareWorker ? null : await import(\"npm:dom-to-semantic-markdown@1.2.4\")\n\n/**\n * converts HTML to markdown\n * @returns markdown in string\n */\nexport async function html2markdown(html: string): Promise {\n if (AgentMarkdownImport) {\n // TurndownService doesn't work on cf\n // Dynamically import AgentMarkdown when running in Cloudflare Worker\n const { AgentMarkdown } = await AgentMarkdownImport;\n return await AgentMarkdown.produce(html);\n } else {\n const ret = (await dom2semanticMarkdown).convertHtmlToMarkdown(html, { overrideDOMParser: new DOMParser() as any });\n console.log(ret);\n return ret;\n }\n}\n\n/**\n * extracts article from html\n * then converts it to md\n * @returns markdown in string\n */\nexport async function readability2markdown(html: string): Promise<{ title: string; markdown: string }> {\n const doc = await (new DOMParser().parseFromString(html, \"text/html\"));\n\n const reader = new Readability(doc);\n const article = reader.parse();\n\n const markdown = await html2markdown(article?.content || \"\");\n return { title: doc.title.textContent || \"\", markdown };\n}\n\nfunction response(message: string, contentType = \"text/markdown\"): Response {\n const headers = new Headers();\n headers.set(\"Access-Control-Allow-Origin\", \"*\");\n headers.set(\"Access-Control-Allow-Methods\", \"GET, POST, PUT, DELETE, OPTIONS\");\n headers.set(\"Access-Control-Allow-Headers\", \"Content-Type, Authorization\");\n headers.set(\"Access-Control-Max-Age\", \"86400\");\n \n // Check if charset is already specified in the contentType\n if (!contentType.toLowerCase().includes(\"charset=\")) {\n contentType = `${contentType}; charset=utf-8`;\n }\n \n headers.set(\"Content-Type\", contentType);\n \n return new Response(message, {\n status: 200,\n headers: headers,\n });\n}\n\nfunction err(msg: string): Response {\n const errorMessage = JSON.stringify({\n error: {\n message: msg,\n code: 400,\n },\n });\n return response(errorMessage, \"application/json\");\n}\n\nfunction fudgeURL(url: string) {\n try {\n return new URL(url);\n } catch (e) {\n // console.log(\"Url parsing failed\", e.stack);\n return new URL(\"https://\" + url);\n }\n}\n\nasync function processInput(req: Request) {\n let ret = {\n url: undefined as undefined | URL,\n response: undefined as undefined | Response,\n htmlContent: undefined as undefined | string,\n };\n\n const myurl = new URL(req.url);\n let pathname = myurl.pathname.substring(1) + myurl.search;\n\n if (req.method === \"POST\") {\n const formData = new URLSearchParams(await req.text());\n const url = formData.get(\"url\");\n const htmlContent = formData.get(\"htmlContent\");\n\n if (htmlContent) {\n ret.htmlContent = htmlContent;\n return ret;\n }\n\n if (url) {\n pathname = url;\n }\n }\n\n if (!pathname.startsWith(\"http\")) {\n const urlAsFormParam = myurl.searchParams.get(\"url\");\n if (urlAsFormParam) {\n pathname = urlAsFormParam;\n } else if (pathname.length < 2) {\n ret.response = response(\n generate_ui(\n \"URL to convert to markdown:\",\n \"https://www.val.town/v/taras/markdown_download\",\n \"markdown.download\",\n ),\n \"text/html\",\n );\n return ret;\n }\n }\n\n ret.url = fudgeURL(pathname);\n return ret;\n}\n\nexport async function youtube2markdown(url: URL): Promise {\n function getYoutubeVideoID(url: URL): string | null {\n const regExp = /(?:youtube\\.com\\/(?:[^/]+\\/.+\\/|(?:v|e(?:mbed)?)\\/|.*[?&]v=)|youtu\\.be\\/)([^\"&?/\\s]{11})/i;\n const match = url.href.match(regExp);\n return match ? match[1] : null;\n }\n const youtubeVideoID = getYoutubeVideoID(url);\n if (!youtubeVideoID)\n return undefined;\n let transcript = \"## Generated Transcription\\n\\n\";\n try {\n const arr = (await getSubtitles({\n videoID: youtubeVideoID,\n })) as { text: string }[];\n transcript += arr.map(({ text }) => text).join(\"\\n\\n\");\n } catch (e) {\n transcript = `Failed to fetch transcript ${e}\\n`;\n }\n let header = \"\";\n try {\n const y = await YouTube.getVideo(url.toString());\n header = \"# \" + y.title + \"\\n\\n\" + y.embedHTML() + `\\n\\n

\\n`\n + y.description + \"\\n

\\n\\n\";\n } catch (e) {\n header = `Failed to fetch youtube metadata: ${e}\\n`;\n }\n return header + \"\\n\" + transcript;\n}\n\n/**\n * Simple UI that takes a url\n */\nexport function generate_ui(input_description: string, link: string, link_text: string): string {\n const html = `\n\n\n\n markdown.download\n \n \n\n\n

\n \n ${link_text}\n

\n\n\n`;\n return html;\n}\n\nexport default async function(req: Request): Promise {\n const action = await processInput(req);\n const url = action.url;\n if (action.response) {\n return action.response!;\n }\n\n let html = action.htmlContent;\n let fetch_response;\n if (!html) {\n fetch_response = await fetch(url.toString(), {\n method: req.method,\n headers: new Headers({\n \"User-Agent\":\n \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36\",\n \"Accept\": \"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8\",\n \"Accept-Language\": \"en-US,en;q=0.5\",\n \"Sec-Fetch-Site\": \"cross-site\",\n \"Sec-Fetch-Mode\": \"navigate\",\n \"Sec-Fetch-User\": \"?1\",\n \"Sec-Fetch-Dest\": \"document\",\n \"Referer\": \"https://www.google.com/\",\n \"sec-ch-ua\": `\"Not A Brand\";v=\"99\", \"Google Chrome\";v=\"91\", \"Chromium\";v=\"91\"`,\n \"sec-ch-ua-mobile\": \"?0\",\n \"sec-ch-ua-platform\": `\"macOS\"`,\n \"Upgrade-Insecure-Requests\": \"1\",\n // Add any other headers you need here\n }),\n });\n\n console.log(fetch_response);\n\n html = await fetch_response.text();\n }\n let title = \"\";\n let markdown = \"\";\n\n const youtube_markdown = url ? await youtube2markdown(url) : undefined;\n if (youtube_markdown) {\n markdown = youtube_markdown;\n } else {\n const r = await html2markdown(html);\n title = \"\";\n markdown = r;\n }\n\n let markdown_extended = markdown + \"\\n\\n\" + url;\n\n if (fetch_response && Math.floor(fetch_response.status / 200) * 200 != 200 || markdown === \"\") {\n const jina_url = `https://r.jina.ai/${url.toString()}`;\n console.log(`jina fallback:${jina_url}`);\n const fallback_fetch_response = await fetch(jina_url);\n markdown_extended = await fallback_fetch_response.text();\n }\n if (req.headers.get(\"Accept\")?.includes(\"text/html\")) {\n const body = await marked.parse(markdown_extended);\n const html = `\n\n \n ${title}\n \n ${body}\n \n `;\n return response(html, \"text/html\");\n } else {\n return response(markdown_extended);\n }\n}",["D",1729152996937],164,{"_134":135,"_136":137},{"_117":161,"_134":135,"_102":103,"_122":159,"_572":-5},"prDisabledHint","markdown.download Handy microservice/library to convert various data sources into markdown. Intended to make it easier to consume the web in ereaders Introductory blog post: https://taras.glek.net/post/markdown.download/ Package: https://jsr.io/@tarasglek/markdown-download Features Apply readability Further convert article into markdown to simplify it Allow webpages to be viewable as markdown via curl Serve markdown converted to html to browsers Extract youtube subtitles Can paste htmlContent in curl 'https://val.markdown.download/' -H 'content-type: application/x-www-form-urlencoded' --data-urlencode \"htmlContent@$HOME/Downloads/oai-reference.html\" Source https://github.com/tarasglek/markdown-download https://www.val.town/v/taras/markdown_download License: MIT Usage:\nhttps://markdown.download/ + URL Dev:\nhttps://val.markdown.download/ + URL ","latestEvaluation","taras/markdown_download",[577,581,585,589,593,597,601,605,609,613],{"_120":578,"_122":569,"_579":580},["D",1729152996937],"n","1",{"_120":582,"_122":583,"_579":584},["D",1723117813114],163,"2",{"_120":586,"_122":587,"_579":588},["D",1722854052507],162,"3",{"_120":590,"_122":591,"_579":592},["D",1722854050623],161,"4",{"_120":594,"_122":595,"_579":596},["D",1722853796538],160,"5",{"_120":598,"_122":599,"_579":600},["D",1722853493437],159,"6",{"_120":602,"_122":603,"_579":604},["D",1722844279502],158,"7",{"_120":606,"_122":607,"_579":608},["D",1722844275012],157,"8",{"_120":610,"_122":611,"_579":612},["D",1722844266542],156,"9",{"_120":614,"_122":615,"_579":616},["D",1722844211043],155,"10","forksCount","valname","page",{"_117":115,"_102":103,"_134":135,"_12":100,"_122":123,"_104":105,"_139":-5,"_118":119,"_108":109,"_116":27,"_141":142},"routes/_app","actionData","errors"]