MD Crawl

Convert websites to Markdown for LLM consumption

\n \n

\n \n \n {error && (\n

\n {error}\n

\n )}\n \n {isLoading && (\n

\n Crawling website...\n

\n )}\n \n {markdown && !isLoading && (\n \n )}\n

\n \n \n

\n );\n}",[],[196,203,210],{"_107":217,"_99":218,"_150":219,"_152":204,"_153":139,"_154":220,"_156":121,"_109":221,"_104":33,"_158":171,"_160":222,"_161":27,"_162":-5,"_163":223,"_102":224},"943344e6-09b2-11f0-92d0-569c3dd06744","CrawlForm.tsx","94334590-09b2-11f0-92d0-569c3dd06744",8,["D",1742932304365],"// @jsxImportSource https://esm.sh/react@18.2.0\nimport { useState } from \"https://esm.sh/react@18.2.0\";\n\ninterface CrawlFormProps {\n onSubmit: (url: string, depth: number) => void;\n isLoading: boolean;\n}\n\nexport function CrawlForm({ onSubmit, isLoading }: CrawlFormProps) {\n const [url, setUrl] = useState(\"\");\n const [depth, setDepth] = useState(3);\n \n const handleSubmit = (e: React.FormEvent) => {\n e.preventDefault();\n if (!url) return;\n \n onSubmit(url, depth);\n };\n \n return (\n

Crawl a Website

\n \n

\n );\n}",[],[196,203,218],{"_107":226,"_99":227,"_150":228,"_152":204,"_153":139,"_154":229,"_156":121,"_109":230,"_104":33,"_158":171,"_160":231,"_161":27,"_162":-5,"_163":232,"_102":233},"9c41d620-09b2-11f0-84e0-569c3dd06744","MarkdownOutput.tsx","9c41d706-09b2-11f0-84e0-569c3dd06744",9,["D",1742932317881],"// @jsxImportSource https://esm.sh/react@18.2.0\nimport { useState, useRef, useEffect } from \"https://esm.sh/react@18.2.0\";\n\ninterface MarkdownOutputProps {\n markdown: string;\n}\n\nexport function MarkdownOutput({ markdown }: MarkdownOutputProps) {\n const [value, setValue] = useState(markdown);\n const [copySuccess, setCopySuccess] = useState(false);\n const textareaRef = useRef(null);\n \n // Update value when markdown prop changes\n useEffect(() => {\n setValue(markdown);\n }, [markdown]);\n \n const handleCopy = async () => {\n if (!textareaRef.current) return;\n \n try {\n await navigator.clipboard.writeText(value);\n setCopySuccess(true);\n \n // Reset success state after 2 seconds\n setTimeout(() => {\n setCopySuccess(false);\n }, 2000);\n } catch (err) {\n console.error(\"Failed to copy text:\", err);\n }\n };\n \n return (\n

Markdown Output

\n \n \n {copySuccess ? (\n <>\n \n Copied!\n \n ) : (\n <>\n \n Copy to Clipboard\n \n )}\n \n

\n \n setValue(e.target.value)}\n className=\"w-full flex-grow p-4 border border-gray-300 rounded-md focus:ring-blue-500 focus:border-blue-500 font-mono text-sm\"\n spellCheck={false}\n />\n \n

\n This markdown is optimized for LLM consumption with a sitemap at the top.\n

\n );\n}",[],[196,203,227],[196,203],{"_107":236,"_99":237,"_150":238,"_152":197,"_153":139,"_154":239,"_156":121,"_109":240,"_104":33,"_158":180,"_160":241,"_161":27,"_162":-5,"_163":242,"_102":243},"a137fd76-09b2-11f0-9c88-569c3dd06744","favicon.svg","a137fe3e-09b2-11f0-9c88-569c3dd06744",11,["D",1742932326204],"",[],[196,237],{"_107":245,"_99":246,"_150":247,"_152":197,"_153":139,"_154":248,"_156":121,"_109":249,"_104":250,"_158":180,"_160":251,"_161":27,"_162":-5,"_163":252,"_102":253},"a534dfde-09b2-11f0-932a-569c3dd06744","index.html","8686f81a-09b2-11f0-bb04-569c3dd06744",12,["D",1742932332893],1,"\n\n\n \n \n MD Crawl - Website to Markdown Crawler\n \n \n \n \n \n\n\n

\n\n",[],[196,246],{"_107":255,"_99":256,"_150":257,"_152":197,"_153":139,"_154":258,"_156":121,"_109":259,"_104":33,"_158":171,"_160":260,"_161":27,"_162":-5,"_163":261,"_102":262},"9ea6f94a-09b2-11f0-940d-569c3dd06744","index.tsx","9ea6fa4e-09b2-11f0-940d-569c3dd06744",10,["D",1742932321900],"// @jsxImportSource https://esm.sh/react@18.2.0\nimport React from \"https://esm.sh/react@18.2.0\";\nimport { createRoot } from \"https://esm.sh/react-dom@18.2.0/client\";\nimport { App } from \"./components/App\";\n\nconst root = createRoot(document.getElementById(\"app\")!);\nroot.render();",[],[196,256],[196],{"_107":265,"_99":266,"_150":267,"_152":-5,"_153":139,"_154":268,"_156":121,"_109":269,"_104":33,"_158":180,"_160":270,"_161":27,"_162":-5,"_163":271,"_102":272},"6ddccf10-09b2-11f0-940d-569c3dd06744","README.md","6ddccff6-09b2-11f0-940d-569c3dd06744",3,["D",1742932240045],"# MD Crawl\n\nA website that crawls URLs recursively and outputs markdown content optimized for LLM consumption.\n\n## Features\n\n- Input a URL to crawl\n- Set crawl depth (default: 3)\n- Crawls only URLs on the same domain\n- Outputs a markdown representation of all pages\n- Includes a sitemap at the top for better context\n- Copy button with success state\n\n## Project Structure\n\n- `backend/` - Server-side code\n - `index.ts` - Main entry point with Hono API\n - `crawler.ts` - Website crawling logic\n- `frontend/` - Client-side code\n - `index.html` - Main HTML template\n - `index.tsx` - Frontend React code\n - `components/` - React components",[],[266],"files",[147,165,175,184,194,201,208,216,225,235,244,254,264],"counts",{"_277":33,"_278":250,"_279":33,"_280":33},"pulls","branches","forks","envVars","contents",[],{"_107":185,"_99":186,"_150":119,"_152":151,"_153":139,"_154":187,"_156":121,"_109":284,"_104":94,"_158":189,"_160":190,"_161":27,"_162":-5,"_102":285},["D",1742932358759],[149,186],"upstream","screenshots","emptyEnvVars","routes/_app.x.$handle.$name.branch.$branch.history_.$",{"_97":98,"_99":100,"_101":43,"_102":103,"_105":291,"_180":295,"_298":299,"_311":312,"_317":-7,"_318":33,"_319":27},{"_107":108,"_99":100,"_109":292,"_111":-5,"_112":27,"_113":114,"_115":-5,"_116":-5,"_12":117,"_118":119,"_120":121,"_122":-5,"_123":33,"_124":125,"_126":125,"_127":128,"_129":128,"_130":128,"_131":125,"_132":27,"_21":293,"_136":294},["D",1742931896492],{"_97":98,"_134":135},{"_104":138,"_99":43,"_107":139,"_140":108,"_141":-5,"_142":-5,"_143":-5,"_144":-5},{"_107":185,"_99":186,"_150":119,"_152":151,"_153":139,"_154":187,"_156":121,"_109":296,"_104":94,"_158":189,"_160":190,"_161":27,"_162":-5,"_102":297},["D",1742932358759],[149,186],"history",[300,302,306],{"_107":185,"_153":139,"_150":119,"_99":186,"_160":190,"_158":189,"_104":94,"_152":151,"_109":301,"_154":187,"_162":-5},["D",1742932358759],{"_107":303,"_153":139,"_150":119,"_99":186,"_160":190,"_158":171,"_104":250,"_152":151,"_109":304,"_154":305,"_162":-5},"a7d34492-09b2-11f0-92d0-569c3dd06744",["D",1742932337287],13,{"_107":307,"_153":139,"_150":119,"_99":186,"_160":308,"_158":171,"_104":33,"_152":151,"_109":309,"_154":310,"_162":-5},"82c6c430-09b2-11f0-856d-569c3dd06744","import { Hono } from \"https://esm.sh/hono@4.0.5\";\nimport { cors } from \"https://esm.sh/hono@4.0.5/cors\";\nimport { readFile, serveFile } from \"https://esm.town/v/std/utils@71-main/index.ts\";\nimport { crawlWebsite } from \"./crawler.ts\";\n\nconst app = new Hono();\n\n// Enable CORS\napp.use(\"/*\", cors());\n\n// Serve static files\napp.get(\"/public/*\", c => serveFile(c.req.path.replace(\"/public\", \"/frontend\"), import.meta.url));\n\n// Serve index.html\napp.get(\"/\", async c => {\n const html = await readFile(\"/frontend/index.html\", import.meta.url);\n return c.html(html);\n});\n\n// API endpoint to crawl a website\napp.post(\"/api/crawl\", async c => {\n try {\n const { url, depth = 3 } = await c.req.json();\n \n if (!url) {\n return c.json({ error: \"URL is required\" }, 400);\n }\n \n // Validate URL\n try {\n new URL(url);\n } catch (error) {\n return c.json({ error: \"Invalid URL\" }, 400);\n }\n \n // Validate depth\n const parsedDepth = parseInt(depth);\n if (isNaN(parsedDepth) || parsedDepth < 0 || parsedDepth > 5) {\n return c.json({ error: \"Depth must be between 0 and 5\" }, 400);\n }\n \n // Crawl website\n const markdown = await crawlWebsite(url, parsedDepth);\n \n return c.json({ markdown });\n } catch (error) {\n console.error(\"Error crawling website:\", error);\n return c.json({ error: \"Failed to crawl website\" }, 500);\n }\n});\n\n// This is the entry point for HTTP vals\nexport default app.fetch;",["D",1742932275097],5,"versions",[313,315,316],{"_180":300,"_314":302},"previous",{"_180":302,"_314":306},{"_180":306,"_314":-5},"versionNumber","page","hasNext","routes/_app","actionData","errors"]

History for index.ts