MD Crawl

Convert websites to Markdown for LLM consumption

\n \n

\n \n \n {error && (\n

\n {error}\n

\n )}\n \n {isLoading && (\n

\n Crawling website...\n

\n )}\n \n {markdown && !isLoading && (\n \n )}\n

\n \n \n

\n );\n}",[],[195,202,209],{"_106":216,"_99":217,"_149":218,"_151":203,"_152":138,"_153":219,"_155":120,"_108":220,"_103":33,"_157":170,"_159":221,"_160":27,"_161":-5,"_162":222,"_102":223},"943344e6-09b2-11f0-92d0-569c3dd06744","CrawlForm.tsx","94334590-09b2-11f0-92d0-569c3dd06744",8,["D",1742932304365],"// @jsxImportSource https://esm.sh/react@18.2.0\nimport { useState } from \"https://esm.sh/react@18.2.0\";\n\ninterface CrawlFormProps {\n onSubmit: (url: string, depth: number) => void;\n isLoading: boolean;\n}\n\nexport function CrawlForm({ onSubmit, isLoading }: CrawlFormProps) {\n const [url, setUrl] = useState(\"\");\n const [depth, setDepth] = useState(3);\n \n const handleSubmit = (e: React.FormEvent) => {\n e.preventDefault();\n if (!url) return;\n \n onSubmit(url, depth);\n };\n \n return (\n

Crawl a Website

\n \n

\n );\n}",[],[195,202,217],{"_106":225,"_99":226,"_149":227,"_151":203,"_152":138,"_153":228,"_155":120,"_108":229,"_103":33,"_157":170,"_159":230,"_160":27,"_161":-5,"_162":231,"_102":232},"9c41d620-09b2-11f0-84e0-569c3dd06744","MarkdownOutput.tsx","9c41d706-09b2-11f0-84e0-569c3dd06744",9,["D",1742932317881],"// @jsxImportSource https://esm.sh/react@18.2.0\nimport { useState, useRef, useEffect } from \"https://esm.sh/react@18.2.0\";\n\ninterface MarkdownOutputProps {\n markdown: string;\n}\n\nexport function MarkdownOutput({ markdown }: MarkdownOutputProps) {\n const [value, setValue] = useState(markdown);\n const [copySuccess, setCopySuccess] = useState(false);\n const textareaRef = useRef(null);\n \n // Update value when markdown prop changes\n useEffect(() => {\n setValue(markdown);\n }, [markdown]);\n \n const handleCopy = async () => {\n if (!textareaRef.current) return;\n \n try {\n await navigator.clipboard.writeText(value);\n setCopySuccess(true);\n \n // Reset success state after 2 seconds\n setTimeout(() => {\n setCopySuccess(false);\n }, 2000);\n } catch (err) {\n console.error(\"Failed to copy text:\", err);\n }\n };\n \n return (\n

Markdown Output

\n \n \n {copySuccess ? (\n <>\n \n Copied!\n \n ) : (\n <>\n \n Copy to Clipboard\n \n )}\n \n

\n \n setValue(e.target.value)}\n className=\"w-full flex-grow p-4 border border-gray-300 rounded-md focus:ring-blue-500 focus:border-blue-500 font-mono text-sm\"\n spellCheck={false}\n />\n \n

\n This markdown is optimized for LLM consumption with a sitemap at the top.\n

\n );\n}",[],[195,202,226],[195,202],{"_106":235,"_99":236,"_149":237,"_151":196,"_152":138,"_153":238,"_155":120,"_108":239,"_103":33,"_157":179,"_159":240,"_160":27,"_161":-5,"_162":241,"_102":242},"a137fd76-09b2-11f0-9c88-569c3dd06744","favicon.svg","a137fe3e-09b2-11f0-9c88-569c3dd06744",11,["D",1742932326204],"",[],[195,236],{"_106":244,"_99":245,"_149":246,"_151":196,"_152":138,"_153":247,"_155":120,"_108":248,"_103":249,"_157":179,"_159":250,"_160":27,"_161":-5,"_162":251,"_102":252},"a534dfde-09b2-11f0-932a-569c3dd06744","index.html","8686f81a-09b2-11f0-bb04-569c3dd06744",12,["D",1742932332893],1,"\n\n\n \n \n MD Crawl - Website to Markdown Crawler\n \n \n \n \n \n\n\n

\n\n",[],[195,245],{"_106":254,"_99":255,"_149":256,"_151":196,"_152":138,"_153":257,"_155":120,"_108":258,"_103":33,"_157":170,"_159":259,"_160":27,"_161":-5,"_162":260,"_102":261},"9ea6f94a-09b2-11f0-940d-569c3dd06744","index.tsx","9ea6fa4e-09b2-11f0-940d-569c3dd06744",10,["D",1742932321900],"// @jsxImportSource https://esm.sh/react@18.2.0\nimport React from \"https://esm.sh/react@18.2.0\";\nimport { createRoot } from \"https://esm.sh/react-dom@18.2.0/client\";\nimport { App } from \"./components/App\";\n\nconst root = createRoot(document.getElementById(\"app\")!);\nroot.render();",[],[195,255],[195],{"_106":264,"_99":265,"_149":266,"_151":-5,"_152":138,"_153":267,"_155":120,"_108":268,"_103":33,"_157":179,"_159":269,"_160":27,"_161":-5,"_162":270,"_102":271},"6ddccf10-09b2-11f0-940d-569c3dd06744","README.md","6ddccff6-09b2-11f0-940d-569c3dd06744",3,["D",1742932240045],"# MD Crawl\n\nA website that crawls URLs recursively and outputs markdown content optimized for LLM consumption.\n\n## Features\n\n- Input a URL to crawl\n- Set crawl depth (default: 3)\n- Crawls only URLs on the same domain\n- Outputs a markdown representation of all pages\n- Includes a sitemap at the top for better context\n- Copy button with success state\n\n## Project Structure\n\n- `backend/` - Server-side code\n - `index.ts` - Main entry point with Hono API\n - `crawler.ts` - Website crawling logic\n- `frontend/` - Client-side code\n - `index.html` - Main HTML template\n - `index.tsx` - Frontend React code\n - `components/` - React components",[],[265],"files",[146,164,174,183,193,200,207,215,224,234,243,253,263],"counts",{"_276":33,"_277":249,"_278":33,"_279":33},"pulls","branches","forks","envVars","contents",[282,285,288],{"_106":147,"_99":148,"_149":150,"_151":-5,"_152":138,"_153":154,"_155":120,"_108":283,"_103":33,"_157":158,"_159":-5,"_160":27,"_161":-5,"_102":284},["D",1742932265577],[148],{"_106":194,"_99":195,"_149":196,"_151":-5,"_152":138,"_153":197,"_155":120,"_108":286,"_103":33,"_157":158,"_159":-5,"_160":27,"_161":-5,"_102":287},["D",1742932281424],[195],{"_106":264,"_99":265,"_149":266,"_151":-5,"_152":138,"_153":267,"_155":120,"_108":289,"_103":33,"_157":179,"_159":269,"_160":27,"_161":-5,"_102":290},["D",1742932240045],[265],"upstream","screenshots","emptyEnvVars","routes/_app.x.$handle.$name.branch.$branch.history",{"_97":98,"_99":100,"_101":43,"_104":296,"_300":301,"_411":33,"_412":27},{"_106":107,"_99":100,"_108":297,"_110":-5,"_111":27,"_112":113,"_114":-5,"_115":-5,"_12":116,"_117":118,"_119":120,"_121":-5,"_122":33,"_123":124,"_125":124,"_126":127,"_128":127,"_129":127,"_130":124,"_131":27,"_21":298,"_135":299},["D",1742931896492],{"_97":98,"_133":134},{"_103":137,"_99":43,"_106":138,"_139":107,"_140":-5,"_141":-5,"_142":-5,"_143":-5},"history",[302,307,312,319,324,331,336,341,346,351,356,364,374,382,392,397,405],{"_149":177,"_103":137,"_108":303,"_152":138,"_272":304},["D",1742932502004],[305],{"_106":175,"_152":138,"_149":177,"_99":176,"_159":180,"_157":179,"_103":33,"_151":150,"_108":303,"_153":137,"_161":-5,"_155":120,"_160":27,"_102":306},[148,176],{"_149":167,"_103":168,"_108":308,"_152":138,"_272":309},["D",1742932448842],[310],{"_106":165,"_152":138,"_149":167,"_99":166,"_159":171,"_157":170,"_103":94,"_151":150,"_108":308,"_153":168,"_161":-5,"_155":120,"_160":27,"_102":311},[148,166],{"_149":167,"_103":313,"_108":314,"_152":138,"_272":315},15,["D",1742932411423],[316],{"_106":317,"_152":138,"_149":167,"_99":166,"_159":180,"_157":170,"_103":249,"_151":150,"_108":314,"_153":313,"_161":-5,"_155":27,"_160":27,"_102":318},"d403bb5a-09b2-11f0-af0b-569c3dd06744",[148,166],{"_149":118,"_103":186,"_108":320,"_152":138,"_272":321},["D",1742932358759],[322],{"_106":184,"_152":138,"_149":118,"_99":185,"_159":189,"_157":188,"_103":94,"_151":150,"_108":320,"_153":186,"_161":-5,"_155":120,"_160":27,"_102":323},[148,185],{"_149":118,"_103":325,"_108":326,"_152":138,"_272":327},13,["D",1742932337287],[328],{"_106":329,"_152":138,"_149":118,"_99":185,"_159":189,"_157":170,"_103":249,"_151":150,"_108":326,"_153":325,"_161":-5,"_155":27,"_160":27,"_102":330},"a7d34492-09b2-11f0-92d0-569c3dd06744",[148,185],{"_149":246,"_103":247,"_108":332,"_152":138,"_272":333},["D",1742932332893],[334],{"_106":244,"_152":138,"_149":246,"_99":245,"_159":250,"_157":179,"_103":249,"_151":196,"_108":332,"_153":247,"_161":-5,"_155":120,"_160":27,"_102":335},[195,245],{"_149":237,"_103":238,"_108":337,"_152":138,"_272":338},["D",1742932326204],[339],{"_106":235,"_152":138,"_149":237,"_99":236,"_159":240,"_157":179,"_103":33,"_151":196,"_108":337,"_153":238,"_161":-5,"_155":120,"_160":27,"_102":340},[195,236],{"_149":256,"_103":257,"_108":342,"_152":138,"_272":343},["D",1742932321900],[344],{"_106":254,"_152":138,"_149":256,"_99":255,"_159":259,"_157":170,"_103":33,"_151":196,"_108":342,"_153":257,"_161":-5,"_155":120,"_160":27,"_102":345},[195,255],{"_149":227,"_103":228,"_108":347,"_152":138,"_272":348},["D",1742932317881],[349],{"_106":225,"_152":138,"_149":227,"_99":226,"_159":230,"_157":170,"_103":33,"_151":203,"_108":347,"_153":228,"_161":-5,"_155":120,"_160":27,"_102":350},[195,202,226],{"_149":218,"_103":219,"_108":352,"_152":138,"_272":353},["D",1742932304365],[354],{"_106":216,"_152":138,"_149":218,"_99":217,"_159":221,"_157":170,"_103":33,"_151":203,"_108":352,"_153":219,"_161":-5,"_155":120,"_160":27,"_102":355},[195,202,217],{"_149":210,"_103":204,"_108":357,"_152":138,"_272":358},["D",1742932293759],[359,361],{"_106":208,"_152":138,"_149":210,"_99":209,"_159":212,"_157":170,"_103":33,"_151":203,"_108":357,"_153":204,"_161":-5,"_155":120,"_160":27,"_102":360},[195,202,209],{"_106":201,"_152":138,"_149":203,"_99":202,"_159":-5,"_157":158,"_103":33,"_151":196,"_108":362,"_153":204,"_161":-5,"_155":120,"_160":27,"_102":363},["D",1742932293759],[195,202],{"_149":246,"_103":197,"_108":365,"_152":138,"_272":366},["D",1742932281424],[367,371],{"_106":368,"_152":138,"_149":246,"_99":245,"_159":369,"_157":179,"_103":33,"_151":196,"_108":365,"_153":197,"_161":-5,"_155":27,"_160":27,"_102":370},"8686f662-09b2-11f0-bb04-569c3dd06744","\n\n\n \n \n MD Crawl - Website to Markdown Crawler\n \n \n \n \n\n\n

\n\n",[195,245],{"_106":194,"_152":138,"_149":196,"_99":195,"_159":-5,"_157":158,"_103":33,"_151":-5,"_108":372,"_153":197,"_161":-5,"_155":120,"_160":27,"_102":373},["D",1742932281424],[195],{"_149":118,"_103":375,"_108":376,"_152":138,"_272":377},5,["D",1742932275097],[378],{"_106":379,"_152":138,"_149":118,"_99":185,"_159":380,"_157":170,"_103":33,"_151":150,"_108":376,"_153":375,"_161":-5,"_155":27,"_160":27,"_102":381},"82c6c430-09b2-11f0-856d-569c3dd06744","import { Hono } from \"https://esm.sh/hono@4.0.5\";\nimport { cors } from \"https://esm.sh/hono@4.0.5/cors\";\nimport { readFile, serveFile } from \"https://esm.town/v/std/utils@71-main/index.ts\";\nimport { crawlWebsite } from \"./crawler.ts\";\n\nconst app = new Hono();\n\n// Enable CORS\napp.use(\"/*\", cors());\n\n// Serve static files\napp.get(\"/public/*\", c => serveFile(c.req.path.replace(\"/public\", \"/frontend\"), import.meta.url));\n\n// Serve index.html\napp.get(\"/\", async c => {\n const html = await readFile(\"/frontend/index.html\", import.meta.url);\n return c.html(html);\n});\n\n// API endpoint to crawl a website\napp.post(\"/api/crawl\", async c => {\n try {\n const { url, depth = 3 } = await c.req.json();\n \n if (!url) {\n return c.json({ error: \"URL is required\" }, 400);\n }\n \n // Validate URL\n try {\n new URL(url);\n } catch (error) {\n return c.json({ error: \"Invalid URL\" }, 400);\n }\n \n // Validate depth\n const parsedDepth = parseInt(depth);\n if (isNaN(parsedDepth) || parsedDepth < 0 || parsedDepth > 5) {\n return c.json({ error: \"Depth must be between 0 and 5\" }, 400);\n }\n \n // Crawl website\n const markdown = await crawlWebsite(url, parsedDepth);\n \n return c.json({ markdown });\n } catch (error) {\n console.error(\"Error crawling website:\", error);\n return c.json({ error: \"Failed to crawl website\" }, 500);\n }\n});\n\n// This is the entry point for HTTP vals\nexport default app.fetch;",[148,185],{"_149":167,"_103":154,"_108":383,"_152":138,"_272":384},["D",1742932265577],[385,389],{"_106":386,"_152":138,"_149":167,"_99":166,"_159":387,"_157":170,"_103":33,"_151":150,"_108":383,"_153":154,"_161":-5,"_155":27,"_160":27,"_102":388},"7d15574a-09b2-11f0-9f07-569c3dd06744","import { DOMParser } from \"https://esm.sh/linkedom@0.16.8\";\n\ninterface CrawlResult {\n url: string;\n title: string;\n content: string;\n links: string[];\n depth: number;\n}\n\ninterface SitemapNode {\n url: string;\n title: string;\n children: SitemapNode[];\n}\n\n/**\n * Crawls a website recursively and returns markdown content\n */\nexport async function crawlWebsite(\n startUrl: string, \n maxDepth: number = 3\n): Promise {\n const visited = new Set();\n const results: CrawlResult[] = [];\n const urlObj = new URL(startUrl);\n const baseDomain = urlObj.hostname;\n \n // Create a queue for BFS traversal\n const queue: { url: string; depth: number }[] = [{ url: startUrl, depth: 0 }];\n \n while (queue.length > 0) {\n const { url, depth } = queue.shift()!;\n \n // Skip if already visited or exceeds max depth\n if (visited.has(url) || depth > maxDepth) {\n continue;\n }\n \n visited.add(url);\n console.log(`Crawling ${url} (depth: ${depth})`);\n \n try {\n const result = await crawlPage(url, baseDomain);\n results.push({ ...result, depth });\n \n // Add links to queue if within same domain and not exceeding max depth\n if (depth < maxDepth) {\n for (const link of result.links) {\n if (!visited.has(link)) {\n queue.push({ url: link, depth: depth + 1 });\n }\n }\n }\n } catch (error) {\n console.error(`Error crawling ${url}:`, error);\n }\n }\n \n // Generate markdown from results\n return generateMarkdown(results, baseDomain);\n}\n\n/**\n * Crawls a single page and extracts content and links\n */\nasync function crawlPage(\n url: string, \n baseDomain: string\n): Promise {\n const response = await fetch(url);\n const html = await response.text();\n const parser = new DOMParser();\n const doc = parser.parseFromString(html, \"text/html\");\n \n // Extract title\n const title = doc.querySelector(\"title\")?.textContent || url;\n \n // Extract main content\n let content = \"\";\n \n // Try to find main content area\n const mainContent = \n doc.querySelector(\"main\") || \n doc.querySelector(\"article\") || \n doc.querySelector(\".content\") || \n doc.querySelector(\"#content\") ||\n doc.body;\n \n if (mainContent) {\n // Remove script and style tags\n const scripts = mainContent.querySelectorAll(\"script, style, iframe, nav, footer, header\");\n scripts.forEach(el => el.remove());\n \n // Get text content\n content = mainContent.textContent || \"\";\n \n // Clean up whitespace\n content = content.replace(/\\s+/g, \" \").trim();\n }\n \n // Extract links on the same domain\n const links: string[] = [];\n const linkElements = doc.querySelectorAll(\"a[href]\");\n \n linkElements.forEach(el => {\n try {\n const href = el.getAttribute(\"href\");\n if (!href) return;\n \n // Handle relative URLs\n let fullUrl: URL;\n try {\n fullUrl = new URL(href, url);\n } catch {\n return; // Invalid URL\n }\n \n // Only include links on the same domain\n if (fullUrl.hostname === baseDomain && \n !fullUrl.pathname.includes(\"#\") && // Exclude anchors\n !fullUrl.pathname.endsWith(\".jpg\") && \n !fullUrl.pathname.endsWith(\".jpeg\") && \n !fullUrl.pathname.endsWith(\".png\") && \n !fullUrl.pathname.endsWith(\".gif\") && \n !fullUrl.pathname.endsWith(\".pdf\") && \n !fullUrl.pathname.endsWith(\".zip\")) {\n links.push(fullUrl.href);\n }\n } catch (error) {\n console.error(\"Error processing link:\", error);\n }\n });\n \n return { url, title, content, links, depth: 0 };\n}\n\n/**\n * Builds a sitemap tree from crawl results\n */\nfunction buildSitemapTree(results: CrawlResult[]): SitemapNode[] {\n const urlToNode = new Map();\n \n // Create nodes for all URLs\n for (const result of results) {\n urlToNode.set(result.url, {\n url: result.url,\n title: result.title,\n children: []\n });\n }\n \n // Build parent-child relationships\n const rootNodes: SitemapNode[] = [];\n \n for (const result of results) {\n const node = urlToNode.get(result.url)!;\n \n if (result.depth === 0) {\n rootNodes.push(node);\n } else {\n // Find parent\n for (const parentResult of results) {\n if (parentResult.depth === result.depth - 1 && \n parentResult.links.includes(result.url)) {\n const parentNode = urlToNode.get(parentResult.url);\n if (parentNode && !parentNode.children.includes(node)) {\n parentNode.children.push(node);\n }\n }\n }\n }\n }\n \n return rootNodes;\n}\n\n/**\n * Generates markdown representation of sitemap\n */\nfunction generateSitemapMarkdown(node: SitemapNode, level: number = 0): string {\n const indent = \" \".repeat(level);\n let markdown = `${indent}- [${node.title}](${node.url})\\n`;\n \n for (const child of node.children) {\n markdown += generateSitemapMarkdown(child, level + 1);\n }\n \n return markdown;\n}\n\n/**\n * Generates markdown from crawl results\n */\nfunction generateMarkdown(results: CrawlResult[], baseDomain: string): string {\n // Sort results by depth for better organization\n results.sort((a, b) => a.depth - b.depth || a.url.localeCompare(b.url));\n \n // Build sitemap\n const sitemapTree = buildSitemapTree(results);\n \n let markdown = `# Website Content: ${baseDomain}\\n\\n`;\n \n // Add sitemap\n markdown += \"## Sitemap\\n\\n\";\n for (const rootNode of sitemapTree) {\n markdown += generateSitemapMarkdown(rootNode);\n }\n \n // Add content for each page\n markdown += \"\\n## Page Contents\\n\\n\";\n \n for (const result of results) {\n markdown += `### ${result.title}\\n`;\n markdown += `URL: ${result.url}\\n\\n`;\n markdown += result.content + \"\\n\\n\";\n markdown += \"---\\n\\n\";\n }\n \n return markdown;\n}",[148,166],{"_106":147,"_152":138,"_149":150,"_99":148,"_159":-5,"_157":158,"_103":33,"_151":-5,"_108":390,"_153":154,"_161":-5,"_155":120,"_160":27,"_102":391},["D",1742932265577],[148],{"_149":266,"_103":267,"_108":393,"_152":138,"_272":394},["D",1742932240045],[395],{"_106":264,"_152":138,"_149":266,"_99":265,"_159":269,"_157":179,"_103":33,"_151":-5,"_108":393,"_153":267,"_161":-5,"_155":120,"_160":27,"_102":396},[265],{"_149":398,"_103":94,"_108":399,"_152":138,"_272":400},"219cd9e2-09b2-11f0-9209-569c3dd06744",["D",1742932153594],[401],{"_106":402,"_152":138,"_149":398,"_99":265,"_159":403,"_157":179,"_103":249,"_151":-5,"_108":399,"_153":94,"_161":-5,"_155":120,"_160":120,"_102":404},"3a57b7a4-09b2-11f0-9fa7-569c3dd06744","# md_crawl\n\nA web application that crawls a website recursively and outputs its content in markdown format, optimized for consumption by LLMs.\n\n## Features\n\n- Input a URL to crawl\n- Set crawl depth (default: 3)\n- Recursively crawls all URLs on the same domain\n- Outputs a markdown string with:\n - A sitemap at the top explaining the overall structure\n - Content from all crawled pages\n- Editable textarea for the markdown output\n- Copy button with success state\n\n## Project Structure\n\n- `backend/`: Contains the Hono API server and crawling logic\n- `frontend/`: Contains the HTML, CSS, and client-side JavaScript\n- `shared/`: Contains shared types and utilities\n\n## How to Use\n\n1. Enter a URL to crawl\n2. Set the desired crawl depth (default: 3)\n3. Click \"Crawl\" to start the process\n4. View and edit the generated markdown in the textarea\n5. Click \"Copy\" to copy the markdown to your clipboard",[265],{"_149":398,"_103":249,"_108":406,"_152":138,"_272":407},["D",1742932112119],[408],{"_106":409,"_152":138,"_149":398,"_99":265,"_159":403,"_157":179,"_103":33,"_151":-5,"_108":406,"_153":249,"_161":-5,"_155":27,"_160":27,"_102":410},"219cd92e-09b2-11f0-9209-569c3dd06744",[265],"page","hasNext","routes/_app","actionData","errors"]

History