tag content\n const mainRegex = /]*>([\\s\\S]*?)<\\/main>/i;\n const mainMatch = mainRegex.exec(html);\n \n if (mainMatch && mainMatch[1]) {\n mainContent = mainMatch[1];\n } else {\n // Fallback: try to extract content div\n const contentRegex = /]*class=\"[^\"]*content[^\"]*\"[^>]*>([\\s\\S]*?)<\\/div>/i;\n const contentMatch = contentRegex.exec(html);\n \n if (contentMatch && contentMatch[1]) {\n mainContent = contentMatch[1];\n } else {\n // Last resort: use the body\n const bodyRegex = /]*>([\\s\\S]*?)<\\/body>/i;\n const bodyMatch = bodyRegex.exec(html);\n \n if (bodyMatch && bodyMatch[1]) {\n mainContent = bodyMatch[1];\n } else {\n mainContent = html;\n }\n }\n }\n \n return mainContent;\n}\n\n/**\n * Extract links from HTML\n */\nfunction extractLinks(html: string): string[] {\n const linkRegex = /]*?\\s+)?href=\"([^\"]*)\"[^>]*>(.*?)<\\/a>/gi;\n const links: string[] = [];\n let match;\n \n while ((match = linkRegex.exec(html)) !== null) {\n const href = match[1];\n if (href && href.startsWith('/') && !href.includes('#') && !href.includes('://')) {\n links.push(href);\n }\n }\n \n return [...new Set(links)]; // Remove duplicates\n}\n\n/**\n * Crawl a page and extract its content\n */\nasync function crawlPage(path: string, visited: Set = new Set()): Promise<{ markdown: string, links: string[] }> {\n // Skip if already visited\n if (visited.has(path)) {\n return { markdown: '', links: [] };\n }\n \n // Mark as visited\n visited.add(path);\n \n // Fetch the page\n const fullUrl = `https://docs.val.town${path.startsWith(\"/\") ? path : \"/\" + path}`;\n const response = await fetch(fullUrl);\n \n if (!response.ok) {\n return { \n markdown: `> Error fetching ${fullUrl}: ${response.status} ${response.statusText}\\n\\n`, \n links: [] \n };\n }\n \n const html = await response.text();\n \n // Extract main content\n const mainContent = extractMainContent(html);\n \n // Convert to markdown\n const markdown = `# ${path}\\n\\n${htmlToMarkdown(mainContent)}`;\n \n // Extract links\n const links = extractLinks(html);\n \n return { markdown, links };\n}\n\n/**\n * Scrapes the Val Town documentation and converts it to markdown\n */\nexport default async function(req: Request): Promise {\n try {\n const url = new URL(req.url);\n const singlePage = url.searchParams.get(\"path\") || \"\";\n const format = url.searchParams.get(\"format\") || \"html\";\n \n // If a specific page is requested, just show that page\n if (singlePage) {\n const { markdown } = await crawlPage(singlePage);\n \n if (format === \"markdown\") {\n return new Response(markdown, {\n headers: {\n \"Content-Type\": \"text/markdown\",\n },\n });\n }\n \n // Create HTML response for a single page\n return new Response(createHtmlResponse(markdown, singlePage), {\n headers: {\n \"Content-Type\": \"text/html\",\n },\n });\n }\n \n // Otherwise, crawl all pages\n const allMarkdown: string[] = [];\n const visited = new Set();\n const toVisit: string[] = [\"/\"];\n \n // Limit the number of pages to crawl to avoid timeouts\n const MAX_PAGES = 100;\n let pagesVisited = 0;\n \n while (toVisit.length > 0 && pagesVisited < MAX_PAGES) {\n const currentPath = toVisit.shift()!;\n \n // Skip if already visited\n if (visited.has(currentPath)) {\n continue;\n }\n \n console.log(`Crawling ${currentPath}`);\n const { markdown, links } = await crawlPage(currentPath, visited);\n \n if (markdown) {\n allMarkdown.push(markdown);\n }\n \n // Add new links to the queue\n for (const link of links) {\n if (!visited.has(link)) {\n toVisit.push(link);\n }\n }\n \n pagesVisited++;\n }\n \n const combinedMarkdown = allMarkdown.join(\"\\n\\n---\\n\\n\");\n \n if (format === \"markdown\") {\n return new Response(combinedMarkdown, {\n headers: {\n \"Content-Type\": \"text/markdown\",\n },\n });\n }\n \n // Create HTML response for all pages\n return new Response(createHtmlResponse(combinedMarkdown, \"All Documentation\"), {\n headers: {\n \"Content-Type\": \"text/html\",\n },\n });\n } catch (error) {\n console.error(\"Error:\", error);\n return new Response(`Error: ${error.message}`, { status: 500 });\n }\n}\n\n/**\n * Create HTML response\n */\nfunction createHtmlResponse(markdown: string, title: string): string {\n return `\n \n \n \n \n \n Val Town Docs - ${title}\n \n \n \n \n \n

\n \n Download Markdown\n \n

\n \n

Val Town Documentation

Current view: ${title}

\n All Documentation\n Original Docs\n

\n \n

\n \n \n

\n \n \n \n \n \n `;\n}","deleted","label",[98],"permalink","https://www.val.town/x/w3b/untitled-1234/code/index.ts","extension","routes/embed","actionData","errors"]