[{"_1":2,"_173":-5,"_174":-5},"loaderData",{"_3":4,"_91":92},"root",{"_5":6,"_34":35,"_64":65},"ctx",{"_7":8,"_23":-5,"_24":25,"_26":27,"_28":20,"_29":20,"_30":20,"_31":32,"_33":20},"auth",{"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20,"_21":22},"sessionClaims","sessionId","sessionStatus","userId","actor","orgId","orgRole","orgSlug","orgPermissions","factorVerificationAge","isAuthenticated",false,"tokenType","session_token","user","preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","notificationCount",0,"flagDenols","ENV",{"_36":37,"_38":39,"_40":41,"_42":43,"_44":45,"_46":47,"_48":49,"_50":51,"_52":53,"_54":55,"_56":57,"_58":59,"_60":61,"_62":63},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","VAL_RUNTIME_NAME","prod","RENDER_GIT_COMMIT","a31a1550ad486cf340b818a53bc6ddc8c7db0295","RENDER_GIT_BRANCH","main","CLERK_PUBLISHABLE_KEY","pk_live_Y2xlcmsudmFsLnRvd24k","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","API_URL","https://api.val.town","clerkState",{"_66":67},"__internal_clerk_state",{"_68":69,"_70":45,"_71":72,"_73":72,"_74":20,"_75":72,"_76":72,"_77":72,"_78":72,"_79":80,"_89":20,"_90":20},"__clerk_ssr_state",{"_21":22,"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20},"__publishableKey","__proxyUrl","","__domain","__isSatellite","__signInUrl","__signUpUrl","__afterSignInUrl","__afterSignUpUrl","__clerk_debug",{"_81":20,"_19":20,"_82":72,"_83":84,"_85":72,"_86":45,"_87":20,"_88":72},"isSignedIn","proxyUrl","reason","session-token-and-uat-missing","message","publishableKey","isSatellite","domain","__telemetryDisabled","__telemetryDebug","routes/embed.x.$handle.$name.$",{"_93":94,"_95":96,"_97":98,"_99":100,"_149":150,"_170":171,"_172":-5},"handle","diegoivo","name","todo-app-conversion","path","backend/index.ts","project",{"_101":102,"_95":96,"_103":104,"_105":-5,"_106":20,"_107":108,"_109":-5,"_110":-5,"_12":111,"_112":113,"_114":115,"_116":115,"_117":-5,"_118":32,"_14":119,"_120":121,"_122":121,"_123":124,"_125":124,"_126":124,"_127":121,"_128":121,"_23":129,"_132":133,"_135":136,"_148":20},"id","f9c7981e-3da3-11f0-93a2-76b3cceeab13","createdAt",["D",1748643492802],"deletedAt","legacyEnvVars","privacy","public","description","imageUrl","add95710-0ddb-4dc2-90f2-bdcb16df89be","pinnedFileId","35d64666-3da4-11f0-9c2e-76b3cceeab13","pinPreview",true,"pinMigrated","screenshotVersion","likeCount","39c8580e-ee92-40a4-b9a2-165f574b6ae2","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm",{"_93":94,"_130":131},"profileImageUrl","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18ycWNtOXpnaDJrU25hbkZMNW15WjN5MlhldEUifQ","org",{"_93":94,"_134":131,"_12":111},"avatar","branch",{"_137":138,"_95":43,"_101":139,"_140":102,"_141":-5,"_142":-5,"_143":-5,"_144":-5,"_145":-5,"_146":96,"_147":94},"version",41,"f9c7ebf2-3da3-11f0-93a2-76b3cceeab13","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","isOwned","file",{"_101":151,"_95":152,"_153":113,"_154":155,"_156":139,"_157":158,"_159":115,"_103":160,"_137":161,"_162":163,"_164":165,"_166":20,"_167":-5,"_97":168},"c49cf868-460f-11f0-ade7-76b3cceeab13","index.ts","fileId","parentId","3ca415fd-99e5-4540-93f7-a2049e55b1e2","branchId","branchVersion",31,"latest",["D",1749569398544],9,"fileType","http","content","/** @jsxImportSource https://esm.sh/react@18.2.0 */\nimport { Hono } from \"https://esm.sh/hono@3.11.7\";\nimport { readFile, serveFile } from \"https://esm.town/v/std/utils@85-main/index.ts\";\nimport type { CrawlRequest, CrawlSession, CrawlResult, CrawlStatus } from \"../shared/types.ts\";\n\nconst app = new Hono();\n\n// In-memory storage for crawl sessions\nconst crawlSessions = new Map();\n\n// Unwrap Hono errors to see original error details\napp.onError((err, c) => {\n throw err;\n});\n\n// Serve static files\napp.get(\"/frontend/*\", c => serveFile(c.req.path, import.meta.url));\napp.get(\"/shared/*\", c => serveFile(c.req.path, import.meta.url));\n\n// Serve main page\napp.get(\"/\", async c => {\n let html = await readFile(\"/frontend/index.html\", import.meta.url);\n return c.html(html);\n});\n\n// Start crawling\napp.post(\"/api/crawl\", async c => {\n try {\n const request: CrawlRequest = await c.req.json();\n \n // Validate request\n if (!request.urls || request.urls.length === 0) {\n return c.json({ error: \"At least one URL is required\" }, 400);\n }\n\n // Validate URLs\n const validUrls = [];\n for (const url of request.urls) {\n try {\n new URL(url);\n validUrls.push(url);\n } catch {\n // Skip invalid URLs\n }\n }\n\n if (validUrls.length === 0) {\n return c.json({ error: \"No valid URLs provided\" }, 400);\n }\n\n const sessionId = crypto.randomUUID();\n const session: CrawlSession = {\n id: sessionId,\n status: 'pending',\n startTime: new Date().toISOString(),\n totalUrls: 0,\n processedUrls: 0,\n results: [],\n errors: [],\n config: {\n ...request,\n urls: validUrls,\n maxDepth: request.maxDepth || 1,\n maxPages: request.maxPages || 50,\n respectRobots: request.respectRobots ?? true,\n delay: request.delay || 1000\n }\n };\n\n crawlSessions.set(sessionId, session);\n\n // Start crawling asynchronously\n crawlUrls(sessionId);\n\n return c.json({ sessionId, status: 'started' });\n } catch (error) {\n return c.json({ error: \"Invalid request format\" }, 400);\n }\n});\n\n// Get crawl status\napp.get(\"/api/crawl/status/:id\", c => {\n const sessionId = c.req.param(\"id\");\n const session = crawlSessions.get(sessionId);\n \n if (!session) {\n return c.json({ error: \"Session not found\" }, 404);\n }\n\n const status: CrawlStatus = {\n id: session.id,\n status: session.status,\n progress: session.totalUrls > 0 ? Math.round((session.processedUrls / session.totalUrls) * 100) : 0,\n totalUrls: session.totalUrls,\n processedUrls: session.processedUrls,\n errors: session.errors\n };\n\n return c.json(status);\n});\n\n// Get crawl results\napp.get(\"/api/crawl/results/:id\", c => {\n const sessionId = c.req.param(\"id\");\n const session = crawlSessions.get(sessionId);\n \n if (!session) {\n return c.json({ error: \"Session not found\" }, 404);\n }\n\n return c.json({\n id: session.id,\n status: session.status,\n results: session.results,\n startTime: session.startTime,\n endTime: session.endTime,\n totalUrls: session.totalUrls,\n processedUrls: session.processedUrls,\n errors: session.errors\n });\n});\n\n// Download CSV report\napp.get(\"/api/crawl/download/:id\", c => {\n const sessionId = c.req.param(\"id\");\n const session = crawlSessions.get(sessionId);\n \n if (!session) {\n return c.json({ error: \"Session not found\" }, 404);\n }\n\n const csv = generateCSV(session.results);\n \n return new Response(csv, {\n headers: {\n 'Content-Type': 'text/csv',\n 'Content-Disposition': `attachment; filename=\"crawl-results-${sessionId}.csv\"`\n }\n });\n});\n\n// Crawling logic\nasync function crawlUrls(sessionId: string) {\n const session = crawlSessions.get(sessionId);\n if (!session) return;\n\n session.status = 'running';\n const urlsToProcess = new Set();\n const processedUrls = new Set();\n \n // Add initial URLs\n session.config.urls.forEach(url => urlsToProcess.add(url));\n session.totalUrls = urlsToProcess.size;\n\n try {\n for (const url of urlsToProcess) {\n if (processedUrls.has(url)) continue;\n if (session.processedUrls >= (session.config.maxPages || 50)) break;\n\n try {\n const result = await crawlSingleUrl(url, 0);\n session.results.push(result);\n processedUrls.add(url);\n session.processedUrls++;\n\n // Add found links for deeper crawling if within depth limit\n if (session.config.maxDepth && session.config.maxDepth > 1) {\n const foundLinks = await extractLinks(url);\n foundLinks.forEach(link => {\n if (!processedUrls.has(link) && session.results.length < (session.config.maxPages || 50)) {\n urlsToProcess.add(link);\n session.totalUrls++;\n }\n });\n }\n\n // Respect delay\n if (session.config.delay) {\n await new Promise(resolve => setTimeout(resolve, session.config.delay));\n }\n\n } catch (error) {\n session.errors.push(`Error crawling ${url}: ${error.message}`);\n session.processedUrls++;\n }\n }\n\n session.status = 'completed';\n session.endTime = new Date().toISOString();\n } catch (error) {\n session.status = 'failed';\n session.errors.push(`Crawling failed: ${error.message}`);\n session.endTime = new Date().toISOString();\n }\n}\n\nasync function crawlSingleUrl(url: string, depth: number): Promise {\n const startTime = Date.now();\n \n try {\n const response = await fetch(url, {\n headers: {\n 'User-Agent': 'WebCrawler/1.0'\n }\n });\n \n const responseTime = Date.now() - startTime;\n const html = await response.text();\n const contentLength = html.length;\n \n // Parse HTML to extract metadata\n const title = extractTitle(html);\n const metaDescription = extractMetaDescription(html);\n const linksFound = countLinks(html);\n const imagesFound = countImages(html);\n\n return {\n url,\n title,\n metaDescription,\n statusCode: response.status,\n responseTime,\n contentLength,\n linksFound,\n imagesFound,\n crawlTimestamp: new Date().toISOString(),\n depth\n };\n } catch (error) {\n return {\n url,\n statusCode: 0,\n responseTime: Date.now() - startTime,\n contentLength: 0,\n linksFound: 0,\n imagesFound: 0,\n crawlTimestamp: new Date().toISOString(),\n error: error.message,\n depth\n };\n }\n}\n\nasync function extractLinks(url: string): Promise {\n try {\n const response = await fetch(url);\n const html = await response.text();\n const linkRegex = /]+href=[\"']([^\"']+)[\"'][^>]*>/gi;\n const links = [];\n let match;\n \n while ((match = linkRegex.exec(html)) !== null) {\n try {\n const link = new URL(match[1], url).href;\n links.push(link);\n } catch {\n // Skip invalid links\n }\n }\n \n return [...new Set(links)]; // Remove duplicates\n } catch {\n return [];\n }\n}\n\nfunction extractTitle(html: string): string | undefined {\n const titleMatch = html.match(/]*>([^<]+)<\\/title>/i);\n return titleMatch ? titleMatch[1].trim() : undefined;\n}\n\nfunction extractMetaDescription(html: string): string | undefined {\n const metaMatch = html.match(/]+name=[\"']description[\"'][^>]+content=[\"']([^\"']+)[\"'][^>]*>/i);\n return metaMatch ? metaMatch[1].trim() : undefined;\n}\n\nfunction countLinks(html: string): number {\n const linkMatches = html.match(/]+href=/gi);\n return linkMatches ? linkMatches.length : 0;\n}\n\nfunction countImages(html: string): number {\n const imgMatches = html.match(/]+src=/gi);\n return imgMatches ? imgMatches.length : 0;\n}\n\nfunction generateCSV(results: CrawlResult[]): string {\n const headers = [\n 'URL',\n 'Title',\n 'Meta Description',\n 'Status Code',\n 'Response Time (ms)',\n 'Content Length',\n 'Links Found',\n 'Images Found',\n 'Crawl Timestamp',\n 'Depth',\n 'Error'\n ];\n\n const rows = results.map(result => [\n result.url,\n result.title || '',\n result.metaDescription || '',\n result.statusCode.toString(),\n result.responseTime.toString(),\n result.contentLength.toString(),\n result.linksFound.toString(),\n result.imagesFound.toString(),\n result.crawlTimestamp,\n result.depth.toString(),\n result.error || ''\n ]);\n\n const csvContent = [headers, ...rows]\n .map(row => row.map(cell => `\"${cell.replace(/\"/g, '\"\"')}\"`).join(','))\n .join('\\n');\n\n return csvContent;\n}\n\nexport default app.fetch;","deleted","label",[169,152],"backend","permalink","https://www.val.town/x/diegoivo/todo-app-conversion/code/backend/index.ts","extension","actionData","errors"]