[{"_1":2,"_403":-5,"_404":-5},"loaderData",{"_3":4,"_92":93,"_336":337},"root",{"_5":6,"_35":36,"_65":66},"ctx",{"_7":8,"_23":-5,"_24":25,"_26":27,"_28":22,"_29":22,"_30":22,"_31":32,"_33":22,"_34":22},"auth",{"_9":10,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":-5,"_20":-5,"_21":22},"tokenType","session_token","sessionClaims","sessionId","sessionStatus","userId","actor","orgId","orgRole","orgSlug","orgPermissions","factorVerificationAge","isAuthenticated",false,"user","preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","notificationCount",0,"flagDenols","isTeamOrgMember","ENV",{"_37":38,"_39":40,"_41":42,"_43":44,"_45":46,"_47":48,"_49":50,"_51":52,"_53":54,"_55":56,"_57":58,"_59":60,"_61":62,"_63":64},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","VAL_RUNTIME_NAME","prod","RENDER_GIT_COMMIT","5c349ef9e7d7333839a0634dd567aae2d49623a6","RENDER_GIT_BRANCH","main","CLERK_PUBLISHABLE_KEY","pk_live_Y2xlcmsudmFsLnRvd24k","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","API_URL","https://api.val.town","clerkState",{"_67":68},"__internal_clerk_state",{"_69":70,"_71":46,"_72":73,"_74":73,"_75":22,"_76":73,"_77":73,"_78":73,"_79":73,"_80":81,"_90":22,"_91":22},"__clerk_ssr_state",{"_9":10,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":-5,"_20":-5,"_21":22},"__publishableKey","__proxyUrl","","__domain","__isSatellite","__signInUrl","__signUpUrl","__afterSignInUrl","__afterSignUpUrl","__clerk_debug",{"_82":22,"_21":22,"_83":73,"_84":85,"_86":73,"_87":46,"_88":22,"_89":73},"isSignedIn","proxyUrl","reason","session-token-and-uat-missing","message","publishableKey","isSatellite","domain","__telemetryDisabled","__telemetryDebug","routes/_app.x.$handle.$name",{"_94":95,"_96":97,"_98":99,"_100":101,"_102":103,"_104":105,"_153":154,"_305":306,"_307":308,"_319":320,"_321":322,"_325":-5,"_326":327,"_332":22,"_333":22,"_334":335},"handle","willthereader","name","failedBrokenUrlChecker","branchName","urlCrawler-top-down-programming","path","URLReceiver","version",134,"project",{"_106":107,"_96":97,"_108":109,"_110":-5,"_111":22,"_112":113,"_114":-5,"_115":-5,"_14":116,"_117":-5,"_118":22,"_119":-5,"_120":-5,"_121":32,"_16":122,"_123":124,"_125":124,"_126":127,"_128":127,"_129":127,"_130":124,"_131":124,"_23":132,"_135":136,"_138":139,"_152":22},"id","95beb4d2-d9d6-11ef-80a3-e6cdfca9ef9f","createdAt",["D",1737670113028],"deletedAt","legacyEnvVars","privacy","public","description","imageUrl","e5615e9f-07dd-4d28-aaa1-9ee23a41135a","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount","76badc02-5066-4776-b1f9-963b50ea816c","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm",{"_94":95,"_133":134},"profileImageUrl","https://img.clerk.com/eyJ0eXBlIjoiZGVmYXVsdCIsImlpZCI6Imluc18yTm10cGhQUklwU0tqZHNiOHNnNGd2ZjJURFUiLCJyaWQiOiJ1c2VyXzJObXZ0bE1rbVBEMkhHNzR3SmVmMHc3eldBQiJ9","org",{"_94":95,"_137":134,"_14":116},"avatar","branch",{"_102":140,"_96":99,"_106":141,"_142":107,"_143":144,"_145":146,"_147":113,"_148":107,"_149":122,"_150":97,"_151":95},323,"9b4920dc-e0ba-11ef-8ff3-e6cdfca9ef9f","projectId","forkedBranchId","95bf0900-d9d6-11ef-80a3-e6cdfca9ef9f","forkedBranchVersion",127,"forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","isOwned","tree",[155,254,263,270,278,287,297],{"_106":156,"_96":157,"_158":159,"_160":-5,"_161":141,"_162":163,"_164":22,"_108":165,"_102":166,"_167":168,"_169":-5,"_170":22,"_171":-5,"_172":173,"_100":253},"78757fe2-e49d-11ef-8612-e6cdfca9ef9f","crawlerFolder","fileId","1ee3eabf-6ea4-46ca-91bd-ccee4f73c781","parentId","branchId","branchVersion",110,"latest",["D",1738855045365],1,"fileType","directory","content","deleted","label","children",[174,186,195,205,215,225,235,244],{"_106":175,"_96":176,"_158":177,"_160":159,"_161":141,"_162":178,"_164":179,"_108":180,"_102":181,"_167":182,"_169":183,"_170":22,"_171":-5,"_172":184,"_100":185},"abd9e2a4-e0bc-11ef-977c-e6cdfca9ef9f","CrawlQueue","313265ae-e0bb-11ef-9b33-e6cdfca9ef9f",27,true,["D",1738428641334],2,"script","type QueueItem = {\n url: string;\n sourceUrl: string | null;\n depth: number;\n priority: number;\n discovered: Date;\n};\n\ntype QueueStats = {\n totalEnqueued: number;\n totalProcessed: number;\n currentQueueSize: number;\n averageWaitTime: number;\n maxDepthSeen: number;\n};\n\nexport class CrawlQueue {\n private queue: QueueItem[] = [];\n private processed: Set = new Set();\n private stats: QueueStats;\n\n constructor() {\n this.stats = {\n totalEnqueued: 0,\n totalProcessed: 0,\n currentQueueSize: 0,\n averageWaitTime: 0,\n maxDepthSeen: 0,\n };\n }\n\n initialize(startUrl: string): void {\n this.queue = [{\n url: startUrl,\n sourceUrl: null,\n depth: 0,\n priority: 1,\n discovered: new Date(),\n }];\n this.stats.totalEnqueued = 1;\n this.stats.currentQueueSize = 1;\n }\n\n enqueue(url: string, sourceUrl: string | null, depth: number): boolean {\n if (this.isProcessed(url) || this.isQueued(url)) {\n return false;\n }\n\n const priority = this.calculatePriority(depth, url);\n\n const queueItem: QueueItem = {\n url,\n sourceUrl,\n depth,\n priority,\n discovered: new Date(),\n };\n\n this.insertWithPriority(queueItem);\n\n this.stats.totalEnqueued++;\n this.stats.currentQueueSize = this.queue.length;\n this.stats.maxDepthSeen = Math.max(this.stats.maxDepthSeen, depth);\n\n return true;\n }\n\n dequeue(): QueueItem | null {\n if (this.isEmpty()) {\n return null;\n }\n\n const item = this.queue.shift()!;\n this.processed.add(item.url);\n\n this.stats.totalProcessed++;\n this.stats.currentQueueSize = this.queue.length;\n this.updateAverageWaitTime(item);\n\n return item;\n }\n\n private calculatePriority(depth: number, url: string): number {\n let priority = 1;\n\n // Prioritize shallower depths\n priority += (10 - depth) * 0.1;\n\n // Prioritize URLs that might be important pages\n if (this.isLikelyImportantPage(url)) {\n priority += 0.2;\n }\n\n // Prioritize URLs with fewer query parameters\n const queryParams = new URL(url).searchParams.size;\n priority += (10 - queryParams) * 0.05;\n\n return Math.max(0, Math.min(1, priority));\n }\n\n private isLikelyImportantPage(url: string): boolean {\n const lowercaseUrl = url.toLowerCase();\n const importantPatterns = [\n \"/index.\",\n \"/home.\",\n \"/main.\",\n \"/about.\",\n \"/contact.\",\n \"/sitemap.\",\n ];\n\n return importantPatterns.some(pattern => lowercaseUrl.includes(pattern));\n }\n\n private insertWithPriority(item: QueueItem): void {\n const insertIndex = this.queue.findIndex(\n existing => existing.priority < item.priority,\n );\n\n if (insertIndex === -1) {\n this.queue.push(item);\n } else {\n this.queue.splice(insertIndex, 0, item);\n }\n }\n\n private updateAverageWaitTime(item: QueueItem): void {\n const waitTime = Date.now() - item.discovered.getTime();\n this.stats.averageWaitTime = (\n this.stats.averageWaitTime * (this.stats.totalProcessed - 1) + waitTime\n ) / this.stats.totalProcessed;\n }\n\n isEmpty(): boolean {\n return this.queue.length === 0;\n }\n\n isQueued(url: string): boolean {\n return this.queue.some(item => item.url === url);\n }\n\n isProcessed(url: string): boolean {\n return this.processed.has(url);\n }\n\n getQueueLength(): number {\n return this.queue.length;\n }\n\n getStats(): QueueStats {\n return { ...this.stats };\n }\n\n peek(): QueueItem | null {\n return this.queue[0] || null;\n }\n\n clear(): void {\n this.queue = [];\n this.processed.clear();\n this.stats = {\n totalEnqueued: 0,\n totalProcessed: 0,\n currentQueueSize: 0,\n averageWaitTime: 0,\n maxDepthSeen: 0,\n };\n }\n}",[],[157,176],{"_106":187,"_96":188,"_158":189,"_160":159,"_161":141,"_162":103,"_164":22,"_108":190,"_102":191,"_167":182,"_169":192,"_170":22,"_171":-5,"_172":193,"_100":194},"0f4d570a-e4b7-11ef-a050-e6cdfca9ef9f","CrawlStats","1214f40c-e0bb-11ef-acc3-e6cdfca9ef9f",["D",1738866035856],5,"type CrawlProgress = {\n urlsFound: number;\n urlsProcessed: number;\n brokenUrlsFound: number;\n inProgress: boolean;\n startTime: number;\n lastUpdateTime: number;\n};\n\ntype BrokenLink = {\n url: string;\n sourceUrl: string | null;\n statusCode: number | null;\n errorMessage: string;\n timestamp: Date;\n possibleCause: string;\n};\n\ntype CrawlSummary = {\n totalURLsChecked: number;\n brokenURLsFound: number;\n brokenURLs: BrokenLink[];\n scanDuration: number;\n averageProcessingTime: number;\n crawlDepthReached: number;\n};\n\nexport class CrawlStats {\n private progress: CrawlProgress;\n private brokenLinks: BrokenLink[];\n private urlDepths: Map;\n private processingTimes: number[];\n\n constructor() {\n this.progress = {\n urlsFound: 0,\n urlsProcessed: 0,\n brokenUrlsFound: 0,\n inProgress: false,\n startTime: 0,\n lastUpdateTime: 0,\n };\n this.brokenLinks = [];\n this.urlDepths = new Map();\n this.processingTimes = [];\n }\n\n startCrawl(): void {\n this.reset();\n this.progress.startTime = Date.now();\n this.progress.inProgress = true;\n }\n\n urlFound(url: string, depth: number): void {\n this.progress.urlsFound++;\n this.urlDepths.set(url, depth);\n this.progress.lastUpdateTime = Date.now();\n }\n\n urlProcessed(url: string, processingTime: number): void {\n this.progress.urlsProcessed++;\n this.processingTimes.push(processingTime);\n this.progress.lastUpdateTime = Date.now();\n }\n\n recordError(url: string, sourceUrl: string | null, statusCode: number | null, errorMessage: string): void {\n this.progress.brokenUrlsFound++;\n\n const brokenLink: BrokenLink = {\n url,\n sourceUrl,\n statusCode,\n errorMessage,\n timestamp: new Date(),\n possibleCause: this.determineCause(statusCode, errorMessage),\n };\n\n this.brokenLinks.push(brokenLink);\n this.progress.lastUpdateTime = Date.now();\n }\n\n private determineCause(statusCode: number | null, error: string): string {\n if (statusCode) {\n switch (statusCode) {\n case 404:\n return \"Page not found - The URL may have been moved or deleted\";\n case 403:\n return \"Access forbidden - Authentication may be required\";\n case 500:\n return \"Server error - The website's server encountered an error\";\n case 502:\n return \"Bad gateway - There may be an issue with the server configuration\";\n case 503:\n return \"Service unavailable - The server may be overloaded or down for maintenance\";\n default:\n return `Server returned status code ${statusCode}`;\n }\n }\n\n if (error.includes(\"ETIMEDOUT\")) return \"Connection timed out - The server took too long to respond\";\n if (error.includes(\"ECONNREFUSED\")) return \"Connection refused - The server may be down\";\n if (error.includes(\"CERT_\")) return \"SSL/TLS certificate issue - There may be a security configuration problem\";\n\n return error || \"Unknown error occurred\";\n }\n\n getProgress(): CrawlProgress {\n return { ...this.progress };\n }\n\n generateSummary(): CrawlSummary {\n console.log(\"\\n=== GENERATING SUMMARY ===\");\n console.log(`📊 Total URLs checked: ${this.progress.urlsProcessed}`);\n console.log(`❌ Broken URLs found: ${this.progress.brokenUrlsFound}`);\n\n const scanDuration = Date.now() - this.progress.startTime;\n const crawlDepthReached = Math.max(...Array.from(this.urlDepths.values()), 0);\n\n console.log(`⏱️ Scan duration: ${scanDuration}ms`);\n console.log(`📏 Max depth reached: ${crawlDepthReached}`);\n\n const averageProcessingTime = this.processingTimes.length > 0\n ? this.processingTimes.reduce((a, b) => a + b, 0) / this.processingTimes.length\n : 0;\n\n console.log(\"🏁 Summary Generation Complete\");\n\n return {\n totalURLsChecked: this.progress.urlsProcessed,\n brokenURLsFound: this.progress.brokenUrlsFound,\n brokenURLs: [...this.brokenLinks],\n scanDuration,\n averageProcessingTime,\n crawlDepthReached,\n };\n }\n\n reset(): void {\n this.progress = {\n urlsFound: 0,\n urlsProcessed: 0,\n brokenUrlsFound: 0,\n inProgress: false,\n startTime: 0,\n lastUpdateTime: 0,\n };\n this.brokenLinks = [];\n this.urlDepths.clear();\n this.processingTimes = [];\n }\n}",[],[157,188],{"_106":196,"_96":197,"_158":198,"_160":159,"_161":141,"_162":199,"_164":179,"_108":200,"_102":201,"_167":182,"_169":202,"_170":22,"_171":-5,"_172":203,"_100":204},"b2303536-e0bc-11ef-977c-e6cdfca9ef9f","ErrorHandler","211b0f36-e0bb-11ef-9b79-e6cdfca9ef9f",28,["D",1738428651968],3,"type ErrorCategory =\n | \"network\"\n | \"http\"\n | \"parsing\"\n | \"validation\"\n | \"system\"\n | \"unknown\";\n\ntype ErrorDetails = {\n category: ErrorCategory;\n code: string;\n message: string;\n timestamp: Date;\n url?: string;\n sourceUrl?: string | null;\n statusCode?: number | null;\n attempts?: number;\n};\n\nexport class ErrorHandler {\n private errors: ErrorDetails[] = [];\n private errorCallbacks: ((error: ErrorDetails) => void)[] = [];\n\n handleNetworkError(error: Error, url: string, sourceUrl: string | null = null): ErrorDetails {\n const details = this.categorizeNetworkError(error, url, sourceUrl);\n this.logError(details);\n return details;\n }\n\n handleHttpError(\n statusCode: number,\n url: string,\n sourceUrl: string | null = null,\n ): ErrorDetails {\n const details = this.categorizeHttpError(statusCode, url, sourceUrl);\n this.logError(details);\n return details;\n }\n\n handleParsingError(\n error: Error,\n url: string,\n sourceUrl: string | null = null,\n ): ErrorDetails {\n const details: ErrorDetails = {\n category: \"parsing\",\n code: \"PARSE_ERROR\",\n message: `Failed to parse content: ${error.message}`,\n timestamp: new Date(),\n url,\n sourceUrl,\n };\n this.logError(details);\n return details;\n }\n\n handleValidationError(\n message: string,\n url: string,\n sourceUrl: string | null = null,\n ): ErrorDetails {\n const details: ErrorDetails = {\n category: \"validation\",\n code: \"VALIDATION_ERROR\",\n message,\n timestamp: new Date(),\n url,\n sourceUrl,\n };\n this.logError(details);\n return details;\n }\n\n handleCrawlError(error: unknown): ErrorDetails {\n const details: ErrorDetails = {\n category: \"system\",\n code: \"CRAWL_ERROR\",\n message: error instanceof Error ? error.message : \"Unknown crawl error occurred\",\n timestamp: new Date(),\n };\n this.logError(details);\n return details;\n }\n\n private categorizeNetworkError(\n error: Error,\n url: string,\n sourceUrl: string | null,\n ): ErrorDetails {\n const details: ErrorDetails = {\n category: \"network\",\n timestamp: new Date(),\n url,\n sourceUrl,\n };\n\n if (error.message.includes(\"ETIMEDOUT\")) {\n details.code = \"TIMEOUT\";\n details.message = \"Connection timed out\";\n } else if (error.message.includes(\"ECONNREFUSED\")) {\n details.code = \"CONNECTION_REFUSED\";\n details.message = \"Connection refused\";\n } else if (error.message.includes(\"CERT_\")) {\n details.code = \"SSL_ERROR\";\n details.message = \"SSL/TLS certificate error\";\n } else {\n details.code = \"NETWORK_ERROR\";\n details.message = error.message;\n }\n\n return details;\n }\n\n private categorizeHttpError(\n statusCode: number,\n url: string,\n sourceUrl: string | null,\n ): ErrorDetails {\n const details: ErrorDetails = {\n category: \"http\",\n timestamp: new Date(),\n url,\n sourceUrl,\n statusCode,\n };\n\n switch (statusCode) {\n case 404:\n details.code = \"NOT_FOUND\";\n details.message = \"Page not found\";\n break;\n case 403:\n details.code = \"FORBIDDEN\";\n details.message = \"Access forbidden\";\n break;\n case 500:\n details.code = \"SERVER_ERROR\";\n details.message = \"Internal server error\";\n break;\n case 502:\n details.code = \"BAD_GATEWAY\";\n details.message = \"Bad gateway\";\n break;\n case 503:\n details.code = \"SERVICE_UNAVAILABLE\";\n details.message = \"Service unavailable\";\n break;\n default:\n details.code = \"HTTP_ERROR\";\n details.message = `HTTP error ${statusCode}`;\n }\n\n return details;\n }\n\n private logError(error: ErrorDetails): void {\n this.errors.push(error);\n this.notifyErrorCallbacks(error);\n }\n\n onError(callback: (error: ErrorDetails) => void): void {\n this.errorCallbacks.push(callback);\n }\n\n private notifyErrorCallbacks(error: ErrorDetails): void {\n this.errorCallbacks.forEach(callback => {\n try {\n callback(error);\n } catch (callbackError) {\n console.error(\"Error in error callback:\", callbackError);\n }\n });\n }\n\n getErrors(): ErrorDetails[] {\n return [...this.errors];\n }\n\n clearErrors(): void {\n this.errors = [];\n }\n}",[],[157,197],{"_106":206,"_96":207,"_158":208,"_160":159,"_161":141,"_162":209,"_164":179,"_108":210,"_102":211,"_167":182,"_169":212,"_170":22,"_171":-5,"_172":213,"_100":214},"db94ddc6-e4b0-11ef-9bb1-e6cdfca9ef9f","HTTPClient","c9cca258-e0ba-11ef-9716-e6cdfca9ef9f",130,["D",1738863372106],10,"export class HTTPClient {\n private userAgent: string = \"Mozilla/5.0 (compatible; URLCrawlerBot/1.0;)\";\n\n async fetchUrl(url: string, sourceUrl: string | null = null): Promise {\n try {\n const controller = new AbortController();\n const timeoutId = setTimeout(() => controller.abort(), 10000); // 10 second timeout\n\n const response = await fetch(url, {\n method: \"GET\",\n headers: {\n \"User-Agent\": this.userAgent,\n \"Accept\": \"text/html,application/xhtml+xml,application/xml;q=0.9\",\n \"Accept-Language\": \"en-US,en;q=0.5\",\n },\n signal: controller.signal,\n });\n\n clearTimeout(timeoutId);\n\n const content = await response.text();\n\n if (!response.ok) {\n console.log(`Failed URL (${response.status}): ${url}`);\n console.log(`From page: ${sourceUrl || \"Starting URL\"}`);\n return {\n url,\n sourceUrl,\n status: response.status,\n content: null,\n error: `HTTP ${response.status} - ${response.statusText}`,\n timestamp: new Date(),\n };\n }\n\n if (response.status === 403 || response.status === 401) {\n const isGoogleDocsAccessDenied = content.includes(\"You need access\") || content.includes(\"Request access\");\n const isAccessDenied = isGoogleDocsAccessDenied || content.includes(\"Sign in\");\n\n if (isAccessDenied) {\n console.log(`Access denied: ${url}`);\n return {\n url,\n sourceUrl,\n status: 403,\n content: null,\n error: \"Access denied or requires authentication\",\n timestamp: new Date(),\n };\n }\n }\n\n return {\n url,\n sourceUrl,\n status: response.status,\n content,\n error: null,\n timestamp: new Date(),\n };\n } catch (error) {\n console.log(`Error checking URL: ${url}`);\n console.log(`Error type: ${error instanceof Error ? error.message : String(error)}`);\n console.log(`From page: ${sourceUrl || \"Starting URL\"}`);\n\n return {\n url,\n sourceUrl,\n status: null,\n content: null,\n error: error instanceof Error ? error.message : String(error),\n timestamp: new Date(),\n };\n }\n }\n\n setUserAgent(userAgent: string): void {\n this.userAgent = userAgent;\n }\n}",[],[157,207],{"_106":216,"_96":217,"_158":218,"_160":159,"_161":141,"_162":219,"_164":179,"_108":220,"_102":221,"_167":182,"_169":222,"_170":22,"_171":-5,"_172":223,"_100":224},"295c9b48-e330-11ef-8113-e6cdfca9ef9f","URLHandler","e5125468-e0ba-11ef-8c30-e6cdfca9ef9f",96,["D",1738698146328],23,"export class URLExtractor {\n validate(url: string): { isValid: boolean; error?: string } {\n try {\n new URL(url);\n return { isValid: true };\n } catch {\n return { isValid: false, error: \"Invalid URL format\" };\n }\n }\n\n getBaseUrl(url: string): string {\n try {\n const urlObj = new URL(url);\n return urlObj.origin;\n } catch {\n throw new Error(\"Invalid URL\");\n }\n }\n\n async extractUrls(html: string, baseUrl: string): Promise {\n // console.log(\"\\n=== URL Extraction ===\");\n // console.log(`Base URL: ${baseUrl}`);\n // console.log(`Content length: ${html.length} characters`);\n\n const extractedUrls = new Set();\n\n const patterns = [\n /href=[\"']([^\"']+)[\"']/gi,\n /src=[\"']([^\"']+)[\"']/gi,\n /data-href=[\"']([^\"']+)[\"']/gi,\n /data-url=[\"']([^\"']+)[\"']/gi,\n /url\\([\"']?([^\"'\\)]+)[\"']?\\)/gi,\n /[\"']([^\"']*?\\.(?:html|php|asp|jsp))[\"']/gi,\n ];\n\n for (const pattern of patterns) {\n const matches = html.matchAll(pattern);\n for (const match of matches) {\n try {\n const urlAttr = match[1];\n let fullUrl: string;\n\n if (urlAttr.startsWith(\"/\")) {\n const baseUrlObj = new URL(baseUrl);\n fullUrl = `${baseUrlObj.origin}${urlAttr}`;\n } else if (urlAttr.startsWith(\"//\")) {\n fullUrl = `https:${urlAttr}`;\n } else if (urlAttr.startsWith(\"http\")) {\n fullUrl = urlAttr;\n } else if (!urlAttr.includes(\"://\")) {\n const baseUrlObj = new URL(baseUrl);\n const urlObject = new URL(urlAttr, baseUrlObj.origin);\n fullUrl = urlObject.toString();\n } else {\n fullUrl = urlAttr;\n }\n\n if (fullUrl.startsWith(\"http://\") || fullUrl.startsWith(\"https://\")) {\n extractedUrls.add(fullUrl);\n }\n } catch (error) {\n console.log(`Failed to parse URL: ${match[1]}, Error: ${error}`);\n }\n }\n }\n\n const urls = [...extractedUrls];\n console.log(`Extracted ${urls.length} unique URLs`);\n return urls;\n }\n\n shouldCrawl(url: string, baseUrl: string, depth: number, maxDepth: number): boolean {\n try {\n if (depth > maxDepth) {\n console.log(`Depth limit exceeded: Current depth ${depth} > max depth ${maxDepth}`);\n return false;\n }\n\n const urlObj = new URL(url);\n const baseUrlObj = new URL(baseUrl);\n const isExternalLink = urlObj.hostname !== baseUrlObj.hostname;\n\n if (isExternalLink && depth > maxDepth) {\n console.log(`External link at depth ${depth} > max depth ${maxDepth}`);\n return false;\n }\n\n return true;\n } catch {\n console.log(`URL parsing failed: \"${url}\" is not a valid URL`);\n return false;\n }\n }\n}\n\n// console.log(new URLExtractor().validate(\"https://willkrouse.com\"));",[],[157,217],{"_106":226,"_96":227,"_158":228,"_160":159,"_161":141,"_162":229,"_164":22,"_108":230,"_102":231,"_167":182,"_169":232,"_170":22,"_171":-5,"_172":233,"_100":234},"099715ee-e4b2-11ef-8fcf-e6cdfca9ef9f","crawler","cca02940-e0bb-11ef-991e-e6cdfca9ef9f",131,["D",1738863878791],34,"import { CrawlStats } from \"./CrawlStats\";\nimport { ErrorHandler } from \"./ErrorHandler\";\nimport { HTTPClient } from \"./HTTPClient\";\nimport { URLExtractor } from \"./URLHandler\";\n\ntype CrawlJob = {\n url: string;\n sourceUrl: string | null;\n depth: number;\n};\n\nexport class ParallelCrawler {\n private visitedUrls: Set;\n private processingUrls: Set;\n private urlBatch: CrawlJob[];\n private httpClient: HTTPClient;\n private urlExtractor: URLExtractor;\n private crawlStats: CrawlStats;\n private errorHandler: ErrorHandler;\n private maxDepth: number;\n private baseUrl: string;\n private maxConcurrent: number;\n private rateLimitDelay: number;\n\n constructor(\n maxDepth: number = 5,\n maxConcurrent: number = 5,\n rateLimitDelay: number = 100,\n ) {\n this.maxDepth = maxDepth;\n this.maxConcurrent = maxConcurrent;\n this.rateLimitDelay = rateLimitDelay;\n this.httpClient = new HTTPClient();\n this.urlExtractor = new URLExtractor();\n this.crawlStats = new CrawlStats();\n this.errorHandler = new ErrorHandler();\n this.visitedUrls = new Set();\n this.processingUrls = new Set();\n this.urlBatch = [];\n this.baseUrl = \"\";\n }\n\n private async delay(ms: number): Promise {\n return new Promise(resolve => setTimeout(resolve, ms));\n }\n\n private async processBatch(jobs: CrawlJob[]): Promise {\n const promises = jobs.map(job => this.processUrl(job));\n await Promise.all(promises);\n }\n\n private async processUrl({ url, sourceUrl, depth }: CrawlJob): Promise {\n if (this.visitedUrls.has(url) || this.processingUrls.has(url)) {\n // console.log(`Skipping duplicate URL: ${url}`);\n return;\n }\n\n this.processingUrls.add(url);\n\n try {\n const currentUrlObj = new URL(url);\n const isExternalLink = currentUrlObj.hostname !== new URL(this.baseUrl).hostname;\n\n if (isExternalLink) {\n // console.log(`Processing external link: ${url}`);\n }\n\n if (!isExternalLink) {\n this.visitedUrls.add(url);\n }\n\n const response = await this.httpClient.fetchUrl(url, sourceUrl);\n await this.delay(this.rateLimitDelay);\n\n if (response.error || (response.status && response.status >= 400)) {\n console.log(`Error processing URL: ${url}`);\n console.log(`Source URL: ${sourceUrl || \"Starting URL\"}`);\n console.log(`Status: ${response.status || \"No status\"}`);\n console.log(`Error: ${response.error || \"No error message\"}`);\n\n if (response.status) {\n this.errorHandler.handleHttpError(response.status, url, sourceUrl);\n } else {\n this.errorHandler.handleNetworkError(\n new Error(response.error || \"Unknown error\"),\n url,\n sourceUrl,\n );\n }\n return;\n }\n\n if (!isExternalLink && response.content && depth < this.maxDepth) {\n const newUrls = await this.urlExtractor.extractUrls(response.content, url);\n\n for (const newUrl of newUrls) {\n if (!this.visitedUrls.has(newUrl) && !this.processingUrls.has(newUrl)) {\n if (this.urlExtractor.shouldCrawl(newUrl, this.baseUrl, depth + 1, this.maxDepth)) {\n this.urlBatch.push({\n url: newUrl,\n sourceUrl: url,\n depth: depth + 1,\n });\n this.crawlStats.urlFound(newUrl, depth + 1);\n } else {\n console.log(`Not crawling URL due to constraints: ${newUrl}`);\n console.log(`Current depth: ${depth + 1}, Max depth: ${this.maxDepth}`);\n }\n }\n }\n }\n\n this.crawlStats.urlProcessed(url, Date.now());\n } catch (error) {\n console.log(`Critical error processing URL: ${url}`);\n console.log(`Error details: ${error instanceof Error ? error.message : String(error)}`);\n this.errorHandler.handleCrawlError(error);\n } finally {\n this.processingUrls.delete(url);\n }\n }\n\n async crawl(startUrl: string): Promise {\n console.log(\"\\n=== CRAWLER STATUS ===\");\n try {\n const validation = this.urlExtractor.validate(startUrl);\n if (!validation.isValid) {\n console.log(\"❌ URL Validation Failed\");\n throw new Error(validation.error || \"Invalid URL format\");\n }\n console.log(\"✅ URL Validation Passed\");\n\n this.baseUrl = this.urlExtractor.getBaseUrl(startUrl);\n this.urlBatch = [{ url: startUrl, sourceUrl: null, depth: 0 }];\n this.crawlStats.startCrawl();\n console.log(\"🚀 Starting Crawl Process\");\n\n while (this.urlBatch.length > 0 || this.processingUrls.size > 0) {\n console.log(\"\\n--- Crawl Loop Status ---\");\n console.log(`📦 Batch Queue: ${this.urlBatch.length} URLs`);\n console.log(`⚙️ Currently Processing: ${this.processingUrls.size} URLs`);\n console.log(\"🔄 Processing URLs:\", Array.from(this.processingUrls));\n\n if (this.urlBatch.length > 0) {\n const currentBatch = this.urlBatch.splice(0, this.maxConcurrent);\n console.log(`\\n🔍 Processing New Batch: ${currentBatch.map(job => job.url).join(\", \")}`);\n await this.processBatch(currentBatch);\n } else {\n console.log(\"⏳ Waiting for in-progress crawls...\");\n await this.delay(100);\n }\n }\n\n console.log(\"\\n✨ Crawl Complete - Generating Summary\");\n return this.crawlStats.generateSummary();\n } catch (error) {\n console.log(\"\\n❌ Crawl Error:\", error);\n this.errorHandler.handleCrawlError(error);\n return this.crawlStats.generateSummary();\n }\n }\n\n reset(): void {\n this.visitedUrls.clear();\n this.processingUrls.clear();\n this.urlBatch = [];\n this.crawlStats.reset();\n this.errorHandler.clearErrors();\n this.baseUrl = \"\";\n }\n\n getProgress(): CrawlProgress {\n return this.crawlStats.getProgress();\n }\n}\n\n// Example usage:\nconst crawler = new ParallelCrawler(5, 3, 100);\nconst summary = await crawler.crawl(\"https://dateme.directory\");",[],[157,227],{"_106":236,"_96":237,"_158":238,"_160":159,"_161":141,"_162":239,"_164":22,"_108":240,"_102":199,"_167":182,"_169":241,"_170":22,"_171":-5,"_172":242,"_100":243},"6cdcb2c0-e4ae-11ef-b710-e6cdfca9ef9f","urlCrawlerMainFunction","ac3aab2c-e0ba-11ef-92ca-e6cdfca9ef9f",126,["D",1738862327356],"import { ParallelCrawler } from \"./crawler\";\nimport { CrawlQueue } from \"./CrawlQueue\";\nimport { CrawlStats } from \"./CrawlStats\";\nimport { ErrorHandler } from \"./ErrorHandler\";\nimport { HTTPClient } from \"./HTTPClient\";\nimport { URLExtractor } from \"./URLHandler\";\n\nexport async function startCrawl(startUrl: string, maxDepth: number = 5): Promise {\n const urlExtractor = new URLExtractor();\n const httpClient = new HTTPClient();\n const crawlStats = new CrawlStats();\n const errorHandler = new ErrorHandler();\n const crawlQueue = new CrawlQueue();\n\n // Try to validate URL by attempting to construct a URL object\n try {\n new URL(startUrl);\n } catch (error) {\n crawlStats.recordError(startUrl, null, null, \"Invalid URL format\");\n return crawlStats.generateSummary();\n }\n\n const crawler = new ParallelCrawler(\n maxDepth,\n httpClient,\n urlExtractor,\n crawlStats,\n errorHandler,\n );\n\n try {\n crawlQueue.initialize(startUrl);\n crawlStats.startCrawl();\n const summary = await crawler.crawl(startUrl);\n return summary;\n } catch (error) {\n errorHandler.handleCrawlError(error);\n return crawlStats.generateSummary();\n }\n}\n\n// Test the crawler\nconst summary = await startCrawl(\"https://dateme.directory\");\nconsole.log(JSON.stringify(summary, null, 2));",[],[157,237],{"_106":245,"_96":246,"_158":247,"_160":159,"_161":141,"_162":248,"_164":179,"_108":249,"_102":201,"_167":182,"_169":250,"_170":22,"_171":-5,"_172":251,"_100":252},"e51c15c6-e26c-11ef-927b-e6cdfca9ef9f","urlCrawlerTests","6c92405a-704d-41dc-9a37-eed12c239dec",45,["D",1738614279990],"import { expect } from \"jsr:@std/expect\";\nimport { startCrawl } from \"./urlCrawlerMainFunction\";\n\nconsole.log(\"Starting Enhanced URL Crawler Tests\");\n\nconst TEST_URLS = [\n // \"https://willkrouse.com\",\n \"https://dateme.directory\",\n // \"https://nonexistent-test-url-123.com\",\n // \"https://willkrouse.com/page1\",\n // \"https://willkrouse.com/page2\",\n // \"https://external-domain.com\",\n];\n\ninterface TestCase {\n description: string;\n inputs: Record;\n expectedBehavior: string;\n function: Function;\n}\n\nconst crawler = new URLCrawler(2);\n\nconst tests: TestCase[] = [\n {\n description: \"initialize crawler with default settings\",\n inputs: {},\n expectedBehavior: \"should return initial progress state\",\n function: () => {\n crawler.reset();\n const progress = crawler.getProgress();\n console.log(\"Initial crawler state:\", progress);\n expect(progress.urlsFound).toEqual(0);\n expect(progress.urlsProcessed).toEqual(0);\n expect(progress.brokenUrlsFound).toEqual(0);\n expect(progress.inProgress).toEqual(false);\n },\n },\n {\n description: \"verify maxDepth enforcement\",\n inputs: { url: TEST_URLS[0] },\n expectedBehavior: \"should not crawl beyond specified depth\",\n function: async (inputs) => {\n // Create mock URLs with known depths\n const baseUrl = inputs.url;\n const depth1Url = `${baseUrl}/page1`;\n const depth2Url = `${baseUrl}/page1/subpage`;\n const depth3Url = `${baseUrl}/page1/subpage/deeppage`;\n\n // Create a crawler with maxDepth of 2\n const crawler = new URLCrawler(2);\n\n // Track processed URLs and their depths\n const processedUrls = new Map();\n\n // Override extractUrlsFromPage to return known structure\n crawler[\"extractUrlsFromPage\"] = async (url: string) => {\n if (url === baseUrl) {\n return [depth1Url];\n }\n if (url === depth1Url) {\n return [depth2Url];\n }\n if (url === depth2Url) {\n return [depth3Url];\n }\n return [];\n };\n\n // Override checkUrl to track processed URLs\n crawler[\"checkUrl\"] = async (url: string, sourceUrl: string | null) => {\n const currentDepth = crawler[\"urlQueue\"].find(item => item.url === url)?.depth || 0;\n processedUrls.set(url, currentDepth);\n return {\n url,\n sourceUrl,\n status: 200,\n error: null,\n timestamp: new Date(),\n };\n };\n\n await crawler.crawl(baseUrl);\n\n console.log(\"Processed URLs and their depths:\");\n processedUrls.forEach((depth, url) => {\n console.log(`${url}: depth ${depth}`);\n });\n\n // Verify URLs at or below maxDepth were processed\n expect(processedUrls.has(baseUrl)).toBe(true);\n expect(processedUrls.has(depth1Url)).toBe(true);\n expect(processedUrls.has(depth2Url)).toBe(true);\n\n // Verify URLs beyond maxDepth were not processed\n expect(processedUrls.has(depth3Url)).toBe(false);\n\n // Verify no URLs were processed beyond maxDepth\n const maxProcessedDepth = Math.max(...processedUrls.values());\n expect(maxProcessedDepth).toBeLessThanOrEqual(2);\n },\n },\n {\n description: \"test domain boundary enforcement\",\n inputs: { url: TEST_URLS[0] },\n expectedBehavior: \"should only crawl URLs from the same domain\",\n function: async (inputs) => {\n const result = await crawler.crawl(inputs.url);\n\n const crawledDomains = new Set(\n [...result.brokenURLs, { url: inputs.url }].map(url => {\n try {\n return new URL(url.url).hostname;\n } catch {\n return null;\n }\n }).filter(Boolean),\n );\n\n console.log(\"Crawled domains:\", [...crawledDomains]);\n const expectedDomain = new URL(inputs.url).hostname;\n expect(crawledDomains.has(expectedDomain)).toBe(true);\n },\n },\n {\n description: \"verify URL normalization\",\n inputs: {\n testUrls: [\n \"https://willkrouse.com/path/\",\n \"https://willkrouse.com/path#fragment\",\n \"https://willkrouse.com/path?param=value\",\n ],\n },\n expectedBehavior: \"should normalize URLs consistently\",\n function: async (inputs) => {\n const results = await Promise.all(\n inputs.testUrls.map(url => crawler.crawl(url)),\n );\n\n const processedUrls = results.map(r => r.totalURLsChecked);\n console.log(\"Processed URLs per variation:\", processedUrls);\n\n // All variations should process the same number of URLs\n const uniqueCounts = new Set(processedUrls);\n console.log(\"Unique URL counts:\", [...uniqueCounts]);\n expect(uniqueCounts.size).toBeGreaterThanOrEqual(1);\n },\n },\n {\n description: \"handle various error scenarios\",\n inputs: {\n testUrls: [\n \"https://timeout.example.com\",\n \"https://ssl-error.example.com\",\n \"https://rate-limited.example.com\",\n ],\n },\n expectedBehavior: \"should properly categorize different errors\",\n function: async (inputs) => {\n const results = await Promise.all(\n inputs.testUrls.map(url => crawler.crawl(url)),\n );\n\n results.forEach((result, index) => {\n console.log(`Error handling for ${inputs.testUrls[index]}:`, {\n brokenCount: result.brokenURLsFound,\n error: result.brokenURLs[0]?.errorMessage,\n cause: result.brokenURLs[0]?.possibleCause,\n });\n\n expect(result.brokenURLsFound).toBeGreaterThanOrEqual(1);\n expect(result.brokenURLs[0]?.errorMessage).toBeTruthy();\n expect(result.brokenURLs[0]?.possibleCause).toBeTruthy();\n });\n },\n },\n];\n\nasync function runTests() {\n console.log(\"\\nStarting enhanced test suite...\");\n let passed = 0;\n let failed = 0;\n\n for (const test of tests) {\n console.log(`\\nExecuting test: ${test.description}`);\n console.log(`Expected behavior: ${test.expectedBehavior}\\n`);\n\n try {\n await test.function(test.inputs);\n console.log(\"✅ Test passed\\n\");\n passed++;\n } catch (error) {\n console.error(\"❌ Test failed:\", error, \"\\n\");\n failed++;\n }\n }\n\n console.log(\"\\nTest Execution Summary\");\n console.log(\"---------------------\");\n console.log(`Total tests: ${tests.length}`);\n console.log(`Passed: ${passed}`);\n console.log(`Failed: ${failed}`);\n}\n\nrunTests().catch(error => {\n console.error(\"Test suite execution failed:\", error);\n});",[],[157,246],[157],{"_106":255,"_96":256,"_158":257,"_160":-5,"_161":141,"_162":258,"_164":22,"_108":259,"_102":166,"_167":182,"_169":260,"_170":22,"_171":-5,"_172":261,"_100":262},"969ef930-e4a2-11ef-91f2-e6cdfca9ef9f","SummaryURLReceiver","b3508e3c-6c4c-46c9-a857-cc09cb06b6bd",113,["D",1738857243445],"type URLCheckResult = {\n url: string;\n sourceUrl: string | null;\n status: number | null;\n error: string | null;\n timestamp: Date;\n};\n\ntype BrokenURLReport = {\n url: string;\n sourceUrl: string | null;\n statusCode: number | null;\n errorMessage: string;\n timestamp: Date;\n possibleCause: string;\n};\n\ntype CrawlSummary = {\n totalURLsChecked: number;\n brokenURLsFound: number;\n brokenURLs: BrokenURLReport[];\n scanDuration: number;\n};\n\nexport class SummaryURLReceiver {\n private startTime: number = 0;\n private urlCheckResults: URLCheckResult[] = [];\n\n startNewScan(): void {\n this.startTime = Date.now();\n this.urlCheckResults = [];\n }\n\n addResult(result: URLCheckResult): void {\n this.urlCheckResults.push(result);\n }\n\n private determineErrorCause(status: number | null, error: string | null): string {\n if (status) {\n switch (status) {\n case 404:\n return \"Page not found - The URL may have been moved or deleted\";\n case 403:\n return \"Access forbidden - Authentication may be required\";\n case 500:\n return \"Server error - The website's server encountered an error\";\n case 502:\n return \"Bad gateway - There may be an issue with the server configuration\";\n case 503:\n return \"Service unavailable - The server may be overloaded or down for maintenance\";\n default:\n return `Server returned status code ${status}`;\n }\n }\n\n if (error) {\n if (error.includes(\"ETIMEDOUT\")) {\n return \"Connection timed out - The server took too long to respond\";\n }\n if (error.includes(\"ECONNREFUSED\")) {\n return \"Connection refused - The server may be down\";\n }\n if (error.includes(\"CERT_\")) {\n return \"SSL/TLS certificate issue - There may be a security configuration problem\";\n }\n return error;\n }\n\n return \"Unknown error occurred\";\n }\n\n generateSummary(): CrawlSummary {\n const scanDuration = Date.now() - this.startTime;\n const domainErrors = new Map>();\n const errorTypeCounts = new Map();\n const domainCounts = new Map>();\n\n // Group URLs by domain and error type\n this.urlCheckResults.forEach(result => {\n if (result.error || (result.status && result.status >= 400)) {\n const url = new URL(result.url);\n const domain = url.hostname;\n const errorType = this.determineErrorCause(result.status, result.error);\n\n // Count error types\n errorTypeCounts.set(errorType, (errorTypeCounts.get(errorType) || 0) + 1);\n\n // Count domain errors\n if (!domainCounts.has(domain)) {\n domainCounts.set(domain, new Map());\n }\n const domainErrorCount = domainCounts.get(domain)!;\n domainErrorCount.set(errorType, (domainErrorCount.get(errorType) || 0) + 1);\n\n // Group URLs\n if (!domainErrors.has(errorType)) {\n domainErrors.set(errorType, new Map());\n }\n const errorGroup = domainErrors.get(errorType)!;\n\n if (!errorGroup.has(domain)) {\n errorGroup.set(domain, []);\n }\n\n errorGroup.get(domain)!.push({\n url: result.url,\n sourceUrl: result.sourceUrl,\n statusCode: result.status,\n errorMessage: result.error || \"Unknown error\",\n timestamp: result.timestamp,\n possibleCause: errorType,\n });\n }\n });\n\n // Create summary text\n let summaryText = \"Error Type Summary:\\n\";\n errorTypeCounts.forEach((count, errorType) => {\n summaryText += `${errorType}: ${count} URLs\\n`;\n });\n\n summaryText += \"\\nDomain Error Summary:\\n\";\n domainCounts.forEach((errorCounts, domain) => {\n summaryText += `${domain}:\\n`;\n errorCounts.forEach((count, errorType) => {\n summaryText += ` ${errorType}: ${count} URLs\\n`;\n });\n });\n\n // Convert grouped errors to flat array for existing interface\n const brokenURLs: BrokenURLReport[] = [];\n domainErrors.forEach((domains, errorType) => {\n domains.forEach((urls) => {\n brokenURLs.push(...urls);\n });\n });\n\n return {\n totalURLsChecked: this.urlCheckResults.length,\n brokenURLsFound: brokenURLs.length,\n brokenURLs,\n scanDuration,\n summaryText,\n errorGroups: Array.from(domainErrors.entries()).map(([errorType, domains]) => ({\n errorType,\n domains: Array.from(domains.entries()).map(([domain, urls]) => ({\n domain,\n urls,\n })),\n })),\n };\n }\n\n reset(): void {\n this.startTime = 0;\n this.urlCheckResults = [];\n }\n}",[],[256],{"_106":264,"_96":101,"_158":265,"_160":-5,"_161":141,"_162":221,"_164":22,"_108":266,"_102":181,"_167":182,"_169":267,"_170":22,"_171":-5,"_172":268,"_100":269},"201864e8-e0bc-11ef-904d-e6cdfca9ef9f","45f1db7c-5c80-42d0-9edf-e299b4e64671",["D",1738428406866],"// This file handles receiving and validating URLs before they are processed\n\n// Type definition for validation result\ntype ValidationResult = {\n isValid: boolean;\n error?: string;\n};\n\n// Type definition for a validated URL\ntype ValidatedURL = {\n url: string;\n timestamp: Date;\n};\n\n// Class to handle URL receiving and validation\nexport class URLReceiver {\n private urlHistory: ValidatedURL[] = [];\n\n // Main method to receive and validate a URL\n receiveURL(inputURL: string): ValidationResult {\n // Trim any whitespace from the input\n const trimmedURL = inputURL.trim();\n\n // Check if URL is empty\n if (!trimmedURL) {\n return {\n isValid: false,\n error: \"URL cannot be empty\"\n };\n }\n\n // Try to create a URL object to validate format\n try {\n const urlObject = new URL(trimmedURL);\n \n // Check if protocol is http or https\n if (!urlObject.protocol.match(/^https?:$/)) {\n return {\n isValid: false,\n error: \"URL must use HTTP or HTTPS protocol\"\n };\n }\n\n // If we reach here, URL is valid\n const validatedURL: ValidatedURL = {\n url: trimmedURL,\n timestamp: new Date()\n };\n\n // Store in history\n this.urlHistory.push(validatedURL);\n\n return {\n isValid: true\n };\n } catch (error) {\n return {\n isValid: false,\n error: \"Invalid URL format\"\n };\n }\n }\n\n // Method to get validation history\n getURLHistory(): ValidatedURL[] {\n return [...this.urlHistory];\n }\n\n // Method to clear history\n clearHistory(): void {\n this.urlHistory = [];\n }\n}",[],[101],{"_106":271,"_96":272,"_158":273,"_160":-5,"_161":141,"_162":166,"_164":22,"_108":274,"_102":32,"_167":182,"_169":275,"_170":22,"_171":-5,"_172":276,"_100":277},"9b4b2cec-e0ba-11ef-8ff3-e6cdfca9ef9f","URLReceiverTest","3f11bab5-4709-468f-919a-25334ff4155b",["D",1738427754563],"import { URLReceiver } from \"https://esm.town/v/willthereader/urlReceiver\";\nimport { expect } from \"jsr:@std/expect\";\n\nconsole.log(\"Starting URL Receiver test execution\");\n\n// Store test URLs in an array for easy maintenance\nconst TEST_URLS = [\n \"https://willkrouse.com\",\n \"https://dateme.directory\",\n];\n\n// Store test inputs that will be applied to each test URL\nconst TEST_MODIFICATIONS = {\n withHTTP: (url: string) => url.replace(\"https://\", \"http://\"),\n withFTP: (url: string) => url.replace(\"https://\", \"ftp://\"),\n withoutProtocol: (url: string) => url.replace(\"https://\", \"\"),\n empty: () => \"\",\n};\n\ninterface TestCase {\n description: string;\n inputs: Record;\n expectedResult: unknown;\n function: Function;\n}\n\nconst urlReceiver = new URLReceiver();\n\n// Create test cases by combining URLs with modifications\nfunction createURLTests(): TestCase[] {\n const tests: TestCase[] = [];\n\n TEST_URLS.forEach(baseURL => {\n tests.push({\n description: `validate HTTPS URL: ${baseURL}`,\n inputs: { url: baseURL },\n expectedResult: { isValid: true },\n function: urlReceiver.receiveURL.bind(urlReceiver),\n });\n\n tests.push({\n description: `validate HTTP version: ${TEST_MODIFICATIONS.withHTTP(baseURL)}`,\n inputs: { url: TEST_MODIFICATIONS.withHTTP(baseURL) },\n expectedResult: { isValid: true },\n function: urlReceiver.receiveURL.bind(urlReceiver),\n });\n\n tests.push({\n description: `reject FTP version: ${TEST_MODIFICATIONS.withFTP(baseURL)}`,\n inputs: { url: TEST_MODIFICATIONS.withFTP(baseURL) },\n expectedResult: {\n isValid: false,\n error: \"URL must use HTTP or HTTPS protocol\",\n },\n function: urlReceiver.receiveURL.bind(urlReceiver),\n });\n\n tests.push({\n description: `reject without protocol: ${TEST_MODIFICATIONS.withoutProtocol(baseURL)}`,\n inputs: { url: TEST_MODIFICATIONS.withoutProtocol(baseURL) },\n expectedResult: {\n isValid: false,\n error: \"Invalid URL format\",\n },\n function: urlReceiver.receiveURL.bind(urlReceiver),\n });\n });\n\n // Start with history-related tests\n tests.push({\n description: \"clear URL history before testing\",\n inputs: {},\n expectedResult: undefined,\n function: urlReceiver.clearHistory.bind(urlReceiver),\n });\n\n tests.push({\n description: \"get empty URL history\",\n inputs: {},\n expectedResult: [],\n function: urlReceiver.getURLHistory.bind(urlReceiver),\n });\n\n tests.push({\n description: \"clear URL history\",\n inputs: {},\n expectedResult: undefined,\n function: urlReceiver.clearHistory.bind(urlReceiver),\n });\n\n return tests;\n}\n\nconst tests = createURLTests();\n\nfor (const testCase of tests) {\n console.log(`Running test: ${testCase.description}`);\n\n const result = testCase.function(testCase.inputs.url);\n console.log(`Input: ${JSON.stringify(testCase.inputs)}`);\n console.log(`Expected result: ${JSON.stringify(testCase.expectedResult)}`);\n console.log(`Actual result: ${JSON.stringify(result)}\\n`);\n\n if (testCase.description.includes(\"validate\") || testCase.description.includes(\"reject\")) {\n expect(result.isValid).toEqual(testCase.expectedResult.isValid);\n if (!result.isValid) {\n expect(result.error).toEqual(testCase.expectedResult.error);\n }\n } else {\n expect(result).toEqual(testCase.expectedResult);\n }\n}\n\nconsole.log(\"All tests completed\");",[],[272],{"_106":279,"_96":280,"_158":281,"_160":-5,"_161":141,"_162":282,"_164":22,"_108":283,"_102":166,"_167":182,"_169":284,"_170":22,"_171":-5,"_172":285,"_100":286},"57dff84a-e0bd-11ef-bf72-e6cdfca9ef9f","mainFunctionTest","f6faf654-87b4-4377-87d6-406098f10e04",32,["D",1738428929944],"import { expect } from \"jsr:@std/expect\";\nimport { checkWebsite } from \"./mainFunctionURLReciver2\";\n\nconsole.log(\"Starting Website Checker Tests\");\n\nconst TEST_URLS = [\n // \"https://willkrouse.com\",\n \"https://dateme.directory\",\n // \"https://nonexistent-test-url-123.com\",\n // \"invalid-url-format\",\n // \"ftp://invalid-protocol.com\",\n];\n\nasync function runTests() {\n console.log(\"\\nTesting valid website URL\");\n let result = await checkWebsite(TEST_URLS[0]);\n if (result.success) {\n console.log(\"\\nScan Results:\");\n console.log(`Total URLs checked: ${result.result.totalChecked}`);\n console.log(`Scan duration: ${result.result.scanDuration}`);\n console.log(`Number of broken links: ${result.result.brokenLinksCount}`);\n\n if (result.result.brokenLinks.length > 0) {\n console.log(\"\\nBroken Links Found:\");\n result.result.brokenLinks.forEach((link, index) => {\n console.log(`\\nBroken Link ${index + 1}:`);\n console.log(`URL: ${link.url}`);\n console.log(`Found on: ${link.foundOn}`);\n console.log(`Error: ${link.error}`);\n console.log(`Likely cause: ${link.cause}`);\n console.log(`Time detected: ${link.time}`);\n });\n }\n } else {\n console.error(`Error scanning website: ${result.error}`);\n }\n expect(result.success).toBe(true);\n\n console.log(\"\\nTesting nonexistent website URL\");\n result = await checkWebsite(TEST_URLS[2]);\n if (!result.success) {\n console.log(\"Test passed: Correctly handled nonexistent URL\");\n }\n expect(result.success).toBe(false);\n\n console.log(\"\\nTesting invalid URL format\");\n result = await checkWebsite(TEST_URLS[3]);\n if (!result.success) {\n console.log(\"Test passed: Correctly rejected invalid URL format\");\n }\n expect(result.success).toBe(false);\n\n console.log(\"\\nTesting invalid protocol\");\n result = await checkWebsite(TEST_URLS[4]);\n if (!result.success) {\n console.log(\"Test passed: Correctly rejected invalid protocol\");\n }\n expect(result.success).toBe(false);\n\n console.log(\"\\nAll tests completed\");\n}\n\nrunTests().catch(error => {\n console.error(\"Test execution failed:\", error);\n});",[],[280],{"_106":288,"_96":289,"_158":290,"_160":-5,"_161":141,"_162":291,"_164":22,"_108":292,"_102":293,"_167":182,"_169":294,"_170":22,"_171":-5,"_172":295,"_100":296},"cdcf6f32-e4ae-11ef-bdfb-e6cdfca9ef9f","mainFunctionURLReciver2","5388b4c6-83c0-44a7-a29f-20a4c23509f5",129,["D",1738862490006],7,"import { URLReceiver } from \"https://esm.town/v/willthereader/urlReceiver\";\nimport { SummaryURLReceiver } from \"https://esm.town/v/willthereader/URLReceiver/SummaryURLReceiver\";\nimport { startCrawl } from \"./crawlerFolder/urlCrawlerMainFunction\";\n\ntype WebsiteCheckResult = {\n success: boolean;\n error: string | null;\n result: {\n totalChecked: number;\n scanDuration: string;\n brokenLinksCount: number;\n brokenLinks: {\n url: string;\n foundOn: string;\n error: string;\n cause: string;\n time: string;\n }[];\n } | null;\n};\n\nexport async function checkWebsite(inputURL: string, maxDepth: number = 3): Promise {\n const urlReceiver = new URLReceiver();\n const summaryGenerator = new SummaryURLReceiver();\n\n // Initial URL validation\n const validationResult = urlReceiver.receiveURL(inputURL);\n if (!validationResult.isValid) {\n return {\n success: false,\n error: validationResult.error || \"Invalid URL\",\n result: null,\n };\n }\n\n try {\n // Start new scan and get crawl results\n summaryGenerator.startNewScan();\n const crawlSummary = await startCrawl(inputURL, maxDepth);\n\n // Process broken URLs\n crawlSummary.brokenURLs.forEach(brokenURL => {\n summaryGenerator.addResult({\n url: brokenURL.url,\n sourceUrl: brokenURL.sourceUrl,\n status: brokenURL.statusCode,\n error: brokenURL.errorMessage,\n timestamp: brokenURL.timestamp,\n });\n });\n\n // Generate final report\n const finalReport = summaryGenerator.generateSummary();\n const scanDurationSeconds = (finalReport.scanDuration / 1000).toFixed(2);\n\n // Clear resources\n summaryGenerator.reset();\n urlReceiver.clearHistory();\n\n // Format response\n return {\n success: true,\n error: null,\n result: {\n totalChecked: finalReport.totalURLsChecked,\n scanDuration: `${scanDurationSeconds} seconds`,\n brokenLinksCount: finalReport.brokenURLsFound,\n brokenLinks: finalReport.brokenURLs.map(link => ({\n url: link.url,\n foundOn: link.sourceUrl || \"Starting URL\",\n error: link.errorMessage,\n cause: link.possibleCause,\n time: link.timestamp.toISOString(),\n })),\n },\n };\n } catch (error) {\n // Error handling\n const errorMessage = error instanceof Error ? error.message : \"An unknown error occurred\";\n return {\n success: false,\n error: errorMessage,\n result: null,\n };\n }\n}\n\n// Example usage:\nawait checkWebsite(\"https://dateme.directory/browse\");",[],[289],{"_106":298,"_96":299,"_158":300,"_160":-5,"_161":141,"_162":166,"_164":22,"_108":301,"_102":32,"_167":182,"_169":302,"_170":22,"_171":-5,"_172":303,"_100":304},"9b4b3606-e0ba-11ef-8ff3-e6cdfca9ef9f","summaryURLReceiverTest","8bd356b6-eb91-469e-b3ed-efe8f6a0a214",["D",1738427754563],"import { SummaryURLReceiver } from \"https://esm.town/v/willthereader/URLReceiver/SummaryURLReceiver\";\nimport { expect } from \"jsr:@std/expect\";\n\nconsole.log(\"\\nSUMMARY GENERATOR TESTS\\n\");\n\nconst generator = new SummaryURLReceiver();\n\nconst TEST_URLS = [\n // \"https://willkrouse.com\",\n \"https://dateme.directory\",\n \"https://nonexistent.example.com\",\n \"https://restricted.example.com\",\n];\n\n// Test data with different URLs and status codes\nconst sampleResults = [\n {\n url: TEST_URLS[0],\n sourceUrl: null,\n status: 200,\n error: null,\n timestamp: new Date(),\n },\n {\n url: TEST_URLS[1],\n sourceUrl: TEST_URLS[0],\n status: 200,\n error: null,\n timestamp: new Date(),\n },\n {\n url: TEST_URLS[2],\n sourceUrl: TEST_URLS[0],\n status: 404,\n error: \"Not Found\",\n timestamp: new Date(),\n },\n {\n url: TEST_URLS[3],\n sourceUrl: TEST_URLS[1],\n status: 403,\n error: \"Forbidden\",\n timestamp: new Date(),\n },\n];\n\ninterface TestCase {\n description: string;\n function: Function;\n expectedBehavior: string;\n}\n\nfunction logTestInfo(message: string) {\n console.log(` ${message}`);\n}\n\nconst tests: TestCase[] = [\n {\n description: \"Initialize generator\",\n expectedBehavior: \"Should start with empty state\",\n function: () => {\n console.log(\"\\nTEST: Initialize generator\");\n console.log(\"Expected: Should start with no URLs processed\\n\");\n\n generator.reset();\n generator.startNewScan();\n const summary = generator.generateSummary();\n\n logTestInfo(\"Initial state:\");\n logTestInfo(`- URLs processed: ${summary.totalURLsChecked}`);\n logTestInfo(`- Problems found: ${summary.brokenURLsFound}`);\n logTestInfo(`- Time running: ${summary.scanDuration}ms`);\n\n expect(summary.totalURLsChecked).toEqual(0);\n expect(summary.brokenURLsFound).toEqual(0);\n expect(summary.brokenURLs).toEqual([]);\n expect(summary.scanDuration).toBeGreaterThanOrEqual(0);\n\n return summary;\n },\n },\n {\n description: \"Process test URLs\",\n expectedBehavior: \"Should identify working and broken URLs\",\n function: () => {\n console.log(\"\\nTEST: Processing multiple URLs\");\n console.log(\"Expected: Should correctly identify good and bad URLs\\n\");\n\n generator.startNewScan();\n\n logTestInfo(\"Processing URLs:\");\n sampleResults.forEach((result, index) => {\n logTestInfo(`\\nURL ${index + 1}:`);\n logTestInfo(`- Address: ${result.url}`);\n logTestInfo(`- Status code: ${result.status || \"None\"}`);\n logTestInfo(`- Error: ${result.error || \"None\"}`);\n generator.addResult(result);\n });\n\n const summary = generator.generateSummary();\n\n logTestInfo(\"\\nResults:\");\n logTestInfo(`- Total URLs checked: ${summary.totalURLsChecked}`);\n logTestInfo(`- Problems found: ${summary.brokenURLsFound}`);\n\n if (summary.brokenURLs.length > 0) {\n logTestInfo(\"\\nProblem details:\");\n summary.brokenURLs.forEach((problem, index) => {\n logTestInfo(`\\nProblem ${index + 1}:`);\n logTestInfo(`- URL: ${problem.url}`);\n logTestInfo(`- Status: ${problem.statusCode || \"None\"}`);\n logTestInfo(`- Error: ${problem.errorMessage}`);\n logTestInfo(`- Likely cause: ${problem.possibleCause}`);\n });\n }\n\n expect(summary.totalURLsChecked).toEqual(4);\n expect(summary.brokenURLsFound).toEqual(2);\n\n return summary;\n },\n },\n {\n description: \"Test error handling\",\n expectedBehavior: \"Should handle different types of errors\",\n function: () => {\n console.log(\"\\nTEST: Error handling\");\n console.log(\"Expected: Should identify and explain different errors\\n\");\n\n generator.startNewScan();\n\n const errorCases = [\n {\n url: \"https://timeout.example.com\",\n sourceUrl: null,\n status: null,\n error: \"ETIMEDOUT\",\n timestamp: new Date(),\n },\n {\n url: \"https://cert-error.example.com\",\n sourceUrl: null,\n status: null,\n error: \"CERT_HAS_EXPIRED\",\n timestamp: new Date(),\n },\n {\n url: \"https://server-error.example.com\",\n sourceUrl: null,\n status: 500,\n error: \"Internal Server Error\",\n timestamp: new Date(),\n },\n ];\n\n logTestInfo(\"Testing error cases:\");\n errorCases.forEach((errorCase, index) => {\n logTestInfo(`\\nError test ${index + 1}:`);\n logTestInfo(`- URL: ${errorCase.url}`);\n logTestInfo(`- Type: ${errorCase.error}`);\n generator.addResult(errorCase);\n });\n\n const summary = generator.generateSummary();\n\n logTestInfo(\"\\nResults:\");\n logTestInfo(`- URLs checked: ${summary.totalURLsChecked}`);\n logTestInfo(`- Errors found: ${summary.brokenURLsFound}`);\n\n summary.brokenURLs.forEach((error, index) => {\n logTestInfo(`\\nError ${index + 1} details:`);\n logTestInfo(`- URL: ${error.url}`);\n logTestInfo(`- Problem: ${error.errorMessage}`);\n logTestInfo(`- Explanation: ${error.possibleCause}`);\n });\n\n expect(summary.brokenURLsFound).toEqual(3);\n\n return summary;\n },\n },\n];\n\nasync function runTests() {\n console.log(\"Starting test run...\\n\");\n let passed = 0;\n let failed = 0;\n\n for (const test of tests) {\n try {\n await test.function();\n console.log(\"\\n✅ TEST PASSED\");\n passed++;\n } catch (error) {\n console.log(\"\\n❌ TEST FAILED\");\n console.log(`Error details: ${error}`);\n failed++;\n }\n console.log(\"----------------------------------------\");\n }\n\n console.log(\"\\nTEST SUMMARY\");\n console.log(`Total tests run: ${tests.length}`);\n console.log(`Passed: ${passed}`);\n console.log(`Failed: ${failed}`);\n}\n\nrunTests().catch(console.error);",[],[299],"files",[155,174,186,195,205,215,225,235,244,254,263,270,278,287,297],"counts",{"_309":32,"_310":181,"_311":32,"_312":313,"_316":317},"pulls","branches","forks","envVars",{"_314":32,"_315":32},"count","emptyCount","connections",{"_314":32,"_318":32},"unfilledCount","contents",[],"file",{"_106":264,"_96":101,"_158":265,"_160":-5,"_161":141,"_162":221,"_164":22,"_108":323,"_102":181,"_167":182,"_169":267,"_170":22,"_171":-5,"_100":324},["D",1738428406866],[101],"upstreamProject","upstreamBranch",{"_106":144,"_108":328,"_329":330,"_96":44,"_142":107,"_102":146,"_143":-5,"_145":-5,"_331":-5,"_110":-5,"_152":22,"_112":113,"_151":95,"_16":122,"_150":97},["D",1737670113028],"updatedAt",["D",1738427708787],"lockfileKey","emptyEnvVars","isNew","flags",[],"routes/_app.x.$handle.$name.branch.$branch.version.$version.code.$",{"_94":95,"_96":97,"_98":99,"_102":103,"_100":101,"_104":338,"_153":343,"_305":389,"_321":390,"_319":393,"_394":395,"_398":-5,"_399":181,"_400":32,"_401":402,"_312":-5},{"_106":107,"_96":97,"_108":339,"_110":-5,"_111":22,"_112":113,"_114":-5,"_115":-5,"_14":116,"_117":-5,"_118":22,"_119":-5,"_120":-5,"_121":32,"_16":122,"_123":124,"_125":124,"_126":127,"_128":127,"_129":127,"_130":124,"_131":124,"_23":340,"_135":341,"_138":342,"_152":22},["D",1737670113028],{"_94":95,"_133":134},{"_94":95,"_137":134,"_14":116},{"_102":140,"_96":99,"_106":141,"_142":107,"_143":144,"_145":146,"_147":113,"_148":107,"_149":122,"_150":97,"_151":95},[344,371,374,377,380,383,386],{"_106":156,"_96":157,"_158":159,"_160":-5,"_161":141,"_162":163,"_164":22,"_108":165,"_102":166,"_167":168,"_169":-5,"_170":22,"_171":-5,"_172":345,"_100":370},[346,349,352,355,358,361,364,367],{"_106":175,"_96":176,"_158":177,"_160":159,"_161":141,"_162":178,"_164":179,"_108":180,"_102":181,"_167":182,"_169":183,"_170":22,"_171":-5,"_172":347,"_100":348},[],[157,176],{"_106":187,"_96":188,"_158":189,"_160":159,"_161":141,"_162":103,"_164":22,"_108":190,"_102":191,"_167":182,"_169":192,"_170":22,"_171":-5,"_172":350,"_100":351},[],[157,188],{"_106":196,"_96":197,"_158":198,"_160":159,"_161":141,"_162":199,"_164":179,"_108":200,"_102":201,"_167":182,"_169":202,"_170":22,"_171":-5,"_172":353,"_100":354},[],[157,197],{"_106":206,"_96":207,"_158":208,"_160":159,"_161":141,"_162":209,"_164":179,"_108":210,"_102":211,"_167":182,"_169":212,"_170":22,"_171":-5,"_172":356,"_100":357},[],[157,207],{"_106":216,"_96":217,"_158":218,"_160":159,"_161":141,"_162":219,"_164":179,"_108":220,"_102":221,"_167":182,"_169":222,"_170":22,"_171":-5,"_172":359,"_100":360},[],[157,217],{"_106":226,"_96":227,"_158":228,"_160":159,"_161":141,"_162":229,"_164":22,"_108":230,"_102":231,"_167":182,"_169":232,"_170":22,"_171":-5,"_172":362,"_100":363},[],[157,227],{"_106":236,"_96":237,"_158":238,"_160":159,"_161":141,"_162":239,"_164":22,"_108":240,"_102":199,"_167":182,"_169":241,"_170":22,"_171":-5,"_172":365,"_100":366},[],[157,237],{"_106":245,"_96":246,"_158":247,"_160":159,"_161":141,"_162":248,"_164":179,"_108":249,"_102":201,"_167":182,"_169":250,"_170":22,"_171":-5,"_172":368,"_100":369},[],[157,246],[157],{"_106":255,"_96":256,"_158":257,"_160":-5,"_161":141,"_162":258,"_164":22,"_108":259,"_102":166,"_167":182,"_169":260,"_170":22,"_171":-5,"_172":372,"_100":373},[],[256],{"_106":264,"_96":101,"_158":265,"_160":-5,"_161":141,"_162":221,"_164":22,"_108":266,"_102":181,"_167":182,"_169":267,"_170":22,"_171":-5,"_172":375,"_100":376},[],[101],{"_106":271,"_96":272,"_158":273,"_160":-5,"_161":141,"_162":166,"_164":22,"_108":274,"_102":32,"_167":182,"_169":275,"_170":22,"_171":-5,"_172":378,"_100":379},[],[272],{"_106":279,"_96":280,"_158":281,"_160":-5,"_161":141,"_162":282,"_164":22,"_108":283,"_102":166,"_167":182,"_169":284,"_170":22,"_171":-5,"_172":381,"_100":382},[],[280],{"_106":288,"_96":289,"_158":290,"_160":-5,"_161":141,"_162":291,"_164":22,"_108":292,"_102":293,"_167":182,"_169":294,"_170":22,"_171":-5,"_172":384,"_100":385},[],[289],{"_106":298,"_96":299,"_158":300,"_160":-5,"_161":141,"_162":166,"_164":22,"_108":301,"_102":32,"_167":182,"_169":302,"_170":22,"_171":-5,"_172":387,"_100":388},[],[299],[344,346,349,352,355,358,361,364,367,371,374,377,380,383,386],{"_106":264,"_161":141,"_158":265,"_96":101,"_169":267,"_167":182,"_102":181,"_160":-5,"_108":391,"_162":221,"_171":-5,"_100":392},["D",1738428406866],[101],[],"upstream",{"_106":144,"_108":396,"_329":397,"_96":44,"_142":107,"_102":146,"_143":-5,"_145":-5,"_331":-5,"_110":-5,"_152":22,"_112":113,"_151":95,"_16":122,"_150":97},["D",1737670113028],["D",1738427708787],"pull","ahead","behind","templates",[],"actionData","errors"]