Broken Link Checker

\n setUrl(e.target.value)}\n placeholder=\"Enter website URL\"\n required\n />\n \n \n {error && (\n

Error

{error}

\n )}\n {progress && (\n

Progress

Phase: {progress.phase}

\n {progress.phase === \"fetching\" && (\n

\n Processed Pages: {progress.processedPages}\n
\n Total Links: {progress.totalLinks}\n
\n Remaining Pages: {progress.remainingPages}\n

\n )}\n {progress.phase === \"checking\" && (\n

\n Checked Links: {progress.checkedLinks} / {progress.totalLinks}\n
\n Broken Links: {progress.brokenLinks?.length || 0}\n
\n Unchecked Links: {progress.uncheckedLinks?.length || 0}\n
\n Progress: {progress.progress?.toFixed(2)}%\n

\n )}\n

\n )}\n {(progress?.currentBrokenLinks?.length > 0 || results) && (\n

Results

\n {progress?.phase === \"checking\" && (\n

Found {progress.currentBrokenLinks?.length || 0} broken links so far...

\n )}\n {results &&

{results.summary}

}\n

Broken Links

\n \n \n \n \n \n \n \n \n \n \n \n \n {renderBrokenLinks(\n progress?.currentBrokenLinks\n || (results?.brokenLinks?.internal?.concat(results?.brokenLinks?.external) || []),\n )}\n \n

#	Found On	Broken Link	Link Text	Status Code	Probable Cause

\n {(progress?.uncheckedLinks?.length > 0 || (results && results.uncheckedLinks)) && (\n <>\n

Unchecked Links

\n \n \n \n \n \n \n \n \n \n \n \n {(progress?.uncheckedLinks || results?.uncheckedLinks || []).map((link, index) => (\n \n \n \n \n \n \n \n ))}\n \n

#	Found On	Unchecked Link	Link Text	Reason
{`${index + 1}`}	\n {link.foundOn}\n	\n {link.url}\n	{link.text}	{link.reason}

\n \n )}\n

\n )}\n

\n );\n}\n\nfunction client() {\n createRoot(document.getElementById(\"root\")).render();\n}\n\nif (typeof document !== \"undefined\") {\n client();\n}\n\nlet currentProgress = null;\nlet lastProgressUpdate = null;\nlet progressUpdateCount = 0;\n\nexport default async function server(request: Request): Promise {\n const url = new URL(request.url);\n\n if (url.pathname === \"/api/progress\" && request.method === \"GET\") {\n const now = new Date();\n if (lastProgressUpdate && currentProgress) {\n const timeSinceLastUpdate = now.getTime() - lastProgressUpdate.getTime();\n if (timeSinceLastUpdate < 1000) {\n progressUpdateCount++;\n } else {\n progressUpdateCount = 0;\n }\n }\n lastProgressUpdate = now;\n\n return new Response(JSON.stringify(currentProgress), {\n headers: { \"Content-Type\": \"application/json\" },\n });\n }\n\n if (url.pathname === \"/api/check-links\" && request.method === \"POST\") {\n let requestBody;\n try {\n requestBody = await request.json();\n } catch (error) {\n console.error(\"Error parsing request body:\", error);\n return new Response(JSON.stringify({ error: \"Invalid request body\" }), {\n status: 400,\n headers: { \"Content-Type\": \"application/json\" },\n });\n }\n\n const { url: websiteUrl } = requestBody;\n if (!websiteUrl) {\n return new Response(JSON.stringify({ error: \"Missing URL in request body\" }), {\n status: 400,\n headers: { \"Content-Type\": \"application/json\" },\n });\n }\n\n try {\n const maxDepth = 5;\n currentProgress = {\n phase: \"fetching\",\n processedPages: 0,\n totalLinks: 0,\n remainingPages: 1,\n depth: 0,\n };\n\n const result = await findBrokenLinks(websiteUrl, maxDepth);\n\n return new Response(JSON.stringify(result), {\n headers: { \"Content-Type\": \"application/json\" },\n });\n } catch (error) {\n console.log(`\\n=== Detailed Error Information ===`);\n console.log(`Error name: ${error.name}`);\n console.log(`Error message: ${error.message}`);\n console.log(`Error stack: ${error.stack}`);\n console.log(`Current progress state: ${JSON.stringify(currentProgress, null, 2)}`);\n console.log(`Last progress update: ${lastProgressUpdate}`);\n console.log(`Progress update count: ${progressUpdateCount}`);\n console.log(`=== End Error Information ===\\n`);\n\n currentProgress = null;\n lastProgressUpdate = null;\n progressUpdateCount = 0;\n\n return new Response(\n JSON.stringify({\n error: error.message,\n errorName: error.name,\n errorStack: error.stack,\n lastKnownProgress: currentProgress,\n }),\n {\n status: 500,\n headers: {\n \"Content-Type\": \"application/json\",\n \"Access-Control-Allow-Origin\": \"*\",\n },\n },\n );\n }\n }\n\n return new Response(\n `\n \n \n Broken Link Checker\n \n \n \n

\n \n \n \n \n `,\n {\n headers: {\n \"content-type\": \"text/html\",\n },\n },\n );\n}\n\nconst css = `\nbody {\n font-family: Arial, sans-serif;\n line-height: 1.6;\n margin: 0;\n padding: 20px;\n background-color: #f4f4f4;\n}\n\nh1, h2, h3 {\n color: #333;\n}\n\nform {\n margin-bottom: 20px;\n}\n\ninput[type=\"url\"] {\n width: 300px;\n padding: 5px;\n margin-right: 10px;\n}\n\nbutton {\n padding: 5px 10px;\n background-color: #333;\n color: #fff;\n border: none;\n cursor: pointer;\n}\n\nbutton:disabled {\n background-color: #999;\n}\n\ntable {\n width: 100%;\n border-collapse: collapse;\n margin-top: 20px;\n}\n\nth, td {\n padding: 10px;\n border: 1px solid #ddd;\n text-align: left;\n}\n\nth {\n background-color: #f2f2f2;\n font-weight: bold;\n}\n\ntr:nth-child(even) {\n background-color: #f9f9f9;\n}\n\na {\n color: #0066cc;\n text-decoration: none;\n}\n\na:hover {\n text-decoration: underline;\n}\n`;",[],[321,328],{"_105":336,"_96":337,"_157":338,"_159":322,"_160":140,"_161":339,"_163":164,"_107":340,"_102":341,"_167":342,"_169":343,"_170":22,"_171":-5,"_172":344,"_100":345},"4e9a8374-f0a1-11ef-8f3c-e6cdfca9ef9f","website","650c74f8-ee3c-11ef-83ea-e6cdfca9ef9f",317,["D",1740176107082],47,"http","/** @jsxImportSource https://esm.sh/react */\nimport { renderToString } from \"npm:react-dom/server\";\n\nexport const website = (request: Request) => {\n // Log the URL parameter if it's in the request URL\n const url = new URL(request.url);\n const urlParam = url.searchParams.get(\"url\");\n if (urlParam) {\n console.log(\"URL parameter received:\", urlParam);\n }\n\n const renderedContent = renderToString(\n \n \n Page Title\n \n \n \n link:\n \n
\n \n \n \n ,\n );\n\n return new Response(renderedContent, {\n headers: { \"Content-Type\": \"text/html\" },\n });\n};",[],[321,337],[321],{"_105":348,"_96":349,"_157":350,"_159":-5,"_160":140,"_161":351,"_163":164,"_107":352,"_102":353,"_167":342,"_169":354,"_170":22,"_171":-5,"_172":355,"_100":356},"ae44f510-f0a2-11ef-ac61-e6cdfca9ef9f","interfacebetweenfrontendandbackend","de444536-ebbb-11ef-a6a0-e6cdfca9ef9f",321,["D",1740176697079],20,"import { checkWebsite } from \"./Backend/mainFunctionURLReciver2\";\nimport { website } from \"./Frontend/website\";\nexport default async function(req: Request): Promise {\n if (req.method == \"POST\" && req.url == \"/checkWebsite\") {\n const input = await req.text();\n return new Response(JSON.stringify(await checkWebsite(input)), { headers: { \"content-type\": \"application/json\" } });\n } else {\n return new Response(website, { headers: { \"content-type\": \"text/html;charset=UTF-8\" } });\n }\n}",[],[349],"files",[154,174,182,193,203,213,223,233,243,253,263,272,281,290,299,309,319,326,335,347],"counts",{"_361":32,"_362":188,"_363":32,"_364":365,"_368":369},"pulls","branches","forks","envVars",{"_366":32,"_367":32},"count","emptyCount","connections",{"_366":32,"_370":32},"unfilledCount","contents",[],"file",{"_105":194,"_96":195,"_157":196,"_159":177,"_160":140,"_161":197,"_163":164,"_107":375,"_102":199,"_167":189,"_169":200,"_170":22,"_171":-5,"_100":376},["D",1738882103836],[156,176,195],"upstreamProject","upstreamBranch",{"_105":143,"_107":380,"_381":382,"_96":44,"_141":106,"_102":145,"_142":-5,"_144":-5,"_383":-5,"_109":-5,"_151":22,"_111":112,"_150":95,"_16":121,"_149":97},["D",1737670113028],"updatedAt",["D",1738427708787],"lockfileKey","emptyEnvVars","isNew","flags",[],"routes/_app.x.$handle.$name.branch.$branch.history_.$",{"_94":95,"_96":97,"_98":99,"_100":101,"_103":390,"_373":395,"_398":399,"_432":433,"_442":-7,"_443":32,"_444":22},{"_105":106,"_96":97,"_107":391,"_109":-5,"_110":22,"_111":112,"_113":-5,"_114":-5,"_14":115,"_116":-5,"_117":22,"_118":-5,"_119":-5,"_120":32,"_16":121,"_122":123,"_124":123,"_125":126,"_127":126,"_128":126,"_129":123,"_130":123,"_23":392,"_134":393,"_137":394,"_151":22},["D",1737670113028],{"_94":95,"_132":133},{"_94":95,"_136":133,"_14":115},{"_102":139,"_96":99,"_105":140,"_141":106,"_142":143,"_144":145,"_146":112,"_147":106,"_148":121,"_149":97,"_150":95},{"_105":194,"_96":195,"_157":196,"_159":177,"_160":140,"_161":197,"_163":164,"_107":396,"_102":199,"_167":189,"_169":200,"_170":22,"_171":-5,"_100":397},["D",1738882103836],[156,176,195],"history",[400,402,408,413,418,423,427],{"_105":194,"_160":140,"_157":196,"_96":195,"_169":200,"_167":189,"_102":199,"_159":177,"_107":401,"_161":197,"_171":-5},["D",1738882103836],{"_105":403,"_160":140,"_157":196,"_96":195,"_169":404,"_167":189,"_102":405,"_159":177,"_107":406,"_161":407,"_171":-5},"0f4d570a-e4b7-11ef-a050-e6cdfca9ef9f","type CrawlProgress = {\n urlsFound: number;\n urlsProcessed: number;\n brokenUrlsFound: number;\n inProgress: boolean;\n startTime: number;\n lastUpdateTime: number;\n};\n\ntype BrokenLink = {\n url: string;\n sourceUrl: string | null;\n statusCode: number | null;\n errorMessage: string;\n timestamp: Date;\n possibleCause: string;\n};\n\ntype CrawlSummary = {\n totalURLsChecked: number;\n brokenURLsFound: number;\n brokenURLs: BrokenLink[];\n scanDuration: number;\n averageProcessingTime: number;\n crawlDepthReached: number;\n};\n\nexport class CrawlStats {\n private progress: CrawlProgress;\n private brokenLinks: BrokenLink[];\n private urlDepths: Map;\n private processingTimes: number[];\n\n constructor() {\n this.progress = {\n urlsFound: 0,\n urlsProcessed: 0,\n brokenUrlsFound: 0,\n inProgress: false,\n startTime: 0,\n lastUpdateTime: 0,\n };\n this.brokenLinks = [];\n this.urlDepths = new Map();\n this.processingTimes = [];\n }\n\n startCrawl(): void {\n this.reset();\n this.progress.startTime = Date.now();\n this.progress.inProgress = true;\n }\n\n urlFound(url: string, depth: number): void {\n this.progress.urlsFound++;\n this.urlDepths.set(url, depth);\n this.progress.lastUpdateTime = Date.now();\n }\n\n urlProcessed(url: string, processingTime: number): void {\n this.progress.urlsProcessed++;\n this.processingTimes.push(processingTime);\n this.progress.lastUpdateTime = Date.now();\n }\n\n recordError(url: string, sourceUrl: string | null, statusCode: number | null, errorMessage: string): void {\n this.progress.brokenUrlsFound++;\n\n const brokenLink: BrokenLink = {\n url,\n sourceUrl,\n statusCode,\n errorMessage,\n timestamp: new Date(),\n possibleCause: this.determineCause(statusCode, errorMessage),\n };\n\n this.brokenLinks.push(brokenLink);\n this.progress.lastUpdateTime = Date.now();\n }\n\n private determineCause(statusCode: number | null, error: string): string {\n if (statusCode) {\n switch (statusCode) {\n case 404:\n return \"Page not found - The URL may have been moved or deleted\";\n case 403:\n return \"Access forbidden - Authentication may be required\";\n case 500:\n return \"Server error - The website's server encountered an error\";\n case 502:\n return \"Bad gateway - There may be an issue with the server configuration\";\n case 503:\n return \"Service unavailable - The server may be overloaded or down for maintenance\";\n default:\n return `Server returned status code ${statusCode}`;\n }\n }\n\n if (error.includes(\"ETIMEDOUT\")) return \"Connection timed out - The server took too long to respond\";\n if (error.includes(\"ECONNREFUSED\")) return \"Connection refused - The server may be down\";\n if (error.includes(\"CERT_\")) return \"SSL/TLS certificate issue - There may be a security configuration problem\";\n\n return error || \"Unknown error occurred\";\n }\n\n getProgress(): CrawlProgress {\n return { ...this.progress };\n }\n\n generateSummary(): CrawlSummary {\n console.log(\"\\n=== GENERATING SUMMARY ===\");\n console.log(`📊 Total URLs checked: ${this.progress.urlsProcessed}`);\n console.log(`❌ Broken URLs found: ${this.progress.brokenUrlsFound}`);\n\n const scanDuration = Date.now() - this.progress.startTime;\n const crawlDepthReached = Math.max(...Array.from(this.urlDepths.values()), 0);\n\n console.log(`⏱️ Scan duration: ${scanDuration}ms`);\n console.log(`📏 Max depth reached: ${crawlDepthReached}`);\n\n const averageProcessingTime = this.processingTimes.length > 0\n ? this.processingTimes.reduce((a, b) => a + b, 0) / this.processingTimes.length\n : 0;\n\n console.log(\"🏁 Summary Generation Complete\");\n\n return {\n totalURLsChecked: this.progress.urlsProcessed,\n brokenURLsFound: this.progress.brokenUrlsFound,\n brokenURLs: [...this.brokenLinks],\n scanDuration,\n averageProcessingTime,\n crawlDepthReached,\n };\n }\n\n reset(): void {\n this.progress = {\n urlsFound: 0,\n urlsProcessed: 0,\n brokenUrlsFound: 0,\n inProgress: false,\n startTime: 0,\n lastUpdateTime: 0,\n };\n this.brokenLinks = [];\n this.urlDepths.clear();\n this.processingTimes = [];\n }\n}",5,["D",1738866035856],134,{"_105":409,"_160":140,"_157":196,"_96":195,"_169":410,"_167":189,"_102":180,"_159":177,"_107":411,"_161":412,"_171":-5},"2640ecf4-e4b4-11ef-8ded-e6cdfca9ef9f","type CrawlProgress = {\n urlsFound: number;\n urlsProcessed: number;\n brokenUrlsFound: number;\n inProgress: boolean;\n startTime: number;\n lastUpdateTime: number;\n};\n\ntype BrokenLink = {\n url: string;\n sourceUrl: string | null;\n statusCode: number | null;\n errorMessage: string;\n timestamp: Date;\n possibleCause: string;\n};\n\ntype CrawlSummary = {\n totalURLsChecked: number;\n brokenURLsFound: number;\n brokenURLs: BrokenLink[];\n scanDuration: number;\n averageProcessingTime: number;\n crawlDepthReached: number;\n};\n\nexport class CrawlStats {\n private progress: CrawlProgress;\n private brokenLinks: BrokenLink[];\n private urlDepths: Map;\n private processingTimes: number[];\n\n constructor() {\n this.progress = {\n urlsFound: 0,\n urlsProcessed: 0,\n brokenUrlsFound: 0,\n inProgress: false,\n startTime: 0,\n lastUpdateTime: 0,\n };\n this.brokenLinks = [];\n this.urlDepths = new Map();\n this.processingTimes = [];\n }\n\n startCrawl(): void {\n this.reset();\n this.progress.startTime = Date.now();\n this.progress.inProgress = true;\n }\n\n urlFound(url: string, depth: number): void {\n this.progress.urlsFound++;\n this.urlDepths.set(url, depth);\n this.progress.lastUpdateTime = Date.now();\n }\n\n urlProcessed(url: string, processingTime: number): void {\n this.progress.urlsProcessed++;\n this.processingTimes.push(processingTime);\n this.progress.lastUpdateTime = Date.now();\n }\n\n recordError(\n url: string,\n sourceUrl: string | null,\n statusCode: number | null,\n errorMessage: string,\n ): void {\n this.progress.brokenUrlsFound++;\n\n const brokenLink: BrokenLink = {\n url,\n sourceUrl,\n statusCode,\n errorMessage,\n timestamp: new Date(),\n possibleCause: this.determineCause(statusCode, errorMessage),\n };\n\n this.brokenLinks.push(brokenLink);\n this.progress.lastUpdateTime = Date.now();\n }\n\n private determineCause(statusCode: number | null, error: string): string {\n if (statusCode) {\n switch (statusCode) {\n case 404:\n return \"Page not found - The URL may have been moved or deleted\";\n case 403:\n return \"Access forbidden - Authentication may be required\";\n case 500:\n return \"Server error - The website's server encountered an error\";\n case 502:\n return \"Bad gateway - There may be an issue with the server configuration\";\n case 503:\n return \"Service unavailable - The server may be overloaded or down for maintenance\";\n default:\n return `Server returned status code ${statusCode}`;\n }\n }\n\n if (error.includes(\"ETIMEDOUT\")) return \"Connection timed out - The server took too long to respond\";\n if (error.includes(\"ECONNREFUSED\")) return \"Connection refused - The server may be down\";\n if (error.includes(\"CERT_\")) return \"SSL/TLS certificate issue - There may be a security configuration problem\";\n\n return error || \"Unknown error occurred\";\n }\n\n getProgress(): CrawlProgress {\n return { ...this.progress };\n }\n\n generateSummary(): CrawlSummary {\n console.log(\"\\n=== SUMMARY GENERATION DEBUG ===\");\n console.log(\"Current Progress:\", this.progress);\n console.log(\"URL Depths Map Size:\", this.urlDepths.size);\n console.log(\"Processing Times Length:\", this.processingTimes.length);\n console.log(\"Broken Links Count:\", this.brokenLinks.length);\n\n const scanDuration = Date.now() - this.progress.startTime;\n console.log(\"Calculated Duration:\", scanDuration);\n\n const crawlDepthReached = Math.max(...Array.from(this.urlDepths.values()), 0);\n console.log(\"Max Depth:\", crawlDepthReached);\n\n const averageProcessingTime = this.processingTimes.length > 0\n ? this.processingTimes.reduce((a, b) => a + b, 0) / this.processingTimes.length\n : 0;\n console.log(\"Average Processing Time:\", averageProcessingTime);\n\n return {\n totalURLsChecked: this.progress.urlsProcessed,\n brokenURLsFound: this.progress.brokenUrlsFound,\n brokenURLs: [...this.brokenLinks],\n scanDuration,\n averageProcessingTime,\n crawlDepthReached,\n };\n }\n\n reset(): void {\n this.progress = {\n urlsFound: 0,\n urlsProcessed: 0,\n brokenUrlsFound: 0,\n inProgress: false,\n startTime: 0,\n lastUpdateTime: 0,\n };\n this.brokenLinks = [];\n this.urlDepths.clear();\n this.processingTimes = [];\n }\n}",["D",1738864785874],133,{"_105":414,"_160":140,"_157":196,"_96":195,"_169":415,"_167":189,"_102":209,"_159":177,"_107":416,"_161":417,"_171":-5},"1614b588-e4b2-11ef-8fc7-e6cdfca9ef9f","type CrawlProgress = {\n urlsFound: number;\n urlsProcessed: number;\n brokenUrlsFound: number;\n inProgress: boolean;\n startTime: number;\n lastUpdateTime: number;\n};\n\ntype BrokenLink = {\n url: string;\n sourceUrl: string | null;\n statusCode: number | null;\n errorMessage: string;\n timestamp: Date;\n possibleCause: string;\n};\n\ntype CrawlSummary = {\n totalURLsChecked: number;\n brokenURLsFound: number;\n brokenURLs: BrokenLink[];\n scanDuration: number;\n averageProcessingTime: number;\n crawlDepthReached: number;\n};\n\nexport class CrawlStats {\n private progress: CrawlProgress;\n private brokenLinks: BrokenLink[];\n private urlDepths: Map;\n private processingTimes: number[];\n\n constructor() {\n this.progress = {\n urlsFound: 0,\n urlsProcessed: 0,\n brokenUrlsFound: 0,\n inProgress: false,\n startTime: 0,\n lastUpdateTime: 0,\n };\n this.brokenLinks = [];\n this.urlDepths = new Map();\n this.processingTimes = [];\n }\n\n startCrawl(): void {\n this.reset();\n this.progress.startTime = Date.now();\n this.progress.inProgress = true;\n }\n\n urlFound(url: string, depth: number): void {\n this.progress.urlsFound++;\n this.urlDepths.set(url, depth);\n this.progress.lastUpdateTime = Date.now();\n }\n\n urlProcessed(url: string, processingTime: number): void {\n this.progress.urlsProcessed++;\n this.processingTimes.push(processingTime);\n this.progress.lastUpdateTime = Date.now();\n }\n\n recordError(\n url: string,\n sourceUrl: string | null,\n statusCode: number | null,\n errorMessage: string,\n ): void {\n this.progress.brokenUrlsFound++;\n\n const brokenLink: BrokenLink = {\n url,\n sourceUrl,\n statusCode,\n errorMessage,\n timestamp: new Date(),\n possibleCause: this.determineCause(statusCode, errorMessage),\n };\n\n this.brokenLinks.push(brokenLink);\n this.progress.lastUpdateTime = Date.now();\n }\n\n private determineCause(statusCode: number | null, error: string): string {\n if (statusCode) {\n switch (statusCode) {\n case 404:\n return \"Page not found - The URL may have been moved or deleted\";\n case 403:\n return \"Access forbidden - Authentication may be required\";\n case 500:\n return \"Server error - The website's server encountered an error\";\n case 502:\n return \"Bad gateway - There may be an issue with the server configuration\";\n case 503:\n return \"Service unavailable - The server may be overloaded or down for maintenance\";\n default:\n return `Server returned status code ${statusCode}`;\n }\n }\n\n if (error.includes(\"ETIMEDOUT\")) return \"Connection timed out - The server took too long to respond\";\n if (error.includes(\"ECONNREFUSED\")) return \"Connection refused - The server may be down\";\n if (error.includes(\"CERT_\")) return \"SSL/TLS certificate issue - There may be a security configuration problem\";\n\n return error || \"Unknown error occurred\";\n }\n\n getProgress(): CrawlProgress {\n return { ...this.progress };\n }\n\n generateSummary(): CrawlSummary {\n console.log(\"\\n=== GENERATING SUMMARY ===\");\n console.log(`📊 Total URLs checked: ${this.progress.urlsProcessed}`);\n console.log(`❌ Broken URLs found: ${this.progress.brokenUrlsFound}`);\n\n const scanDuration = Date.now() - this.progress.startTime;\n const crawlDepthReached = Math.max(...Array.from(this.urlDepths.values()), 0);\n\n console.log(`⏱️ Scan duration: ${scanDuration}ms`);\n console.log(`📏 Max depth reached: ${crawlDepthReached}`);\n\n const averageProcessingTime = this.processingTimes.length > 0\n ? this.processingTimes.reduce((a, b) => a + b, 0) / this.processingTimes.length\n : 0;\n\n console.log(\"🏁 Summary Generation Complete\");\n\n return {\n totalURLsChecked: this.progress.urlsProcessed,\n brokenURLsFound: this.progress.brokenUrlsFound,\n brokenURLs: [...this.brokenLinks],\n scanDuration,\n averageProcessingTime,\n crawlDepthReached,\n };\n }\n\n reset(): void {\n this.progress = {\n urlsFound: 0,\n urlsProcessed: 0,\n brokenUrlsFound: 0,\n inProgress: false,\n startTime: 0,\n lastUpdateTime: 0,\n };\n this.brokenLinks = [];\n this.urlDepths.clear();\n this.processingTimes = [];\n }\n}",["D",1738863899748],132,{"_105":419,"_160":140,"_157":196,"_96":195,"_169":420,"_167":189,"_102":188,"_159":177,"_107":421,"_161":422,"_171":-5},"9ab68f2c-e0bc-11ef-b511-e6cdfca9ef9f","type CrawlProgress = {\n urlsFound: number;\n urlsProcessed: number;\n brokenUrlsFound: number;\n inProgress: boolean;\n startTime: number;\n lastUpdateTime: number;\n};\n\ntype BrokenLink = {\n url: string;\n sourceUrl: string | null;\n statusCode: number | null;\n errorMessage: string;\n timestamp: Date;\n possibleCause: string;\n};\n\ntype CrawlSummary = {\n totalURLsChecked: number;\n brokenURLsFound: number;\n brokenURLs: BrokenLink[];\n scanDuration: number;\n averageProcessingTime: number;\n crawlDepthReached: number;\n};\n\nexport class CrawlStats {\n private progress: CrawlProgress;\n private brokenLinks: BrokenLink[];\n private urlDepths: Map;\n private processingTimes: number[];\n\n constructor() {\n this.progress = {\n urlsFound: 0,\n urlsProcessed: 0,\n brokenUrlsFound: 0,\n inProgress: false,\n startTime: 0,\n lastUpdateTime: 0,\n };\n this.brokenLinks = [];\n this.urlDepths = new Map();\n this.processingTimes = [];\n }\n\n startCrawl(): void {\n this.reset();\n this.progress.startTime = Date.now();\n this.progress.inProgress = true;\n }\n\n urlFound(url: string, depth: number): void {\n this.progress.urlsFound++;\n this.urlDepths.set(url, depth);\n this.progress.lastUpdateTime = Date.now();\n }\n\n urlProcessed(url: string, processingTime: number): void {\n this.progress.urlsProcessed++;\n this.processingTimes.push(processingTime);\n this.progress.lastUpdateTime = Date.now();\n }\n\n recordError(\n url: string,\n sourceUrl: string | null,\n statusCode: number | null,\n errorMessage: string,\n ): void {\n this.progress.brokenUrlsFound++;\n\n const brokenLink: BrokenLink = {\n url,\n sourceUrl,\n statusCode,\n errorMessage,\n timestamp: new Date(),\n possibleCause: this.determineCause(statusCode, errorMessage),\n };\n\n this.brokenLinks.push(brokenLink);\n this.progress.lastUpdateTime = Date.now();\n }\n\n private determineCause(statusCode: number | null, error: string): string {\n if (statusCode) {\n switch (statusCode) {\n case 404:\n return \"Page not found - The URL may have been moved or deleted\";\n case 403:\n return \"Access forbidden - Authentication may be required\";\n case 500:\n return \"Server error - The website's server encountered an error\";\n case 502:\n return \"Bad gateway - There may be an issue with the server configuration\";\n case 503:\n return \"Service unavailable - The server may be overloaded or down for maintenance\";\n default:\n return `Server returned status code ${statusCode}`;\n }\n }\n\n if (error.includes(\"ETIMEDOUT\")) return \"Connection timed out - The server took too long to respond\";\n if (error.includes(\"ECONNREFUSED\")) return \"Connection refused - The server may be down\";\n if (error.includes(\"CERT_\")) return \"SSL/TLS certificate issue - There may be a security configuration problem\";\n\n return error || \"Unknown error occurred\";\n }\n\n getProgress(): CrawlProgress {\n return { ...this.progress };\n }\n\n generateSummary(): CrawlSummary {\n const scanDuration = Date.now() - this.progress.startTime;\n const crawlDepthReached = Math.max(...Array.from(this.urlDepths.values()), 0);\n\n const averageProcessingTime = this.processingTimes.length > 0\n ? this.processingTimes.reduce((a, b) => a + b, 0) / this.processingTimes.length\n : 0;\n\n return {\n totalURLsChecked: this.progress.urlsProcessed,\n brokenURLsFound: this.progress.brokenUrlsFound,\n brokenURLs: [...this.brokenLinks],\n scanDuration,\n averageProcessingTime,\n crawlDepthReached,\n };\n }\n\n reset(): void {\n this.progress = {\n urlsFound: 0,\n urlsProcessed: 0,\n brokenUrlsFound: 0,\n inProgress: false,\n startTime: 0,\n lastUpdateTime: 0,\n };\n this.brokenLinks = [];\n this.urlDepths.clear();\n this.processingTimes = [];\n }\n}",["D",1738428612582],26,{"_105":424,"_160":140,"_157":196,"_96":195,"_169":73,"_167":189,"_102":166,"_159":177,"_107":425,"_161":426,"_171":-5},"17ab035c-e0bb-11ef-9b79-e6cdfca9ef9f",["D",1738427963231],13,{"_105":428,"_160":140,"_157":196,"_96":429,"_169":73,"_167":189,"_102":32,"_159":177,"_107":430,"_161":431,"_171":-5},"1214f36c-e0bb-11ef-acc3-e6cdfca9ef9f","transcendentJadeEagle",["D",1738427953862],12,"versions",[434,436,437,438,439,440,441],{"_373":400,"_435":402},"previous",{"_373":402,"_435":408},{"_373":408,"_435":413},{"_373":413,"_435":418},{"_373":418,"_435":423},{"_373":423,"_435":427},{"_373":427,"_435":-5},"versionNumber","page","hasNext","actionData","errors"]

willthereader

failedBrokenUrlChecker

History for CrawlStats