[{"_1":2,"_176":-5,"_177":-5},"loaderData",{"_3":4,"_92":93},"root",{"_5":6,"_35":36,"_65":66},"ctx",{"_7":8,"_23":-5,"_24":25,"_26":27,"_28":20,"_29":20,"_30":20,"_31":32,"_33":20,"_34":20},"auth",{"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20,"_21":22},"sessionClaims","sessionId","sessionStatus","userId","actor","orgId","orgRole","orgSlug","orgPermissions","factorVerificationAge","isAuthenticated",false,"tokenType","session_token","user","preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","notificationCount",0,"flagDenols","isTeamOrgMember","ENV",{"_37":38,"_39":40,"_41":42,"_43":44,"_45":46,"_47":48,"_49":50,"_51":52,"_53":54,"_55":56,"_57":58,"_59":60,"_61":62,"_63":64},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","VAL_RUNTIME_NAME","prod","RENDER_GIT_COMMIT","abb8403524a4061c15f290c317905fd5206b7d64","RENDER_GIT_BRANCH","main","CLERK_PUBLISHABLE_KEY","pk_live_Y2xlcmsudmFsLnRvd24k","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","API_URL","https://api.val.town","clerkState",{"_67":68},"__internal_clerk_state",{"_69":70,"_71":46,"_72":73,"_74":73,"_75":20,"_76":73,"_77":73,"_78":73,"_79":73,"_80":81,"_90":20,"_91":20},"__clerk_ssr_state",{"_21":22,"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20},"__publishableKey","__proxyUrl","","__domain","__isSatellite","__signInUrl","__signUpUrl","__afterSignInUrl","__afterSignUpUrl","__clerk_debug",{"_82":20,"_19":20,"_83":73,"_84":85,"_86":73,"_87":46,"_88":20,"_89":73},"isSignedIn","proxyUrl","reason","session-token-and-uat-missing","message","publishableKey","isSatellite","domain","__telemetryDisabled","__telemetryDebug","routes/embed.x.$handle.$name.$",{"_94":95,"_96":97,"_98":99,"_100":101,"_151":152,"_173":174,"_175":-5},"handle","pinsar","name","feistyAmberRhinoceros","path","backend/monitoring/healthChecker.ts","project",{"_102":103,"_96":97,"_104":105,"_106":-5,"_107":20,"_108":109,"_110":-5,"_111":-5,"_12":112,"_113":114,"_115":116,"_117":-5,"_118":119,"_120":32,"_14":121,"_122":123,"_124":123,"_125":126,"_127":126,"_128":126,"_129":123,"_130":123,"_23":131,"_134":135,"_137":138,"_150":20},"id","2149c16a-277d-11f0-b46e-569c3dd06744","createdAt",["D",1746207883134],"deletedAt","legacyEnvVars","privacy","public","description","imageUrl","851f6760-f7af-11ef-9322-0fd6f1c36e70","pinnedFileId","214e03d8-277d-11f0-b46e-569c3dd06744","pinPreview",true,"pinMigrated","screenshotVersion",19,"likeCount","094b6920-183c-466c-985a-fe1516b5d507","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm",{"_94":95,"_132":133},"profileImageUrl","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18ydG1LWVR3QzZpNUhHQkNmbzF5VlRvOGZweUQifQ","org",{"_94":95,"_136":133,"_12":112},"avatar","branch",{"_139":140,"_96":44,"_102":141,"_142":103,"_143":-5,"_144":-5,"_145":-5,"_146":-5,"_147":-5,"_148":97,"_149":95},"version",27,"214a9ba8-277d-11f0-b46e-569c3dd06744","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","isOwned","file",{"_102":153,"_96":154,"_155":156,"_157":158,"_159":141,"_160":161,"_162":116,"_104":163,"_139":32,"_164":165,"_166":167,"_168":20,"_169":-5,"_98":170},"b99e7dfe-3a04-11f0-aeac-9e149126039e","healthChecker.ts","fileId","b99e7e8a-3a04-11f0-aeac-9e149126039e","parentId","17c4688b-835b-4794-a45a-0600aee83343","branchId","branchVersion",16,"latest",["D",1748245241689],"fileType","script","content","import { MetricsCollector, SystemMetrics } from \"./metricsCollector.ts\";\n\nexport interface HealthStatus {\n status: 'healthy' | 'degraded' | 'critical';\n timestamp: number;\n uptime: number;\n checks: HealthCheck[];\n summary: {\n total: number;\n passing: number;\n warning: number;\n critical: number;\n };\n}\n\nexport interface HealthCheck {\n name: string;\n status: 'pass' | 'warn' | 'fail';\n message: string;\n duration: number;\n timestamp: number;\n metadata?: Record;\n}\n\nexport class HealthChecker {\n private metricsCollector: MetricsCollector;\n private startTime: number;\n private lastHealthCheck: number = 0;\n private healthHistory: HealthStatus[] = [];\n\n constructor(metricsCollector: MetricsCollector) {\n this.metricsCollector = metricsCollector;\n this.startTime = Date.now();\n }\n\n async getSystemHealth(): Promise {\n const startTime = performance.now();\n const timestamp = Date.now();\n \n // Run all health checks\n const checks = await Promise.all([\n this.checkEventIngestion(),\n this.checkProcessingLatency(),\n this.checkErrorRate(),\n this.checkThroughput(),\n this.checkSystemResources(),\n this.checkDataStorage(),\n this.checkProcessingPipeline()\n ]);\n \n // Calculate overall status\n const summary = this.calculateSummary(checks);\n const overallStatus = this.determineOverallStatus(summary);\n \n const healthStatus: HealthStatus = {\n status: overallStatus,\n timestamp,\n uptime: timestamp - this.startTime,\n checks,\n summary\n };\n \n // Store in history (keep last 100 checks)\n this.healthHistory.push(healthStatus);\n if (this.healthHistory.length > 100) {\n this.healthHistory.shift();\n }\n \n this.lastHealthCheck = timestamp;\n \n return healthStatus;\n }\n\n async checkEventIngestion(): Promise {\n const startTime = performance.now();\n \n try {\n const metrics = await this.metricsCollector.getCurrentMetrics();\n const eventsPerSecond = metrics.throughput.eventsPerSecond;\n const totalEvents = metrics.throughput.totalEvents;\n \n let status: HealthCheck['status'] = 'pass';\n let message = `Event ingestion healthy: ${eventsPerSecond.toFixed(2)} events/sec`;\n \n if (eventsPerSecond === 0 && totalEvents > 0) {\n status = 'warn';\n message = 'No recent event ingestion detected';\n } else if (eventsPerSecond > 50000) {\n status = 'warn';\n message = `Very high ingestion rate: ${eventsPerSecond.toFixed(2)} events/sec`;\n }\n \n return {\n name: 'event_ingestion',\n status,\n message,\n duration: performance.now() - startTime,\n timestamp: Date.now(),\n metadata: {\n eventsPerSecond,\n totalEvents\n }\n };\n \n } catch (error) {\n return {\n name: 'event_ingestion',\n status: 'fail',\n message: `Event ingestion check failed: ${error.message}`,\n duration: performance.now() - startTime,\n timestamp: Date.now()\n };\n }\n }\n\n async checkProcessingLatency(): Promise {\n const startTime = performance.now();\n \n try {\n const metrics = await this.metricsCollector.getCurrentMetrics();\n const avgLatency = metrics.latency.avgProcessingTime;\n const p99Latency = metrics.latency.p99ProcessingTime;\n \n let status: HealthCheck['status'] = 'pass';\n let message = `Processing latency healthy: avg ${avgLatency.toFixed(2)}ms, p99 ${p99Latency.toFixed(2)}ms`;\n \n if (avgLatency > 100) {\n status = 'warn';\n message = `Elevated average latency: ${avgLatency.toFixed(2)}ms`;\n }\n \n if (avgLatency > 500 || p99Latency > 1000) {\n status = 'fail';\n message = `Critical latency: avg ${avgLatency.toFixed(2)}ms, p99 ${p99Latency.toFixed(2)}ms`;\n }\n \n return {\n name: 'processing_latency',\n status,\n message,\n duration: performance.now() - startTime,\n timestamp: Date.now(),\n metadata: {\n avgLatency,\n p99Latency,\n p95Latency: metrics.latency.p95ProcessingTime\n }\n };\n \n } catch (error) {\n return {\n name: 'processing_latency',\n status: 'fail',\n message: `Latency check failed: ${error.message}`,\n duration: performance.now() - startTime,\n timestamp: Date.now()\n };\n }\n }\n\n async checkErrorRate(): Promise {\n const startTime = performance.now();\n \n try {\n const metrics = await this.metricsCollector.getCurrentMetrics();\n const errorRate = metrics.errors.errorRate;\n const totalErrors = metrics.errors.totalErrors;\n \n let status: HealthCheck['status'] = 'pass';\n let message = `Error rate healthy: ${(errorRate * 100).toFixed(2)}%`;\n \n if (errorRate > 0.01) {\n status = 'warn';\n message = `Elevated error rate: ${(errorRate * 100).toFixed(2)}%`;\n }\n \n if (errorRate > 0.05) {\n status = 'fail';\n message = `Critical error rate: ${(errorRate * 100).toFixed(2)}%`;\n }\n \n return {\n name: 'error_rate',\n status,\n message,\n duration: performance.now() - startTime,\n timestamp: Date.now(),\n metadata: {\n errorRate,\n totalErrors,\n errorsByType: metrics.errors.errorsByType\n }\n };\n \n } catch (error) {\n return {\n name: 'error_rate',\n status: 'fail',\n message: `Error rate check failed: ${error.message}`,\n duration: performance.now() - startTime,\n timestamp: Date.now()\n };\n }\n }\n\n async checkThroughput(): Promise {\n const startTime = performance.now();\n \n try {\n const metrics = await this.metricsCollector.getCurrentMetrics();\n const eventsPerSecond = metrics.throughput.eventsPerSecond;\n const expectedMinThroughput = 100; // Minimum expected events/sec\n \n let status: HealthCheck['status'] = 'pass';\n let message = `Throughput healthy: ${eventsPerSecond.toFixed(2)} events/sec`;\n \n if (eventsPerSecond < expectedMinThroughput && metrics.throughput.totalEvents > 1000) {\n status = 'warn';\n message = `Low throughput: ${eventsPerSecond.toFixed(2)} events/sec`;\n }\n \n if (eventsPerSecond === 0 && metrics.throughput.totalEvents > 1000) {\n status = 'fail';\n message = 'Zero throughput detected';\n }\n \n return {\n name: 'throughput',\n status,\n message,\n duration: performance.now() - startTime,\n timestamp: Date.now(),\n metadata: {\n eventsPerSecond,\n batchesPerSecond: metrics.throughput.batchesPerSecond,\n totalEvents: metrics.throughput.totalEvents\n }\n };\n \n } catch (error) {\n return {\n name: 'throughput',\n status: 'fail',\n message: `Throughput check failed: ${error.message}`,\n duration: performance.now() - startTime,\n timestamp: Date.now()\n };\n }\n }\n\n async checkSystemResources(): Promise {\n const startTime = performance.now();\n \n try {\n const metrics = await this.metricsCollector.getCurrentMetrics();\n const memoryUsage = metrics.system.memoryUsage;\n const cpuUsage = metrics.system.cpuUsage;\n \n let status: HealthCheck['status'] = 'pass';\n let message = `System resources healthy: CPU ${cpuUsage.toFixed(1)}%, Memory ${memoryUsage}MB`;\n \n if (cpuUsage > 80 || memoryUsage > 1000) {\n status = 'warn';\n message = `High resource usage: CPU ${cpuUsage.toFixed(1)}%, Memory ${memoryUsage}MB`;\n }\n \n if (cpuUsage > 95 || memoryUsage > 2000) {\n status = 'fail';\n message = `Critical resource usage: CPU ${cpuUsage.toFixed(1)}%, Memory ${memoryUsage}MB`;\n }\n \n return {\n name: 'system_resources',\n status,\n message,\n duration: performance.now() - startTime,\n timestamp: Date.now(),\n metadata: {\n cpuUsage,\n memoryUsage,\n processingInstances: metrics.system.processingInstances\n }\n };\n \n } catch (error) {\n return {\n name: 'system_resources',\n status: 'fail',\n message: `System resources check failed: ${error.message}`,\n duration: performance.now() - startTime,\n timestamp: Date.now()\n };\n }\n }\n\n async checkDataStorage(): Promise {\n const startTime = performance.now();\n \n try {\n // Test database connectivity by attempting a simple query\n const { sqlite } = await import(\"https://esm.town/v/stevekrouse/sqlite\");\n await sqlite.execute(\"SELECT 1\");\n \n return {\n name: 'data_storage',\n status: 'pass',\n message: 'Data storage healthy',\n duration: performance.now() - startTime,\n timestamp: Date.now()\n };\n \n } catch (error) {\n return {\n name: 'data_storage',\n status: 'fail',\n message: `Data storage check failed: ${error.message}`,\n duration: performance.now() - startTime,\n timestamp: Date.now()\n };\n }\n }\n\n async checkProcessingPipeline(): Promise {\n const startTime = performance.now();\n \n try {\n const metrics = await this.metricsCollector.getCurrentMetrics();\n const queueDepth = metrics.system.queueDepth;\n const processingInstances = metrics.system.processingInstances;\n \n let status: HealthCheck['status'] = 'pass';\n let message = `Processing pipeline healthy: ${processingInstances} instances, queue depth ${queueDepth}`;\n \n if (queueDepth > 1000) {\n status = 'warn';\n message = `High queue depth: ${queueDepth}`;\n }\n \n if (queueDepth > 10000) {\n status = 'fail';\n message = `Critical queue depth: ${queueDepth}`;\n }\n \n if (processingInstances === 0) {\n status = 'fail';\n message = 'No processing instances running';\n }\n \n return {\n name: 'processing_pipeline',\n status,\n message,\n duration: performance.now() - startTime,\n timestamp: Date.now(),\n metadata: {\n queueDepth,\n processingInstances\n }\n };\n \n } catch (error) {\n return {\n name: 'processing_pipeline',\n status: 'fail',\n message: `Processing pipeline check failed: ${error.message}`,\n duration: performance.now() - startTime,\n timestamp: Date.now()\n };\n }\n }\n\n private calculateSummary(checks: HealthCheck[]): HealthStatus['summary'] {\n const summary = {\n total: checks.length,\n passing: 0,\n warning: 0,\n critical: 0\n };\n \n checks.forEach(check => {\n switch (check.status) {\n case 'pass':\n summary.passing++;\n break;\n case 'warn':\n summary.warning++;\n break;\n case 'fail':\n summary.critical++;\n break;\n }\n });\n \n return summary;\n }\n\n private determineOverallStatus(summary: HealthStatus['summary']): HealthStatus['status'] {\n if (summary.critical > 0) {\n return 'critical';\n } else if (summary.warning > 0) {\n return 'degraded';\n } else {\n return 'healthy';\n }\n }\n\n getHealthHistory(): HealthStatus[] {\n return [...this.healthHistory];\n }\n\n getLastHealthCheck(): HealthStatus | null {\n return this.healthHistory.length > 0 \n ? this.healthHistory[this.healthHistory.length - 1] \n : null;\n }\n}","deleted","label",[171,172,154],"backend","monitoring","permalink","https://www.val.town/x/pinsar/feistyAmberRhinoceros/code/backend/monitoring/healthChecker.ts","extension","actionData","errors"]