[{"_1":2,"_176":-5,"_177":-5},"loaderData",{"_3":4,"_92":93},"root",{"_5":6,"_35":36,"_65":66},"ctx",{"_7":8,"_23":-5,"_24":25,"_26":27,"_28":20,"_29":20,"_30":20,"_31":32,"_33":20,"_34":20},"auth",{"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20,"_21":22},"sessionClaims","sessionId","sessionStatus","userId","actor","orgId","orgRole","orgSlug","orgPermissions","factorVerificationAge","isAuthenticated",false,"tokenType","session_token","user","preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","notificationCount",0,"flagDenols","isTeamOrgMember","ENV",{"_37":38,"_39":40,"_41":42,"_43":44,"_45":46,"_47":48,"_49":50,"_51":52,"_53":54,"_55":56,"_57":58,"_59":60,"_61":62,"_63":64},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","VAL_RUNTIME_NAME","prod","RENDER_GIT_COMMIT","5f8f00a127b1349c32de089cedf4e10bc7855654","RENDER_GIT_BRANCH","main","CLERK_PUBLISHABLE_KEY","pk_live_Y2xlcmsudmFsLnRvd24k","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","API_URL","https://api.val.town","clerkState",{"_67":68},"__internal_clerk_state",{"_69":70,"_71":46,"_72":73,"_74":73,"_75":20,"_76":73,"_77":73,"_78":73,"_79":73,"_80":81,"_90":20,"_91":20},"__clerk_ssr_state",{"_21":22,"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20},"__publishableKey","__proxyUrl","","__domain","__isSatellite","__signInUrl","__signUpUrl","__afterSignInUrl","__afterSignUpUrl","__clerk_debug",{"_82":20,"_19":20,"_83":73,"_84":85,"_86":73,"_87":46,"_88":20,"_89":73},"isSignedIn","proxyUrl","reason","session-token-and-uat-missing","message","publishableKey","isSatellite","domain","__telemetryDisabled","__telemetryDebug","routes/embed.x.$handle.$name.$",{"_94":95,"_96":97,"_98":99,"_100":101,"_151":152,"_173":174,"_175":-5},"handle","pinsar","name","feistyAmberRhinoceros","path","backend/ingestion/eventIngestion.ts","project",{"_102":103,"_96":97,"_104":105,"_106":-5,"_107":20,"_108":109,"_110":-5,"_111":-5,"_12":112,"_113":114,"_115":116,"_117":-5,"_118":119,"_120":32,"_14":121,"_122":123,"_124":123,"_125":126,"_127":126,"_128":126,"_129":123,"_130":123,"_23":131,"_134":135,"_137":138,"_150":20},"id","2149c16a-277d-11f0-b46e-569c3dd06744","createdAt",["D",1746207883134],"deletedAt","legacyEnvVars","privacy","public","description","imageUrl","851f6760-f7af-11ef-9322-0fd6f1c36e70","pinnedFileId","214e03d8-277d-11f0-b46e-569c3dd06744","pinPreview",true,"pinMigrated","screenshotVersion",19,"likeCount","094b6920-183c-466c-985a-fe1516b5d507","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm",{"_94":95,"_132":133},"profileImageUrl","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18ydG1LWVR3QzZpNUhHQkNmbzF5VlRvOGZweUQifQ","org",{"_94":95,"_136":133,"_12":112},"avatar","branch",{"_139":140,"_96":44,"_102":141,"_142":103,"_143":-5,"_144":-5,"_145":-5,"_146":-5,"_147":-5,"_148":97,"_149":95},"version",27,"214a9ba8-277d-11f0-b46e-569c3dd06744","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","isOwned","file",{"_102":153,"_96":154,"_155":156,"_157":158,"_159":141,"_160":161,"_162":116,"_104":163,"_139":32,"_164":165,"_166":167,"_168":20,"_169":-5,"_98":170},"65619208-3a04-11f0-82da-9e149126039e","eventIngestion.ts","fileId","6561929e-3a04-11f0-82da-9e149126039e","parentId","6174dc11-1e3b-4527-aaaa-759af49e7ae4","branchId","branchVersion",13,"latest",["D",1748245100361],"fileType","script","content","import { EventStorage, Event } from \"../storage/database.ts\";\nimport { MetricsCollector } from \"../monitoring/metricsCollector.ts\";\n\nexport interface IngestResult {\n eventId: string;\n processingTime: number;\n success: boolean;\n error?: string;\n}\n\nexport interface BatchIngestResult {\n processed: number;\n failed: number;\n totalProcessingTime: number;\n errors: string[];\n}\n\nexport class EventIngestionService {\n private eventStorage: EventStorage;\n private metricsCollector: MetricsCollector;\n private rateLimiter: RateLimiter;\n private loadTestActive = false;\n\n constructor(metricsCollector: MetricsCollector) {\n this.eventStorage = new EventStorage();\n this.metricsCollector = metricsCollector;\n this.rateLimiter = new RateLimiter(10000); // 10k events per second default\n }\n\n async ingestEvent(eventData: any): Promise {\n const startTime = performance.now();\n \n try {\n // Rate limiting check\n if (!this.rateLimiter.allowRequest()) {\n throw new Error(\"Rate limit exceeded\");\n }\n\n // Validate and normalize event\n const event = this.validateAndNormalizeEvent(eventData);\n \n // Store event\n await this.eventStorage.storeEvent(event);\n \n const processingTime = performance.now() - startTime;\n \n // Update metrics\n await this.metricsCollector.recordEventProcessed(processingTime);\n await this.metricsCollector.incrementCounter('events_ingested_total');\n \n return {\n eventId: event.id,\n processingTime,\n success: true\n };\n \n } catch (error) {\n const processingTime = performance.now() - startTime;\n \n await this.metricsCollector.incrementCounter('events_failed_total');\n await this.metricsCollector.recordEventProcessed(processingTime, false);\n \n return {\n eventId: eventData.id || 'unknown',\n processingTime,\n success: false,\n error: error.message\n };\n }\n }\n\n async ingestBatch(eventsData: any[]): Promise {\n const startTime = performance.now();\n const results: IngestResult[] = [];\n const errors: string[] = [];\n \n try {\n // Validate batch size\n if (eventsData.length > 1000) {\n throw new Error(\"Batch size exceeds maximum limit of 1000 events\");\n }\n\n // Process events in parallel batches for better performance\n const batchSize = 100;\n const batches = this.chunkArray(eventsData, batchSize);\n \n for (const batch of batches) {\n const batchPromises = batch.map(eventData => this.ingestEvent(eventData));\n const batchResults = await Promise.all(batchPromises);\n results.push(...batchResults);\n }\n \n const processed = results.filter(r => r.success).length;\n const failed = results.filter(r => !r.success).length;\n const totalProcessingTime = performance.now() - startTime;\n \n // Collect errors\n results.forEach(result => {\n if (!result.success && result.error) {\n errors.push(result.error);\n }\n });\n \n // Update batch metrics\n await this.metricsCollector.recordBatchProcessed(processed, failed, totalProcessingTime);\n \n return {\n processed,\n failed,\n totalProcessingTime,\n errors\n };\n \n } catch (error) {\n const totalProcessingTime = performance.now() - startTime;\n \n await this.metricsCollector.incrementCounter('batch_ingestion_failed_total');\n \n return {\n processed: 0,\n failed: eventsData.length,\n totalProcessingTime,\n errors: [error.message]\n };\n }\n }\n\n private validateAndNormalizeEvent(eventData: any): Event {\n // Generate ID if not provided\n const id = eventData.id || this.generateEventId();\n \n // Use current timestamp if not provided\n const timestamp = eventData.timestamp || Date.now();\n \n // Validate required fields\n if (!eventData.type) {\n throw new Error(\"Event type is required\");\n }\n \n if (!eventData.source) {\n throw new Error(\"Event source is required\");\n }\n \n // Generate partition key for distributed processing\n const partition_key = this.generatePartitionKey(eventData);\n \n return {\n id,\n timestamp,\n type: eventData.type,\n source: eventData.source,\n data: eventData.data || {},\n partition_key\n };\n }\n\n private generateEventId(): string {\n return `evt_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;\n }\n\n private generatePartitionKey(eventData: any): string {\n // Use user_id for user-based partitioning, otherwise use event type\n const key = eventData.data?.user_id || eventData.type || 'default';\n return `partition_${this.hashString(key) % 16}`; // 16 partitions\n }\n\n private hashString(str: string): number {\n let hash = 0;\n for (let i = 0; i < str.length; i++) {\n const char = str.charCodeAt(i);\n hash = ((hash << 5) - hash) + char;\n hash = hash & hash; // Convert to 32-bit integer\n }\n return Math.abs(hash);\n }\n\n private chunkArray(array: T[], chunkSize: number): T[][] {\n const chunks: T[][] = [];\n for (let i = 0; i < array.length; i += chunkSize) {\n chunks.push(array.slice(i, i + chunkSize));\n }\n return chunks;\n }\n\n // Load testing functionality\n async startLoadTest(eventsPerSecond: number, durationSeconds: number): Promise {\n if (this.loadTestActive) {\n throw new Error(\"Load test already active\");\n }\n\n this.loadTestActive = true;\n const intervalMs = 1000 / eventsPerSecond;\n const endTime = Date.now() + (durationSeconds * 1000);\n \n console.log(`Starting load test: ${eventsPerSecond} events/sec for ${durationSeconds} seconds`);\n \n const generateTestEvent = () => ({\n type: this.getRandomEventType(),\n source: 'load_test',\n data: {\n user_id: Math.floor(Math.random() * 10000).toString(),\n action: this.getRandomAction(),\n value: Math.random() * 100,\n metadata: {\n test: true,\n batch_id: Math.floor(Date.now() / 1000)\n }\n }\n });\n\n const sendEvent = async () => {\n if (Date.now() >= endTime) {\n this.loadTestActive = false;\n console.log(\"Load test completed\");\n return;\n }\n\n try {\n await this.ingestEvent(generateTestEvent());\n } catch (error) {\n console.error(\"Load test event failed:\", error);\n }\n\n if (this.loadTestActive) {\n setTimeout(sendEvent, intervalMs);\n }\n };\n\n // Start multiple concurrent streams for higher throughput\n const concurrentStreams = Math.min(10, Math.ceil(eventsPerSecond / 100));\n for (let i = 0; i < concurrentStreams; i++) {\n setTimeout(sendEvent, i * (intervalMs / concurrentStreams));\n }\n }\n\n private getRandomEventType(): string {\n const types = ['user_action', 'page_view', 'purchase', 'signup', 'login', 'api_call'];\n return types[Math.floor(Math.random() * types.length)];\n }\n\n private getRandomAction(): string {\n const actions = ['click', 'scroll', 'hover', 'submit', 'navigate', 'search'];\n return actions[Math.floor(Math.random() * actions.length)];\n }\n}\n\nclass RateLimiter {\n private maxRequestsPerSecond: number;\n private requests: number[] = [];\n\n constructor(maxRequestsPerSecond: number) {\n this.maxRequestsPerSecond = maxRequestsPerSecond;\n }\n\n allowRequest(): boolean {\n const now = Date.now();\n const oneSecondAgo = now - 1000;\n \n // Remove requests older than 1 second\n this.requests = this.requests.filter(timestamp => timestamp > oneSecondAgo);\n \n // Check if we're under the limit\n if (this.requests.length < this.maxRequestsPerSecond) {\n this.requests.push(now);\n return true;\n }\n \n return false;\n }\n\n getCurrentRate(): number {\n const now = Date.now();\n const oneSecondAgo = now - 1000;\n return this.requests.filter(timestamp => timestamp > oneSecondAgo).length;\n }\n}","deleted","label",[171,172,154],"backend","ingestion","permalink","https://www.val.town/x/pinsar/feistyAmberRhinoceros/code/backend/ingestion/eventIngestion.ts","extension","actionData","errors"]