NEXUS Collaborative Hand Tracking v6.0

[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_152":153,"_154":155},"routes/_app.x.$handle.$name",{"_11":12,"_13":14,"_15":16,"_18":19,"_17":-7,"_115":-5,"_20":116,"_22":117,"_118":119,"_120":121,"_25":122,"_24":-5,"_123":-5,"_124":-5,"_125":53,"_126":127,"_128":44,"_129":-5,"_130":-5,"_131":132},"routes/_app.x.$handle.$name.branch.$branch.code.$",{"_11":12,"_13":14,"_15":16,"_17":-7,"_18":19,"_20":21,"_22":23,"_24":-7,"_25":26,"_27":-5,"_28":-5,"_29":-5,"_30":-5,"_31":-5},"handle","volt","name","collab","branchName","main","version","path","","project",{"_33":61,"_13":14,"_45":62,"_63":-5,"_64":53,"_65":66,"_67":66,"_68":-5,"_69":-5,"_70":71,"_72":37,"_73":44,"_74":44,"_75":-5,"_76":77,"_78":79,"_80":-5,"_81":-5,"_82":-5,"_83":-5,"_84":85,"_86":85,"_87":88,"_89":88,"_90":88,"_91":85,"_92":85,"_93":94,"_95":96,"_97":98,"_99":53},"tree",[56],"file","contents",[32],"upstream","pull","ahead","behind","envVars",{"_33":34,"_13":35,"_36":37,"_38":-5,"_39":40,"_41":42,"_43":44,"_45":46,"_17":47,"_48":49,"_50":51,"_52":53,"_54":-5,"_18":55},"id","594bca44-af6e-11f0-a0f6-0224a6c84d84","main.ts","fileId","4c60eff2-af6a-11f0-b193-0224a6c84d84","parentId","branchId","4c5d3cf4-af6a-11f0-b193-0224a6c84d84","branchVersion",14,"latest",true,"createdAt",["D",1761154892774],13,"fileType","http","content","// @ts-nocheck\n/*\n NEXUS Collaborative Hand Tracking v6.0\n A Val Town application by ValleyGirl\n\n This application provides a real-time, multi-user 3D hand tracking experience.\n It utilizes MediaPipe for client-side hand detection and a Val Town backend\n for state management and AI-driven gesture interpretation.\n\n Core Architecture:\n 1. Frontend (Three.js): Renders the 3D scene and captures hand data via MediaPipe.\n 2. Backend (Hono Router): A serverless API running on Val Town.\n 3. State (std/blob): A simple key-value store (`std/blob`) is used to maintain\n the state of each \"room\" (participant hand data, gesture events).\n 4. Real-time (HTTP Polling): The WebSocket logic from the original file has been\n refactored to use a serverless-friendly, short-polling mechanism. Clients\n POST updates and GET the latest room state every 500ms.\n 5. AI (std/openai): The novel cornerstone of this application. When a user's\n hand is detected, the raw 3D landmarks are periodically sent to the\n backend. An OpenAI model (GPT-4o) classifies this spatial data into a\n specific gesture (e.g., 'PEACE', 'FIST'). This classification is then\n broadcast to all participants, triggering a 3D visual effect.\n*/\n\n// --- VAL TOWN BACKEND ---\n\nimport { OpenAI } from \"https://esm.town/v/std/openai?v=4\";\nimport { blob } from \"https://esm.town/v/std/blob?v=11\";\nimport { Hono } from \"npm:hono@4.4.12\";\n\n// --- Type Definitions ---\n\ninterface Landmark {\n x: number;\n y: number;\n z: number;\n}\ninterface HandData {\n handedness: \"left\" | \"right\";\n landmarks: Landmark[];\n}\ninterface UserState {\n userId: string;\n hands: HandData[];\n lastUpdate: number;\n}\ninterface GestureEvent {\n eventId: string;\n userId: string;\n gesture: string;\n reason: string;\n timestamp: number;\n}\ninterface RoomState {\n users: Record;\n events: GestureEvent[];\n}\n\n// --- AI Behavioral Guidelines & Prompts ---\n\nconst GENERATION_STYLE_GUIDE =\n \"You are an expert in kinesiology and human-computer interaction, functioning as a system-level classifier. Your analysis must be precise, objective, and based on spatial relationships (e..g., 'Landmark 8 is close to Landmark 12, indicating a closed finger'). Avoid conversational language. The goal is to provide an accurate classification and justification for a software system's logs.\";\n\nconst CLASSIFY_GESTURE_PROMPT = `\nYou are an expert in kinesiology and gesture recognition, analyzing 3D hand landmark data from MediaPipe.\nThe user will provide a JSON array of 21 {x, y, z} landmarks.\nYour task is to analyze this spatial data and classify the gesture.\nThe classification MUST be one of: ['PEACE', 'POINT', 'FIST', 'OPEN_PALM', 'THUMBS_UP', 'OTHER'].\nYou MUST also provide a 'reason' field justifying your classification based on landmark positions (e.g., \"Fingers (12, 16, 20) are flexed towards the palm (0), while index (8) and thumb (4) are extended.\").\nRespond STRICTLY with a single, minified JSON object: {\"classification\": \"GESTURE_NAME\", \"reason\": \"Your analysis...\"}\n${GENERATION_STYLE_GUIDE}\n`;\n\n// --- Constants ---\nconst ROOM_BLOB_PREFIX = \"nexus_room_v6:\";\nconst ROOM_EXPIRATION_MS = 1000 * 60 * 60; // 1 hour\nconst USER_TIMEOUT_MS = 1000 * 30; // 30 seconds\nconst EVENT_TIMEOUT_MS = 1000 * 10; // 10 seconds\n\n// --- Backend Initialization ---\nconst app = new Hono();\nlet openai: OpenAI;\n\n// Initialization middleware to ensure OpenAI client is ready.\n// This gates requests until the API key is confirmed.\napp.use(\"*\", async (c, next) => {\n if (!openai) {\n openai = new OpenAI();\n\n await next();\n }\n});\n\n/**\n * Safely extracts a JSON object from a string,\n * ignoring conversational chatter from the AI.\n */\nfunction robustJsonParse(text: string): any {\n const match = text.match(/{.*}/s);\n if (!match) {\n throw new Error(\"AI response did not contain a valid JSON object.\");\n }\n try {\n return JSON.parse(match[0]);\n } catch (e) {\n console.error(\"Failed to parse AI JSON:\", e.message, \"Raw text:\", text);\n throw new Error(\"AI returned malformed JSON.\");\n }\n}\n\n/**\n * Retrieves and sanitizes the state for a given room.\n * This function prunes inactive users and expired events.\n */\nasync function getSaneRoomState(code: string): Promise {\n const roomKey = ROOM_BLOB_PREFIX + code;\n const room = await blob.getJSON(roomKey) as RoomState;\n if (!room) {\n return null;\n }\n\n const now = Date.now();\n let stateChanged = false;\n\n // Prune inactive users\n for (const userId in room.users) {\n if (now - room.users[userId].lastUpdate > USER_TIMEOUT_MS) {\n delete room.users[userId];\n stateChanged = true;\n }\n }\n\n // Prune old events\n const freshEvents = room.events.filter((e) =>\n now - e.timestamp < EVENT_TIMEOUT_MS\n );\n if (freshEvents.length < room.events.length) {\n room.events = freshEvents;\n stateChanged = true;\n }\n\n // If changes were made, write the sanitized state back to the blob.\n if (stateChanged) {\n await blob.setJSON(roomKey, room, { expiresIn: ROOM_EXPIRATION_MS });\n }\n\n return room;\n}\n\n// --- API Routes ---\n\n/**\n * [POST] /create_room\n * Generates a new unique room code and initializes its state in the blob.\n */\napp.post(\"/create_room\", async (c) => {\n try {\n let roomCode: string;\n let roomKey: string;\n let attempts = 0;\n\n do {\n roomCode = Math.random().toString(36).substr(2, 6).toUpperCase();\n roomKey = ROOM_BLOB_PREFIX + roomCode;\n if (attempts++ > 10) {\n return c.json({ error: \"Failed to generate a unique room\" }, 500);\n }\n } while (await blob.getJSON(roomKey)); // This can fail\n\n const initialState: RoomState = { users: {}, events: [] };\n await blob.setJSON(roomKey, initialState, {\n expiresIn: ROOM_EXPIRATION_MS,\n }); // This can fail\n\n return c.json({ roomCode });\n } catch (error) {\n console.error(\"Failed to create room:\", error.message);\n return c.json({ error: \"Internal server error on room creation.\" }, 500);\n }\n});\n\n/**\n * [POST] /join_room/:code\n * Adds a new user to an existing room.\n */\napp.post(\"/join_room/:code\", async (c) => {\n const code = c.req.param(\"code\").toUpperCase();\n const roomKey = ROOM_BLOB_PREFIX + code;\n const room = await blob.getJSON(roomKey) as RoomState;\n\n if (!room) {\n return c.json({ error: \"Room not found\" }, 404);\n }\n\n const { userId } = await c.req.json();\n if (!userId) {\n return c.json({ error: \"Invalid user ID\" }, 400);\n }\n\n room.users[userId] = { userId, hands: [], lastUpdate: Date.now() };\n await blob.setJSON(roomKey, room, { expiresIn: ROOM_EXPIRATION_MS });\n\n return c.json({ success: true, roomCode: code });\n});\n\n/**\n * [POST] /update_hands/:code\n * Receives hand landmark data from a user and updates the room state.\n */\napp.post(\"/update_hands/:code\", async (c) => {\n const code = c.req.param(\"code\").toUpperCase();\n const roomKey = ROOM_BLOB_PREFIX + code;\n const room = await blob.getJSON(roomKey) as RoomState;\n\n if (!room) {\n return c.json({ error: \"Room not found\" }, 404);\n }\n\n const { userId, hands } = await c.req.json();\n if (!userId || !Array.isArray(hands)) {\n return c.json({ error: \"Invalid payload\" }, 400);\n }\n\n if (room.users[userId]) {\n room.users[userId].hands = hands;\n room.users[userId].lastUpdate = Date.now();\n } else {\n // User is not in the room, perhaps joined but state not fully consistent.\n // Add them now.\n room.users[userId] = { userId, hands, lastUpdate: Date.now() };\n }\n\n await blob.setJSON(roomKey, room, { expiresIn: ROOM_EXPIRATION_MS });\n return c.json({ success: true });\n});\n\n/**\n * [GET] /sync_room/:code\n * The main polling endpoint for clients to get the latest room state.\n */\napp.get(\"/sync_room/:code\", async (c) => {\n try {\n const code = c.req.param(\"code\").toUpperCase();\n const room = await getSaneRoomState(code); // Pruning happens here\n\n if (!room) {\n return c.json({ error: \"Room not found\" }, 404);\n }\n\n return c.json(room);\n } catch (error) {\n console.error(\n `Error in /sync_room/${c.req.param(\"code\")}: \\${error.message}`,\n );\n return c.json({ error: \"Internal server error\" }, 500);\n }\n});\n\n/**\n * [POST] /interpret_gesture/:code\n * The novel AI endpoint. Classifies raw 3D landmarks and adds the\n * resulting gesture as an event to the room state.\n */\napp.post(\"/interpret_gesture/:code\", async (c) => {\n const code = c.req.param(\"code\").toUpperCase();\n const roomKey = ROOM_BLOB_PREFIX + code;\n\n const { userId, landmarks } = await c.req.json();\n if (!userId || !Array.isArray(landmarks) || landmarks.length !== 21) {\n return c.json({ error: \"Invalid gesture data\" }, 400);\n }\n\n const room = await blob.getJSON(roomKey) as RoomState;\n if (!room || !room.users[userId]) {\n return c.json({ error: \"Room or user not found\" }, 404);\n }\n\n try {\n const completion = await openai.chat.completions.create({\n model: \"gpt-4o\",\n messages: [\n { role: \"system\", content: CLASSIFY_GESTURE_PROMPT },\n { role: \"user\", content: JSON.stringify(landmarks) },\n ],\n response_format: { type: \"json_object\" },\n temperature: 0.2,\n });\n\n const content = completion.choices[0].message.content;\n const aiResponse = robustJsonParse(content);\n\n if (!aiResponse.classification || !aiResponse.reason) {\n throw new Error(\"AI response was missing required fields.\");\n }\n\n // Log the AI's justification for transparency and debugging.\n console.log(\n `AI Gesture [${code}]: ${userId} -> ${aiResponse.classification} (Reason: ${aiResponse.reason})`,\n );\n\n // Add the gesture as a persistent event\n const event: GestureEvent = {\n eventId: `evt_${Date.now()}_${Math.random().toString(36).substr(2, 5)}`,\n userId: userId,\n gesture: aiResponse.classification,\n reason: aiResponse.reason, // Store reason for potential debugging\n timestamp: Date.now(),\n };\n room.events.push(event);\n\n await blob.setJSON(roomKey, room, { expiresIn: ROOM_EXPIRATION_MS });\n\n return c.json({ classification: aiResponse.classification });\n } catch (error) {\n console.error(\"AI Gesture Interpretation Error:\", error.message);\n return c.json({ error: \"AI interpretation failed\" }, 500);\n }\n});\n\n/**\n * [GET] /\n * Serves the main HTML, CSS, and JavaScript for the application.\n */\napp.get(\"/\", (c) => {\n // Ensure the base URL always ends with a '/'\n let sourceUrl = `https://${new URL(c.req.url).hostname}${\n new URL(c.req.url).pathname\n }`;\n if (!sourceUrl.endsWith(\"/\")) {\n sourceUrl += \"/\";\n }\n return c.html(generateHtml(sourceUrl));\n});\n\n// Export the Hono app fetch handler for the Val Town runtime\nexport default app.fetch;\n\n// --- FRONTEND (HTML/CSS/JS) ---\n\n/**\n * Generates the complete single-page application.\n * @param sourceUrl The base URL of this Val Town val, used for API calls.\n */\nfunction generateHtml(sourceUrl: string) {\n return `\n\n\n\n \n \n NEXUS Collaborative Hand Tracking v6.0\n \n \n \n \n\n\n

◬ NEXUS COLLAB

\n Connecting...\n

\n Camera Off\n

\n 0 Hands Detected\n

\n FPS\n 60\n

\n Latency\n 0ms\n

\n Users Online\n 0\n

⚡ Control Center

\n \n

Or Join Existing

\n \n \n

\n\n

Room Code

------

Click to copy

\n \n

View Mode

\n \n

Standard

\n \n

\n\n \n \n \n t\n\n\n`;\n}","deleted",false,"label",[35],{"_33":34,"_13":35,"_36":37,"_38":-5,"_39":40,"_41":42,"_43":44,"_45":57,"_17":47,"_48":49,"_50":51,"_52":53,"_54":-5,"_58":59,"_18":60},["D",1761154892774],"children",[],[35],"4c5cdbce-af6a-11f0-b193-0224a6c84d84",["D",1761153153098],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","imageUrl","userId","0199b749-1e1c-7399-b093-a7ba99382572","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","0199b749-1e1c-7399-b093-a97ac9fff69e","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_11":12,"_114":110},"org",{"_11":12,"_108":-5,"_109":110,"_70":71,"_111":112,"_113":53},"branch",{"_17":42,"_13":16,"_33":40,"_100":61,"_101":-5,"_102":-5,"_103":-5,"_104":-5,"_105":-5,"_106":14,"_107":12},"isOwned","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoiZGVmYXVsdCIsImlpZCI6Imluc18yTm10cGhQUklwU0tqZHNiOHNnNGd2ZjJURFUiLCJyaWQiOiJ1c2VyXzMzZm1qMllGY2JhQmhhWFYwVXVvQ05aT2tJciJ9","tier","free","canRestrictVals","profileImageUrl","adoptOrgId",{"_33":61,"_13":14,"_45":148,"_63":-5,"_64":53,"_65":66,"_67":66,"_68":-5,"_69":-5,"_70":71,"_72":37,"_73":44,"_74":44,"_75":-5,"_76":77,"_78":79,"_80":-5,"_81":-5,"_82":-5,"_83":-5,"_84":85,"_86":85,"_87":88,"_89":88,"_90":88,"_91":85,"_92":85,"_93":149,"_95":150,"_97":151,"_99":53},[145],"files",[145],"counts",{"_135":77,"_136":137,"_138":77,"_31":139,"_140":141},[133],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_33":34,"_13":35,"_36":37,"_38":-5,"_39":40,"_41":42,"_43":44,"_45":46,"_17":47,"_48":49,"_50":51,"_52":53,"_54":-5,"_18":134},[35],"pulls","branches",1,"forks",{"_142":77,"_144":77},"connections",{"_142":77,"_143":77},"count","unfilledCount","emptyCount",{"_33":34,"_13":35,"_36":37,"_38":-5,"_39":40,"_41":42,"_43":44,"_45":57,"_17":47,"_48":49,"_50":51,"_52":53,"_54":-5,"_58":146,"_18":147},[],[35],["D",1761153153098],{"_11":12,"_114":110},{"_11":12,"_108":-5,"_109":110,"_70":71,"_111":112,"_113":53},{"_17":42,"_13":16,"_33":40,"_100":61,"_101":-5,"_102":-5,"_103":-5,"_104":-5,"_105":-5,"_106":14,"_107":12},"ctx",{"_93":-5,"_130":185,"_186":187,"_188":189,"_190":53,"_191":53,"_192":53,"_193":44,"_194":-5,"_195":-7,"_196":-7,"_197":77,"_198":53,"_199":53,"_200":-5,"_201":53,"_202":53,"_203":53,"_204":-5},"ENV",{"_156":157,"_158":159,"_160":16,"_161":162,"_163":164,"_165":166,"_167":168,"_169":170,"_171":172,"_173":174,"_175":176,"_177":178,"_179":180,"_181":182,"_183":184},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]