Face Texture Mapper (Fixed Tracking)

\n \n

\n Initializing...\n

\n \n

Waiting for face...

\n \n

\n\n \n

\n\n

\n ℹ️\n Welcome\n

\n`;\n\n// ============================================================================\n// 3. CLIENT-SIDE LOGIC\n// ============================================================================\nconst CLIENT_SCRIPT = `\nimport * as THREE from 'three';\nimport { MindARThree } from 'mindar-face-three';\n\n// ============================================================================\n// TEXTURE CAPTURE SYSTEM (TRIANGLE PROJECTION)\n// ============================================================================\n\nclass TextureCaptureSystem {\n constructor(uvMapSize = 2048) {\n this.uvMapSize = uvMapSize;\n \n this.uvCanvas = document.createElement('canvas');\n this.uvCanvas.width = uvMapSize;\n this.uvCanvas.height = uvMapSize;\n this.uvCtx = this.uvCanvas.getContext('2d', { willReadFrequently: true });\n \n this.capturedAngles = new Set();\n this.angleResolution = 15;\n this.captureCount = 0;\n \n this.initializeCanvas();\n }\n\n initializeCanvas() {\n this.uvCtx.fillStyle = '#e0c6aa'; \n this.uvCtx.fillRect(0, 0, this.uvMapSize, this.uvMapSize);\n \n this.uvCtx.strokeStyle = 'rgba(0, 0, 0, 0.1)';\n this.uvCtx.lineWidth = 1;\n this.uvCtx.beginPath();\n for (let i = 0; i <= 10; i++) {\n const pos = (i / 10) * this.uvMapSize;\n this.uvCtx.moveTo(pos, 0);\n this.uvCtx.lineTo(pos, this.uvMapSize);\n this.uvCtx.moveTo(0, pos);\n this.uvCtx.lineTo(this.uvMapSize, pos);\n }\n this.uvCtx.stroke();\n }\n\n clear() {\n this.capturedAngles.clear();\n this.captureCount = 0;\n this.initializeCanvas();\n }\n\n quantizeAngle(yaw, pitch) {\n const qYaw = Math.round(yaw / this.angleResolution) * this.angleResolution;\n const qPitch = Math.round(pitch / this.angleResolution) * this.angleResolution;\n return qYaw + ',' + qPitch;\n }\n\n isAngleCaptured(yaw, pitch) {\n return this.capturedAngles.has(this.quantizeAngle(yaw, pitch));\n }\n\n captureFrame(videoElement, faceMesh, camera, yaw, pitch) {\n if (!videoElement || !faceMesh) return false;\n\n const angleKey = this.quantizeAngle(yaw, pitch);\n if (this.capturedAngles.has(angleKey)) return false;\n\n const geometry = faceMesh.geometry;\n const positions = geometry.attributes.position.array;\n const uvs = geometry.attributes.uv.array;\n const indices = geometry.index.array;\n\n camera.updateMatrixWorld(true);\n faceMesh.updateMatrixWorld(true);\n const viewMatrix = camera.matrixWorldInverse;\n const projectionMatrix = camera.projectionMatrix;\n const worldMatrix = faceMesh.matrixWorld;\n\n const vidW = videoElement.videoWidth;\n const vidH = videoElement.videoHeight;\n \n const videoCanvas = document.createElement('canvas');\n videoCanvas.width = vidW;\n videoCanvas.height = vidH;\n const videoCtx = videoCanvas.getContext('2d');\n \n videoCtx.translate(vidW, 0);\n videoCtx.scale(-1, 1);\n videoCtx.drawImage(videoElement, 0, 0, vidW, vidH);\n videoCtx.setTransform(1, 0, 0, 1, 0, 0);\n\n const vA = new THREE.Vector3(), vB = new THREE.Vector3(), vC = new THREE.Vector3();\n const uvA = {x:0, y:0}, uvB = {x:0, y:0}, uvC = {x:0, y:0};\n const scA = {x:0, y:0}, scB = {x:0, y:0}, scC = {x:0, y:0};\n\n this.uvCtx.globalAlpha = 0.7; \n this.uvCtx.globalCompositeOperation = 'source-over';\n\n let triangleCount = 0;\n\n for (let i = 0; i < indices.length; i += 3) {\n const idxA = indices[i];\n const idxB = indices[i+1];\n const idxC = indices[i+2];\n\n vA.set(positions[idxA*3], positions[idxA*3+1], positions[idxA*3+2]).applyMatrix4(worldMatrix);\n vB.set(positions[idxB*3], positions[idxB*3+1], positions[idxB*3+2]).applyMatrix4(worldMatrix);\n vC.set(positions[idxC*3], positions[idxC*3+1], positions[idxC*3+2]).applyMatrix4(worldMatrix);\n\n const ab = new THREE.Vector3().subVectors(vB, vA);\n const ac = new THREE.Vector3().subVectors(vC, vA);\n const normal = new THREE.Vector3().crossVectors(ab, ac).normalize();\n \n const viewDir = new THREE.Vector3(0, 0, 1).applyQuaternion(camera.quaternion);\n const facingRatio = normal.dot(viewDir);\n\n if (facingRatio < 0.2) continue;\n\n vA.applyMatrix4(viewMatrix).applyMatrix4(projectionMatrix);\n vB.applyMatrix4(viewMatrix).applyMatrix4(projectionMatrix);\n vC.applyMatrix4(viewMatrix).applyMatrix4(projectionMatrix);\n\n // NO FLIP needed here because video canvas is already flipped\n scA.x = ((vA.x + 1) / 2) * vidW; scA.y = ((-vA.y + 1) / 2) * vidH;\n scB.x = ((vB.x + 1) / 2) * vidW; scB.y = ((-vB.y + 1) / 2) * vidH;\n scC.x = ((vC.x + 1) / 2) * vidW; scC.y = ((-vC.y + 1) / 2) * vidH;\n\n uvA.x = uvs[idxA*2] * this.uvMapSize; uvA.y = (1 - uvs[idxA*2+1]) * this.uvMapSize;\n uvB.x = uvs[idxB*2] * this.uvMapSize; uvB.y = (1 - uvs[idxB*2+1]) * this.uvMapSize;\n uvC.x = uvs[idxC*2] * this.uvMapSize; uvC.y = (1 - uvs[idxC*2+1]) * this.uvMapSize;\n\n this.drawTexturedTriangle(\n this.uvCtx, videoCanvas,\n scA, scB, scC, \n uvA, uvB, uvC\n );\n \n triangleCount++;\n }\n\n this.capturedAngles.add(angleKey);\n this.captureCount++;\n console.log(\\`Captured \\${triangleCount} triangles at Yaw:\\${yaw.toFixed(0)}\\`);\n return true;\n }\n\n drawTexturedTriangle(ctx, img, p0, p1, p2, t0, t1, t2) {\n ctx.save();\n ctx.beginPath();\n ctx.moveTo(t0.x, t0.y);\n ctx.lineTo(t1.x, t1.y);\n ctx.lineTo(t2.x, t2.y);\n ctx.closePath();\n ctx.clip();\n\n const u0 = p0.x, v0 = p0.y;\n const u1 = p1.x, v1 = p1.y;\n const u2 = p2.x, v2 = p2.y;\n\n const denom = u0 * (v2 - v1) - u1 * (v2 - v0) + u2 * (v1 - v0);\n if (Math.abs(denom) < 0.1) { ctx.restore(); return; }\n\n const m11 = -(v0 * (t1.x - t2.x) - v1 * (t0.x - t2.x) + v2 * (t0.x - t1.x)) / denom;\n const m12 = (v0 * (t1.y - t2.y) - v1 * (t0.y - t2.y) + v2 * (t0.y - t1.y)) / denom;\n const m21 = (u0 * (t1.x - t2.x) - u1 * (t0.x - t2.x) + u2 * (t0.x - t1.x)) / denom;\n const m22 = -(u0 * (t1.y - t2.y) - u1 * (t0.y - t2.y) + u2 * (t0.y - t1.y)) / denom;\n const dx = (u0 * (v2 * t1.x - v1 * t2.x) + u1 * (v0 * t2.x - v2 * t0.x) + u2 * (v1 * t0.x - v0 * t1.x)) / denom;\n const dy = (u0 * (v2 * t1.y - v1 * t2.y) + u1 * (v0 * t2.y - v2 * t0.y) + u2 * (v1 * t0.y - v0 * t1.y)) / denom;\n\n ctx.transform(m11, m12, m21, m22, dx, dy);\n ctx.drawImage(img, 0, 0);\n ctx.restore();\n }\n\n getCoverage() {\n return this.capturedAngles.size / (5 * 3);\n }\n\n getAngleGrid() {\n const yawRange = [-45, -30, -15, 0, 15, 30, 45];\n const pitchRange = [-30, -15, 0, 15, 30];\n const grid = [];\n pitchRange.forEach(pitch => {\n yawRange.forEach(yaw => {\n grid.push({\n yaw, pitch,\n captured: this.capturedAngles.has(this.quantizeAngle(yaw, pitch))\n });\n });\n });\n return grid;\n }\n\n exportPNG() {\n return this.uvCanvas.toDataURL('image/png', 1.0);\n }\n\n getPreviewCanvas() {\n return this.uvCanvas;\n }\n}\n\n// ============================================================================\n// CONFIGURATION\n// ============================================================================\n\nconst CONFIG = {\n autoCapture: true,\n autoCaptureInterval: 250, \n minAngleChange: 5,\n uvMapSize: 2048\n};\n\n// ============================================================================\n// UI HELPERS\n// ============================================================================\n\nfunction showToast(message) {\n const toast = document.getElementById('toast');\n if (!toast) return;\n const msgEl = document.getElementById('toast-message');\n if(msgEl) msgEl.textContent = message;\n toast.classList.add('show');\n setTimeout(() => toast.classList.remove('show'), 2500);\n}\n\nfunction updateAngleGrid(currentYaw, currentPitch) {\n if (!captureSystem) return;\n const grid = captureSystem.getAngleGrid();\n const gridEl = document.getElementById('angle-grid');\n if (!gridEl) return;\n gridEl.innerHTML = '';\n grid.forEach(cell => {\n const div = document.createElement('div');\n div.className = 'angle-cell';\n if (cell.captured) { div.classList.add('captured'); div.textContent = '✓'; }\n const isCurrentYaw = Math.abs(cell.yaw - currentYaw) < CONFIG.minAngleChange + 5;\n const isCurrentPitch = Math.abs(cell.pitch - currentPitch) < CONFIG.minAngleChange + 5;\n if (isCurrentYaw && isCurrentPitch && !cell.captured) div.classList.add('current');\n gridEl.appendChild(div);\n });\n}\n\nfunction updateProgress() {\n if (!captureSystem) return;\n const coverage = captureSystem.getCoverage();\n const percent = Math.min(100, Math.round(coverage * 100));\n \n const percentEl = document.getElementById('progress-percent');\n if (percentEl) percentEl.textContent = percent + '%';\n \n const circle = document.getElementById('progress-circle');\n if (circle) {\n const circumference = 2 * Math.PI * 21;\n const offset = circumference - (coverage * circumference);\n circle.style.strokeDashoffset = offset;\n if (coverage >= 0.8) circle.style.stroke = '#22c55e';\n else if (coverage >= 0.5) circle.style.stroke = '#f59e0b';\n else circle.style.stroke = '#6366f1';\n }\n const countEl = document.getElementById('capture-count');\n if (countEl) countEl.textContent = captureSystem.captureCount + ' frames';\n}\n\nfunction updatePreview() {\n if (!captureSystem) return;\n const previewCanvas = document.getElementById('uv-preview-canvas');\n if (!previewCanvas) return;\n const previewCtx = previewCanvas.getContext('2d');\n const sourceCanvas = captureSystem.getPreviewCanvas();\n previewCtx.clearRect(0, 0, previewCanvas.width, previewCanvas.height);\n previewCtx.drawImage(sourceCanvas, 0, 0, previewCanvas.width, previewCanvas.height);\n}\n\n// ============================================================================\n// MAIN LOGIC\n// ============================================================================\n\nlet captureSystem = null;\nlet mindarThree = null;\nlet video = null;\n\n// Function to calculate pose from landmarks (ROBUST method)\n// This ignores local quaternion issues and uses raw vertex positions\nfunction estimatePoseFromLandmarks(face, camera) {\n const positions = face.geometry.attributes.position.array;\n \n // Transform key points to World Space\n const getVertexWorld = (index) => {\n const v = new THREE.Vector3(\n positions[index*3], \n positions[index*3+1], \n positions[index*3+2]\n );\n v.applyMatrix4(face.matrixWorld);\n return v;\n };\n \n // Landmarks (MediaPipe indices)\n const nose = getVertexWorld(1);\n const leftCheek = getVertexWorld(454); // Outer left\n const rightCheek = getVertexWorld(234); // Outer right\n const forehead = getVertexWorld(10);\n const chin = getVertexWorld(152);\n \n // Convert to Camera Space (relative to view)\n // This makes calculations work regardless of how user holds phone\n const viewMatrix = camera.matrixWorldInverse;\n nose.applyMatrix4(viewMatrix);\n leftCheek.applyMatrix4(viewMatrix);\n rightCheek.applyMatrix4(viewMatrix);\n forehead.applyMatrix4(viewMatrix);\n chin.applyMatrix4(viewMatrix);\n \n // 1. Calculate YAW (Horizontal rotation)\n // Compare nose X to center of cheeks\n const faceWidth = Math.abs(leftCheek.x - rightCheek.x);\n const faceCenterX = (leftCheek.x + rightCheek.x) / 2;\n const noseOffsetX = nose.x - faceCenterX;\n \n // Normalize: when looking 45deg, nose offset is approx 25% of width\n let yaw = (noseOffsetX / (faceWidth * 0.5)) * 90; \n yaw = -yaw; // Flip to match intuition (Turn right = positive)\n \n // 2. Calculate PITCH (Vertical rotation)\n // Compare nose Y to center of forehead/chin\n const faceHeight = Math.abs(forehead.y - chin.y);\n const faceCenterY = (forehead.y + chin.y) / 2;\n const noseOffsetY = nose.y - faceCenterY;\n \n // Normalize\n let pitch = (noseOffsetY / (faceHeight * 0.5)) * 90;\n pitch = -pitch; // Flip to match intuition (Look up = positive)\n \n // Clamp\n yaw = Math.max(-60, Math.min(60, yaw));\n pitch = Math.max(-45, Math.min(45, pitch));\n \n return { yaw, pitch };\n}\n\n// Project a 3D point to 2D Canvas space for drawing debug lines\nfunction projectPoint(v, camera, w, h) {\n const p = v.clone();\n p.project(camera);\n return {\n x: ((p.x + 1) / 2) * w,\n y: ((-p.y + 1) / 2) * h\n };\n}\n\nasync function initTextureMapper() {\n console.log('=== Initializing Texture Mapper (Triangle Mode + Geo Tracking) ===');\n \n const container = document.getElementById('container');\n const statusEl = document.getElementById('tracking-status');\n const statusDot = document.querySelector('.status-dot');\n const overlayCanvas = document.getElementById('overlay-canvas');\n const overlayCtx = overlayCanvas.getContext('2d');\n \n // Setup overlay size\n const resizeOverlay = () => {\n overlayCanvas.width = container.clientWidth;\n overlayCanvas.height = container.clientHeight;\n };\n window.addEventListener('resize', resizeOverlay);\n resizeOverlay();\n\n captureSystem = new TextureCaptureSystem(CONFIG.uvMapSize);\n \n try {\n mindarThree = new MindARThree({ container });\n const { renderer, scene, camera } = mindarThree;\n \n // Add face mesh\n const faceMesh = mindarThree.addFaceMesh();\n faceMesh.material = new THREE.MeshBasicMaterial({ \n color: 0xffffff, \n wireframe: true,\n transparent: true,\n opacity: 0.05\n });\n scene.add(faceMesh);\n \n // UI Bindings\n document.getElementById('export-texture').onclick = () => {\n if (!captureSystem) return;\n const link = document.createElement('a');\n link.download = 'face-texture.png';\n link.href = captureSystem.exportPNG();\n link.click();\n showToast('Downloaded PNG');\n };\n \n document.getElementById('clear-captures').onclick = () => {\n captureSystem.clear();\n updateAngleGrid(0, 0);\n updatePreview();\n updateProgress();\n showToast('Cleared');\n };\n \n const autoToggle = document.getElementById('toggle-auto-capture');\n autoToggle.onclick = () => {\n CONFIG.autoCapture = !CONFIG.autoCapture;\n autoToggle.classList.toggle('active', CONFIG.autoCapture);\n showToast(CONFIG.autoCapture ? 'Auto: ON' : 'Auto: OFF');\n };\n\n await mindarThree.start();\n video = container.querySelector('video');\n \n // STATE\n let lastCaptureTime = 0;\n let lastYaw = -999;\n let lastPitch = -999;\n let currentYaw = 0;\n let currentPitch = 0;\n \n statusEl.textContent = 'Ready';\n statusDot.classList.remove('inactive');\n\n // MAIN LOOP\n renderer.setAnimationLoop(() => {\n const now = performance.now();\n const face = mindarThree.faceMeshes[0];\n \n // Clear debug canvas\n overlayCtx.clearRect(0, 0, overlayCanvas.width, overlayCanvas.height);\n \n if (face && face.visible) {\n statusEl.textContent = 'Tracking';\n statusDot.classList.remove('inactive');\n \n // 1. UPDATE MATRICES\n face.updateMatrixWorld(true);\n camera.updateMatrixWorld(true);\n \n // 2. CALCULATE POSE (Robust Method)\n const pose = estimatePoseFromLandmarks(face, camera);\n currentYaw = pose.yaw;\n currentPitch = pose.pitch;\n \n // 3. DRAW DEBUG VISUALS (Red/Green Crosshair on Nose)\n const positions = face.geometry.attributes.position.array;\n const noseLocal = new THREE.Vector3(positions[1*3], positions[1*3+1], positions[1*3+2]);\n noseLocal.applyMatrix4(face.matrixWorld);\n const noseScreen = projectPoint(noseLocal, camera, overlayCanvas.width, overlayCanvas.height);\n \n // Draw crosshair\n overlayCtx.lineWidth = 3;\n // Yaw indicator (Green horizontal)\n overlayCtx.strokeStyle = '#22c55e';\n overlayCtx.beginPath();\n overlayCtx.moveTo(noseScreen.x - (currentYaw * 2), noseScreen.y); // Exaggerate movement\n overlayCtx.lineTo(noseScreen.x + (currentYaw * 2), noseScreen.y);\n overlayCtx.stroke();\n \n // Pitch indicator (Red vertical)\n overlayCtx.strokeStyle = '#ef4444';\n overlayCtx.beginPath();\n overlayCtx.moveTo(noseScreen.x, noseScreen.y - (currentPitch * 2));\n overlayCtx.lineTo(noseScreen.x, noseScreen.y + (currentPitch * 2));\n overlayCtx.stroke();\n \n // Center dot\n overlayCtx.fillStyle = '#fff';\n overlayCtx.beginPath();\n overlayCtx.arc(noseScreen.x, noseScreen.y, 3, 0, Math.PI*2);\n overlayCtx.fill();\n\n // 4. UPDATE UI\n const pill = document.getElementById('angle-pill');\n if(pill) pill.textContent = \\`Y:\\${currentYaw.toFixed(0)}° P:\\${currentPitch.toFixed(0)}°\\`;\n \n // 5. CAPTURE LOGIC\n if (CONFIG.autoCapture && now - lastCaptureTime > CONFIG.autoCaptureInterval) {\n const angleChanged = (Math.abs(currentYaw - lastYaw) + Math.abs(currentPitch - lastPitch)) > CONFIG.minAngleChange;\n const newAngle = !captureSystem.isAngleCaptured(currentYaw, currentPitch);\n \n if ((angleChanged || newAngle) && video && video.readyState >= 2) {\n const success = captureSystem.captureFrame(video, face, camera, currentYaw, currentPitch);\n if (success) {\n lastYaw = currentYaw;\n lastPitch = currentPitch;\n updateAngleGrid(currentYaw, currentPitch);\n updateProgress();\n updatePreview();\n \n // Flash feedback\n overlayCtx.fillStyle = 'rgba(255, 255, 255, 0.3)';\n overlayCtx.fillRect(0,0, overlayCanvas.width, overlayCanvas.height);\n }\n }\n lastCaptureTime = now;\n }\n \n } else {\n statusEl.textContent = 'Searching...';\n statusDot.classList.add('inactive');\n }\n \n renderer.render(scene, camera);\n });\n \n } catch (err) {\n console.error(err);\n statusEl.textContent = 'Error';\n }\n}\n\nif (document.readyState === 'loading') {\n document.addEventListener('DOMContentLoaded', initTextureMapper);\n} else {\n initTextureMapper();\n}\n`;\n\n// ============================================================================\n// 4. SERVER HANDLER\n// ============================================================================\n\nexport default async function handler(req: Request): Promise {\n const html = `\n\n\n Face Texture Mapper (Fixed Tracking)\n \n \n \n \n \n \n \n\n \n\n\n ${UI_HTML}\n \n\n`;\n\n return new Response(html, {\n headers: {\n \"Content-Type\": \"text/html; charset=utf-8\",\n \"Cache-Control\": \"no-cache\",\n },\n });\n}","deleted","label",[99],"permalink","https://www.val.town/x/banks/facelabtexture/code/main.ts","extension","actionData","errors"]