{status.message} {status.progress}%

;\n * }\n *\n * const response = await sendMessage([{ role: \"user\", content: \"Hello!\" }]);\n * ```\n *\n * @param selectedModel - Model ID from AVAILABLE_MODELS\n * @returns Hook interface with status, isReady flag, and sendMessage function\n */\nexport function useWebLLM(selectedModel: string) {\n const [status, setStatus] = useState({\n state: \"idle\",\n message: \"Initializing...\",\n });\n const engineRef = useRef(null);\n const retryCountRef = useRef(0);\n const retryTimeoutRef = useRef(null);\n\n useEffect(() => {\n let cancelled = false;\n // Reset retry count when model changes\n retryCountRef.current = 0;\n\n async function loadModel() {\n const modelInfo = getModelInfo(selectedModel);\n const modelName = modelInfo ? modelInfo.name : selectedModel;\n\n console.log(\"[useWebLLM] Starting with model:\", selectedModel);\n\n if (!navigator.gpu) {\n setStatus({\n state: \"error\",\n message: \"WebGPU not supported - need Chrome/Edge\",\n });\n return;\n }\n\n try {\n // Dispose of previous engine if it exists\n if (engineRef.current) {\n console.log(\"[useWebLLM] Disposing previous engine...\");\n const prevEngine = engineRef.current;\n engineRef.current = null;\n\n try {\n await prevEngine.unload();\n } catch (disposeError) {\n console.warn(\n \"[useWebLLM] Error unloading previous engine:\",\n disposeError,\n );\n }\n }\n\n setStatus({\n state: \"loading\",\n message: \"Loading WebLLM library...\",\n progress: 0,\n });\n\n const webllm = await import(WEBLLM_CONFIG.cdnUrl) as WebLLMModule;\n\n if (cancelled) return;\n\n setStatus({\n state: \"loading\",\n message: `Downloading ${modelName}...`,\n progress: 0,\n });\n\n const engine = await webllm.CreateMLCEngine(selectedModel, {\n initProgressCallback: (progress: WebLLMProgress) => {\n if (cancelled) return;\n const pct = Math.round(progress.progress * 100);\n console.log(`[useWebLLM] Progress: ${pct}%`);\n setStatus({\n state: \"loading\",\n message: progress.text || `Downloading ${modelName}...`,\n progress: pct,\n });\n },\n });\n\n if (cancelled) return;\n\n console.log(\"[useWebLLM] Ready with\", modelName);\n engineRef.current = engine;\n setStatus({\n state: \"ready\",\n message: `${modelName} Ready`,\n progress: 100,\n });\n } catch (error: unknown) {\n const errorMessage = error instanceof Error\n ? error.message\n : \"Unknown error occurred\";\n console.error(\"[useWebLLM] Error:\", error);\n\n // If it's a cache network error and we haven't retried yet, clear cache and retry\n if (isCacheNetworkError(error) && retryCountRef.current === 0) {\n console.log(\n \"[useWebLLM] Detected cache corruption, clearing and retrying...\",\n );\n retryCountRef.current = 1;\n setStatus({\n state: \"loading\",\n message: \"Clearing corrupted cache and retrying...\",\n progress: 0,\n });\n\n await clearWebLLMCache();\n\n if (!cancelled) {\n // Retry loading after a short delay\n retryTimeoutRef.current = window.setTimeout(() => {\n if (!cancelled) loadModel();\n }, WEBLLM_CONFIG.retryDelayMs);\n }\n } else {\n setStatus({ state: \"error\", message: errorMessage });\n }\n }\n }\n\n loadModel();\n\n return () => {\n cancelled = true;\n if (retryTimeoutRef.current !== null) {\n clearTimeout(retryTimeoutRef.current);\n retryTimeoutRef.current = null;\n }\n };\n }, [selectedModel]);\n\n /**\n * Run a single inference call (internal helper)\n */\n const runSingleInference = async (\n chatMessages: Array<{ role: string; content: string }>,\n maxTokens: number,\n ): Promise => {\n if (!engineRef.current) {\n throw new Error(\"Engine not ready\");\n }\n\n const response = await engineRef.current.chat.completions.create({\n messages: chatMessages,\n temperature: WEBLLM_CONFIG.temperature,\n max_tokens: maxTokens,\n }) as WebLLMCompletionResponse;\n\n if (!response.choices || response.choices.length === 0) {\n throw new Error(\"No response choices returned from model\");\n }\n\n const content = response.choices[0]?.message?.content;\n if (!content) {\n throw new Error(\"Response message content is empty\");\n }\n\n // Store the complete raw response for deep mode\n const rawResponse = content;\n\n // Parse thinking tags\n const thinkingMatch = content.match(/([\\s\\S]*?)<\\/thinking>/);\n const thinking = thinkingMatch ? thinkingMatch[1].trim() : undefined;\n const cleanContent = content.replace(/[\\s\\S]*?<\\/thinking>/g, \"\").trim();\n\n return { content: cleanContent, thinking, rawResponse };\n };\n\n /**\n * Pick the best result from multiple runs using simple voting\n */\n const pickBestResult = (results: SendMessageResult[]): SendMessageResult => {\n // For deep mode: combine ALL raw responses to show complete transparency\n const allRawResponses = results\n .map((r, i) => {\n // Use raw response if available (deep mode), otherwise reconstruct from thinking + content\n if (r.rawResponse) {\n return `[Run ${i + 1}]\\n${r.rawResponse}`;\n }\n // Fallback for compatibility with non-raw results\n const parts = [];\n if (r.thinking) parts.push(`\\n${r.thinking}\\n`);\n if (r.content) parts.push(r.content);\n return `[Run ${i + 1}]\\n${parts.join('\\n\\n')}`;\n })\n .join('\\n\\n');\n\n // Voting logic: find the most common answer (by first line or first sentence)\n const getAnswerKey = (content: string) => {\n const firstLine = content.split('\\n')[0].trim().toLowerCase();\n // Extract key parts (numbers, short phrases)\n return firstLine.slice(0, 100);\n };\n\n const votes: Map = new Map();\n\n for (const result of results) {\n const key = getAnswerKey(result.content);\n const existing = votes.get(key) || { count: 0, results: [] };\n existing.count++;\n existing.results.push(result);\n votes.set(key, existing);\n }\n\n // Find the answer with most votes\n let bestKey = \"\";\n let bestCount = 0;\n for (const [key, data] of votes) {\n if (data.count > bestCount) {\n bestCount = data.count;\n bestKey = key;\n }\n }\n\n const winners = votes.get(bestKey)!.results;\n // Return the one with the longest thinking (most detailed)\n const best = winners.reduce((a, b) =>\n (a.thinking?.length || 0) > (b.thinking?.length || 0) ? a : b\n );\n\n return {\n content: best.content,\n thinking: allRawResponses, // Show ALL raw responses in thinking field\n };\n };\n\n /**\n * Send messages to the LLM and get a response.\n * Automatically prepends system prompt to conversation.\n * Supports streaming and thinking modes (off, think, deep).\n *\n * @param messages - Array of user/assistant messages\n * @param options - Optional settings for thinking mode and streaming\n * @returns Promise resolving to result with content and optional thinking\n * @throws Error if engine not ready or generation fails\n */\n const sendMessage = useCallback(\n async (messages: Message[], options?: SendMessageOptions): Promise => {\n console.log(\"[useWebLLM] Sending messages:\", messages);\n\n if (!engineRef.current) {\n throw new Error(\"Engine not ready\");\n }\n\n const thinkingMode: ThinkingMode = options?.thinkingMode ?? \"off\";\n const onStream = options?.onStream;\n const onProgress = options?.onProgress;\n\n // Choose system prompt based on thinking mode\n let systemPrompt = WEBLLM_CONFIG.systemPrompt;\n let maxTokens = WEBLLM_CONFIG.maxTokens;\n\n if (thinkingMode === \"think\") {\n systemPrompt = THINK_PROMPT;\n maxTokens = 1024;\n } else if (thinkingMode === \"deep\") {\n systemPrompt = DEEP_THINK_PROMPT;\n maxTokens = 1536; // More tokens for detailed reasoning\n }\n\n // Build messages array with system prompt\n const chatMessages = [\n { role: \"system\" as const, content: systemPrompt },\n ...messages.map((m) => ({ role: m.role, content: m.content })),\n ];\n\n console.log(\"[useWebLLM] Full chat context:\", chatMessages);\n console.log(\"[useWebLLM] Thinking mode:\", thinkingMode);\n\n // Deep mode: run 3 times with self-consistency (no streaming for individual runs)\n if (thinkingMode === \"deep\") {\n const numRuns = 3;\n const results: (SendMessageResult & { rawResponse?: string })[] = [];\n\n for (let i = 0; i < numRuns; i++) {\n console.log(`[useWebLLM] Deep think run ${i + 1}/${numRuns}`);\n onProgress?.(i + 1, numRuns);\n\n try {\n const result = await runSingleInference(chatMessages, maxTokens);\n results.push(result);\n } catch (error) {\n console.error(`[useWebLLM] Run ${i + 1} failed:`, error);\n // Continue with remaining runs\n }\n }\n\n if (results.length === 0) {\n throw new Error(\"All deep thinking runs failed\");\n }\n\n const best = results.length === 1 ? results[0] : pickBestResult(results);\n\n // Send final result to stream callback if provided\n if (onStream) {\n if (best.thinking) {\n onStream({ phase: \"thinking\", content: best.thinking, isComplete: true });\n }\n onStream({ phase: \"response\", content: best.content, isComplete: true });\n }\n\n return best;\n }\n\n // Standard mode or Think mode: use streaming if callback provided\n if (onStream && thinkingMode !== \"off\") {\n const parser = createThinkingParser();\n\n try {\n const stream = await engineRef.current.chat.completions.create({\n messages: chatMessages,\n temperature: WEBLLM_CONFIG.temperature,\n max_tokens: maxTokens,\n stream: true,\n }) as AsyncIterable;\n\n for await (const chunk of stream) {\n const deltaContent = chunk.choices[0]?.delta?.content || \"\";\n if (deltaContent) {\n const parsed = parser.feed(deltaContent);\n onStream({\n phase: parsed.phase,\n content: parsed.phase === \"thinking\" ? parsed.thinking : parsed.response,\n isComplete: false,\n });\n }\n }\n\n const final = parser.finalize();\n onStream({\n phase: \"response\",\n content: final.response,\n isComplete: true,\n });\n\n return {\n content: final.response,\n thinking: final.thinking || undefined,\n };\n } catch (streamError) {\n console.error(\"[useWebLLM] Stream error:\", streamError);\n const partial = parser.finalize();\n return {\n content: partial.response || \"Response interrupted. Please try again.\",\n thinking: partial.thinking || undefined,\n };\n }\n }\n\n // Non-streaming fallback (off mode or no callback)\n const response = await engineRef.current.chat.completions.create({\n messages: chatMessages,\n temperature: WEBLLM_CONFIG.temperature,\n max_tokens: maxTokens,\n }) as WebLLMCompletionResponse;\n\n console.log(\"[useWebLLM] Response:\", response);\n\n if (!response.choices || response.choices.length === 0) {\n throw new Error(\"No response choices returned from model\");\n }\n\n const content = response.choices[0]?.message?.content;\n if (!content) {\n throw new Error(\"Response message content is empty\");\n }\n\n // Parse thinking tags if in thinking mode\n if (thinkingMode !== \"off\") {\n const thinkingMatch = content.match(/([\\s\\S]*?)<\\/thinking>/);\n const thinking = thinkingMatch ? thinkingMatch[1].trim() : undefined;\n const cleanContent = content.replace(/[\\s\\S]*?<\\/thinking>/g, \"\").trim();\n return { content: cleanContent, thinking };\n }\n\n return { content };\n },\n [],\n );\n\n return { status, isReady: status.state === \"ready\", sendMessage };\n}\n","deleted","label",[178,162],"hooks","permalink","https://www.val.town/x/Prada/Locals-AI-Deep1/code/hooks/useWebLLM.ts","extension","actionData","errors"]