AI Logic Trap Scorecard

1. Configure Evaluation Run

\n \n

2. Scorecard

\n\n\n`;\n}\n\n// --- BACKEND LOGIC ---\nexport default async function (req: Request): Promise {\n const url = new URL(req.url);\n const action = url.searchParams.get(\"action\");\n\n try {\n if (req.method === \"POST\" && action === \"runAutomatedEval\") {\n const openai = new OpenAI();\n const { num_questions, prompt_types } = await req.json();\n\n if (num_questions > 15 || num_questions < 1) {\n throw new Error(\"Number of questions must be between 1 and 15.\");\n }\n\n // 1. Generation Phase\n const generationPromises = Array(num_questions).fill(null).map(() =>\n openai.chat.completions.create({\n model: \"gpt-4o\",\n messages: [{ role: \"system\", content: TRAP_DESIGNER_PROMPT }],\n response_format: { type: \"json_object\" },\n })\n );\n const generatedTraps = (await Promise.all(generationPromises)).map(\n (c) =>\n c.choices?.[0] ? extractJson(c.choices[0].message.content) : null,\n ).filter(Boolean);\n\n // 2. Evaluation Phase\n const evaluationResults = await Promise.all(\n generatedTraps.map(async (trap) => {\n const evals = {};\n for (const promptKey of prompt_types) {\n const completion = await openai.chat.completions.create({\n model: \"gpt-4o\",\n messages: [\n { role: \"system\", content: SOLVER_PROMPTS[promptKey] },\n { role: \"user\", content: trap.question },\n ],\n });\n const answerText = completion.choices?.[0]?.message?.content || \"\";\n const finalAnswer = extractFinalAnswer(answerText);\n const is_correct =\n finalAnswer.toLowerCase() === trap.correct_answer.toLowerCase();\n evals[promptKey] = { answer: finalAnswer, is_correct };\n }\n return { trap, evals };\n }),\n );\n\n // 3. Scoring & Packaging Phase\n const summary = {};\n prompt_types.forEach((p) => summary[p] = { correct: 0, total: 0 });\n evaluationResults.forEach((res) => {\n prompt_types.forEach((p) => {\n summary[p].total++;\n if (res.evals[p].is_correct) {\n summary[p].correct++;\n }\n });\n });\n\n const scorecard = {\n id: `scorecard_${Date.now()}`,\n run_date: new Date().toISOString(),\n settings: { num_questions, prompt_types },\n summary,\n results: evaluationResults,\n };\n\n // 4. Persistence (Optional, for this version we just return the result)\n // await blob.setJSON(scorecard.id, scorecard);\n // ... update index ...\n\n return new Response(JSON.stringify(scorecard), {\n headers: { \"Content-Type\": \"application/json\" },\n });\n }\n } catch (error) {\n console.error(\"API Error:\", error);\n return new Response(JSON.stringify({ error: error.message }), {\n status: 500,\n headers: { \"Content-Type\": \"application/json\" },\n });\n }\n\n // Serve HTML for GET requests\n return new Response(generateHtml(req.url), {\n headers: { \"Content-Type\": \"text/html\" },\n });\n}","deleted","label",[98],"permalink","https://www.val.town/x/realtime/scorecard/code/main.ts","extension","actionData","errors"]