[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_152":153,"_154":155},"routes/_app.x.$handle.$name",{"_11":12,"_13":14,"_15":16,"_18":19,"_17":-7,"_115":-5,"_20":116,"_22":117,"_118":119,"_120":121,"_25":122,"_24":-5,"_123":-5,"_124":-5,"_125":53,"_126":127,"_128":44,"_129":-5,"_130":-5,"_131":132},"routes/_app.x.$handle.$name.branch.$branch.code.$",{"_11":12,"_13":14,"_15":16,"_17":-7,"_18":19,"_20":21,"_22":23,"_24":-7,"_25":26,"_27":-5,"_28":-5,"_29":-5,"_30":-5,"_31":-5},"handle","real","name","Evals","branchName","main","version","path","","project",{"_33":61,"_13":14,"_45":62,"_63":-5,"_64":53,"_65":66,"_67":66,"_68":-5,"_69":-5,"_70":71,"_72":37,"_73":44,"_74":44,"_75":-5,"_76":77,"_78":79,"_80":-5,"_81":-5,"_82":-5,"_83":-5,"_84":85,"_86":85,"_87":88,"_89":88,"_90":88,"_91":85,"_92":85,"_93":94,"_95":96,"_97":98,"_99":53},"tree",[56],"file","contents",[32],"upstream","pull","ahead","behind","envVars",{"_33":34,"_13":35,"_36":37,"_38":-5,"_39":40,"_41":42,"_43":44,"_45":46,"_17":47,"_48":49,"_50":51,"_52":53,"_54":-5,"_18":55},"id","2c27836c-7c03-11f0-90e8-0224a6c84d84","main.tsx","fileId","092ddd58-7c02-11f0-a9db-0224a6c84d84","parentId","branchId","092b6a96-7c02-11f0-a9db-0224a6c84d84","branchVersion",6,"latest",true,"createdAt",["D",1755501351588],5,"fileType","http","content","// @ts-ignore\nimport { OpenAI } from \"https://esm.town/v/std/openai?v=4\";\n\n// --- TYPE DEFINITIONS ---\ninterface TestCase {\n id: string;\n sourceText: string; // The problem statement\n}\n\ninterface SolverOutput {\n answer: string;\n chain_of_thought: string;\n}\n\ninterface VerifierDiff {\n critique_of_reasoning: string;\n is_answer_correct: boolean;\n corrected_answer?: string;\n refined_reasoning: string;\n}\n\ninterface Evaluation {\n draft_score: {\n correctness: number;\n logical_soundness: number;\n explanation_clarity: number;\n };\n final_score: {\n correctness: number;\n logical_soundness: number;\n explanation_clarity: number;\n };\n justification: string;\n}\n\ninterface TestResult {\n id: string;\n sourceText: string;\n solverOutput: SolverOutput;\n verifierDiff: VerifierDiff;\n finalOutput: {\n answer: string;\n reasoning: string;\n };\n evaluation: Evaluation;\n}\n\n// --- AI BEHAVIORAL GUIDELINES ---\nconst GENERATION_STYLE_GUIDE =\n \"Adopt the persona of a senior logician and analyst. Articulate your output using precise, unambiguous language suitable for a formal proof or technical report. Avoid clichés and informalities. The objective is to produce structured data that is insightful and appears meticulously prepared by a human expert.\";\n\nconst SOLVER_PROMPT = `\nYou are a \"Solver\" AI, a raw, powerful logical reasoning engine.\nGiven the problem, provide a final answer and a step-by-step 'chain_of_thought' explaining how you arrived at it. The chain of thought can be your raw, unedited internal monologue.\nYour response MUST be a single, minified JSON object: { \"answer\": \"...\", \"chain_of_thought\": \"Step 1: ..., Step 2: ...\" }.\n${GENERATION_STYLE_GUIDE}\n`;\n\nconst VERIFIER_PROMPT = `\nYou are a \"Verifier\" AI, a meticulous logician and communication expert. You are given a problem and a proposed solution from a \"Solver\" AI.\nYour task is to CRITIQUE, VERIFY, and REFINE the Solver's output.\n1. **Critique**: Provide a brief, professional critique of the Solver's logical process.\n2. **Verify**: State if the Solver's final answer is correct. If not, provide the corrected answer.\n3. **Refine**: Rewrite the 'chain_of_thought' into a 'refined_reasoning' that is clearer, more structured, and logically rigorous. This is the most important step.\n\nYou MUST respond ONLY with a single, minified JSON \"diff\" object:\n{\n \"critique_of_reasoning\": \"The solver's approach was sound but failed to account for...\",\n \"is_answer_correct\": false,\n \"corrected_answer\": \"The correct answer is...\",\n \"refined_reasoning\": \"1. First, we establish the premises... 2. Next, by applying deductive reasoning...\"\n}\nIf the answer is correct, omit the \"corrected_answer\" field.\n${GENERATION_STYLE_GUIDE}\n`;\n\nconst EVALUATOR_PROMPT = `\nYou are an \"Evaluator\" AI, an impartial professor of logic. You will be given a problem, a \"draft_solution\" (from a Solver AI), and a \"final_solution\" (refined by a Verifier AI).\n\nYour task is to score BOTH solutions on a scale of 1 to 10 across three criteria:\n1. **Correctness**: Is the final answer to the problem correct? (Score 10 for correct, 1 for incorrect).\n2. **Logical Soundness**: Is the chain of reasoning free of fallacies and logical errors?\n3. **Explanation Clarity**: How easy is the reasoning to follow for a human?\n\nProvide a final justification explaining the score differences.\n\nYou MUST respond ONLY with a single, minified JSON object:\n{\n \"draft_score\": { \"correctness\": , \"logical_soundness\": , \"explanation_clarity\": },\n \"final_score\": { \"correctness\": , \"logical_soundness\": , \"explanation_clarity\": },\n \"justification\": \"The final solution is superior because its reasoning was restructured...\"\n}\n${GENERATION_STYLE_GUIDE}\n`;\n\n// --- BACKEND LOGIC ---\nfunction applyVerifierDiff(\n solverOutput: SolverOutput,\n verifierDiff: VerifierDiff,\n): TestResult[\"finalOutput\"] {\n return {\n answer: verifierDiff.is_answer_correct\n ? solverOutput.answer\n : verifierDiff.corrected_answer,\n reasoning: verifierDiff.refined_reasoning,\n };\n}\n\nasync function processTestCase(\n testCase: TestCase,\n openai: OpenAI,\n): Promise {\n // Phase 1: Solver\n const solverCompletion = await openai.chat.completions.create({\n model: \"gpt-4o\",\n messages: [{ role: \"system\", content: SOLVER_PROMPT }, {\n role: \"user\",\n content: testCase.sourceText,\n }],\n response_format: { type: \"json_object\" },\n });\n const solverOutput: SolverOutput = JSON.parse(\n solverCompletion.choices[0].message.content,\n );\n\n // Phase 2: Verifier\n const verifierCompletion = await openai.chat.completions.create({\n model: \"gpt-4o\",\n messages: [\n { role: \"system\", content: VERIFIER_PROMPT },\n {\n role: \"user\",\n content: `Problem:\\n${testCase.sourceText}\\n\\nSolver's Draft:\\n${\n JSON.stringify(solverOutput)\n }`,\n },\n ],\n response_format: { type: \"json_object\" },\n });\n const verifierDiff: VerifierDiff = JSON.parse(\n verifierCompletion.choices[0].message.content,\n );\n\n // Phase 3: Apply Diff\n const finalOutput = applyVerifierDiff(solverOutput, verifierDiff);\n\n // Phase 4: Evaluator\n const draftForEval = {\n answer: solverOutput.answer,\n reasoning: solverOutput.chain_of_thought,\n };\n const evaluatorCompletion = await openai.chat.completions.create({\n model: \"gpt-4o-2024-05-13\",\n messages: [\n { role: \"system\", content: EVALUATOR_PROMPT },\n {\n role: \"user\",\n content: `Problem:\\n${testCase.sourceText}\\n\\nDraft Solution:\\n${\n JSON.stringify(draftForEval)\n }\\n\\nFinal Solution:\\n${JSON.stringify(finalOutput)}`,\n },\n ],\n response_format: { type: \"json_object\" },\n });\n const evaluation: Evaluation = JSON.parse(\n evaluatorCompletion.choices[0].message.content,\n );\n\n return {\n id: testCase.id,\n sourceText: testCase.sourceText,\n solverOutput,\n verifierDiff,\n finalOutput,\n evaluation,\n };\n}\n\n// --- HTML & FRONTEND ---\nfunction generateHtml(sourceUrl: string) {\n return `\n\n\n \n \n Reasoning & Problem-Solving LLM Eval Suite\n \n \n \n \n\n\n\n

Reasoning & Problem-Solving LLM Eval Suite

A Solver/Verifier/Evaluator pipeline to test an LLM's logical capabilities.

Running evaluation...

\n \n

Average Score Improvement

Detailed Results

\n \n \n \n

Problem	Solver Score	Final Score	Details

\n\n\n\n\n`;\n}\n\nexport default async function (req: Request) {\n if (req.method === \"GET\") {\n return new Response(\n generateHtml(new URL(req.url).origin + new URL(req.url).pathname),\n {\n headers: { \"Content-Type\": \"text/html\" },\n },\n );\n }\n\n if (req.method === \"POST\") {\n try {\n const { testCases } = await req.json();\n if (!Array.isArray(testCases) || testCases.length === 0) {\n throw new Error(\"Invalid or empty test cases provided.\");\n }\n const openai = new OpenAI();\n const results = await Promise.all(\n testCases.map((testCase) => processTestCase(testCase, openai)),\n );\n return new Response(JSON.stringify(results), {\n headers: { \"Content-Type\": \"application/json\" },\n });\n } catch (error) {\n console.error(\"Evaluation Suite Error:\", error);\n return new Response(\n JSON.stringify({\n error: error.message || \"An unknown error occurred.\",\n }),\n {\n status: 500,\n headers: { \"Content-Type\": \"application/json\" },\n },\n );\n }\n }\n\n return new Response(\"Method Not Allowed\", { status: 405 });\n}","deleted",false,"label",[35],{"_33":34,"_13":35,"_36":37,"_38":-5,"_39":40,"_41":42,"_43":44,"_45":57,"_17":47,"_48":49,"_50":51,"_52":53,"_54":-5,"_58":59,"_18":60},["D",1755501351588],"children",[],[35],"092b0646-7c02-11f0-a9db-0224a6c84d84",["D",1755500863403],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","imageUrl","userId","63f04ac0-562b-11f0-a849-2f0fd58d5bdc","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","0abcaa5c-2531-4e34-967b-6f68bd9904a8","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_11":12,"_114":110},"org",{"_11":12,"_108":-5,"_109":110,"_70":71,"_111":112,"_113":53},"branch",{"_17":42,"_13":16,"_33":40,"_100":61,"_101":-5,"_102":-5,"_103":-5,"_104":-5,"_105":-5,"_106":14,"_107":12},"isOwned","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoiZGVmYXVsdCIsImlpZCI6Imluc18yTm10cGhQUklwU0tqZHNiOHNnNGd2ZjJURFUiLCJyaWQiOiJ1c2VyXzJ6Rng5NEh2b3dPMmVwbThwVm1keXA3aTk5bCJ9","tier","free","canRestrictVals","profileImageUrl","adoptOrgId",{"_33":61,"_13":14,"_45":148,"_63":-5,"_64":53,"_65":66,"_67":66,"_68":-5,"_69":-5,"_70":71,"_72":37,"_73":44,"_74":44,"_75":-5,"_76":77,"_78":79,"_80":-5,"_81":-5,"_82":-5,"_83":-5,"_84":85,"_86":85,"_87":88,"_89":88,"_90":88,"_91":85,"_92":85,"_93":149,"_95":150,"_97":151,"_99":53},[145],"files",[145],"counts",{"_135":77,"_136":137,"_138":77,"_31":139,"_140":141},[133],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_33":34,"_13":35,"_36":37,"_38":-5,"_39":40,"_41":42,"_43":44,"_45":46,"_17":47,"_48":49,"_50":51,"_52":53,"_54":-5,"_18":134},[35],"pulls","branches",1,"forks",{"_142":77,"_144":77},"connections",{"_142":77,"_143":77},"count","unfilledCount","emptyCount",{"_33":34,"_13":35,"_36":37,"_38":-5,"_39":40,"_41":42,"_43":44,"_45":57,"_17":47,"_48":49,"_50":51,"_52":53,"_54":-5,"_58":146,"_18":147},[],[35],["D",1755500863403],{"_11":12,"_114":110},{"_11":12,"_108":-5,"_109":110,"_70":71,"_111":112,"_113":53},{"_17":42,"_13":16,"_33":40,"_100":61,"_101":-5,"_102":-5,"_103":-5,"_104":-5,"_105":-5,"_106":14,"_107":12},"ctx",{"_93":-5,"_130":185,"_186":187,"_188":189,"_190":53,"_191":53,"_192":53,"_193":44,"_194":-5,"_195":-7,"_196":-7,"_197":77,"_198":53,"_199":53,"_200":-5,"_201":53,"_202":53,"_203":53,"_204":-5},"ENV",{"_156":157,"_158":159,"_160":16,"_161":162,"_163":164,"_165":166,"_167":168,"_169":170,"_171":172,"_173":174,"_175":176,"_177":178,"_179":180,"_181":182,"_183":184},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e8c4b7030cb7600e4eb0736b15f422c8b8bd130f","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Code