[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_153":154,"_155":156},"routes/_app.x.$handle.$name",{"_95":94,"_32":45,"_103":85,"_27":104,"_13":14,"_105":-5,"_11":106,"_107":108,"_109":110,"_111":112,"_113":114,"_20":-5,"_115":-5,"_116":-5,"_117":25,"_118":119,"_120":57,"_121":-5,"_122":-5,"_123":124},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_43":44,"_32":45,"_38":46,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":-5,"_53":-5,"_18":54,"_55":31,"_56":57,"_58":57,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":78,"_79":80,"_81":82,"_83":25},"version",5,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_33":34,"_35":37,"_38":39,"_40":14,"_41":42,"_24":25},"previous",{"_30":31,"_32":28,"_33":34,"_35":36},"deleted",false,"deletedHere","path","main.tsx","latestPath","fileId","092ddd58-7c02-11f0-a9db-0224a6c84d84","name","fileType","http","content","// @ts-ignore\nimport { OpenAI } from \"https://esm.town/v/std/openai?v=4\";\n\n// --- TYPE DEFINITIONS ---\ninterface TestCase {\n id: string;\n sourceText: string;\n}\n\ninterface WriterOutput {\n title: string;\n summary: string;\n key_points: string[];\n}\n\ninterface ReaderDiff {\n critique: string;\n property_additions: Record;\n array_enrichments: Record;\n corrections: {\n path: string;\n corrected_value: string;\n reason: string;\n }[];\n}\n\ninterface Evaluation {\n draft_score: {\n completeness: number;\n accuracy: number;\n insight: number;\n };\n final_score: {\n completeness: number;\n accuracy: number;\n insight: number;\n };\n justification: string;\n}\n\ninterface TestResult {\n id: string;\n sourceText: string;\n writerOutput: WriterOutput;\n readerDiff: ReaderDiff;\n finalOutput: any;\n evaluation: Evaluation;\n}\n\n// --- AI BEHAVIORAL GUIDELINES ---\nconst GENERATION_STYLE_GUIDE =\n \"Adopt the persona of a senior analyst. Articulate your output using precise, intelligent language suitable for a professional report. Avoid clichés and informalities. The objective is to produce structured data that is insightful and appears meticulously prepared by a human expert.\";\n\nconst WRITER_PROMPT = `\nYou are a \"Writer\" AI. Perform an initial analysis of the provided text, extracting the main title, a concise summary, and three primary key points.\nYour response MUST be a single, minified JSON object: { \"title\": \"...\", \"summary\": \"...\", \"key_points\": [\"...\", \"...\", \"...\"] }.\nThis is a first draft; do not add extra metadata.\n${GENERATION_STYLE_GUIDE}\n`;\n\nconst READER_PROMPT = `\nYou are a \"Reader\" AI, a meticulous editor. You are given an original text and a \"first draft\" JSON from a Writer AI.\nYour task is to CRITIQUE and ENRICH the Writer's output, adding significant net-new value.\n1. **Critique**: Provide a brief, professional critique.\n2. **Add Properties**: Add missing properties like sentiment, content category, or target audience.\n3. **Enrich Arrays**: Add at least two more relevant items to the 'key_points' array.\n4. **Correct Fields**: If original fields are weak, provide a corrected version and a reason.\n\nYou MUST respond ONLY with a single, minified JSON \"diff\" object:\n{\n \"critique\": \"...\",\n \"property_additions\": { \"new_property\": \"value\" },\n \"array_enrichments\": { \"key_points\": [\"point 4\", \"point 5\"] },\n \"corrections\": [ { \"path\": \"/summary\", \"corrected_value\": \"...\", \"reason\": \"...\" } ]\n}\n${GENERATION_STYLE_GUIDE}\n`;\n\nconst EVALUATOR_PROMPT = `\nYou are an \"Evaluator\" AI, an impartial Quality Assurance analyst. You will be given a source text and two JSON analyses of it: \"draft_output\" (from a Writer AI) and \"final_output\" (from a Writer+Reader AI).\n\nYour task is to score BOTH outputs on a scale of 1 to 10 across three criteria:\n1. **Completeness**: Does it capture all key information and nuances from the source?\n2. **Accuracy**: Is the extracted and inferred information factually correct?\n3. **Insight**: Does it add valuable metadata, categorization, or connections not immediately obvious in the source? (The draft should score low here by design).\n\nProvide a final justification explaining the score differences.\n\nYou MUST respond ONLY with a single, minified JSON object:\n{\n \"draft_score\": { \"completeness\": , \"accuracy\": , \"insight\": },\n \"final_score\": { \"completeness\": , \"accuracy\": , \"insight\": },\n \"justification\": \"The final output is superior because...\"\n}\n${GENERATION_STYLE_GUIDE}\n`;\n\n// --- BACKEND LOGIC ---\nfunction applyDiff(writerOutput: WriterOutput, readerDiff: ReaderDiff): any {\n let finalOutput = { ...writerOutput, ...readerDiff.property_additions };\n for (const key in readerDiff.array_enrichments) {\n if (finalOutput[key] && Array.isArray(finalOutput[key])) {\n finalOutput[key] = [\n ...finalOutput[key],\n ...readerDiff.array_enrichments[key],\n ];\n }\n }\n readerDiff.corrections.forEach((correction) => {\n const key = correction.path.substring(1);\n if (finalOutput.hasOwnProperty(key)) {\n finalOutput[key] = correction.corrected_value;\n }\n });\n return finalOutput;\n}\n\nasync function processTestCase(\n testCase: TestCase,\n openai: OpenAI,\n): Promise {\n // Phase 1: Writer\n const writerCompletion = await openai.chat.completions.create({\n model: \"gpt-4o\",\n messages: [{ role: \"system\", content: WRITER_PROMPT }, {\n role: \"user\",\n content: testCase.sourceText,\n }],\n response_format: { type: \"json_object\" },\n });\n const writerOutput: WriterOutput = JSON.parse(\n writerCompletion.choices[0].message.content,\n );\n\n // Phase 2: Reader\n const readerCompletion = await openai.chat.completions.create({\n model: \"gpt-4o\",\n messages: [\n { role: \"system\", content: READER_PROMPT },\n {\n role: \"user\",\n content: `Original Text:\\n${testCase.sourceText}\\n\\nWriter's Draft:\\n${\n JSON.stringify(writerOutput)\n }`,\n },\n ],\n response_format: { type: \"json_object\" },\n });\n const readerDiff: ReaderDiff = JSON.parse(\n readerCompletion.choices[0].message.content,\n );\n\n // Phase 3: Merge\n const finalOutput = applyDiff(writerOutput, readerDiff);\n\n // Phase 4: Evaluator\n const evaluatorCompletion = await openai.chat.completions.create({\n model: \"gpt-4o\",\n messages: [\n { role: \"system\", content: EVALUATOR_PROMPT },\n {\n role: \"user\",\n content: `Source Text:\\n${testCase.sourceText}\\n\\nDraft Output:\\n${\n JSON.stringify(writerOutput)\n }\\n\\nFinal Output:\\n${JSON.stringify(finalOutput)}`,\n },\n ],\n response_format: { type: \"json_object\" },\n });\n const evaluation: Evaluation = JSON.parse(\n evaluatorCompletion.choices[0].message.content,\n );\n\n return {\n id: testCase.id,\n sourceText: testCase.sourceText,\n writerOutput,\n readerDiff,\n finalOutput,\n evaluation,\n };\n}\n\n// --- HTML & FRONTEND ---\nfunction generateHtml(sourceUrl: string) {\n return `\n\n\n \n \n Read/Write LLM Evaluation Suite\n \n \n \n \n\n\n\n

Read/Write LLM Evaluation Suite

Running evaluation...

\n \n

Average Score Improvement

Detailed Results

\n \n \n \n

Test Case	Draft Score	Final Score	Details

\n\n\n\n\n`;\n}\n\nexport default async function (req: Request) {\n if (req.method === \"GET\") {\n return new Response(\n generateHtml(new URL(req.url).origin + new URL(req.url).pathname),\n {\n headers: { \"Content-Type\": \"text/html\" },\n },\n );\n }\n\n if (req.method === \"POST\") {\n try {\n const { testCases } = await req.json();\n if (!Array.isArray(testCases) || testCases.length === 0) {\n throw new Error(\"Invalid or empty test cases provided.\");\n }\n const openai = new OpenAI();\n const results = await Promise.all(\n testCases.map((testCase) => processTestCase(testCase, openai)),\n );\n return new Response(JSON.stringify(results), {\n headers: { \"Content-Type\": \"application/json\" },\n });\n } catch (error) {\n console.error(\"Evaluation Suite Error:\", error);\n return new Response(\n JSON.stringify({\n error: error.message || \"An unknown error occurred.\",\n }),\n {\n status: 500,\n headers: { \"Content-Type\": \"application/json\" },\n },\n );\n }\n }\n\n return new Response(\"Method Not Allowed\", { status: 405 });\n}","// @ts-ignore\nimport { OpenAI } from \"https://esm.town/v/std/openai?v=4\";\n\n// --- TYPE DEFINITIONS ---\ninterface TestCase {\n id: string;\n sourceText: string;\n}\n\ninterface WriterOutput {\n title: string;\n summary: string;\n key_points: string[];\n}\n\ninterface ReaderDiff {\n critique: string;\n property_additions: Record;\n array_enrichments: Record;\n corrections: {\n path: string;\n corrected_value: string;\n reason: string;\n }[];\n}\n\ninterface Evaluation {\n draft_score: {\n completeness: number;\n accuracy: number;\n insight: number;\n };\n final_score: {\n completeness: number;\n accuracy: number;\n insight: number;\n };\n justification: string;\n}\n\ninterface TestResult {\n id: string;\n sourceText: string;\n writerOutput: WriterOutput;\n readerDiff: ReaderDiff;\n finalOutput: any;\n evaluation: Evaluation;\n}\n\n// --- AI BEHAVIORAL GUIDELINES ---\nconst GENERATION_STYLE_GUIDE =\n \"Adopt the persona of a senior analyst. Articulate your output using precise, intelligent language suitable for a professional report. Avoid clichés and informalities. The objective is to produce structured data that is insightful and appears meticulously prepared by a human expert.\";\n\nconst WRITER_PROMPT = `\nYou are a \"Writer\" AI. Perform an initial analysis of the provided text, extracting the main title, a concise summary, and three primary key points.\nYour response MUST be a single, minified JSON object: { \"title\": \"...\", \"summary\": \"...\", \"key_points\": [\"...\", \"...\", \"...\"] }.\nThis is a first draft; do not add extra metadata.\n${GENERATION_STYLE_GUIDE}\n`;\n\nconst READER_PROMPT = `\nYou are a \"Reader\" AI, a meticulous editor. You are given an original text and a \"first draft\" JSON from a Writer AI.\nYour task is to CRITIQUE and ENRICH the Writer's output, adding significant net-new value.\n1. **Critique**: Provide a brief, professional critique.\n2. **Add Properties**: Add missing properties like sentiment, content category, or target audience.\n3. **Enrich Arrays**: Add at least two more relevant items to the 'key_points' array.\n4. **Correct Fields**: If original fields are weak, provide a corrected version and a reason.\n\nYou MUST respond ONLY with a single, minified JSON \"diff\" object:\n{\n \"critique\": \"...\",\n \"property_additions\": { \"new_property\": \"value\" },\n \"array_enrichments\": { \"key_points\": [\"point 4\", \"point 5\"] },\n \"corrections\": [ { \"path\": \"/summary\", \"corrected_value\": \"...\", \"reason\": \"...\" } ]\n}\n${GENERATION_STYLE_GUIDE}\n`;\n\nconst EVALUATOR_PROMPT = `\nYou are an \"Evaluator\" AI, an impartial Quality Assurance analyst. You will be given a source text and two JSON analyses of it: \"draft_output\" (from a Writer AI) and \"final_output\" (from a Writer+Reader AI).\n\nYour task is to score BOTH outputs on a scale of 1 to 10 across three criteria:\n1. **Completeness**: Does it capture all key information and nuances from the source?\n2. **Accuracy**: Is the extracted and inferred information factually correct?\n3. **Insight**: Does it add valuable metadata, categorization, or connections not immediately obvious in the source? (The draft should score low here by design).\n\nProvide a final justification explaining the score differences.\n\nYou MUST respond ONLY with a single, minified JSON object:\n{\n \"draft_score\": { \"completeness\": , \"accuracy\": , \"insight\": },\n \"final_score\": { \"completeness\": , \"accuracy\": , \"insight\": },\n \"justification\": \"The final output is superior because...\"\n}\n${GENERATION_STYLE_GUIDE}\n`;\n\n// --- BACKEND LOGIC ---\nfunction applyDiff(writerOutput: WriterOutput, readerDiff: ReaderDiff): any {\n let finalOutput = { ...writerOutput, ...readerDiff.property_additions };\n for (const key in readerDiff.array_enrichments) {\n if (finalOutput[key] && Array.isArray(finalOutput[key])) {\n finalOutput[key] = [\n ...finalOutput[key],\n ...readerDiff.array_enrichments[key],\n ];\n }\n }\n readerDiff.corrections.forEach((correction) => {\n const key = correction.path.substring(1);\n if (finalOutput.hasOwnProperty(key)) {\n finalOutput[key] = correction.corrected_value;\n }\n });\n return finalOutput;\n}\n\nasync function processTestCase(\n testCase: TestCase,\n openai: OpenAI,\n): Promise {\n // Phase 1: Writer\n const writerCompletion = await openai.chat.completions.create({\n model: \"gpt-4o\",\n messages: [{ role: \"system\", content: WRITER_PROMPT }, {\n role: \"user\",\n content: testCase.sourceText,\n }],\n response_format: { type: \"json_object\" },\n });\n const writerOutput: WriterOutput = JSON.parse(\n writerCompletion.choices[0].message.content,\n );\n\n // Phase 2: Reader\n const readerCompletion = await openai.chat.completions.create({\n model: \"gpt-4o\",\n messages: [\n { role: \"system\", content: READER_PROMPT },\n {\n role: \"user\",\n content: `Original Text:\\n${testCase.sourceText}\\n\\nWriter's Draft:\\n${\n JSON.stringify(writerOutput)\n }`,\n },\n ],\n response_format: { type: \"json_object\" },\n });\n const readerDiff: ReaderDiff = JSON.parse(\n readerCompletion.choices[0].message.content,\n );\n\n // Phase 3: Merge\n const finalOutput = applyDiff(writerOutput, readerDiff);\n\n // Phase 4: Evaluator\n const evaluatorCompletion = await openai.chat.completions.create({\n model: \"gpt-4o\",\n messages: [\n { role: \"system\", content: EVALUATOR_PROMPT },\n {\n role: \"user\",\n content: `Source Text:\\n${testCase.sourceText}\\n\\nDraft Output:\\n${\n JSON.stringify(writerOutput)\n }\\n\\nFinal Output:\\n${JSON.stringify(finalOutput)}`,\n },\n ],\n response_format: { type: \"json_object\" },\n });\n const evaluation: Evaluation = JSON.parse(\n evaluatorCompletion.choices[0].message.content,\n );\n\n return {\n id: testCase.id,\n sourceText: testCase.sourceText,\n writerOutput,\n readerDiff,\n finalOutput,\n evaluation,\n };\n}\n\n// --- HTML & FRONTEND ---\nfunction generateHtml(sourceUrl: string) {\n return `\n\n\n \n \n Read/Write LLM Evaluation Suite\n \n \n \n \n\n\n\n

Read/Write LLM Evaluation Suite

Running evaluation...

\n \n

Average Score Improvement

Detailed Results

\n \n \n \n

Test Case	Draft Score	Final Score	Details

\n\n\n\n\n`;\n}\n\nexport default async function (req: Request) {\n if (req.method === \"GET\") {\n return new Response(\n generateHtml(new URL(req.url).origin + new URL(req.url).pathname),\n {\n headers: { \"Content-Type\": \"text/html\" },\n },\n );\n }\n\n if (req.method === \"POST\") {\n try {\n const { testCases } = await req.json();\n if (!Array.isArray(testCases) || testCases.length === 0) {\n throw new Error(\"Invalid or empty test cases provided.\");\n }\n const openai = new OpenAI();\n const results = await Promise.all(\n testCases.map((testCase) => processTestCase(testCase, openai)),\n );\n return new Response(JSON.stringify(results), {\n headers: { \"Content-Type\": \"application/json\" },\n });\n } catch (error) {\n console.error(\"Evaluation Suite Error:\", error);\n return new Response(\n JSON.stringify({\n error: error.message || \"An unknown error occurred.\",\n }),\n {\n status: 500,\n headers: { \"Content-Type\": \"application/json\" },\n },\n );\n }\n }\n\n return new Response(\"Method Not Allowed\", { status: 405 });\n}","createdAt",["D",1755501087810],"branchVersion","branchId","092b6a96-7c02-11f0-a9db-0224a6c84d84","id","092b0646-7c02-11f0-a9db-0224a6c84d84","Evals",["D",1755500863403],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","imageUrl","63f04ac0-562b-11f0-a849-2f0fd58d5bdc","pinnedFileId","pinPreview",true,"pinMigrated","screenshotVersion","likeCount",0,"orgId","0abcaa5c-2531-4e34-967b-6f68bd9904a8","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_95":94,"_102":98},"org",{"_95":94,"_96":-5,"_97":98,"_18":54,"_99":100,"_101":25},"branch",{"_13":84,"_32":85,"_43":42,"_86":44,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":45,"_93":94},"isOwned",6,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","real","handle","displayName","avatar","https://img.clerk.com/eyJ0eXBlIjoiZGVmYXVsdCIsImlpZCI6Imluc18yTm10cGhQUklwU0tqZHNiOHNnNGd2ZjJURFUiLCJyaWQiOiJ1c2VyXzJ6Rng5NEh2b3dPMmVwbThwVm1keXA3aTk5bCJ9","tier","free","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_43":44,"_32":45,"_38":149,"_47":-5,"_48":25,"_49":50,"_51":50,"_52":-5,"_53":-5,"_18":54,"_55":31,"_56":57,"_58":57,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":150,"_79":151,"_81":152,"_83":25},"tree",[144],"files",[144],"counts",{"_133":61,"_134":135,"_136":61,"_137":138,"_139":140},"contents",[125],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_43":126,"_32":28,"_30":31,"_127":-5,"_41":42,"_40":14,"_128":25,"_38":129,"_13":130,"_33":34,"_35":37,"_24":25,"_131":-5,"_27":132},"8eee0080-7c02-11f0-a72e-0224a6c84d84","parentId","latest",["D",1755501087810],4,"label",[28],"pulls","branches",1,"forks","envVars",{"_141":61,"_143":61},"connections",{"_141":61,"_142":61},"count","unfilledCount","emptyCount",{"_43":126,"_32":28,"_30":31,"_127":-5,"_41":42,"_40":14,"_128":25,"_38":145,"_13":130,"_33":34,"_35":37,"_24":25,"_131":-5,"_146":147,"_27":148},["D",1755501087810],"children",[],[28],["D",1755500863403],{"_95":94,"_102":98},{"_95":94,"_96":-5,"_97":98,"_18":54,"_99":100,"_101":25},{"_13":84,"_32":85,"_43":42,"_86":44,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":45,"_93":94},"ctx",{"_77":-5,"_122":186,"_187":188,"_189":190,"_191":25,"_192":25,"_193":25,"_194":57,"_195":-5,"_196":-7,"_197":-7,"_198":61,"_199":25,"_200":25,"_201":-5,"_202":25,"_203":25,"_204":25,"_205":-5},"ENV",{"_157":158,"_159":160,"_161":85,"_162":163,"_164":165,"_166":167,"_168":169,"_170":171,"_172":173,"_174":175,"_176":177,"_178":179,"_180":181,"_182":183,"_184":185},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","71c1b20804b3232a7e513be260cfffc1932a9e97","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v5