LLM Evaluation Matrix

\n \n \n \n \n \n {contextWindowSizes.map(size => )}\n \n \n \n {models.map(model => (\n \n \n {contextWindowSizes.map(size => )}\n \n ))}\n \n

Model / Context Window Size	{size}
{model}	Queued

\n\n {\n window.results[model] = {};\n contextWindowSizes.forEach(size => {\n window.results[model][size] = new Array(runs).fill(null);\n });\n });\n\n function updateStatus() {\n document.getElementById('status').textContent = \n \\`Running context window size \\${contextWindowSizes[currentSize]} (Run \\${currentRun + 1}/\\${runs})\\`;\n }\n\n function updateTable(model, size, score, status) {\n const table = document.getElementById('resultTable');\n const row = Array.from(table.rows).find(row => row.cells[0].textContent === model);\n const cell = Array.from(row.cells).find((_, index) => table.rows[0].cells[index].textContent == size);\n if (status === 'running') {\n cell.className = 'running';\n cell.textContent = 'Running...';\n } else if (status === 'done') {\n cell.className = '';\n window.results[model][size][currentRun] = score;\n cell.innerHTML = window.results[model][size].map(s => s !== null ? s.toFixed(2) : '-').join('
');\n }\n }\n\n async function runEvaluation() {\n if (currentRun >= runs) {\n document.getElementById('status').textContent = 'Evaluation complete!';\n document.getElementById('startButton').disabled = false;\n return;\n }\n\n updateStatus();\n\n const size = contextWindowSizes[currentSize];\n const frameContainer = document.getElementById('evaluationFrames');\n frameContainer.innerHTML = '';\n\n const promises = models.map(model => {\n updateTable(model, size, 0, 'running');\n const iframe = document.createElement('iframe');\n iframe.className = 'evalFrame';\n iframe.src = \\`https://janpaul123-evalluator.web.val.run/?model=\\${encodeURIComponent(model)}&contextWindowSize=\\${size}\\`;\n frameContainer.appendChild(iframe);\n\n return new Promise(resolve => {\n window.addEventListener('message', function listener(event) {\n if (event.data.messageType === 'evalsFinished' && event.source === iframe.contentWindow) {\n updateTable(model, size, event.data.totalScore, 'done');\n window.removeEventListener('message', listener);\n resolve();\n }\n });\n });\n });\n\n await Promise.all(promises);\n\n currentSize++;\n if (currentSize >= contextWindowSizes.length) {\n currentSize = 0;\n currentRun++;\n }\n\n runEvaluation();\n }\n\n document.getElementById('startButton').addEventListener('click', function() {\n this.disabled = true;\n currentSize = 0;\n currentRun = 0;\n runEvaluation();\n });\n `,\n }}\n />\n \n \n);\n\nexport default async function main(req: Request): Promise {\n const html = renderToString();\n return new Response(\"\" + html, {\n headers: { \"Content-Type\": \"text/html\" },\n });\n}","deleted","label",[98],"permalink","https://www.val.town/x/janpaul123/metaEVALLuator/code/main.tsx","extension","actionData","errors"]