[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_157":158,"_159":160},"routes/_app.x.$handle.$name",{"_95":94,"_32":44,"_104":85,"_27":53,"_13":14,"_105":-5,"_11":106,"_107":108,"_109":110,"_111":112,"_113":114,"_20":-5,"_115":-5,"_116":-5,"_117":48,"_118":119,"_120":25,"_121":-5,"_122":-5,"_123":124},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_42":43,"_32":44,"_37":45,"_46":-5,"_47":48,"_49":50,"_51":50,"_52":53,"_54":-5,"_18":55,"_56":-5,"_57":48,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":78,"_79":80,"_81":82,"_83":48},"version",22,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_33":34,"_35":36,"_37":38,"_39":14,"_40":41,"_24":25},"previous",{"_30":31,"_32":28,"_33":34,"_35":36},"deleted",true,"deletedHere","path","report.ts","latestPath","fileId","045e97dc-52b1-11f1-b210-ee650bb23af1","name","fileType","script","content","// Claude Code Effort Benchmark — Phase 1+SWE\n// Auto-generated by scripts/generate_report.py\nexport default async function(req: Request): Promise {\n const html = `\n\n\n\n\nClaude Code Effort Benchmark\n\n\n\n\n

Agent Effort Benchmark Phase 1+SWE

Empirické měření dopadu reasoning effort napříč providery a modely

372

celkem runů

úlohy

202/372

verify pass

\\$128.3

celkem náklady

nekompletní runy

Přehled výsledků

Úloha	low	medium	high	max	Effort gradient

Pass rate per effort

Průměrná cena per effort

Tokeny per effort

Iterace (turns) per effort

Effort	n	Pass%	Avg score	Avg cost	Avg wall	Avg tools	Avg iter	Tokens	Tool breakdown

Efektivita: cena vs. kvalita

Každý bod = jeden run. Osy: cena (USD) vs. skóre (0–1). Ideální bod = levý horní roh (levné a kvalitní). Pareto frontier = zelená čára.

Tokeny a iterace napříč úlohami

Token consumption per effort (stacked)

Průměrné iterace per effort

Čas a efektivita

Průměrný wall time per effort

Cena per úspěšný run

Success rate podle typu úlohy

SWE-bench výsledky

Reálné bugy z open-source projektů.

Instance	low	medium	high	max	Finding

Syntetické úlohy

Na dobře specifikovaných úlohách jsou všechny effort úrovně funkčně ekvivalentní.

Pass rate — 06_strict_scope

Průměrná cena — 01_rename

Klíčová zjištění

💡 SWE-bench diferenciuje effort

Na reálných bugech (django-16910) low/medium konzistentně selávají, high uspívá. Na jednoduchých bugech i low stačí.

💡 Max ≠ lepší výsledek

Max effort stojí 2–4× více než high, ale výsledek je stejný. High je sweet spot pro reálné bugy.

⚠️ Debug bez testu = nulový efekt

03_debug_order Bug B nebyl nalezen ani max effortem. Failing testy jsou nezbytný guidance.

⚠️ Python 3.14 limituje SWE-bench

Pre-2022 projekty jsou nekompatibilní (odstraňeny moduly cgi, distutils). Používejte Django 4.2+ / Sympy 1.11+.

Doporučení

Use case	Effort	Proč

\n\n\n`;\n return new Response(html, { headers: {'content-type': 'text/html;charset=utf-8'} });\n}\n","createdAt",["D",1779113832946],"branchVersion","branchId","7683ffee-501f-11f1-afc4-ee650bb23af1","id","76834068-501f-11f1-afc4-ee650bb23af1","effortBenchmark",["D",1778823148771],"deletedAt","legacyEnvVars",false,"privacy","public","httpPrivacy","description","","imageUrl","7241cb49-222a-4e9a-af99-25b2615eda07","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","019e0b3d-7f60-7428-8330-1beb489e3416","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_95":94,"_103":99},"org",{"_95":94,"_96":97,"_98":99,"_18":55,"_100":101,"_102":48},"branch",{"_13":84,"_32":85,"_42":41,"_86":43,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":44,"_93":94},"isOwned",23,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","patrickzandl","handle","displayName","Patrick Zandl","avatar","https://lh3.googleusercontent.com/a/ACg8ocK47-vrp0IIPUC8DK3BcjE57lcyPd28xqviX34gk6fkTziKkbep=s96-c","tier","free","canRestrictVals","profileImageUrl","branchName","adoptOrgId",{"_42":43,"_32":44,"_37":153,"_46":-5,"_47":48,"_49":50,"_51":50,"_52":53,"_54":-5,"_18":55,"_56":-5,"_57":48,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":154,"_79":155,"_81":156,"_83":48},"tree",[148],"files",[148],"counts",{"_137":61,"_138":139,"_140":61,"_141":142,"_143":144},"contents",[125],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_42":126,"_32":127,"_30":128,"_129":-5,"_40":41,"_39":130,"_131":48,"_37":132,"_13":133,"_33":134,"_35":36,"_24":48,"_135":-5,"_27":136},"1d4f7c94-52c4-11f1-9943-ee650bb23af1","main.http.ts","7768c20a-501f-11f1-afc4-ee650bb23af1","parentId",21,"latest",["D",1779113768338],13,"http","label",[127],"pulls","branches",1,"forks","envVars",{"_145":61,"_147":61},"connections",{"_145":61,"_146":61},"count","unfilledCount","emptyCount",{"_42":126,"_32":127,"_30":128,"_129":-5,"_40":41,"_39":130,"_131":48,"_37":149,"_13":133,"_33":134,"_35":36,"_24":48,"_135":-5,"_150":151,"_27":152},["D",1779113768338],"children",[],[127],["D",1778823148771],{"_95":94,"_103":99},{"_95":94,"_96":97,"_98":99,"_18":55,"_100":101,"_102":48},{"_13":84,"_32":85,"_42":41,"_86":43,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":44,"_93":94},"ctx",{"_77":-5,"_122":190,"_191":192,"_193":194,"_195":48,"_196":48,"_197":48,"_198":25,"_199":-5,"_200":-7,"_201":-7,"_202":61,"_203":48,"_204":48,"_205":-5,"_206":48,"_207":48,"_208":48,"_209":-5},"ENV",{"_161":162,"_163":164,"_165":85,"_166":167,"_168":169,"_170":171,"_172":173,"_174":175,"_176":177,"_178":179,"_180":181,"_182":183,"_184":185,"_186":187,"_188":189},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","5015b8df46402d8bba4f485ecc37aec202b0e06b","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v22