[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_144":145,"_146":147},"routes/_app.x.$handle.$name",{"_16":17,"_20":21,"_83":65,"_84":31,"_63":-7,"_85":-5,"_13":86,"_87":88,"_11":89,"_90":91,"_92":93,"_94":-5,"_95":-5,"_96":-5,"_97":26,"_98":99,"_100":101,"_102":-5,"_103":-5,"_104":105},"routes/_app.x.$handle.$name.search.$(branch)",{"_11":12,"_13":14,"_15":-5,"_16":17},"files",[],"project",{"_18":19,"_20":21,"_22":23,"_24":-5,"_25":26,"_27":28,"_29":28,"_30":31,"_32":-5,"_33":34,"_35":-5,"_36":26,"_37":-5,"_38":-5,"_39":40,"_41":42,"_43":-5,"_44":-5,"_45":-5,"_46":-5,"_47":48,"_49":48,"_50":51,"_52":51,"_53":51,"_54":48,"_55":48,"_56":57,"_58":59,"_60":61,"_62":26},"query","handle","patrickzandl","id","76834068-501f-11f1-afc4-ee650bb23af1","name","effortBenchmark","createdAt",["D",1778823148771],"deletedAt","legacyEnvVars",false,"privacy","public","httpPrivacy","description","","imageUrl","userId","7241cb49-222a-4e9a-af99-25b2615eda07","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","019e0b3d-7f60-7428-8330-1beb489e3416","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_16":17,"_82":78},"org",{"_16":17,"_75":76,"_77":78,"_33":34,"_79":80,"_81":26},"branch",{"_63":64,"_20":65,"_18":66,"_67":19,"_68":-5,"_69":-5,"_70":-5,"_71":-5,"_72":-5,"_73":21,"_74":17},"isOwned","version",23,"main","7683ffee-501f-11f1-afc4-ee650bb23af1","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","displayName","Patrick Zandl","avatar","https://lh3.googleusercontent.com/a/ACg8ocK47-vrp0IIPUC8DK3BcjE57lcyPd28xqviX34gk6fkTziKkbep=s96-c","tier","free","canRestrictVals","profileImageUrl","branchName","path","adoptOrgId",{"_18":19,"_20":21,"_22":140,"_24":-5,"_25":26,"_27":28,"_29":28,"_30":31,"_32":-5,"_33":34,"_35":-5,"_36":26,"_37":-5,"_38":-5,"_39":40,"_41":42,"_43":-5,"_44":-5,"_45":-5,"_46":-5,"_47":48,"_49":48,"_50":51,"_52":51,"_53":51,"_54":48,"_55":48,"_56":141,"_58":142,"_60":143,"_62":26},"tree",[135],[135],"counts",{"_124":40,"_125":126,"_127":40,"_128":129,"_130":131},"contents",[106],"file","upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles",true,"grants","orgs","customDomains",{},{"_18":107,"_20":108,"_109":110,"_111":-5,"_112":66,"_113":64,"_114":101,"_22":115,"_63":116,"_117":118,"_119":120,"_121":26,"_122":-5,"_84":123},"f585fdba-52e0-11f1-a0ff-ee650bb23af1","main.http.ts","fileId","7768c20a-501f-11f1-afc4-ee650bb23af1","parentId","branchId","branchVersion","latest",["D",1779126157018],14,"fileType","http","content","// Claude Code Effort Benchmark — Phase 1+SWE\n// Auto-generated by scripts/generate_report.py\nexport default async function(req: Request): Promise {\n const html = `\n\n\n\n\nClaude Code Effort Benchmark\n\n\n\n\n

Agent Effort Benchmark Phase 1+SWE

Empirické měření dopadu reasoning effort napříč providery a modely

336

celkem runů

úlohy

202/336

verify pass

\\$128.3

celkem náklady

nekompletní runy

Interpretace výsledků

Kimi runy s exit_code=75 jsou z tohoto reportu vyřazené jako infrastrukturní výpadek, ne jako selhání modelu. Vyřazeno: 37 runů.

Opus sweet spot

Opus medium je pracovní default: proti low zvedá úspěšnost zhruba z 53 % na 75 %. Max má smysl pro SWE, security, auth, latentní chyby a větší refaktoringy, kde cena ustoupí riziku.

Sonnet sweet spot

Sonnet low nebo medium je vhodný pro rutinní implementace, scope compliance a async chyby. Vyšší effort v této sadě nepřinesl stabilní zlepšení, proto ho nedávám jako default.

Kde použít Kimi

Kimi se hodí na jasně testovatelné úlohy: implementace, scope, async, flaky, API contract a performance. U API contract a performance se ukazuje práh na medium.

Kde Kimi nepoužívat

Kimi zatím není vhodná náhrada pro auth/permission, security audit, frontend state a latentní debugging. Legacy refaktoring nelze vyhodnotit, protože validní Kimi data chybí.

Přehled výsledků

Úloha	low	medium	high	max	Effort gradient

Pass rate per effort

Průměrná cena per effort

Tokeny per effort

Iterace (turns) per effort

Effort	n	Pass%	Avg score	Avg cost	Avg wall	Avg tools	Avg iter	Tokens	Tool breakdown

Efektivita: cena vs. kvalita

Každý bod = jeden run. Osy: cena (USD) vs. skóre (0–1). Ideální bod = levý horní roh (levné a kvalitní). Pareto frontier = zelená čára.

Tokeny a iterace napříč úlohami

Token consumption per effort (stacked)

Průměrné iterace per effort

Čas a efektivita

Průměrný wall time per effort

Cena per úspěšný run

Success rate podle typu úlohy

SWE-bench výsledky

Reálné bugy z open-source projektů.

Instance	low	medium	high	max	Finding

Syntetické úlohy

Na dobře specifikovaných úlohách jsou všechny effort úrovně funkčně ekvivalentní.

Pass rate — 06_strict_scope

Průměrná cena — 01_rename

Klíčová zjištění

Opus profituje z medium

Opus low je vhodný pro jednoduché změny. Medium výrazně zvedá úspěšnost při stále rozumné ceně; max patří na rizikové a dlouhé úlohy.

Sonnet držet na low/medium

Sonnet vyšší effort v této sadě nedává stabilní zlepšení. Pro běžné úlohy je vhodný low nebo medium, pro rizikové chyby přepnout na Opus.

⚠️ Debug bez testu = nulový efekt

03_debug_order Bug B nebyl nalezen ani max effortem. Failing testy jsou nezbytný guidance.

Kimi je kategoriální volba

Kimi se hodí pro levné, dobře měřitelné tasky. U auth, security, frontend state a latentních chyb zatím není vhodná náhrada.

Finální doporučovací tabulka

Claude Code sloupec počítá s uživateli, kteří mohou přepínat jen Opus/Sonnet a effort. Kimi sloupec říká, kde dává smysl levná náhrada.

Kategorie	Naměřený signál	Claude Code volba	Kimi volba	Verdikt
Mechanický rename/refactor	Opus 100 %, Sonnet low/medium 100 %, Kimi high 2/2	Sonnet low nebo Opus low	Kimi high jen s kontrolou diffu	podmíněně
Jednoduchá implementace	Opus/Sonnet/Kimi 100 %	Sonnet low	Kimi low	lze nahradit
Scope compliance	Sonnet low/medium/max 100 %, Kimi 100 %, Opus max 100 %	Sonnet low; Opus max pokud musí být Opus	Kimi low	lze nahradit
Async/Promise chyby	Sonnet 100 %, Kimi 100 %	Sonnet low	Kimi low	lze nahradit
API contract regrese	Kimi low 0/2, medium+ 6/6	Sonnet/Opus medium; pro kritické contracty Opus medium	Kimi medium	podmíněně
Performance regrese	Kimi low 1/2, medium+ 6/6	Sonnet/Opus medium	Kimi medium	podmíněně
Flaky/race condition	Kimi 8/8 i na low	Sonnet low/medium	Kimi low	lze nahradit
Frontend state bug	Kimi 0/8, průměrné score 0.75	Opus high/max	Nepoužívat jako náhradu	nenahrazovat
Auth/permission bug	Kimi 0/8, hidden permission část selhává	Opus high/max	Nepoužívat jako náhradu	nenahrazovat
Security audit	Sonnet 0 %, Kimi 0 %, Opus low 1/1	Opus medium/high	Nepoužívat jako náhradu	nenahrazovat
Build/config	Kimi low 1/2, medium 1/1, vyšší effort nevalidní	Sonnet/Opus medium	Zatím neprokázáno	málo dat
Legacy PHP/Java refactor	Kimi data nevalidní kvůli connection errorům	Opus high/max; Sonnet high jen pro menší refaktor	Nelze hodnotit	málo dat
SWE Django	Opus medium často stačí, max pomáhá u těžších případů	Opus medium; při vyšším riziku Opus max	Zatím nenahrazovat	nenahrazovat
SWE Sympy	Opus max řeší část případů, Sonnet/Kimi slabé	Opus max	Nepoužívat jako náhradu	nenahrazovat
Latent debugging	03_debug_order nevyřešil žádný model/effort	Nepočítat jen s effortem; doplnit testy a diagnostiku	Nepoužívat jako náhradu	nenahrazovat

Doporučení

Use case	Effort	Proč

\n\n\n`;\n return new Response(html, { headers: {'content-type': 'text/html;charset=utf-8'} });\n}\n","deleted","label",[108],"pulls","branches",1,"forks","envVars",{"_132":40,"_134":40},"connections",{"_132":40,"_133":40},"count","unfilledCount","emptyCount",{"_18":107,"_20":108,"_109":110,"_111":-5,"_112":66,"_113":64,"_114":101,"_22":136,"_63":116,"_117":118,"_119":120,"_121":26,"_122":-5,"_137":138,"_84":139},["D",1779126157018],"children",[],[108],["D",1778823148771],{"_16":17,"_82":78},{"_16":17,"_75":76,"_77":78,"_33":34,"_79":80,"_81":26},{"_63":64,"_20":65,"_18":66,"_67":19,"_68":-5,"_69":-5,"_70":-5,"_71":-5,"_72":-5,"_73":21,"_74":17},"ctx",{"_56":-5,"_103":177,"_178":179,"_180":181,"_182":26,"_183":26,"_184":26,"_185":101,"_186":-5,"_187":-7,"_188":-7,"_189":40,"_190":26,"_191":26,"_192":-5,"_193":26,"_194":26,"_195":26,"_196":-5},"ENV",{"_148":149,"_150":151,"_152":65,"_153":154,"_155":156,"_157":158,"_159":160,"_161":162,"_163":164,"_165":166,"_167":168,"_169":170,"_171":172,"_173":174,"_175":176},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Search