[{"_1":2,"_183":-5,"_184":-5},"loaderData",{"_3":4,"_95":96},"root",{"_5":6,"_36":37,"_68":69},"ctx",{"_7":8,"_23":-5,"_24":25,"_26":27,"_28":20,"_29":20,"_30":20,"_31":-5,"_32":33,"_34":20,"_35":20},"auth",{"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20,"_21":22},"sessionClaims","sessionId","sessionStatus","userId","actor","orgId","orgRole","orgSlug","orgPermissions","factorVerificationAge","isAuthenticated",false,"tokenType","session_token","user","preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesChatPanelState","notificationCount",0,"flagDenols","isPaidOrgMember","ENV",{"_38":39,"_40":41,"_42":43,"_44":45,"_46":47,"_48":49,"_50":51,"_52":53,"_54":55,"_56":57,"_58":59,"_60":61,"_62":63,"_64":65,"_66":67},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","VAL_RUNTIME_NAME","prod","RENDER_GIT_COMMIT","d1375c05ce15aa00f0f5fc1d56cfa8d009f00f8a","RENDER_GIT_BRANCH","main","CLERK_PUBLISHABLE_KEY","pk_live_Y2xlcmsudmFsLnRvd24k","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","API_URL","https://api.val.town","clerkState",{"_70":71},"__internal_clerk_state",{"_72":73,"_74":47,"_75":76,"_77":76,"_78":20,"_79":76,"_80":76,"_81":76,"_82":76,"_83":84,"_93":20,"_94":20},"__clerk_ssr_state",{"_21":22,"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20},"__publishableKey","__proxyUrl","","__domain","__isSatellite","__signInUrl","__signUpUrl","__afterSignInUrl","__afterSignUpUrl","__clerk_debug",{"_85":20,"_19":20,"_86":76,"_87":88,"_89":76,"_90":47,"_91":20,"_92":76},"isSignedIn","proxyUrl","reason","session-token-and-uat-missing","message","publishableKey","isSatellite","domain","__telemetryDisabled","__telemetryDebug","routes/embed.x.$handle.$name.$",{"_97":98,"_99":100,"_101":102,"_103":104,"_157":158,"_180":181,"_182":-5},"handle","c15r","name","microgpt","path","model/backward.ts","project",{"_105":106,"_99":100,"_107":108,"_109":-5,"_110":20,"_111":112,"_113":114,"_115":-5,"_12":116,"_117":-5,"_118":20,"_119":-5,"_120":-5,"_121":33,"_14":122,"_123":-5,"_124":-5,"_125":-5,"_126":127,"_128":127,"_129":130,"_131":130,"_132":130,"_133":127,"_134":127,"_23":135,"_138":139,"_143":144,"_156":20},"id","a0fa6194-27db-11f1-89a2-42dde27851f2","createdAt",["D",1774395967725],"deletedAt","legacyEnvVars","privacy","public","description","Online learning micro-GPT: byte-level decoder transformer","imageUrl","21f2e37e-8066-4730-9dc7-d27cf8d3c9f3","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount","0630d977-f678-453e-926c-52043293f59f","tursoName","tursoHostname","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm",{"_97":98,"_136":137},"profileImageUrl","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18yb2RJNlc1czZ1NlpZRmoxdGJ2eFVNNVhmeTYifQ","org",{"_97":98,"_140":137,"_12":116,"_141":142},"avatar","tier","pro","branch",{"_145":146,"_99":45,"_105":147,"_148":106,"_149":-5,"_150":-5,"_151":-5,"_152":-5,"_153":-5,"_154":100,"_155":98},"version",37,"a0faf3a2-27db-11f1-89a2-42dde27851f2","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","isOwned","file",{"_105":159,"_99":160,"_161":162,"_163":164,"_165":147,"_166":167,"_168":169,"_107":170,"_145":171,"_172":173,"_174":175,"_176":20,"_177":-5,"_101":178},"589f2030-27e3-11f1-8571-42dde27851f2","backward.ts","fileId","043dba6c-27dc-11f1-9cf4-42dde27851f2","parentId","a3727b82-27db-11f1-918a-42dde27851f2","branchId","branchVersion",35,"latest",true,["D",1774399282298],2,"fileType","script","content","/**\n * model/backward.ts — Full backward pass through causal decoder block\n *\n * Reads dimensions from p.cfg. Same code works for any model size.\n */\nimport { rmsnormBwd, type ModelParams, type ModelGrads, type ForwardCache } from \"./core.ts\";\n\nexport function backward(p: ModelParams, g: ModelGrads, cache: ForwardCache, target: number): number {\n const { D, N_HEAD, HD, MLP_HID, VS, N_LAYER } = p.cfg;\n const { T, probs, z } = cache;\n const loss = -Math.log(Math.max(probs[target], 1e-10));\n\n const dl = probs.map((pr, i) => pr - (i === target ? 1 : 0));\n\n // d_z, d_lm_head, d_b_out\n const dz = new Array(D).fill(0);\n for (let j = 0; j < D; j++) {\n for (let i = 0; i < VS; i++) {\n g.lm_head[j][i] += z[j] * dl[i];\n dz[j] += dl[i] * p.lm_head[j][i];\n }\n }\n for (let i = 0; i < VS; i++) g.b_out[i] += dl[i];\n\n const { dx: dxf, dgain: dgf } = rmsnormBwd(dz, cache.fn.x_in, p.gn_f, cache.fn.rms);\n for (let j = 0; j < D; j++) g.gn_f[j] += dgf[j];\n\n const dx = Array.from({ length: T }, () => new Array(D).fill(0));\n for (let j = 0; j < D; j++) dx[T - 1][j] = dxf[j];\n\n for (let l = N_LAYER - 1; l >= 0; l--) {\n const ly = p.layers[l], gl = g.layers[l], lc = cache.layer_caches[l];\n\n // MLP residual backward\n const dxm = Array.from({ length: T }, (_, t) => [...dx[t]]);\n for (let t = 0; t < T; t++) {\n const dr = [...dx[t]];\n const dh = new Array(MLP_HID).fill(0);\n for (let k = 0; k < MLP_HID; k++) for (let j = 0; j < D; j++) {\n gl.W2[k][j] += lc.h[t][k] * dr[j]; dh[k] += dr[j] * ly.W2[k][j];\n }\n for (let j = 0; j < D; j++) gl.b2[j] += dr[j];\n const dhp = dh.map((d: number, k: number) => lc.h_pre[t][k] > 0 ? d : 0);\n for (let j = 0; j < MLP_HID; j++) gl.b1[j] += dhp[j];\n const dm = new Array(D).fill(0);\n for (let k = 0; k < D; k++) for (let j = 0; j < MLP_HID; j++) {\n gl.W1[k][j] += lc.m[t][k] * dhp[j]; dm[k] += dhp[j] * ly.W1[k][j];\n }\n const { dx: dn, dgain: dg } = rmsnormBwd(dm, lc.norm_m[t].x_in, ly.gn_m, lc.norm_m[t].rms);\n for (let j = 0; j < D; j++) { gl.gn_m[j] += dg[j]; dxm[t][j] += dn[j]; }\n }\n\n // Attention residual backward\n const dxi = Array.from({ length: T }, (_, t) => [...dxm[t]]);\n const dao = Array.from({ length: T }, () => new Array(D).fill(0));\n for (let t = 0; t < T; t++) for (let m = 0; m < D; m++) for (let j = 0; j < D; j++) {\n gl.Wo[m][j] += lc.attn_out[t][m] * dxm[t][j];\n dao[t][m] += dxm[t][j] * ly.Wo[m][j];\n }\n\n // Multi-head attention backward\n const dQ = Array.from({ length: T }, () => new Array(D).fill(0));\n const dK = Array.from({ length: T }, () => new Array(D).fill(0));\n const dV = Array.from({ length: T }, () => new Array(D).fill(0));\n for (let t = 0; t < T; t++) for (let h = 0; h < N_HEAD; h++) {\n const ho = h * HD, attn = lc.attn_weights[t][h];\n const da = new Array(t + 1).fill(0);\n for (let s = 0; s <= t; s++) for (let d = 0; d < HD; d++) {\n da[s] += dao[t][ho + d] * lc.V[s][ho + d];\n dV[s][ho + d] += attn[s] * dao[t][ho + d];\n }\n let dot = 0;\n for (let s = 0; s <= t; s++) dot += attn[s] * da[s];\n const scale = 1 / Math.sqrt(HD);\n for (let s = 0; s <= t; s++) {\n const ds = attn[s] * (da[s] - dot);\n for (let d = 0; d < HD; d++) {\n dQ[t][ho + d] += ds * lc.K[s][ho + d] * scale;\n dK[s][ho + d] += ds * lc.Q[t][ho + d] * scale;\n }\n }\n }\n\n // QKV projection backward\n const du = Array.from({ length: T }, () => new Array(D).fill(0));\n for (let t = 0; t < T; t++) for (let m = 0; m < D; m++) for (let j = 0; j < D; j++) {\n gl.Wq[m][j] += lc.u[t][m] * dQ[t][j]; du[t][m] += dQ[t][j] * ly.Wq[m][j];\n gl.Wk[m][j] += lc.u[t][m] * dK[t][j]; du[t][m] += dK[t][j] * ly.Wk[m][j];\n gl.Wv[m][j] += lc.u[t][m] * dV[t][j]; du[t][m] += dV[t][j] * ly.Wv[m][j];\n }\n\n // Attention pre-norm backward\n for (let t = 0; t < T; t++) {\n const { dx: dn, dgain: dg } = rmsnormBwd(du[t], lc.norm_a[t].x_in, ly.gn_a, lc.norm_a[t].rms);\n for (let j = 0; j < D; j++) { gl.gn_a[j] += dg[j]; dxi[t][j] += dn[j]; }\n }\n for (let t = 0; t < T; t++) for (let j = 0; j < D; j++) dx[t][j] = dxi[t][j];\n }\n\n // Embedding gradients\n const toks = cache.tokens;\n for (let t = 0; t < T; t++) for (let j = 0; j < D; j++) {\n g.te[toks[t]][j] += dx[t][j]; g.pe[t][j] += dx[t][j];\n }\n return loss;\n}\n","deleted","label",[179,160],"model","permalink","https://www.val.town/x/c15r/microgpt/code/model/backward.ts","extension","actionData","errors"]