[{"_1":2,"_183":-5,"_184":-5},"loaderData",{"_3":4,"_95":96},"root",{"_5":6,"_36":37,"_68":69},"ctx",{"_7":8,"_23":-5,"_24":25,"_26":27,"_28":20,"_29":20,"_30":20,"_31":-5,"_32":33,"_34":20,"_35":20},"auth",{"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20,"_21":22},"sessionClaims","sessionId","sessionStatus","userId","actor","orgId","orgRole","orgSlug","orgPermissions","factorVerificationAge","isAuthenticated",false,"tokenType","session_token","user","preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesChatPanelState","notificationCount",0,"flagDenols","isPaidOrgMember","ENV",{"_38":39,"_40":41,"_42":43,"_44":45,"_46":47,"_48":49,"_50":51,"_52":53,"_54":55,"_56":57,"_58":59,"_60":61,"_62":63,"_64":65,"_66":67},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","VAL_RUNTIME_NAME","prod","RENDER_GIT_COMMIT","d1375c05ce15aa00f0f5fc1d56cfa8d009f00f8a","RENDER_GIT_BRANCH","main","CLERK_PUBLISHABLE_KEY","pk_live_Y2xlcmsudmFsLnRvd24k","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","API_URL","https://api.val.town","clerkState",{"_70":71},"__internal_clerk_state",{"_72":73,"_74":47,"_75":76,"_77":76,"_78":20,"_79":76,"_80":76,"_81":76,"_82":76,"_83":84,"_93":20,"_94":20},"__clerk_ssr_state",{"_21":22,"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20},"__publishableKey","__proxyUrl","","__domain","__isSatellite","__signInUrl","__signUpUrl","__afterSignInUrl","__afterSignUpUrl","__clerk_debug",{"_85":20,"_19":20,"_86":76,"_87":88,"_89":76,"_90":47,"_91":20,"_92":76},"isSignedIn","proxyUrl","reason","session-token-and-uat-missing","message","publishableKey","isSatellite","domain","__telemetryDisabled","__telemetryDebug","routes/embed.x.$handle.$name.$",{"_97":98,"_99":100,"_101":102,"_103":104,"_157":158,"_180":181,"_182":-5},"handle","c15r","name","microgpt","path","model/forward.ts","project",{"_105":106,"_99":100,"_107":108,"_109":-5,"_110":20,"_111":112,"_113":114,"_115":-5,"_12":116,"_117":-5,"_118":20,"_119":-5,"_120":-5,"_121":33,"_14":122,"_123":-5,"_124":-5,"_125":-5,"_126":127,"_128":127,"_129":130,"_131":130,"_132":130,"_133":127,"_134":127,"_23":135,"_138":139,"_143":144,"_156":20},"id","a0fa6194-27db-11f1-89a2-42dde27851f2","createdAt",["D",1774395967725],"deletedAt","legacyEnvVars","privacy","public","description","Online learning micro-GPT: byte-level decoder transformer","imageUrl","21f2e37e-8066-4730-9dc7-d27cf8d3c9f3","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount","0630d977-f678-453e-926c-52043293f59f","tursoName","tursoHostname","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm",{"_97":98,"_136":137},"profileImageUrl","https://img.clerk.com/eyJ0eXBlIjoicHJveHkiLCJzcmMiOiJodHRwczovL2ltYWdlcy5jbGVyay5kZXYvb2F1dGhfZ2l0aHViL2ltZ18yb2RJNlc1czZ1NlpZRmoxdGJ2eFVNNVhmeTYifQ","org",{"_97":98,"_140":137,"_12":116,"_141":142},"avatar","tier","pro","branch",{"_145":146,"_99":45,"_105":147,"_148":106,"_149":-5,"_150":-5,"_151":-5,"_152":-5,"_153":-5,"_154":100,"_155":98},"version",37,"a0faf3a2-27db-11f1-89a2-42dde27851f2","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","isOwned","file",{"_105":159,"_99":160,"_161":162,"_163":164,"_165":147,"_166":167,"_168":169,"_107":170,"_145":171,"_172":173,"_174":175,"_176":20,"_177":-5,"_101":178},"481ae3fc-27e3-11f1-9891-42dde27851f2","forward.ts","fileId","f2c3666a-27db-11f1-9ee9-42dde27851f2","parentId","a3727b82-27db-11f1-918a-42dde27851f2","branchId","branchVersion",34,"latest",true,["D",1774399254573],2,"fileType","script","content","/**\n * model/forward.ts — Full causal self-attention forward pass\n *\n * Reads all dimensions from p.cfg (the config snapshot stored in the model).\n * This means the same code works for any model size.\n */\nimport { rmsnorm, type ModelParams, type ForwardCache } from \"./core.ts\";\n\nexport function forward(p: ModelParams, tokens: number[]): ForwardCache {\n const { D, N_HEAD, HD, MLP_HID, N_LAYER, VS } = p.cfg;\n const T = tokens.length;\n const cache: ForwardCache = {\n tokens: [...tokens], T,\n x_layer_in: [], layer_caches: [],\n x_final: [], fn: { y: [], rms: 0, x_in: [] },\n z: [], logits: [], probs: [], last_attn: [],\n };\n\n // Embeddings: tok + pos\n const x0: number[][] = [];\n for (let t = 0; t < T; t++) {\n const e = new Array(D);\n for (let j = 0; j < D; j++) e[j] = p.te[tokens[t]][j] + p.pe[t][j];\n x0.push(e);\n }\n let x = x0.map(e => [...e]);\n cache.x_layer_in = [x0];\n\n for (let l = 0; l < N_LAYER; l++) {\n const ly = p.layers[l];\n const lc: any = {};\n\n // Attention pre-norm\n lc.norm_a = [];\n const u: number[][] = [];\n for (let t = 0; t < T; t++) {\n const nr = rmsnorm(x[t], ly.gn_a);\n lc.norm_a.push({ rms: nr.rms, x_in: [...x[t]] });\n u.push(nr.y);\n }\n\n // QKV projections\n const Q: number[][] = [], K: number[][] = [], V: number[][] = [];\n for (let t = 0; t < T; t++) {\n const q = new Array(D).fill(0), k = new Array(D).fill(0), v = new Array(D).fill(0);\n for (let j = 0; j < D; j++) for (let m = 0; m < D; m++) {\n q[j] += u[t][m] * ly.Wq[m][j];\n k[j] += u[t][m] * ly.Wk[m][j];\n v[j] += u[t][m] * ly.Wv[m][j];\n }\n Q.push(q); K.push(k); V.push(v);\n }\n lc.u = u; lc.Q = Q; lc.K = K; lc.V = V;\n\n // Multi-head causal self-attention\n const aw: number[][][] = [], ao: number[][] = [];\n for (let t = 0; t < T; t++) {\n const aw_t: number[][] = [];\n const o_t = new Array(D).fill(0);\n for (let h = 0; h < N_HEAD; h++) {\n const ho = h * HD;\n const scores = new Array(t + 1);\n let maxS = -Infinity;\n for (let s = 0; s <= t; s++) {\n let sc = 0;\n for (let d = 0; d < HD; d++) sc += Q[t][ho + d] * K[s][ho + d];\n sc /= Math.sqrt(HD);\n scores[s] = sc;\n if (sc > maxS) maxS = sc;\n }\n let sumE = 0;\n const attn = new Array(t + 1);\n for (let s = 0; s <= t; s++) { attn[s] = Math.exp(scores[s] - maxS); sumE += attn[s]; }\n for (let s = 0; s <= t; s++) attn[s] /= sumE;\n aw_t.push(attn);\n for (let d = 0; d < HD; d++) {\n let acc = 0;\n for (let s = 0; s <= t; s++) acc += attn[s] * V[s][ho + d];\n o_t[ho + d] = acc;\n }\n }\n aw.push(aw_t); ao.push(o_t);\n }\n lc.attn_weights = aw; lc.attn_out = ao;\n\n // Output projection + residual\n const x_mid: number[][] = [];\n for (let t = 0; t < T; t++) {\n const proj = new Array(D).fill(0);\n for (let j = 0; j < D; j++) for (let m = 0; m < D; m++) proj[j] += ao[t][m] * ly.Wo[m][j];\n const xm = new Array(D);\n for (let j = 0; j < D; j++) xm[j] = x[t][j] + proj[j];\n x_mid.push(xm);\n }\n\n // MLP pre-norm\n lc.norm_m = [];\n const m: number[][] = [];\n for (let t = 0; t < T; t++) {\n const nr = rmsnorm(x_mid[t], ly.gn_m);\n lc.norm_m.push({ rms: nr.rms, x_in: [...x_mid[t]] });\n m.push(nr.y);\n }\n lc.m = m;\n\n // MLP: D → MLP_HID (ReLU) → D + residual\n lc.h_pre = []; lc.h = [];\n const x_out: number[][] = [];\n for (let t = 0; t < T; t++) {\n const hp = new Array(MLP_HID).fill(0);\n for (let j = 0; j < MLP_HID; j++) {\n for (let k = 0; k < D; k++) hp[j] += m[t][k] * ly.W1[k][j];\n hp[j] += ly.b1[j];\n }\n const h = hp.map((x: number) => x > 0 ? x : 0);\n const r = new Array(D).fill(0);\n for (let j = 0; j < D; j++) {\n for (let k = 0; k < MLP_HID; k++) r[j] += h[k] * ly.W2[k][j];\n r[j] += ly.b2[j];\n }\n lc.h_pre.push(hp); lc.h.push(h);\n const xo = new Array(D);\n for (let j = 0; j < D; j++) xo[j] = x_mid[t][j] + r[j];\n x_out.push(xo);\n }\n\n lc.x_mid = x_mid;\n cache.layer_caches.push(lc);\n x = x_out;\n if (l < N_LAYER - 1) cache.x_layer_in.push(x.map(e => [...e]));\n }\n\n cache.x_final = x;\n\n // Final norm → lm_head → softmax\n const fn = rmsnorm(x[T - 1], p.gn_f);\n cache.fn = { y: fn.y, rms: fn.rms, x_in: [...x[T - 1]] };\n\n const logits = new Array(VS).fill(0);\n for (let i = 0; i < VS; i++) {\n for (let j = 0; j < D; j++) logits[i] += fn.y[j] * p.lm_head[j][i];\n logits[i] += p.b_out[i];\n }\n const maxL = Math.max(...logits);\n const expL = logits.map((l: number) => Math.exp(l - maxL));\n const sumL = expL.reduce((a: number, b: number) => a + b, 0);\n cache.probs = expL.map((e: number) => e / sumL);\n cache.z = fn.y;\n cache.logits = logits;\n cache.last_attn = cache.layer_caches[N_LAYER - 1].attn_weights[T - 1];\n\n return cache;\n}\n","deleted","label",[179,160],"model","permalink","https://www.val.town/x/c15r/microgpt/code/model/forward.ts","extension","actionData","errors"]