[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_255":256,"_257":258},"routes/_app.x.$handle.$name",{"_11":12,"_13":14,"_15":16,"_69":154,"_41":-7,"_188":-5,"_17":189,"_190":191,"_192":193,"_194":195,"_196":197,"_109":-5,"_198":-5,"_199":200,"_201":49,"_202":203,"_204":49,"_205":-5,"_206":-5,"_207":208},"routes/_app.x.$handle.$name.compare.$branch",{"_11":12,"_13":14,"_15":16,"_17":18,"_19":20,"_21":22,"_23":24,"_25":26,"_27":28,"_29":30,"_31":-5},"handle","stevekrouse","name","microgpt","branchName","karpathy-scalar-autograd","project",{"_32":40,"_13":14,"_34":150,"_47":-5,"_151":49,"_50":51,"_152":51,"_153":154,"_155":154,"_156":157,"_158":-5,"_159":49,"_160":-5,"_161":-5,"_162":22,"_53":54,"_163":-5,"_164":-5,"_165":-5,"_166":-5,"_167":168,"_169":168,"_170":171,"_172":171,"_173":171,"_174":168,"_175":168,"_176":177,"_178":179,"_180":28,"_48":49},"ahead",54,"behind",0,"diff",{"_61":62,"_63":64,"_65":66},"hasChanges",true,"sourceBranch",{"_41":56,"_13":16,"_32":57,"_39":40,"_43":33,"_44":42,"_58":51,"_59":40,"_60":54,"_55":14,"_52":12},"targetBranch",{"_32":33,"_34":35,"_36":37,"_13":38,"_39":40,"_41":42,"_43":-5,"_44":-5,"_45":46,"_47":-5,"_48":49,"_50":51,"_52":12,"_53":54,"_55":14},"pull","id","9c1e152e-0f41-11f1-b466-42dde27851f2","createdAt",["D",1771690989159],"updatedAt",["D",1771692268905],"main","projectId","9c1dc470-0f41-11f1-b466-42dde27851f2","version",17,"forkedBranchId","forkedBranchVersion","lockfileKey","1771692371179-esm.town_v_stevekrouse_microgpt@17-main","deletedAt","isOwned",false,"privacy","public","userHandle","orgId","e81e7335-6d4c-4764-a902-d0962bbfa9f7","projectName",55,"11405b5a-0f4c-11f1-8f75-42dde27851f2","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","added",[119,120,121],"deleted",[],"modified",[67,68],{"_69":101,"_71":102,"_73":103},{"_69":70,"_71":72,"_73":74},"path","main.ts","targetFile",{"_32":93,"_13":70,"_76":94,"_78":-5,"_79":33,"_80":95,"_81":26,"_34":96,"_41":97,"_84":85,"_86":98,"_63":49,"_88":-5,"_89":-7,"_69":99,"_91":100},"sourceFile",{"_32":75,"_13":70,"_76":77,"_78":-5,"_79":57,"_80":56,"_81":26,"_34":82,"_41":83,"_84":85,"_86":87,"_63":49,"_88":-5,"_89":-7,"_69":90,"_91":92},"9d9566f2-0f4e-11f1-9585-42dde27851f2","fileId","019c8147-9f56-743d-9a85-5753000aa122","parentId","branchId","branchVersion","latest",["D",1771696575074],48,"fileType","script","content","import { Value } from \"./value.ts\";\nimport { buildCharTokenizerFromDocs } from \"./tokenizer.ts\";\nimport type { GenOpts, Model, ModelSpec, Params, TrainOpts, Trained } from \"./types.ts\";\n\nfunction makeRng(seed: number) { // xorshift32\n let s = seed | 0 || 1;\n return () => {\n s ^= s << 13;\n s ^= s >> 17;\n s ^= s << 5;\n return (s >>> 0) / 4294967296;\n };\n}\n\nasync function loadDocs(): Promise {\n const text = await (await fetch(\"https://raw.githubusercontent.com/karpathy/makemore/refs/heads/master/names.txt\")).text();\n return text.trim().split(\"\\n\").map((s) => s.trim().toLowerCase()).filter((s) => s.length > 0);\n}\n\nfunction makeTransformerLanguageModel(spec: ModelSpec): Model {\n const { vocabSize, maxLen, dModel, nHeads, nLayers, dFF } = spec;\n const headDim = dModel / nHeads;\n\n function initParams(seed: number): Params {\n const rng = makeRng(seed), params: Params = [];\n const randn = (rows: number, cols: number, scale = 0.02) => {\n const arr = Array.from({ length: rows * cols }, () => {\n const u1 = rng() || 1e-10, u2 = rng();\n return new Value(Math.sqrt(-2 * Math.log(u1)) * Math.cos(6.2831853 * u2) * scale);\n });\n params.push(arr);\n return arr;\n };\n randn(vocabSize, dModel);\n randn(maxLen, dModel); // token embeddings, position embeddings\n for (let _l = 0; _l < nLayers; _l++) {\n randn(dModel, 3 * dModel);\n randn(1, 3 * dModel); // attention QKV weight + bias\n randn(dModel, dModel, 0);\n randn(1, dModel); // attention output projection (init to 0)\n randn(dModel, dFF);\n randn(1, dFF); // feedforward layer 1 weight + bias\n randn(dFF, dModel, 0);\n randn(1, dModel); // feedforward layer 2 (init to 0)\n }\n return params;\n }\n\n function getRow(w: Value[], cols: number, row: number): Value[] {\n return w.slice(row * cols, (row + 1) * cols);\n }\n\n function dot(a: Value[], b: Value[]): Value {\n let s = a[0].mul(b[0]);\n for (let i = 1; i < a.length; i++) s = s.add(a[i].mul(b[i]));\n return s;\n }\n\n // v[rows] @ w[rows, cols] -> result[cols]\n function vecmat(v: Value[], w: Value[], rows: number, cols: number): Value[] {\n const out: Value[] = [];\n for (let c = 0; c < cols; c++) {\n let s = v[0].mul(w[0 * cols + c]);\n for (let r = 1; r < rows; r++) {\n s = s.add(v[r].mul(w[r * cols + c]));\n }\n out.push(s);\n }\n return out;\n }\n\n function rmsnorm(v: Value[]): Value[] {\n let ms = v[0].mul(v[0]);\n for (let i = 1; i < v.length; i++) ms = ms.add(v[i].mul(v[i]));\n const scale = ms.mul(new Value(1 / v.length)).add(new Value(1e-5)).pow(-0.5);\n return v.map((x) => x.mul(scale));\n }\n\n function vadd(a: Value[], b: Value[]): Value[] {\n return a.map((x, i) => x.add(b[i]));\n }\n\n function forward(params: Params, tokens: Int32Array): Value[][] {\n let pi = 0;\n const tokEmb = params[pi++], posEmb = params[pi++], T = tokens.length;\n\n let x: Value[][] = [];\n for (let t = 0; t < T; t++)\n x.push(vadd(getRow(tokEmb, dModel, tokens[t]), getRow(posEmb, dModel, t % maxLen)));\n x = x.map(rmsnorm);\n\n for (let _l = 0; _l < nLayers; _l++) {\n const wQKV = params[pi++], bQKV = params[pi++];\n const wO = params[pi++], bO = params[pi++];\n\n const xNorm = x.map(rmsnorm);\n const qkv: Value[][] = [];\n for (let t = 0; t < T; t++) {\n qkv.push(vadd(vecmat(xNorm[t], wQKV, dModel, 3 * dModel), bQKV));\n }\n\n const attnOut: Value[][] = Array.from({ length: T }, () => new Array(dModel));\n\n for (let h = 0; h < nHeads; h++) {\n const off = h * headDim;\n const scaleVal = 1 / Math.sqrt(headDim);\n\n const Q: Value[][] = [], K: Value[][] = [], V: Value[][] = [];\n for (let t = 0; t < T; t++) {\n Q.push(qkv[t].slice(off, off + headDim));\n K.push(qkv[t].slice(dModel + off, dModel + off + headDim));\n V.push(qkv[t].slice(2 * dModel + off, 2 * dModel + off + headDim));\n }\n\n for (let i = 0; i < T; i++) { // causal attention\n const scores: Value[] = [];\n for (let j = 0; j <= i; j++) {\n scores.push(dot(Q[i], K[j]).mul(new Value(scaleVal)));\n }\n\n let mx = -Infinity;\n for (const s of scores) mx = Math.max(mx, s.data);\n const ex = scores.map((s) => s.add(new Value(-mx)).exp());\n const sum = ex.reduce((a, b) => a.add(b));\n const attnW = ex.map((e) => e.mul(sum.pow(-1)));\n\n for (let d = 0; d < headDim; d++)\n attnOut[i][off + d] = attnW.reduce((s, w, j) => s.add(w.mul(V[j][d])), new Value(0));\n }\n }\n\n for (let t = 0; t < T; t++) // output projection + residual\n x[t] = vadd(x[t], vadd(vecmat(attnOut[t], wO, dModel, dModel), bO));\n\n const wFF1 = params[pi++], bFF1 = params[pi++]; // MLP\n const wFF2 = params[pi++], bFF2 = params[pi++];\n const xNorm2 = x.map(rmsnorm);\n for (let t = 0; t < T; t++) {\n const h = vadd(vecmat(xNorm2[t], wFF1, dModel, dFF), bFF1).map((x) => x.relu());\n x[t] = vadd(x[t], vadd(vecmat(h, wFF2, dFF, dModel), bFF2));\n }\n }\n\n return x.map((xt) => // weight-tied logits: dot(x[t], tokEmb[v])\n Array.from({ length: vocabSize }, (_, v) => dot(xt, getRow(tokEmb, dModel, v)))\n );\n }\n\n return {\n spec,\n initParams,\n forward,\n countParams: (p: Params) => p.reduce((s, t) => s + t.length, 0),\n };\n}\n\nfunction crossEntropyLoss(logits: Value[][], targets: Int32Array): Value {\n let loss = new Value(0);\n for (let t = 0; t < logits.length; t++) {\n let mx = -Infinity;\n for (const v of logits[t]) mx = Math.max(mx, v.data);\n const ex = logits[t].map((v) => v.add(new Value(-mx)).exp());\n loss = loss.add(ex[targets[t]].mul(ex.reduce((a, b) => a.add(b)).pow(-1)).log().neg());\n }\n return loss.mul(new Value(1 / logits.length));\n}\n\nfunction trainNameModel(opts: TrainOpts): Trained {\n const { docs, tokenizer, model, steps, maxLen, seed, learningRate } = opts;\n const params = model.initParams(seed);\n const numParams = model.countParams(params);\n console.log(`num params: ${numParams}`);\n const beta1 = 0.9, beta2 = 0.95, epsAdam = 1e-8; // Adam\n const mBuf = params.map((p) => new Float64Array(p.length));\n const vBuf = params.map((p) => new Float64Array(p.length));\n const rng = makeRng(seed);\n const shuffled = [...docs];\n for (let i = shuffled.length - 1; i > 0; i--) {\n const j = Math.floor(rng() * (i + 1));\n [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];\n }\n\n const t0 = Date.now();\n for (let step = 0; step < steps; step++) {\n for (const p of params) for (const v of p) v.grad = 0; // zero grad\n const allTokens = tokenizer.encode(shuffled[step % shuffled.length]);\n const n = Math.min(maxLen, allTokens.length - 1);\n const logits = model.forward(params, allTokens.slice(0, n));\n const loss = crossEntropyLoss(logits, allTokens.slice(1, n + 1));\n loss.backward();\n const lrT = learningRate * (1 - step / steps); // Adam + LR decay\n for (let pi = 0; pi < params.length; pi++) {\n const p = params[pi], m = mBuf[pi], v = vBuf[pi];\n for (let i = 0; i < p.length; i++) {\n m[i] = beta1 * m[i] + (1 - beta1) * p[i].grad;\n v[i] = beta2 * v[i] + (1 - beta2) * p[i].grad * p[i].grad;\n const mHat = m[i] / (1 - Math.pow(beta1, step + 1));\n const vHat = v[i] / (1 - Math.pow(beta2, step + 1));\n p[i].data -= lrT * mHat / (Math.sqrt(vHat) + epsAdam);\n }\n }\n\n if (step % 100 === 0 || step === steps - 1) {\n const elapsed = ((Date.now() - t0) / 1000).toFixed(1);\n console.log(\n `step ${String(step + 1).padStart(4)} / ${steps} | loss ${\n loss.data.toFixed(4)\n } | ${elapsed}s`,\n );\n }\n }\n\n return { tokenizer, model, params };\n}\n\nfunction generateName(opts: GenOpts): string {\n const { trained: { tokenizer, model, params }, maxNewTokens, temperature, seed } = opts;\n const rng = makeRng(seed);\n let tokens = new Int32Array([tokenizer.BOS]);\n\n for (let i = 0; i < maxNewTokens; i++) {\n const logits = model.forward(params, tokens);\n const T = tokens.length, V = tokenizer.vocabSize;\n const lp = new Float64Array(V);\n for (let v = 0; v < V; v++) lp[v] = logits[T - 1][v].data / temperature;\n let mx = -Infinity;\n for (let v = 0; v < V; v++) mx = Math.max(mx, lp[v]);\n let s = 0;\n for (let v = 0; v < V; v++) { lp[v] = Math.exp(lp[v] - mx); s += lp[v]; }\n for (let v = 0; v < V; v++) lp[v] /= s;\n const r = rng();\n let cum = 0, next = 0;\n for (let v = 0; v < V; v++) { cum += lp[v]; if (r < cum) { next = v; break; } }\n if (next === tokenizer.BOS) break;\n const newTokens = new Int32Array(tokens.length + 1);\n newTokens.set(tokens);\n newTokens[tokens.length] = next;\n tokens = newTokens;\n }\n\n return tokenizer.decode(tokens);\n}\n\n// ============================================================================\n// main script\n// ============================================================================\n\nconst docs = await loadDocs();\nconsole.log(`num docs: ${docs.length}`);\n\nconst tok = buildCharTokenizerFromDocs(docs);\nconsole.log(`vocab size: ${tok.vocabSize}`);\n\nconst model = makeTransformerLanguageModel({\n vocabSize: tok.vocabSize, // 27 (a-z + BOS)\n maxLen: 8, // max sequence length (matches Karpathy's block_size)\n dModel: 16, // embedding dimension (matches Karpathy's n_embd)\n nHeads: 4, // attention heads (matches Karpathy's n_head)\n nLayers: 1, // transformer blocks (matches Karpathy's n_layer)\n dFF: 64, // feedforward hidden dim (4 * dModel, matches Karpathy)\n});\n\nconst trained = trainNameModel({\n docs,\n tokenizer: tok,\n model,\n steps: 1000, // same as Karpathy\n maxLen: 8, // crop sequences to block_size\n seed: 42, // same seed as Karpathy\n learningRate: 0.01, // same as Karpathy\n});\n\nconsole.log(\"\\n--- generation ---\");\nArray.from({ length: 20 }, (_, i) => i + 1)\n .map((i) =>\n generateName({\n trained,\n maxNewTokens: 8, // max name length = block_size\n temperature: 0.7, // balance between variety and quality\n seed: i * 7919, // spread seeds far apart for diversity (7919 is prime)\n })\n )\n .forEach((name, i) =>\n console.log(`sample ${String(i + 1).padStart(2, \" \")}: ${name}`)\n );","label","children",[70],"contentHash","a72b6d771a74bdd637ed80372cff74437410460ea36ecdb5cb08886c15b01307","6396ef34-0f44-11f1-b572-42dde27851f2","9c206342-0f41-11f1-b466-42dde27851f2",15,["D",1771692182733],14,"// ---- Tensor (micrograd-style autograd on flat arrays) ----\ntype Shape = number[];\nclass Tensor {\n data: Float32Array;\n grad: Float32Array;\n shape: Shape;\n _backward: () => void = () => {};\n _parents: Tensor[] = [];\n constructor(data: Float32Array, shape: Shape) {\n this.data = data;\n this.grad = new Float32Array(data.length);\n this.shape = shape;\n }\n backward() {\n const order: Tensor[] = [], visited = new Set();\n const topo = (t: Tensor) => {\n if (visited.has(t)) return;\n visited.add(t);\n t._parents.forEach(topo);\n order.push(t);\n };\n topo(this);\n this.grad.fill(1);\n for (let i = order.length - 1; i >= 0; i--) order[i]._backward();\n }\n}\nfunction zeros(shape: Shape) {\n return new Tensor(new Float32Array(shape.reduce((a, b) => a * b, 1)), shape);\n}\n\nfunction matmul(a: Tensor, b: Tensor): Tensor {\n const [M, K] = a.shape, N = b.shape[1], out = zeros([M, N]);\n for (let i = 0; i < M; i++) {\n for (let j = 0; j < N; j++) {\n let s = 0;\n for (let k = 0; k < K; k++) s += a.data[i * K + k] * b.data[k * N + j];\n out.data[i * N + j] = s;\n }\n }\n out._parents = [a, b];\n out._backward = () => {\n for (let i = 0; i < M; i++) {\n for (let j = 0; j < N; j++) {\n const g = out.grad[i * N + j];\n for (let k = 0; k < K; k++) {\n a.grad[i * K + k] += g * b.data[k * N + j];\n b.grad[k * N + j] += g * a.data[i * K + k];\n }\n }\n }\n };\n return out;\n}\n\nfunction addBias(a: Tensor, b: Tensor): Tensor {\n const [M, N] = a.shape, out = zeros([M, N]);\n for (let i = 0; i < M; i++) {\n for (let j = 0; j < N; j++) {\n out.data[i * N + j] = a.data[i * N + j] + b.data[j];\n }\n }\n out._parents = [a, b];\n out._backward = () => {\n for (let i = 0; i < M * N; i++) a.grad[i] += out.grad[i];\n for (let i = 0; i < M; i++) {\n for (let j = 0; j < N; j++) b.grad[j] += out.grad[i * N + j];\n }\n };\n return out;\n}\n\nfunction relu(a: Tensor): Tensor {\n const n = a.data.length, out = zeros(a.shape);\n for (let i = 0; i < n; i++) out.data[i] = Math.max(0, a.data[i]);\n out._parents = [a];\n out._backward = () => {\n for (let i = 0; i < n; i++) {\n a.grad[i] += out.grad[i] * (a.data[i] > 0 ? 1 : 0);\n }\n };\n return out;\n}\n\nfunction residualAdd(a: Tensor, b: Tensor): Tensor {\n const out = zeros(a.shape);\n for (let i = 0; i < a.data.length; i++) out.data[i] = a.data[i] + b.data[i];\n out._parents = [a, b];\n out._backward = () => {\n for (let i = 0; i < a.data.length; i++) {\n a.grad[i] += out.grad[i];\n b.grad[i] += out.grad[i];\n }\n };\n return out;\n}\n\nfunction embedding(table: Tensor, indices: Int32Array): Tensor {\n const D = table.shape[1], T = indices.length, out = zeros([T, D]);\n for (let t = 0; t < T; t++) {\n for (let d = 0; d < D; d++) {\n out.data[t * D + d] = table.data[indices[t] * D + d];\n }\n }\n out._parents = [table];\n out._backward = () => {\n for (let t = 0; t < T; t++) {\n for (let d = 0; d < D; d++) {\n table.grad[indices[t] * D + d] += out.grad[t * D + d];\n }\n }\n };\n return out;\n}\n\nfunction rmsnorm(a: Tensor): Tensor {\n // RMSNorm: x * rsqrt(mean(x^2) + eps), applied per-row for [T, D]\n const [T, D] = a.shape, out = zeros([T, D]);\n const scales = new Float32Array(T);\n for (let t = 0; t < T; t++) {\n let ms = 0;\n for (let d = 0; d < D; d++) ms += a.data[t * D + d] * a.data[t * D + d];\n ms /= D;\n scales[t] = 1 / Math.sqrt(ms + 1e-5);\n for (let d = 0; d < D; d++) {\n out.data[t * D + d] = a.data[t * D + d] * scales[t];\n }\n }\n out._parents = [a];\n out._backward = () => {\n for (let t = 0; t < T; t++) {\n const s = scales[t];\n let dot = 0;\n for (let d = 0; d < D; d++) {\n dot += a.data[t * D + d] * out.grad[t * D + d];\n }\n for (let d = 0; d < D; d++) {\n a.grad[t * D + d] += s *\n (out.grad[t * D + d] - a.data[t * D + d] * dot * s * s / D);\n }\n }\n };\n return out;\n}\n\nfunction crossEntropyLoss(logits: Tensor, targets: Int32Array): Tensor {\n const [T, V] = logits.shape,\n out = zeros([1]),\n probs = new Float32Array(T * V);\n for (let t = 0; t < T; t++) {\n let mx = -Infinity;\n for (let v = 0; v < V; v++) mx = Math.max(mx, logits.data[t * V + v]);\n let s = 0;\n for (let v = 0; v < V; v++) {\n probs[t * V + v] = Math.exp(logits.data[t * V + v] - mx);\n s += probs[t * V + v];\n }\n for (let v = 0; v < V; v++) probs[t * V + v] /= s;\n out.data[0] -= Math.log(probs[t * V + targets[t]] + 1e-10);\n }\n out.data[0] /= T;\n out._parents = [logits];\n out._backward = () => {\n for (let t = 0; t < T; t++) {\n for (let v = 0; v < V; v++) {\n logits.grad[t * V + v] += out.grad[0] *\n (probs[t * V + v] - (v === targets[t] ? 1 : 0)) / T;\n }\n }\n };\n return out;\n}\n\n// ---- Seeded RNG (xorshift32) ----\nfunction makeRng(seed: number) {\n let s = seed | 0 || 1;\n return () => {\n s ^= s << 13;\n s ^= s >> 17;\n s ^= s << 5;\n return (s >>> 0) / 4294967296;\n };\n}\n\n// ---- Dataset (fetch from Karpathy's repo) ----\nasync function loadDocs(): Promise {\n const resp = await fetch(\n \"https://raw.githubusercontent.com/karpathy/makemore/refs/heads/master/names.txt\",\n );\n const text = await resp.text();\n return text.trim().split(\"\\n\").map((s) => s.trim().toLowerCase()).filter(\n (s) => s.length > 0\n );\n}\n\n// ---- Tokenizer ----\nfunction buildCharTokenizerFromDocs(docs: string[]) {\n const chars = [...new Set(docs.join(\"\"))].sort();\n const BOS = chars.length, vocabSize = chars.length + 1;\n const c2i = new Map(chars.map((c, i) => [c, i]));\n return {\n vocabSize,\n BOS,\n chars,\n encode(text: string): Int32Array {\n const ids = [BOS];\n for (const c of text) ids.push(c2i.get(c)!);\n ids.push(BOS);\n return new Int32Array(ids);\n },\n decode(tokens: Int32Array): string {\n let s = \"\";\n for (const t of tokens) if (t !== BOS && t < chars.length) s += chars[t];\n return s;\n },\n };\n}\n\n// ---- Model ----\ntype Params = Tensor[];\ninterface ModelSpec {\n vocabSize: number;\n maxLen: number;\n dModel: number;\n nHeads: number;\n nLayers: number;\n dFF: number;\n}\ninterface Model {\n spec: ModelSpec;\n initParams: (seed: number) => Params;\n forward: (params: Params, tokens: Int32Array) => Tensor;\n countParams: (p: Params) => number;\n}\n\nfunction makeTransformerLanguageModel(spec: ModelSpec): Model {\n const { vocabSize, maxLen, dModel, nHeads, nLayers, dFF } = spec;\n const headDim = dModel / nHeads;\n\n function initParams(seed: number): Params {\n const rng = makeRng(seed), params: Params = [];\n const randn = (shape: Shape, scale = 0.02) => {\n const t = zeros(shape);\n for (let i = 0; i < t.data.length; i++) {\n const u1 = rng() || 1e-10, u2 = rng();\n t.data[i] = Math.sqrt(-2 * Math.log(u1)) * Math.cos(6.2831853 * u2) *\n scale;\n }\n params.push(t);\n return t;\n };\n randn([vocabSize, dModel]);\n randn([maxLen, dModel]); // token embeddings, position embeddings\n for (let _l = 0; _l < nLayers; _l++) {\n randn([dModel, 3 * dModel]);\n randn([3 * dModel]); // attention QKV weight + bias\n randn([dModel, dModel], 0);\n randn([dModel]); // attention output projection (init to 0 like Karpathy)\n randn([dModel, dFF]);\n randn([dFF]); // feedforward layer 1 weight + bias\n randn([dFF, dModel], 0);\n randn([dModel]); // feedforward layer 2 (init to 0 like Karpathy)\n }\n // No separate lm_head — we'll use weight tying with wte (like Karpathy)\n return params;\n }\n\n function forward(params: Params, tokens: Int32Array): Tensor {\n let pi = 0;\n const tokEmb = params[pi++], posEmb = params[pi++], T = tokens.length;\n const posIds = new Int32Array(T);\n for (let i = 0; i < T; i++) posIds[i] = i % maxLen;\n let x = residualAdd(embedding(tokEmb, tokens), embedding(posEmb, posIds));\n x = rmsnorm(x);\n\n for (let _l = 0; _l < nLayers; _l++) {\n const wQKV = params[pi++],\n bQKV = params[pi++],\n wO = params[pi++],\n bO = params[pi++];\n\n // RMSNorm before attention\n const xNorm = rmsnorm(x);\n const qkv = addBias(matmul(xNorm, wQKV), bQKV); // [T, 3*dModel]\n\n // Attention (all heads, manual forward + backward)\n const attnOut = zeros([T, dModel]);\n const headData: {\n Q: Float32Array;\n K: Float32Array;\n V: Float32Array;\n attnW: Float32Array;\n off: number;\n }[] = [];\n for (let h = 0; h < nHeads; h++) {\n const off = h * headDim, scale = 1 / Math.sqrt(headDim);\n const Q = new Float32Array(T * headDim),\n K = new Float32Array(T * headDim),\n V = new Float32Array(T * headDim);\n for (let t = 0; t < T; t++) {\n for (let d = 0; d < headDim; d++) {\n Q[t * headDim + d] = qkv.data[t * 3 * dModel + off + d];\n K[t * headDim + d] = qkv.data[t * 3 * dModel + dModel + off + d];\n V[t * headDim + d] =\n qkv.data[t * 3 * dModel + 2 * dModel + off + d];\n }\n }\n // Causal attention scores + softmax\n const attnW = new Float32Array(T * T);\n for (let i = 0; i < T; i++) {\n const scores = new Float32Array(T);\n let mx = -Infinity;\n for (let j = 0; j <= i; j++) {\n let s = 0;\n for (let d = 0; d < headDim; d++) {\n s += Q[i * headDim + d] * K[j * headDim + d];\n }\n scores[j] = s * scale;\n mx = Math.max(mx, scores[j]);\n }\n let s = 0;\n for (let j = 0; j <= i; j++) {\n attnW[i * T + j] = Math.exp(scores[j] - mx);\n s += attnW[i * T + j];\n }\n for (let j = 0; j <= i; j++) attnW[i * T + j] /= s;\n }\n // Weighted sum of values\n for (let i = 0; i < T; i++) {\n for (let d = 0; d < headDim; d++) {\n let s = 0;\n for (let j = 0; j <= i; j++) {\n s += attnW[i * T + j] * V[j * headDim + d];\n }\n attnOut.data[i * dModel + off + d] = s;\n }\n }\n headData.push({ Q, K, V, attnW, off });\n }\n attnOut._parents = [qkv];\n attnOut._backward = () => {\n for (const { Q, K, V, attnW, off } of headData) {\n const scale = 1 / Math.sqrt(headDim);\n const dAttnW = new Float32Array(T * T),\n dV = new Float32Array(T * headDim);\n for (let i = 0; i < T; i++) {\n for (let d = 0; d < headDim; d++) {\n const g = attnOut.grad[i * dModel + off + d];\n for (let j = 0; j <= i; j++) {\n dAttnW[i * T + j] += g * V[j * headDim + d];\n dV[j * headDim + d] += g * attnW[i * T + j];\n }\n }\n }\n const dQ = new Float32Array(T * headDim),\n dK = new Float32Array(T * headDim);\n for (let i = 0; i < T; i++) {\n let dot = 0;\n for (let j = 0; j <= i; j++) {\n dot += dAttnW[i * T + j] * attnW[i * T + j];\n }\n for (let j = 0; j <= i; j++) {\n const ds = attnW[i * T + j] * (dAttnW[i * T + j] - dot) * scale;\n for (let d = 0; d < headDim; d++) {\n dQ[i * headDim + d] += ds * K[j * headDim + d];\n dK[j * headDim + d] += ds * Q[i * headDim + d];\n }\n }\n }\n for (let t = 0; t < T; t++) {\n for (let d = 0; d < headDim; d++) {\n qkv.grad[t * 3 * dModel + off + d] += dQ[t * headDim + d];\n qkv.grad[t * 3 * dModel + dModel + off + d] +=\n dK[t * headDim + d];\n qkv.grad[t * 3 * dModel + 2 * dModel + off + d] +=\n dV[t * headDim + d];\n }\n }\n }\n };\n\n x = residualAdd(x, addBias(matmul(attnOut, wO), bO));\n\n // MLP with RMSNorm, ReLU (like Karpathy), no bias needed but keeping for simplicity\n const xNorm2 = rmsnorm(x);\n const wFF1 = params[pi++],\n bFF1 = params[pi++],\n wFF2 = params[pi++],\n bFF2 = params[pi++];\n x = residualAdd(\n x,\n addBias(matmul(relu(addBias(matmul(xNorm2, wFF1), bFF1)), wFF2), bFF2),\n );\n }\n\n // Weight-tied output head: logits = x @ tokEmb^T (reuse token embeddings)\n // tokEmb is [vocabSize, dModel], we need [T, dModel] @ [dModel, vocabSize] = [T, vocabSize]\n // So we need to transpose tokEmb. We'll do this manually.\n const Tout = tokens.length, V = vocabSize;\n const logits = zeros([Tout, V]);\n logits._parents = [x, tokEmb];\n for (let t = 0; t < Tout; t++) {\n for (let v = 0; v < V; v++) {\n let s = 0;\n for (let d = 0; d < dModel; d++) {\n s += x.data[t * dModel + d] * tokEmb.data[v * dModel + d];\n }\n logits.data[t * V + v] = s;\n }\n }\n logits._backward = () => {\n for (let t = 0; t < Tout; t++) {\n for (let v = 0; v < V; v++) {\n const g = logits.grad[t * V + v];\n for (let d = 0; d < dModel; d++) {\n x.grad[t * dModel + d] += g * tokEmb.data[v * dModel + d];\n tokEmb.grad[v * dModel + d] += g * x.data[t * dModel + d];\n }\n }\n }\n };\n return logits;\n }\n\n return {\n spec,\n initParams,\n forward,\n countParams: (p: Params) => p.reduce((s, t) => s + t.data.length, 0),\n };\n}\n\n// ---- Training (Karpathy-style: 1 doc per step, Adam optimizer) ----\ntype Trained = {\n tokenizer: ReturnType;\n model: Model;\n params: Params;\n};\n\nfunction trainNameModel(opts: {\n docs: string[];\n tokenizer: ReturnType;\n model: Model;\n steps: number;\n maxLen: number;\n seed: number;\n learningRate: number;\n}): Trained {\n const { docs, tokenizer, model, steps, maxLen, seed, learningRate } = opts;\n const params = model.initParams(seed);\n const numParams = model.countParams(params);\n console.log(`num params: ${numParams}`);\n\n // Adam optimizer state\n const beta1 = 0.9, beta2 = 0.95, epsAdam = 1e-8;\n const mBuf: Float32Array[] = params.map((p) =>\n new Float32Array(p.data.length)\n );\n const vBuf: Float32Array[] = params.map((p) =>\n new Float32Array(p.data.length)\n );\n\n // Shuffle docs with our seeded RNG\n const rng = makeRng(seed);\n const shuffled = [...docs];\n for (let i = shuffled.length - 1; i > 0; i--) {\n const j = Math.floor(rng() * (i + 1));\n [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];\n }\n\n const t0 = Date.now();\n for (let step = 0; step < steps; step++) {\n // Zero gradients\n for (const p of params) p.grad.fill(0);\n\n // Pick one doc, encode it: [BOS, ...chars, BOS]\n const doc = shuffled[step % shuffled.length];\n const allTokens = tokenizer.encode(doc); // already has BOS on both sides\n // Crop to maxLen\n const n = Math.min(maxLen, allTokens.length - 1);\n const input = allTokens.slice(0, n);\n const target = allTokens.slice(1, n + 1);\n\n // Forward + loss + backward\n const logits = model.forward(params, input);\n const loss = crossEntropyLoss(logits, target);\n loss.backward();\n\n // Adam update with linear LR decay\n const lrT = learningRate * (1 - step / steps);\n for (let pi = 0; pi < params.length; pi++) {\n const p = params[pi], m = mBuf[pi], v = vBuf[pi];\n for (let i = 0; i < p.data.length; i++) {\n m[i] = beta1 * m[i] + (1 - beta1) * p.grad[i];\n v[i] = beta2 * v[i] + (1 - beta2) * p.grad[i] * p.grad[i];\n const mHat = m[i] / (1 - Math.pow(beta1, step + 1));\n const vHat = v[i] / (1 - Math.pow(beta2, step + 1));\n p.data[i] -= lrT * mHat / (Math.sqrt(vHat) + epsAdam);\n }\n }\n\n if (step % 100 === 0 || step === steps - 1) {\n const elapsed = ((Date.now() - t0) / 1000).toFixed(1);\n console.log(\n `step ${String(step + 1).padStart(4)} / ${steps} | loss ${\n loss.data[0].toFixed(4)\n } | ${elapsed}s`,\n );\n }\n }\n\n return { tokenizer, model, params };\n}\n\n// ---- Sampling ----\nfunction generateName(\n opts: {\n trained: Trained;\n maxNewTokens: number;\n temperature: number;\n seed: number;\n },\n): string {\n const {\n trained: { tokenizer, model, params },\n maxNewTokens,\n temperature,\n seed,\n } = opts;\n const rng = makeRng(seed);\n let tokens = new Int32Array([tokenizer.BOS]);\n\n for (let i = 0; i < maxNewTokens; i++) {\n const logits = model.forward(params, tokens);\n const T = tokens.length, V = tokenizer.vocabSize;\n // Take logits from the last position, apply temperature\n const lp = new Float32Array(V);\n for (let v = 0; v < V; v++) {\n lp[v] = logits.data[(T - 1) * V + v] / temperature;\n }\n // Softmax\n let mx = -Infinity;\n for (let v = 0; v < V; v++) mx = Math.max(mx, lp[v]);\n let s = 0;\n for (let v = 0; v < V; v++) {\n lp[v] = Math.exp(lp[v] - mx);\n s += lp[v];\n }\n for (let v = 0; v < V; v++) lp[v] /= s;\n // Sample from distribution\n const r = rng();\n let cum = 0, next = 0;\n for (let v = 0; v < V; v++) {\n cum += lp[v];\n if (r < cum) {\n next = v;\n break;\n }\n }\n if (next === tokenizer.BOS) break; // BOS doubles as end-of-name\n const newTokens = new Int32Array(tokens.length + 1);\n newTokens.set(tokens);\n newTokens[tokens.length] = next;\n tokens = newTokens;\n }\n\n return tokenizer.decode(tokens);\n}\n\n// ============================================================================\n// main script\n// ============================================================================\n\nconst docs = await loadDocs();\nconsole.log(`num docs: ${docs.length}`);\n\nconst tok = buildCharTokenizerFromDocs(docs);\nconsole.log(`vocab size: ${tok.vocabSize}`);\n\nconst model = makeTransformerLanguageModel({\n vocabSize: tok.vocabSize, // 27 (a-z + BOS)\n maxLen: 8, // max sequence length (matches Karpathy's block_size)\n dModel: 16, // embedding dimension (matches Karpathy's n_embd)\n nHeads: 4, // attention heads (matches Karpathy's n_head)\n nLayers: 1, // transformer blocks (matches Karpathy's n_layer)\n dFF: 64, // feedforward hidden dim (4 * dModel, matches Karpathy)\n});\n\nconst trained = trainNameModel({\n docs,\n tokenizer: tok,\n model,\n steps: 1000, // same as Karpathy\n maxLen: 8, // crop sequences to block_size\n seed: 42, // same seed as Karpathy\n learningRate: 0.01, // same as Karpathy\n});\n\nconsole.log(\"\\n--- generation ---\");\nArray.from({ length: 20 }, (_, i) => i + 1)\n .map((i) =>\n generateName({\n trained,\n maxNewTokens: 8, // max name length = block_size\n temperature: 0.7, // balance between variety and quality\n seed: i * 7919, // spread seeds far apart for diversity (7919 is prime)\n })\n )\n .forEach((name, i) =>\n console.log(`sample ${String(i + 1).padStart(2, \" \")}: ${name}`)\n );",[70],"2888e1cefd7dc6481c203d615e7f6848afae8a4d55be3e3f4bc5804457d9fde5","README.md",{"_32":113,"_13":101,"_76":114,"_78":-5,"_79":33,"_80":42,"_81":26,"_34":115,"_41":108,"_84":109,"_86":116,"_63":49,"_88":-5,"_89":-7,"_69":117,"_91":118},{"_32":104,"_13":101,"_76":105,"_78":-5,"_79":57,"_80":106,"_81":26,"_34":107,"_41":108,"_84":109,"_86":110,"_63":49,"_88":-5,"_89":-7,"_69":111,"_91":112},"60b5b608-0f4c-11f1-a98e-42dde27851f2","019c8147-9f56-743d-9a85-51574f3e6a91",3,["D",1771695613949],1,"file","# microGPT.ts (scalar autograd edition)\n\nA direct TypeScript port of [Karpathy's microGPT](https://karpathy.github.io/2026/02/12/microgpt/) — faithful to the original's scalar `Value`-based autograd, not optimized for speed.\n\nKarpathy's microGPT is a single Python file that trains and inferences a GPT with zero dependencies. As he put it: *\"This is the full algorithmic content of what is needed. Everything else is just efficiency.\"* This port preserves that spirit **at the same level of abstraction** — every scalar is a `Value` node in a dynamically-built computation graph, just like the original.\n\n## Scalar Autograd: Same as Karpathy\n\nThe core design choice: **every number is a `Value` object**.\n\n```typescript\nclass Value {\n data: number; // one scalar\n grad: number; // one scalar\n _backward: () => void;\n _prev: Value[]; // parent nodes in the graph\n}\n```\n\nA 16-dim embedding lookup creates 16 `Value` objects. A vector-matrix multiply of `[16] × [16, 48]` creates 48 dot products, each made of 16 multiplies and 15 adds — **over 1,000 new `Value` nodes**. A single forward pass builds a computation graph with **tens of thousands of nodes**, each individually linked and walked during `.backward()`.\n\nThis is the same approach as Karpathy's Python `Value` class. It makes the chain rule visible at every single scalar operation.\n\n## Architecture\n\nSame GPT-2-style transformer as the original:\n\n```mermaid\ngraph TD\n X[Input x] --> NORM1[RMSNorm]\n NORM1 --> QKV[\"Linear → Q, K, V\"]\n QKV --> ATTN[\"Causal Multi-Head Attention
4 heads × 4-dim\"]\n ATTN --> PROJ[\"Output projection\"]\n PROJ --> ADD1[\"+ residual\"]\n X --> ADD1\n\n ADD1 --> NORM2[RMSNorm]\n NORM2 --> FF1[\"Linear → 64-dim\"]\n FF1 --> RELU2[ReLU]\n RELU2 --> FF2[\"Linear → 16-dim\"]\n FF2 --> ADD2[\"+ residual\"]\n ADD1 --> ADD2\n\n ADD2 --> LOGITS[\"Weight-tied logits
dot(x, tokEmb)\"]\n```\n\n## The Seven Components\n\nFollowing Karpathy's decomposition:\n\n1. **Dataset** — ~32k names fetched from Karpathy's `makemore` repo\n2. **Tokenizer** — Character-level: 26 letters + 1 BOS/EOS token (vocab size 27)\n3. **Autograd** — Scalar `Value` class with `add`, `mul`, `pow`, `exp`, `log`, `relu` — each operation creates a node in the graph\n4. **Architecture** — 1-layer GPT-2-style transformer (RMSNorm, causal attention, ReLU MLP, weight tying)\n5. **Loss** — Cross-entropy over next-token predictions\n6. **Optimizer** — Adam with bias correction and linear learning rate decay\n7. **Sampling** — Temperature-controlled autoregressive generation\n\n## Running\n\nThis is a [Val Town](https://val.town) script val. It trains for 1000 steps on CPU. Because every scalar is its own autograd node, it's **much slower** than a tensor-based implementation (~60x slower) — but the code is simpler and more transparent:\n\n```\nnum docs: 32033\nvocab size: 27\nnum params: 4795\nstep 1 / 1000 | loss 3.5062 | 0.0s\nstep 101 / 1000 | loss 2.7573 | ...s\n...\nstep 1000 / 1000 | loss 2.2891 | ...s\n\n--- generation ---\nsample 1: malede\nsample 2: jara\n...\n```\n\n## Hyperparameters\n\nMatched to Karpathy's defaults:\n\n| Parameter | Value | Notes |\n|---|---|---|\n| `dModel` | 16 | Embedding dimension |\n| `nHeads` | 4 | Attention heads (head dim = 4) |\n| `nLayers` | 1 | Transformer blocks |\n| `dFF` | 64 | FF hidden dim (4× `dModel`) |\n| `maxLen` | 8 | Context window |\n| `steps` | 1000 | Training iterations |\n| `learningRate` | 0.01 | With linear decay to 0 |\n| `seed` | 42 | Deterministic initialization |\n\n## Why is this slow?\n\nThis version uses **scalar autograd** — the same approach as Karpathy's original Python. Every addition, multiplication, and activation creates a new `Value` object in the computation graph. A single forward pass creates tens of thousands of nodes, and `backward()` must topologically sort and walk all of them.\n\nThe [main branch](https://www.val.town/x/stevekrouse/microgpt) uses **tensor autograd** instead — a single `matmul` operation creates one graph node with a fused backward closure that loops over all scalars in tight `for` loops. Same math, ~60x faster, but the individual scalar chain rule is hidden inside the closure.\n\n| Factor | This version (scalar Values) | Main branch (Tensors) |\n|---|---|---|\n| **Graph nodes per forward pass** | ~10,000+ `Value` objects | ~30-40 `Tensor` objects |\n| **Object allocation** | One object per scalar op | One TypedArray per tensor op |\n| **Backward traversal** | Topological sort over ~10K nodes | Topological sort over ~30 nodes |\n| **Inner loop execution** | V8 JIT over many small closures | V8 JIT over tight array loops |\n\nKarpathy's choice of scalar `Value` was deliberate: it makes the chain rule visible at the most granular level. This port preserves that choice.\n\n## Why TypeScript?\n\nTypeScript's type system makes the structure visible: `ModelSpec` describes *what* the model is, `Model` is the *behavior*, `Trained` is a frozen snapshot. But the autograd is the same scalar-level approach as the Python original — no tensor tricks, no fused kernels, just `Value.add()` and `Value.mul()` all the way down.\n",[101],"ad93bfc097b6ab4d1d26a67e0add7ba706168d6684b0bc924ceef19baa8f8b4f","96e8594a-0f44-11f1-9efe-42dde27851f2","6a68b658-0f44-11f1-889d-42dde27851f2",["D",1771692268905],"# microGPT.ts\n\nA TypeScript port of [Karpathy's microGPT](https://karpathy.github.io/2026/02/12/microgpt/) — but written in the style of [Conal Elliott's denotational design](https://github.com/conal/talk-2014-bayhac-denotational-design): types first, meanings first, implementation as a consequence.\n\nKarpathy's original microGPT is a single Python file that trains and inferences a GPT with zero dependencies. As he put it: *\"This is the full algorithmic content of what is needed. Everything else is just efficiency.\"* This port preserves that spirit but raises the level of abstraction — leaning into TypeScript's type system to make the *structure* of a language model legible, not just the *math*.\n\n## Denotational Design, Applied\n\nConal Elliott's core idea: give a simple mathematical meaning (denotation) for each type, then define operations as if they work on meanings, not representations. The implementation is free to differ for efficiency, but must be observationally equivalent to the denotation.\n\nHere, the \"meanings\" are:\n\n| Type | Denotation (meaning) |\n|---|---|\n| `Tensor` | A shaped array of scalars with attached gradient and backward function — i.e., a node in a computation graph |\n| `ModelSpec` | The *what* of a transformer: vocab size, dimensions, heads, layers — a pure description with no behavior |\n| `Model` | A triple of `(spec, initParams, forward)` — a model *is* its specification plus two functions |\n| `Trained` | A triple of `(tokenizer, model, params)` — a frozen snapshot: everything needed to generate |\n\nThe key move: `Model` is not a class with hidden state. It's a plain record of functions. `makeTransformerLanguageModel` takes a `ModelSpec` and returns a `Model` — a function from specification to behavior. This is the denotational design pattern: separate the *what* (spec) from the *how* (init + forward), and make the connection between them explicit and total.\n\n## Architecture\n\n```mermaid\ngraph TD\n subgraph \"Denotation Layer — the What\"\n SPEC[\"ModelSpec
{vocabSize, maxLen, dModel, nHeads, nLayers, dFF}\"]\n TOK[\"Tokenizer
encode: string → Int32Array
decode: Int32Array → string\"]\n TRAINED[\"Trained
{tokenizer, model, params}\"]\n end\n\n subgraph \"Autograd Engine — Tensor as computation graph node\"\n TENSOR[\"Tensor
data + grad + shape + _backward\"]\n MATMUL[matmul]\n ADDBIAS[addBias]\n RELU[relu]\n RESADD[residualAdd]\n EMBED[embedding]\n RMSNORM[rmsnorm]\n CELOSS[crossEntropyLoss]\n TENSOR --- MATMUL & ADDBIAS & RELU & RESADD & EMBED & RMSNORM & CELOSS\n end\n\n subgraph \"Model — the How\"\n MAKE[\"makeTransformerLanguageModel\"]\n INIT[\"initParams
seed → Params\"]\n FWD[\"forward
(params, tokens) → logits\"]\n MAKE --> INIT & FWD\n end\n\n subgraph \"Training Loop\"\n ADAM[\"Adam optimizer
β₁=0.9, β₂=0.95, linear LR decay\"]\n LOOP[\"For each step:
1. Pick doc
2. Encode
3. Forward
4. Loss
5. Backward
6. Update\"]\n end\n\n subgraph \"Inference\"\n GEN[\"generateName
autoregressive sampling
with temperature\"]\n end\n\n SPEC -->|\"defines\"| MAKE\n TOK -->|\"encodes docs\"| LOOP\n INIT -->|\"creates\"| LOOP\n FWD -->|\"used in\"| LOOP\n CELOSS -->|\"computes loss\"| LOOP\n ADAM -->|\"updates params\"| LOOP\n LOOP -->|\"produces\"| TRAINED\n TRAINED -->|\"consumed by\"| GEN\n FWD -->|\"used in\"| GEN\n```\n\n## Forward Pass Detail\n\nEach transformer layer follows the now-standard pre-norm pattern:\n\n```mermaid\ngraph TD\n X[Input x] --> NORM1[RMSNorm]\n NORM1 --> QKV[\"Linear → Q, K, V\"]\n QKV --> ATTN[\"Causal Multi-Head Attention
4 heads × 4-dim\"]\n ATTN --> PROJ[\"Output projection\"]\n PROJ --> ADD1[\"+ residual\"]\n X --> ADD1\n\n ADD1 --> NORM2[RMSNorm]\n NORM2 --> FF1[\"Linear → 64-dim\"]\n FF1 --> RELU2[ReLU]\n RELU2 --> FF2[\"Linear → 16-dim\"]\n FF2 --> ADD2[\"+ residual\"]\n ADD1 --> ADD2\n\n ADD2 --> LOGITS[\"Weight-tied logits
x @ tokEmb^T\"]\n```\n\n## The Seven Components\n\nFollowing Karpathy's decomposition — every LLM has exactly these parts, and nothing else:\n\n1. **Dataset** — ~32k names fetched from Karpathy's `makemore` repo\n2. **Tokenizer** — Character-level: 26 letters + 1 BOS/EOS token (vocab size 27)\n3. **Autograd** — Micrograd-style reverse-mode AD on flat `Float32Array` tensors\n4. **Architecture** — 1-layer GPT-2-style transformer (RMSNorm, causal attention, ReLU MLP, weight tying)\n5. **Loss** — Cross-entropy over next-token predictions\n6. **Optimizer** — Adam with bias correction and linear learning rate decay\n7. **Sampling** — Temperature-controlled autoregressive generation\n\n## Running\n\nThis is a [Val Town](https://val.town) script val. Run it directly — it will train for 1000 steps on CPU and generate 20 sample names:\n\n```\nnum docs: 32033\nvocab size: 27\nnum params: 4795\nstep 1 / 1000 | loss 3.5062 | 0.0s\nstep 101 / 1000 | loss 2.7573 | 1.2s\n...\nstep 1000 / 1000 | loss 2.2891 | 11.8s\n\n--- generation ---\nsample 1: malede\nsample 2: jara\nsample 3: kaylin\n...\n```\n\n## Hyperparameters\n\nMatched to Karpathy's defaults:\n\n| Parameter | Value | Notes |\n|---|---|---|\n| `dModel` | 16 | Embedding dimension |\n| `nHeads` | 4 | Attention heads (head dim = 4) |\n| `nLayers` | 1 | Transformer blocks |\n| `dFF` | 64 | FF hidden dim (4× `dModel`) |\n| `maxLen` | 8 | Context window |\n| `steps` | 1000 | Training iterations |\n| `learningRate` | 0.01 | With linear decay to 0 |\n| `seed` | 42 | Deterministic initialization |\n\n## Why is this ~60x faster than Karpathy's?\n\nKarpathy's Python version takes about **1 minute** on a MacBook. This TypeScript version runs in about **1 second**. Same algorithm, same hyperparameters, same 1000 steps. The difference comes down to one architectural decision in the autograd:\n\n### Karpathy: scalar `Value` objects\n\nKarpathy's autograd operates on **individual scalar numbers**, each wrapped in a `Value` object:\n\n```python\nclass Value:\n def __init__(self, data, children=(), local_grads=()):\n self.data = data # one float\n self.grad = 0 # one float\n self._children = children\n self._local_grads = local_grads\n```\n\nA 16-dim embedding lookup creates 16 `Value` objects. A matmul of `[7, 16] × [16, 48]` creates `7 × 48 = 336` new `Value` objects, each individually linked in the graph. Over a single forward pass, the computation graph grows to **thousands of `Value` nodes**, each requiring a Python object allocation, pointer tracking, and a topological sort during `.backward()`.\n\nThe graph must also be walked **one scalar at a time** during backprop. Python's interpreter overhead per operation is ~100ns, and there are tens of thousands of operations per step.\n\n### Ours: `Tensor` with fused backward closures\n\nOur autograd operates on **flat `Float32Array` buffers** with shape metadata:\n\n```typescript\nclass Tensor {\n data: Float32Array; // all scalars in one contiguous buffer\n grad: Float32Array; // same size, contiguous\n shape: Shape;\n _backward: () => void; // one closure for the whole operation\n _parents: Tensor[]; // typically 1-2 parents, not thousands\n}\n```\n\nA matmul of `[7, 16] × [16, 48]` creates **one Tensor node** in the graph, with a single backward closure that loops over all `7 × 48 × 16 = 5,376` multiply-adds in tight `for` loops. The computation graph for an entire forward pass has only **~30-40 nodes** (not thousands).\n\n### Where the speedup comes from\n\n| Factor | Karpathy (Python scalars) | Ours (TS tensors) |\n|---|---|---|\n| **Graph nodes per forward pass** | ~10,000+ `Value` objects | ~30-40 `Tensor` objects |\n| **Object allocation** | One Python object per scalar | One TypedArray per operation |\n| **Backward traversal** | Topological sort over ~10K nodes | Topological sort over ~30 nodes |\n| **Inner loop execution** | Python interpreter (~100ns/op) | V8 JIT-compiled tight loops (~1-2ns/op) |\n| **Memory layout** | Scattered Python `float` objects on heap | Contiguous `Float32Array` buffers |\n| **Per-operation overhead** | Dict lookup + pointer chasing per scalar | One closure call, then raw array math |\n\nThe dominant factor is **graph granularity**: Karpathy has one autograd node per scalar multiplication, we have one autograd node per matrix multiplication. This is the same distinction between micrograd (scalar autograd) and PyTorch (tensor autograd), just implemented within a single file.\n\nKarpathy is fully aware of this — his blog post says: *\"Production systems use tensors (large multi-dimensional arrays of numbers)... The math is identical, just corresponds to many scalars processed in parallel.\"* His choice of scalar `Value` was deliberate: it makes the chain rule visible at the most granular level. Our choice of tensor operations is equally deliberate: it shows that the same algorithm, lifted to operate on arrays, becomes fast enough to run in a second.\n\nThis is the denotational design lens: both implementations *denote* the same thing (a computation graph with reverse-mode AD), but the representations differ. Karpathy's representation makes the calculus legible. Ours makes the linear algebra legible.\n\n## Why TypeScript?\n\nKarpathy's Python version is the *irreducible essence* of a language model. This version asks: what if we took that essence and gave it more structure? TypeScript's interfaces (`ModelSpec`, `Model`, `Trained`) make the *architecture of the architecture* visible — you can see the separation of concerns that's implicit in the Python version.\n\nIn Conal Elliott's terms: the Python version is the implementation, this version tries to also show the denotation.\n",[101],"a6dd86b00dc61cbd87953e154447474011e12a483d7659e755ae5fb92aa93aeb",{"_69":141,"_71":-7,"_73":142},{"_69":131,"_71":-7,"_73":132},{"_69":122,"_71":-7,"_73":123},"value.ts",{"_32":124,"_13":122,"_76":125,"_78":-5,"_79":57,"_80":126,"_81":26,"_34":127,"_41":22,"_84":85,"_86":128,"_63":49,"_88":-5,"_89":-7,"_69":129,"_91":130},"f1f30e5e-0f4c-11f1-89a5-42dde27851f2","f1f30fda-0f4c-11f1-89a5-42dde27851f2",6,["D",1771695857629],"// ---- Value (Karpathy's micrograd-style scalar autograd) ----\n// Every scalar is a Value node in a dynamic computation graph.\n// This is Karpathy's approach: slow but maximally transparent.\nexport class Value {\n data: number;\n grad: number = 0;\n _backward: () => void = () => {};\n _prev: Value[];\n constructor(data: number, children: Value[] = []) {\n this.data = data;\n this._prev = children;\n }\n add(other: Value): Value {\n const out = new Value(this.data + other.data, [this, other]);\n out._backward = () => {\n this.grad += out.grad;\n other.grad += out.grad;\n };\n return out;\n }\n mul(other: Value): Value {\n const out = new Value(this.data * other.data, [this, other]);\n out._backward = () => {\n this.grad += other.data * out.grad;\n other.grad += this.data * out.grad;\n };\n return out;\n }\n pow(n: number): Value {\n const out = new Value(this.data ** n, [this]);\n out._backward = () => {\n this.grad += n * this.data ** (n - 1) * out.grad;\n };\n return out;\n }\n exp(): Value {\n const out = new Value(Math.exp(this.data), [this]);\n out._backward = () => {\n this.grad += out.data * out.grad;\n };\n return out;\n }\n log(): Value {\n const out = new Value(Math.log(this.data + 1e-10), [this]);\n out._backward = () => {\n this.grad += out.grad / (this.data + 1e-10);\n };\n return out;\n }\n relu(): Value {\n const out = new Value(this.data > 0 ? this.data : 0, [this]);\n out._backward = () => {\n this.grad += (this.data > 0 ? 1 : 0) * out.grad;\n };\n return out;\n }\n neg(): Value {\n const out = new Value(-this.data, [this]);\n out._backward = () => {\n this.grad += -out.grad;\n };\n return out;\n }\n backward() {\n const order: Value[] = [], visited = new Set();\n const topo = (t: Value) => {\n if (visited.has(t)) return;\n visited.add(t);\n for (const p of t._prev) topo(p);\n order.push(t);\n };\n topo(this);\n this.grad = 1;\n for (let i = order.length - 1; i >= 0; i--) order[i]._backward();\n }\n}\n",[122],"fbba0f303d0c273c2cbb2ed9cdf50a78d94b84f39097e811b6432a7cb2ef2c84","types.ts",{"_32":133,"_13":131,"_76":134,"_78":-5,"_79":57,"_80":135,"_81":26,"_34":136,"_41":137,"_84":85,"_86":138,"_63":49,"_88":-5,"_89":-7,"_69":139,"_91":140},"7c4545d0-0f4e-11f1-a65e-42dde27851f2","da699e56-0f4c-11f1-982f-42dde27851f2",51,["D",1771696519176],2,"import type { Value } from \"./value.ts\";\nimport type { buildCharTokenizerFromDocs } from \"./tokenizer.ts\";\n\nexport type Params = Value[][];\n\nexport interface ModelSpec {\n vocabSize: number;\n maxLen: number;\n dModel: number;\n nHeads: number;\n nLayers: number;\n dFF: number;\n}\n\nexport interface Model {\n spec: ModelSpec;\n initParams: (seed: number) => Params;\n forward: (params: Params, tokens: Int32Array) => Value[][];\n countParams: (p: Params) => number;\n}\n\nexport type Trained = {\n tokenizer: ReturnType;\n model: Model;\n params: Params;\n};\n\nexport type TrainOpts = {\n docs: string[];\n tokenizer: ReturnType;\n model: Model;\n steps: number;\n maxLen: number;\n seed: number;\n learningRate: number;\n};\n\nexport type GenOpts = {\n trained: Trained;\n maxNewTokens: number;\n temperature: number;\n seed: number;\n};\n",[131],"29b6c81aaa3cd72133c1457b176b8b724092868412021d833ee012f8b350a0ba","tokenizer.ts",{"_32":143,"_13":141,"_76":144,"_78":-5,"_79":57,"_80":145,"_81":26,"_34":146,"_41":22,"_84":85,"_86":147,"_63":49,"_88":-5,"_89":-7,"_69":148,"_91":149},"f8accc26-0f4c-11f1-a8e3-42dde27851f2","f8acccd0-0f4c-11f1-a8e3-42dde27851f2",7,["D",1771695868907],"// ---- Tokenizer ----\nexport function buildCharTokenizerFromDocs(docs: string[]) {\n const chars = [...new Set(docs.join(\"\"))].sort();\n const BOS = chars.length, vocabSize = chars.length + 1;\n const c2i = new Map(chars.map((c, i) => [c, i]));\n return {\n vocabSize,\n BOS,\n chars,\n encode(text: string): Int32Array {\n const ids = [BOS];\n for (const c of text) ids.push(c2i.get(c)!);\n ids.push(BOS);\n return new Int32Array(ids);\n },\n decode(tokens: Int32Array): string {\n let s = \"\";\n for (const t of tokens) if (t !== BOS && t < chars.length) s += chars[t];\n return s;\n },\n };\n}\n",[141],"d8b20e471536851fc8ad190067e0806e3fc4b8d0921cd645bd95e1f8073507c7",["D",1771690989159],"legacyEnvVars","httpPrivacy","description","","imageUrl","userId","a0bf3b31-15a5-4d5c-880e-4b1e22c9bc18","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_11":12,"_187":183},"org",{"_11":12,"_181":-5,"_182":183,"_156":157,"_184":185,"_186":49},"branch","displayName","avatar","https://images.clerk.dev/uploaded/img_2PqHa2Gsy93xQrjh2w78Xu0cChW.jpeg","tier","pro","canRestrictVals","profileImageUrl","adoptOrgId",{"_32":40,"_13":14,"_34":251,"_47":-5,"_151":49,"_50":51,"_152":51,"_153":154,"_155":154,"_156":157,"_158":-5,"_159":49,"_160":-5,"_161":-5,"_162":22,"_53":54,"_163":-5,"_164":-5,"_165":-5,"_166":-5,"_167":168,"_169":168,"_170":171,"_172":171,"_173":171,"_174":168,"_175":168,"_176":252,"_178":253,"_180":254,"_48":49},"tree",[236,237,238,239,240],"files",[236,237,238,239,240],"counts",{"_226":22,"_227":137,"_228":22,"_229":230,"_231":232},"contents",[211,212,213,214,215],"upstreamProject","upstreamBranch",{"_32":33,"_34":209,"_36":210,"_13":38,"_39":40,"_41":42,"_43":-5,"_44":-5,"_45":46,"_47":-5,"_48":49,"_50":51,"_52":12,"_53":54,"_55":14},"emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},["D",1771690989159],["D",1771692268905],{"_32":104,"_13":101,"_76":105,"_78":-5,"_79":57,"_80":106,"_81":26,"_34":224,"_41":108,"_84":109,"_86":110,"_63":49,"_88":-5,"_69":225},{"_32":75,"_13":70,"_76":77,"_78":-5,"_79":57,"_80":56,"_81":26,"_34":222,"_41":83,"_84":85,"_86":87,"_63":49,"_88":-5,"_69":223},{"_32":143,"_13":141,"_76":144,"_78":-5,"_79":57,"_80":145,"_81":26,"_34":220,"_41":22,"_84":85,"_86":147,"_63":49,"_88":-5,"_69":221},{"_32":133,"_13":131,"_76":134,"_78":-5,"_79":57,"_80":135,"_81":26,"_34":218,"_41":137,"_84":85,"_86":138,"_63":49,"_88":-5,"_69":219},{"_32":124,"_13":122,"_76":125,"_78":-5,"_79":57,"_80":126,"_81":26,"_34":216,"_41":22,"_84":85,"_86":128,"_63":49,"_88":-5,"_69":217},["D",1771695857629],[122],["D",1771696519176],[131],["D",1771695868907],[141],["D",1771696575074],[70],["D",1771695613949],[101],"pulls","branches","forks","envVars",{"_233":22,"_235":22},"connections",{"_233":22,"_234":22},"count","unfilledCount","emptyCount",{"_32":104,"_13":101,"_76":105,"_78":-5,"_79":57,"_80":106,"_81":26,"_34":107,"_41":108,"_84":109,"_86":110,"_63":49,"_88":-5,"_89":249,"_69":250},{"_32":75,"_13":70,"_76":77,"_78":-5,"_79":57,"_80":56,"_81":26,"_34":82,"_41":83,"_84":85,"_86":87,"_63":49,"_88":-5,"_89":247,"_69":248},{"_32":143,"_13":141,"_76":144,"_78":-5,"_79":57,"_80":145,"_81":26,"_34":146,"_41":22,"_84":85,"_86":147,"_63":49,"_88":-5,"_89":245,"_69":246},{"_32":133,"_13":131,"_76":134,"_78":-5,"_79":57,"_80":135,"_81":26,"_34":136,"_41":137,"_84":85,"_86":138,"_63":49,"_88":-5,"_89":243,"_69":244},{"_32":124,"_13":122,"_76":125,"_78":-5,"_79":57,"_80":126,"_81":26,"_34":127,"_41":22,"_84":85,"_86":128,"_63":49,"_88":-5,"_89":241,"_69":242},[],[122],[],[131],[],[141],[],[70],[],[101],["D",1771690989159],{"_11":12,"_187":183},{"_11":12,"_181":-5,"_182":183,"_156":157,"_184":185,"_186":49},{"_41":56,"_13":16,"_32":57,"_39":40,"_43":33,"_44":42,"_58":51,"_59":40,"_60":54,"_55":14,"_52":12},"ctx",{"_176":-5,"_206":288,"_289":290,"_291":292,"_293":49,"_294":49,"_295":49,"_296":26,"_297":-5,"_298":-7,"_299":-7,"_300":22,"_301":49,"_302":49,"_303":-5,"_304":49,"_305":49,"_306":49,"_307":-5},"ENV",{"_259":260,"_261":262,"_263":38,"_264":265,"_266":267,"_268":269,"_270":271,"_272":273,"_274":275,"_276":277,"_278":279,"_280":281,"_282":283,"_284":285,"_286":287},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Compare

Mergekarpathy-scalar-autogradintomain

README.md+30-119

main.ts+135-460

tokenizer.ts+22-0

types.ts+43-0

value.ts+76-0