[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_170":171,"_172":173},"routes/_app.x.$handle.$name",{"_11":12,"_13":14,"_15":16,"_18":19,"_17":-7,"_128":-5,"_20":129,"_22":130,"_131":132,"_133":134,"_25":135,"_24":-5,"_136":-5,"_137":-5,"_138":54,"_139":140,"_141":54,"_142":-5,"_143":-5,"_144":145},"routes/_app.x.$handle.$name.branch.$branch.code.$",{"_11":12,"_13":14,"_15":16,"_17":-7,"_18":19,"_20":21,"_22":23,"_24":-7,"_25":26,"_27":-5,"_28":-5,"_29":-5,"_30":-5,"_31":-5},"handle","stevekrouse","name","microgpt","branchName","main","version","path","","project",{"_34":74,"_13":14,"_46":75,"_76":-5,"_77":54,"_78":79,"_80":79,"_81":19,"_82":19,"_83":84,"_85":-5,"_86":54,"_87":-5,"_88":-5,"_89":90,"_91":92,"_93":-5,"_94":-5,"_95":-5,"_96":-5,"_97":98,"_99":98,"_100":101,"_102":101,"_103":101,"_104":98,"_105":98,"_106":107,"_108":109,"_110":111,"_112":54},"tree",[65,66],"file","contents",[32,33],"upstream","pull","ahead","behind","envVars",{"_34":57,"_13":58,"_37":59,"_39":-5,"_40":41,"_42":60,"_44":45,"_46":61,"_17":62,"_49":24,"_51":63,"_53":54,"_55":-5,"_18":64},{"_34":35,"_13":36,"_37":38,"_39":-5,"_40":41,"_42":43,"_44":45,"_46":47,"_17":48,"_49":50,"_51":52,"_53":54,"_55":-5,"_18":56},"id","6396ef34-0f44-11f1-b572-42dde27851f2","main.ts","fileId","9c206342-0f41-11f1-b466-42dde27851f2","parentId","branchId","9c1e152e-0f41-11f1-b466-42dde27851f2","branchVersion",15,"latest",true,"createdAt",["D",1771692182733],14,"fileType","script","content","// ---- Tensor (micrograd-style autograd on flat arrays) ----\ntype Shape = number[];\nclass Tensor {\n data: Float32Array;\n grad: Float32Array;\n shape: Shape;\n _backward: () => void = () => {};\n _parents: Tensor[] = [];\n constructor(data: Float32Array, shape: Shape) {\n this.data = data;\n this.grad = new Float32Array(data.length);\n this.shape = shape;\n }\n backward() {\n const order: Tensor[] = [], visited = new Set();\n const topo = (t: Tensor) => {\n if (visited.has(t)) return;\n visited.add(t);\n t._parents.forEach(topo);\n order.push(t);\n };\n topo(this);\n this.grad.fill(1);\n for (let i = order.length - 1; i >= 0; i--) order[i]._backward();\n }\n}\nfunction zeros(shape: Shape) {\n return new Tensor(new Float32Array(shape.reduce((a, b) => a * b, 1)), shape);\n}\n\nfunction matmul(a: Tensor, b: Tensor): Tensor {\n const [M, K] = a.shape, N = b.shape[1], out = zeros([M, N]);\n for (let i = 0; i < M; i++) {\n for (let j = 0; j < N; j++) {\n let s = 0;\n for (let k = 0; k < K; k++) s += a.data[i * K + k] * b.data[k * N + j];\n out.data[i * N + j] = s;\n }\n }\n out._parents = [a, b];\n out._backward = () => {\n for (let i = 0; i < M; i++) {\n for (let j = 0; j < N; j++) {\n const g = out.grad[i * N + j];\n for (let k = 0; k < K; k++) {\n a.grad[i * K + k] += g * b.data[k * N + j];\n b.grad[k * N + j] += g * a.data[i * K + k];\n }\n }\n }\n };\n return out;\n}\n\nfunction addBias(a: Tensor, b: Tensor): Tensor {\n const [M, N] = a.shape, out = zeros([M, N]);\n for (let i = 0; i < M; i++) {\n for (let j = 0; j < N; j++) {\n out.data[i * N + j] = a.data[i * N + j] + b.data[j];\n }\n }\n out._parents = [a, b];\n out._backward = () => {\n for (let i = 0; i < M * N; i++) a.grad[i] += out.grad[i];\n for (let i = 0; i < M; i++) {\n for (let j = 0; j < N; j++) b.grad[j] += out.grad[i * N + j];\n }\n };\n return out;\n}\n\nfunction relu(a: Tensor): Tensor {\n const n = a.data.length, out = zeros(a.shape);\n for (let i = 0; i < n; i++) out.data[i] = Math.max(0, a.data[i]);\n out._parents = [a];\n out._backward = () => {\n for (let i = 0; i < n; i++) {\n a.grad[i] += out.grad[i] * (a.data[i] > 0 ? 1 : 0);\n }\n };\n return out;\n}\n\nfunction residualAdd(a: Tensor, b: Tensor): Tensor {\n const out = zeros(a.shape);\n for (let i = 0; i < a.data.length; i++) out.data[i] = a.data[i] + b.data[i];\n out._parents = [a, b];\n out._backward = () => {\n for (let i = 0; i < a.data.length; i++) {\n a.grad[i] += out.grad[i];\n b.grad[i] += out.grad[i];\n }\n };\n return out;\n}\n\nfunction embedding(table: Tensor, indices: Int32Array): Tensor {\n const D = table.shape[1], T = indices.length, out = zeros([T, D]);\n for (let t = 0; t < T; t++) {\n for (let d = 0; d < D; d++) {\n out.data[t * D + d] = table.data[indices[t] * D + d];\n }\n }\n out._parents = [table];\n out._backward = () => {\n for (let t = 0; t < T; t++) {\n for (let d = 0; d < D; d++) {\n table.grad[indices[t] * D + d] += out.grad[t * D + d];\n }\n }\n };\n return out;\n}\n\nfunction rmsnorm(a: Tensor): Tensor {\n // RMSNorm: x * rsqrt(mean(x^2) + eps), applied per-row for [T, D]\n const [T, D] = a.shape, out = zeros([T, D]);\n const scales = new Float32Array(T);\n for (let t = 0; t < T; t++) {\n let ms = 0;\n for (let d = 0; d < D; d++) ms += a.data[t * D + d] * a.data[t * D + d];\n ms /= D;\n scales[t] = 1 / Math.sqrt(ms + 1e-5);\n for (let d = 0; d < D; d++) {\n out.data[t * D + d] = a.data[t * D + d] * scales[t];\n }\n }\n out._parents = [a];\n out._backward = () => {\n for (let t = 0; t < T; t++) {\n const s = scales[t];\n let dot = 0;\n for (let d = 0; d < D; d++) {\n dot += a.data[t * D + d] * out.grad[t * D + d];\n }\n for (let d = 0; d < D; d++) {\n a.grad[t * D + d] += s *\n (out.grad[t * D + d] - a.data[t * D + d] * dot * s * s / D);\n }\n }\n };\n return out;\n}\n\nfunction crossEntropyLoss(logits: Tensor, targets: Int32Array): Tensor {\n const [T, V] = logits.shape,\n out = zeros([1]),\n probs = new Float32Array(T * V);\n for (let t = 0; t < T; t++) {\n let mx = -Infinity;\n for (let v = 0; v < V; v++) mx = Math.max(mx, logits.data[t * V + v]);\n let s = 0;\n for (let v = 0; v < V; v++) {\n probs[t * V + v] = Math.exp(logits.data[t * V + v] - mx);\n s += probs[t * V + v];\n }\n for (let v = 0; v < V; v++) probs[t * V + v] /= s;\n out.data[0] -= Math.log(probs[t * V + targets[t]] + 1e-10);\n }\n out.data[0] /= T;\n out._parents = [logits];\n out._backward = () => {\n for (let t = 0; t < T; t++) {\n for (let v = 0; v < V; v++) {\n logits.grad[t * V + v] += out.grad[0] *\n (probs[t * V + v] - (v === targets[t] ? 1 : 0)) / T;\n }\n }\n };\n return out;\n}\n\n// ---- Seeded RNG (xorshift32) ----\nfunction makeRng(seed: number) {\n let s = seed | 0 || 1;\n return () => {\n s ^= s << 13;\n s ^= s >> 17;\n s ^= s << 5;\n return (s >>> 0) / 4294967296;\n };\n}\n\n// ---- Dataset (fetch from Karpathy's repo) ----\nasync function loadDocs(): Promise {\n const resp = await fetch(\n \"https://raw.githubusercontent.com/karpathy/makemore/refs/heads/master/names.txt\",\n );\n const text = await resp.text();\n return text.trim().split(\"\\n\").map((s) => s.trim().toLowerCase()).filter(\n (s) => s.length > 0\n );\n}\n\n// ---- Tokenizer ----\nfunction buildCharTokenizerFromDocs(docs: string[]) {\n const chars = [...new Set(docs.join(\"\"))].sort();\n const BOS = chars.length, vocabSize = chars.length + 1;\n const c2i = new Map(chars.map((c, i) => [c, i]));\n return {\n vocabSize,\n BOS,\n chars,\n encode(text: string): Int32Array {\n const ids = [BOS];\n for (const c of text) ids.push(c2i.get(c)!);\n ids.push(BOS);\n return new Int32Array(ids);\n },\n decode(tokens: Int32Array): string {\n let s = \"\";\n for (const t of tokens) if (t !== BOS && t < chars.length) s += chars[t];\n return s;\n },\n };\n}\n\n// ---- Model ----\ntype Params = Tensor[];\ninterface ModelSpec {\n vocabSize: number;\n maxLen: number;\n dModel: number;\n nHeads: number;\n nLayers: number;\n dFF: number;\n}\ninterface Model {\n spec: ModelSpec;\n initParams: (seed: number) => Params;\n forward: (params: Params, tokens: Int32Array) => Tensor;\n countParams: (p: Params) => number;\n}\n\nfunction makeTransformerLanguageModel(spec: ModelSpec): Model {\n const { vocabSize, maxLen, dModel, nHeads, nLayers, dFF } = spec;\n const headDim = dModel / nHeads;\n\n function initParams(seed: number): Params {\n const rng = makeRng(seed), params: Params = [];\n const randn = (shape: Shape, scale = 0.02) => {\n const t = zeros(shape);\n for (let i = 0; i < t.data.length; i++) {\n const u1 = rng() || 1e-10, u2 = rng();\n t.data[i] = Math.sqrt(-2 * Math.log(u1)) * Math.cos(6.2831853 * u2) *\n scale;\n }\n params.push(t);\n return t;\n };\n randn([vocabSize, dModel]);\n randn([maxLen, dModel]); // token embeddings, position embeddings\n for (let _l = 0; _l < nLayers; _l++) {\n randn([dModel, 3 * dModel]);\n randn([3 * dModel]); // attention QKV weight + bias\n randn([dModel, dModel], 0);\n randn([dModel]); // attention output projection (init to 0 like Karpathy)\n randn([dModel, dFF]);\n randn([dFF]); // feedforward layer 1 weight + bias\n randn([dFF, dModel], 0);\n randn([dModel]); // feedforward layer 2 (init to 0 like Karpathy)\n }\n // No separate lm_head — we'll use weight tying with wte (like Karpathy)\n return params;\n }\n\n function forward(params: Params, tokens: Int32Array): Tensor {\n let pi = 0;\n const tokEmb = params[pi++], posEmb = params[pi++], T = tokens.length;\n const posIds = new Int32Array(T);\n for (let i = 0; i < T; i++) posIds[i] = i % maxLen;\n let x = residualAdd(embedding(tokEmb, tokens), embedding(posEmb, posIds));\n x = rmsnorm(x);\n\n for (let _l = 0; _l < nLayers; _l++) {\n const wQKV = params[pi++],\n bQKV = params[pi++],\n wO = params[pi++],\n bO = params[pi++];\n\n // RMSNorm before attention\n const xNorm = rmsnorm(x);\n const qkv = addBias(matmul(xNorm, wQKV), bQKV); // [T, 3*dModel]\n\n // Attention (all heads, manual forward + backward)\n const attnOut = zeros([T, dModel]);\n const headData: {\n Q: Float32Array;\n K: Float32Array;\n V: Float32Array;\n attnW: Float32Array;\n off: number;\n }[] = [];\n for (let h = 0; h < nHeads; h++) {\n const off = h * headDim, scale = 1 / Math.sqrt(headDim);\n const Q = new Float32Array(T * headDim),\n K = new Float32Array(T * headDim),\n V = new Float32Array(T * headDim);\n for (let t = 0; t < T; t++) {\n for (let d = 0; d < headDim; d++) {\n Q[t * headDim + d] = qkv.data[t * 3 * dModel + off + d];\n K[t * headDim + d] = qkv.data[t * 3 * dModel + dModel + off + d];\n V[t * headDim + d] =\n qkv.data[t * 3 * dModel + 2 * dModel + off + d];\n }\n }\n // Causal attention scores + softmax\n const attnW = new Float32Array(T * T);\n for (let i = 0; i < T; i++) {\n const scores = new Float32Array(T);\n let mx = -Infinity;\n for (let j = 0; j <= i; j++) {\n let s = 0;\n for (let d = 0; d < headDim; d++) {\n s += Q[i * headDim + d] * K[j * headDim + d];\n }\n scores[j] = s * scale;\n mx = Math.max(mx, scores[j]);\n }\n let s = 0;\n for (let j = 0; j <= i; j++) {\n attnW[i * T + j] = Math.exp(scores[j] - mx);\n s += attnW[i * T + j];\n }\n for (let j = 0; j <= i; j++) attnW[i * T + j] /= s;\n }\n // Weighted sum of values\n for (let i = 0; i < T; i++) {\n for (let d = 0; d < headDim; d++) {\n let s = 0;\n for (let j = 0; j <= i; j++) {\n s += attnW[i * T + j] * V[j * headDim + d];\n }\n attnOut.data[i * dModel + off + d] = s;\n }\n }\n headData.push({ Q, K, V, attnW, off });\n }\n attnOut._parents = [qkv];\n attnOut._backward = () => {\n for (const { Q, K, V, attnW, off } of headData) {\n const scale = 1 / Math.sqrt(headDim);\n const dAttnW = new Float32Array(T * T),\n dV = new Float32Array(T * headDim);\n for (let i = 0; i < T; i++) {\n for (let d = 0; d < headDim; d++) {\n const g = attnOut.grad[i * dModel + off + d];\n for (let j = 0; j <= i; j++) {\n dAttnW[i * T + j] += g * V[j * headDim + d];\n dV[j * headDim + d] += g * attnW[i * T + j];\n }\n }\n }\n const dQ = new Float32Array(T * headDim),\n dK = new Float32Array(T * headDim);\n for (let i = 0; i < T; i++) {\n let dot = 0;\n for (let j = 0; j <= i; j++) {\n dot += dAttnW[i * T + j] * attnW[i * T + j];\n }\n for (let j = 0; j <= i; j++) {\n const ds = attnW[i * T + j] * (dAttnW[i * T + j] - dot) * scale;\n for (let d = 0; d < headDim; d++) {\n dQ[i * headDim + d] += ds * K[j * headDim + d];\n dK[j * headDim + d] += ds * Q[i * headDim + d];\n }\n }\n }\n for (let t = 0; t < T; t++) {\n for (let d = 0; d < headDim; d++) {\n qkv.grad[t * 3 * dModel + off + d] += dQ[t * headDim + d];\n qkv.grad[t * 3 * dModel + dModel + off + d] +=\n dK[t * headDim + d];\n qkv.grad[t * 3 * dModel + 2 * dModel + off + d] +=\n dV[t * headDim + d];\n }\n }\n }\n };\n\n x = residualAdd(x, addBias(matmul(attnOut, wO), bO));\n\n // MLP with RMSNorm, ReLU (like Karpathy), no bias needed but keeping for simplicity\n const xNorm2 = rmsnorm(x);\n const wFF1 = params[pi++],\n bFF1 = params[pi++],\n wFF2 = params[pi++],\n bFF2 = params[pi++];\n x = residualAdd(\n x,\n addBias(matmul(relu(addBias(matmul(xNorm2, wFF1), bFF1)), wFF2), bFF2),\n );\n }\n\n // Weight-tied output head: logits = x @ tokEmb^T (reuse token embeddings)\n // tokEmb is [vocabSize, dModel], we need [T, dModel] @ [dModel, vocabSize] = [T, vocabSize]\n // So we need to transpose tokEmb. We'll do this manually.\n const Tout = tokens.length, V = vocabSize;\n const logits = zeros([Tout, V]);\n logits._parents = [x, tokEmb];\n for (let t = 0; t < Tout; t++) {\n for (let v = 0; v < V; v++) {\n let s = 0;\n for (let d = 0; d < dModel; d++) {\n s += x.data[t * dModel + d] * tokEmb.data[v * dModel + d];\n }\n logits.data[t * V + v] = s;\n }\n }\n logits._backward = () => {\n for (let t = 0; t < Tout; t++) {\n for (let v = 0; v < V; v++) {\n const g = logits.grad[t * V + v];\n for (let d = 0; d < dModel; d++) {\n x.grad[t * dModel + d] += g * tokEmb.data[v * dModel + d];\n tokEmb.grad[v * dModel + d] += g * x.data[t * dModel + d];\n }\n }\n }\n };\n return logits;\n }\n\n return {\n spec,\n initParams,\n forward,\n countParams: (p: Params) => p.reduce((s, t) => s + t.data.length, 0),\n };\n}\n\n// ---- Training (Karpathy-style: 1 doc per step, Adam optimizer) ----\ntype Trained = {\n tokenizer: ReturnType;\n model: Model;\n params: Params;\n};\n\nfunction trainNameModel(opts: {\n docs: string[];\n tokenizer: ReturnType;\n model: Model;\n steps: number;\n maxLen: number;\n seed: number;\n learningRate: number;\n}): Trained {\n const { docs, tokenizer, model, steps, maxLen, seed, learningRate } = opts;\n const params = model.initParams(seed);\n const numParams = model.countParams(params);\n console.log(`num params: ${numParams}`);\n\n // Adam optimizer state\n const beta1 = 0.9, beta2 = 0.95, epsAdam = 1e-8;\n const mBuf: Float32Array[] = params.map((p) =>\n new Float32Array(p.data.length)\n );\n const vBuf: Float32Array[] = params.map((p) =>\n new Float32Array(p.data.length)\n );\n\n // Shuffle docs with our seeded RNG\n const rng = makeRng(seed);\n const shuffled = [...docs];\n for (let i = shuffled.length - 1; i > 0; i--) {\n const j = Math.floor(rng() * (i + 1));\n [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];\n }\n\n const t0 = Date.now();\n for (let step = 0; step < steps; step++) {\n // Zero gradients\n for (const p of params) p.grad.fill(0);\n\n // Pick one doc, encode it: [BOS, ...chars, BOS]\n const doc = shuffled[step % shuffled.length];\n const allTokens = tokenizer.encode(doc); // already has BOS on both sides\n // Crop to maxLen\n const n = Math.min(maxLen, allTokens.length - 1);\n const input = allTokens.slice(0, n);\n const target = allTokens.slice(1, n + 1);\n\n // Forward + loss + backward\n const logits = model.forward(params, input);\n const loss = crossEntropyLoss(logits, target);\n loss.backward();\n\n // Adam update with linear LR decay\n const lrT = learningRate * (1 - step / steps);\n for (let pi = 0; pi < params.length; pi++) {\n const p = params[pi], m = mBuf[pi], v = vBuf[pi];\n for (let i = 0; i < p.data.length; i++) {\n m[i] = beta1 * m[i] + (1 - beta1) * p.grad[i];\n v[i] = beta2 * v[i] + (1 - beta2) * p.grad[i] * p.grad[i];\n const mHat = m[i] / (1 - Math.pow(beta1, step + 1));\n const vHat = v[i] / (1 - Math.pow(beta2, step + 1));\n p.data[i] -= lrT * mHat / (Math.sqrt(vHat) + epsAdam);\n }\n }\n\n if (step % 100 === 0 || step === steps - 1) {\n const elapsed = ((Date.now() - t0) / 1000).toFixed(1);\n console.log(\n `step ${String(step + 1).padStart(4)} / ${steps} | loss ${\n loss.data[0].toFixed(4)\n } | ${elapsed}s`,\n );\n }\n }\n\n return { tokenizer, model, params };\n}\n\n// ---- Sampling ----\nfunction generateName(\n opts: {\n trained: Trained;\n maxNewTokens: number;\n temperature: number;\n seed: number;\n },\n): string {\n const {\n trained: { tokenizer, model, params },\n maxNewTokens,\n temperature,\n seed,\n } = opts;\n const rng = makeRng(seed);\n let tokens = new Int32Array([tokenizer.BOS]);\n\n for (let i = 0; i < maxNewTokens; i++) {\n const logits = model.forward(params, tokens);\n const T = tokens.length, V = tokenizer.vocabSize;\n // Take logits from the last position, apply temperature\n const lp = new Float32Array(V);\n for (let v = 0; v < V; v++) {\n lp[v] = logits.data[(T - 1) * V + v] / temperature;\n }\n // Softmax\n let mx = -Infinity;\n for (let v = 0; v < V; v++) mx = Math.max(mx, lp[v]);\n let s = 0;\n for (let v = 0; v < V; v++) {\n lp[v] = Math.exp(lp[v] - mx);\n s += lp[v];\n }\n for (let v = 0; v < V; v++) lp[v] /= s;\n // Sample from distribution\n const r = rng();\n let cum = 0, next = 0;\n for (let v = 0; v < V; v++) {\n cum += lp[v];\n if (r < cum) {\n next = v;\n break;\n }\n }\n if (next === tokenizer.BOS) break; // BOS doubles as end-of-name\n const newTokens = new Int32Array(tokens.length + 1);\n newTokens.set(tokens);\n newTokens[tokens.length] = next;\n tokens = newTokens;\n }\n\n return tokenizer.decode(tokens);\n}\n\n// ============================================================================\n// main script\n// ============================================================================\n\nconst docs = await loadDocs();\nconsole.log(`num docs: ${docs.length}`);\n\nconst tok = buildCharTokenizerFromDocs(docs);\nconsole.log(`vocab size: ${tok.vocabSize}`);\n\nconst model = makeTransformerLanguageModel({\n vocabSize: tok.vocabSize, // 27 (a-z + BOS)\n maxLen: 8, // max sequence length (matches Karpathy's block_size)\n dModel: 16, // embedding dimension (matches Karpathy's n_embd)\n nHeads: 4, // attention heads (matches Karpathy's n_head)\n nLayers: 1, // transformer blocks (matches Karpathy's n_layer)\n dFF: 64, // feedforward hidden dim (4 * dModel, matches Karpathy)\n});\n\nconst trained = trainNameModel({\n docs,\n tokenizer: tok,\n model,\n steps: 1000, // same as Karpathy\n maxLen: 8, // crop sequences to block_size\n seed: 42, // same seed as Karpathy\n learningRate: 0.01, // same as Karpathy\n});\n\nconsole.log(\"\\n--- generation ---\");\nArray.from({ length: 20 }, (_, i) => i + 1)\n .map((i) =>\n generateName({\n trained,\n maxNewTokens: 8, // max name length = block_size\n temperature: 0.7, // balance between variety and quality\n seed: i * 7919, // spread seeds far apart for diversity (7919 is prime)\n })\n )\n .forEach((name, i) =>\n console.log(`sample ${String(i + 1).padStart(2, \" \")}: ${name}`)\n );","deleted",false,"label",[36],"96e8594a-0f44-11f1-9efe-42dde27851f2","README.md","6a68b658-0f44-11f1-889d-42dde27851f2",17,["D",1771692268905],1,"# microGPT.ts\n\nA TypeScript port of [Karpathy's microGPT](https://karpathy.github.io/2026/02/12/microgpt/) — but written in the style of [Conal Elliott's denotational design](https://github.com/conal/talk-2014-bayhac-denotational-design): types first, meanings first, implementation as a consequence.\n\nKarpathy's original microGPT is a single Python file that trains and inferences a GPT with zero dependencies. As he put it: *\"This is the full algorithmic content of what is needed. Everything else is just efficiency.\"* This port preserves that spirit but raises the level of abstraction — leaning into TypeScript's type system to make the *structure* of a language model legible, not just the *math*.\n\n## Denotational Design, Applied\n\nConal Elliott's core idea: give a simple mathematical meaning (denotation) for each type, then define operations as if they work on meanings, not representations. The implementation is free to differ for efficiency, but must be observationally equivalent to the denotation.\n\nHere, the \"meanings\" are:\n\n| Type | Denotation (meaning) |\n|---|---|\n| `Tensor` | A shaped array of scalars with attached gradient and backward function — i.e., a node in a computation graph |\n| `ModelSpec` | The *what* of a transformer: vocab size, dimensions, heads, layers — a pure description with no behavior |\n| `Model` | A triple of `(spec, initParams, forward)` — a model *is* its specification plus two functions |\n| `Trained` | A triple of `(tokenizer, model, params)` — a frozen snapshot: everything needed to generate |\n\nThe key move: `Model` is not a class with hidden state. It's a plain record of functions. `makeTransformerLanguageModel` takes a `ModelSpec` and returns a `Model` — a function from specification to behavior. This is the denotational design pattern: separate the *what* (spec) from the *how* (init + forward), and make the connection between them explicit and total.\n\n## Architecture\n\n```mermaid\ngraph TD\n subgraph \"Denotation Layer — the What\"\n SPEC[\"ModelSpec
{vocabSize, maxLen, dModel, nHeads, nLayers, dFF}\"]\n TOK[\"Tokenizer
encode: string → Int32Array
decode: Int32Array → string\"]\n TRAINED[\"Trained
{tokenizer, model, params}\"]\n end\n\n subgraph \"Autograd Engine — Tensor as computation graph node\"\n TENSOR[\"Tensor
data + grad + shape + _backward\"]\n MATMUL[matmul]\n ADDBIAS[addBias]\n RELU[relu]\n RESADD[residualAdd]\n EMBED[embedding]\n RMSNORM[rmsnorm]\n CELOSS[crossEntropyLoss]\n TENSOR --- MATMUL & ADDBIAS & RELU & RESADD & EMBED & RMSNORM & CELOSS\n end\n\n subgraph \"Model — the How\"\n MAKE[\"makeTransformerLanguageModel\"]\n INIT[\"initParams
seed → Params\"]\n FWD[\"forward
(params, tokens) → logits\"]\n MAKE --> INIT & FWD\n end\n\n subgraph \"Training Loop\"\n ADAM[\"Adam optimizer
β₁=0.9, β₂=0.95, linear LR decay\"]\n LOOP[\"For each step:
1. Pick doc
2. Encode
3. Forward
4. Loss
5. Backward
6. Update\"]\n end\n\n subgraph \"Inference\"\n GEN[\"generateName
autoregressive sampling
with temperature\"]\n end\n\n SPEC -->|\"defines\"| MAKE\n TOK -->|\"encodes docs\"| LOOP\n INIT -->|\"creates\"| LOOP\n FWD -->|\"used in\"| LOOP\n CELOSS -->|\"computes loss\"| LOOP\n ADAM -->|\"updates params\"| LOOP\n LOOP -->|\"produces\"| TRAINED\n TRAINED -->|\"consumed by\"| GEN\n FWD -->|\"used in\"| GEN\n```\n\n## Forward Pass Detail\n\nEach transformer layer follows the now-standard pre-norm pattern:\n\n```mermaid\ngraph TD\n X[Input x] --> NORM1[RMSNorm]\n NORM1 --> QKV[\"Linear → Q, K, V\"]\n QKV --> ATTN[\"Causal Multi-Head Attention
4 heads × 4-dim\"]\n ATTN --> PROJ[\"Output projection\"]\n PROJ --> ADD1[\"+ residual\"]\n X --> ADD1\n\n ADD1 --> NORM2[RMSNorm]\n NORM2 --> FF1[\"Linear → 64-dim\"]\n FF1 --> RELU2[ReLU]\n RELU2 --> FF2[\"Linear → 16-dim\"]\n FF2 --> ADD2[\"+ residual\"]\n ADD1 --> ADD2\n\n ADD2 --> LOGITS[\"Weight-tied logits
x @ tokEmb^T\"]\n```\n\n## The Seven Components\n\nFollowing Karpathy's decomposition — every LLM has exactly these parts, and nothing else:\n\n1. **Dataset** — ~32k names fetched from Karpathy's `makemore` repo\n2. **Tokenizer** — Character-level: 26 letters + 1 BOS/EOS token (vocab size 27)\n3. **Autograd** — Micrograd-style reverse-mode AD on flat `Float32Array` tensors\n4. **Architecture** — 1-layer GPT-2-style transformer (RMSNorm, causal attention, ReLU MLP, weight tying)\n5. **Loss** — Cross-entropy over next-token predictions\n6. **Optimizer** — Adam with bias correction and linear learning rate decay\n7. **Sampling** — Temperature-controlled autoregressive generation\n\n## Running\n\nThis is a [Val Town](https://val.town) script val. Run it directly — it will train for 1000 steps on CPU and generate 20 sample names:\n\n```\nnum docs: 32033\nvocab size: 27\nnum params: 4795\nstep 1 / 1000 | loss 3.5062 | 0.0s\nstep 101 / 1000 | loss 2.7573 | 1.2s\n...\nstep 1000 / 1000 | loss 2.2891 | 11.8s\n\n--- generation ---\nsample 1: malede\nsample 2: jara\nsample 3: kaylin\n...\n```\n\n## Hyperparameters\n\nMatched to Karpathy's defaults:\n\n| Parameter | Value | Notes |\n|---|---|---|\n| `dModel` | 16 | Embedding dimension |\n| `nHeads` | 4 | Attention heads (head dim = 4) |\n| `nLayers` | 1 | Transformer blocks |\n| `dFF` | 64 | FF hidden dim (4× `dModel`) |\n| `maxLen` | 8 | Context window |\n| `steps` | 1000 | Training iterations |\n| `learningRate` | 0.01 | With linear decay to 0 |\n| `seed` | 42 | Deterministic initialization |\n\n## Why is this ~60x faster than Karpathy's?\n\nKarpathy's Python version takes about **1 minute** on a MacBook. This TypeScript version runs in about **1 second**. Same algorithm, same hyperparameters, same 1000 steps. The difference comes down to one architectural decision in the autograd:\n\n### Karpathy: scalar `Value` objects\n\nKarpathy's autograd operates on **individual scalar numbers**, each wrapped in a `Value` object:\n\n```python\nclass Value:\n def __init__(self, data, children=(), local_grads=()):\n self.data = data # one float\n self.grad = 0 # one float\n self._children = children\n self._local_grads = local_grads\n```\n\nA 16-dim embedding lookup creates 16 `Value` objects. A matmul of `[7, 16] × [16, 48]` creates `7 × 48 = 336` new `Value` objects, each individually linked in the graph. Over a single forward pass, the computation graph grows to **thousands of `Value` nodes**, each requiring a Python object allocation, pointer tracking, and a topological sort during `.backward()`.\n\nThe graph must also be walked **one scalar at a time** during backprop. Python's interpreter overhead per operation is ~100ns, and there are tens of thousands of operations per step.\n\n### Ours: `Tensor` with fused backward closures\n\nOur autograd operates on **flat `Float32Array` buffers** with shape metadata:\n\n```typescript\nclass Tensor {\n data: Float32Array; // all scalars in one contiguous buffer\n grad: Float32Array; // same size, contiguous\n shape: Shape;\n _backward: () => void; // one closure for the whole operation\n _parents: Tensor[]; // typically 1-2 parents, not thousands\n}\n```\n\nA matmul of `[7, 16] × [16, 48]` creates **one Tensor node** in the graph, with a single backward closure that loops over all `7 × 48 × 16 = 5,376` multiply-adds in tight `for` loops. The computation graph for an entire forward pass has only **~30-40 nodes** (not thousands).\n\n### Where the speedup comes from\n\n| Factor | Karpathy (Python scalars) | Ours (TS tensors) |\n|---|---|---|\n| **Graph nodes per forward pass** | ~10,000+ `Value` objects | ~30-40 `Tensor` objects |\n| **Object allocation** | One Python object per scalar | One TypedArray per operation |\n| **Backward traversal** | Topological sort over ~10K nodes | Topological sort over ~30 nodes |\n| **Inner loop execution** | Python interpreter (~100ns/op) | V8 JIT-compiled tight loops (~1-2ns/op) |\n| **Memory layout** | Scattered Python `float` objects on heap | Contiguous `Float32Array` buffers |\n| **Per-operation overhead** | Dict lookup + pointer chasing per scalar | One closure call, then raw array math |\n\nThe dominant factor is **graph granularity**: Karpathy has one autograd node per scalar multiplication, we have one autograd node per matrix multiplication. This is the same distinction between micrograd (scalar autograd) and PyTorch (tensor autograd), just implemented within a single file.\n\nKarpathy is fully aware of this — his blog post says: *\"Production systems use tensors (large multi-dimensional arrays of numbers)... The math is identical, just corresponds to many scalars processed in parallel.\"* His choice of scalar `Value` was deliberate: it makes the chain rule visible at the most granular level. Our choice of tensor operations is equally deliberate: it shows that the same algorithm, lifted to operate on arrays, becomes fast enough to run in a second.\n\nThis is the denotational design lens: both implementations *denote* the same thing (a computation graph with reverse-mode AD), but the representations differ. Karpathy's representation makes the calculus legible. Ours makes the linear algebra legible.\n\n## Why TypeScript?\n\nKarpathy's Python version is the *irreducible essence* of a language model. This version asks: what if we took that essence and gave it more structure? TypeScript's interfaces (`ModelSpec`, `Model`, `Trained`) make the *architecture of the architecture* visible — you can see the separation of concerns that's implicit in the Python version.\n\nIn Conal Elliott's terms: the Python version is the implementation, this version tries to also show the denotation.\n",[58],{"_34":57,"_13":58,"_37":59,"_39":-5,"_40":41,"_42":60,"_44":45,"_46":71,"_17":62,"_49":24,"_51":63,"_53":54,"_55":-5,"_68":72,"_18":73},{"_34":35,"_13":36,"_37":38,"_39":-5,"_40":41,"_42":43,"_44":45,"_46":67,"_17":48,"_49":50,"_51":52,"_53":54,"_55":-5,"_68":69,"_18":70},["D",1771692182733],"children",[],[36],["D",1771692268905],[],[58],"9c1dc470-0f41-11f1-b466-42dde27851f2",["D",1771690989159],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","imageUrl","userId","a0bf3b31-15a5-4d5c-880e-4b1e22c9bc18","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","e81e7335-6d4c-4764-a902-d0962bbfa9f7","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_11":12,"_127":123},"org",{"_11":12,"_121":-5,"_122":123,"_83":84,"_124":125,"_126":54},"branch",{"_17":60,"_13":16,"_34":41,"_113":74,"_114":-5,"_115":-5,"_116":-5,"_117":-5,"_118":-5,"_119":14,"_120":12},"isOwned","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","displayName","avatar","https://images.clerk.dev/uploaded/img_2PqHa2Gsy93xQrjh2w78Xu0cChW.jpeg","tier","pro","canRestrictVals","profileImageUrl","adoptOrgId",{"_34":74,"_13":14,"_46":166,"_76":-5,"_77":54,"_78":79,"_80":79,"_81":19,"_82":19,"_83":84,"_85":-5,"_86":54,"_87":-5,"_88":-5,"_89":90,"_91":92,"_93":-5,"_94":-5,"_95":-5,"_96":-5,"_97":98,"_99":98,"_100":101,"_102":101,"_103":101,"_104":98,"_105":98,"_106":167,"_108":168,"_110":169,"_112":54},[160,161],"files",[160,161],"counts",{"_150":90,"_151":152,"_153":90,"_31":154,"_155":156},[146,147],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},{"_34":57,"_13":58,"_37":59,"_39":-5,"_40":41,"_42":60,"_44":45,"_46":61,"_17":62,"_49":24,"_51":63,"_53":54,"_55":-5,"_18":149},{"_34":35,"_13":36,"_37":38,"_39":-5,"_40":41,"_42":43,"_44":45,"_46":47,"_17":48,"_49":50,"_51":52,"_53":54,"_55":-5,"_18":148},[36],[58],"pulls","branches",2,"forks",{"_157":90,"_159":90},"connections",{"_157":90,"_158":90},"count","unfilledCount","emptyCount",{"_34":57,"_13":58,"_37":59,"_39":-5,"_40":41,"_42":60,"_44":45,"_46":71,"_17":62,"_49":24,"_51":63,"_53":54,"_55":-5,"_68":164,"_18":165},{"_34":35,"_13":36,"_37":38,"_39":-5,"_40":41,"_42":43,"_44":45,"_46":67,"_17":48,"_49":50,"_51":52,"_53":54,"_55":-5,"_68":162,"_18":163},[],[36],[],[58],["D",1771690989159],{"_11":12,"_127":123},{"_11":12,"_121":-5,"_122":123,"_83":84,"_124":125,"_126":54},{"_17":60,"_13":16,"_34":41,"_113":74,"_114":-5,"_115":-5,"_116":-5,"_117":-5,"_118":-5,"_119":14,"_120":12},"ctx",{"_106":-5,"_143":203,"_204":205,"_206":207,"_208":54,"_209":54,"_210":54,"_211":45,"_212":-5,"_213":-7,"_214":-7,"_215":90,"_216":54,"_217":54,"_218":-5,"_219":54,"_220":54,"_221":54,"_222":-5},"ENV",{"_174":175,"_176":177,"_178":16,"_179":180,"_181":182,"_183":184,"_185":186,"_187":188,"_189":190,"_191":192,"_193":194,"_195":196,"_197":198,"_199":200,"_201":202},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Code