[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_201":202,"_203":204},"routes/_app.x.$handle.$name",{"_16":17,"_20":21,"_84":65,"_85":31,"_63":-7,"_86":-5,"_13":87,"_88":89,"_11":90,"_91":92,"_93":94,"_95":-5,"_96":-5,"_97":98,"_99":26,"_100":101,"_102":26,"_103":-5,"_104":-5,"_105":106},"routes/_app.x.$handle.$name.search.$(branch)",{"_11":12,"_13":14,"_15":-5,"_16":17},"files",[],"project",{"_18":19,"_20":21,"_22":23,"_24":-5,"_25":26,"_27":28,"_29":28,"_30":31,"_32":31,"_33":34,"_35":-5,"_36":26,"_37":-5,"_38":-5,"_39":40,"_41":42,"_43":-5,"_44":-5,"_45":-5,"_46":-5,"_47":48,"_49":48,"_50":51,"_52":51,"_53":51,"_54":48,"_55":48,"_56":57,"_58":59,"_60":61,"_62":26},"query","handle","stevekrouse","id","9c1dc470-0f41-11f1-b466-42dde27851f2","name","microgpt","createdAt",["D",1771690989159],"deletedAt","legacyEnvVars",false,"privacy","public","httpPrivacy","description","","imageUrl","userId","a0bf3b31-15a5-4d5c-880e-4b1e22c9bc18","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","e81e7335-6d4c-4764-a902-d0962bbfa9f7","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_16":17,"_83":79},"org",{"_16":17,"_77":-5,"_78":79,"_33":34,"_80":81,"_82":26},"branch",{"_63":64,"_20":65,"_18":66,"_67":19,"_68":69,"_70":71,"_72":28,"_73":19,"_74":42,"_75":21,"_76":17},"isOwned","version",55,"karpathy-scalar-autograd","11405b5a-0f4c-11f1-8f75-42dde27851f2","projectId","forkedBranchId","9c1e152e-0f41-11f1-b466-42dde27851f2","forkedBranchVersion",17,"forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","displayName","avatar","https://images.clerk.dev/uploaded/img_2PqHa2Gsy93xQrjh2w78Xu0cChW.jpeg","tier","pro","canRestrictVals","profileImageUrl","branchName","path","adoptOrgId",{"_18":19,"_20":21,"_22":197,"_24":-5,"_25":26,"_27":28,"_29":28,"_30":31,"_32":31,"_33":34,"_35":-5,"_36":26,"_37":-5,"_38":-5,"_39":40,"_41":42,"_43":-5,"_44":-5,"_45":-5,"_46":-5,"_47":48,"_49":48,"_50":51,"_52":51,"_53":51,"_54":48,"_55":48,"_56":198,"_58":199,"_60":200,"_62":26},"tree",[176,177,178,179,180],[176,177,178,179,180],"counts",{"_166":40,"_167":141,"_168":40,"_169":170,"_171":172},"contents",[113,114,115,116,117],"file","upstreamProject","upstreamBranch",{"_18":69,"_22":107,"_108":109,"_20":110,"_67":19,"_63":71,"_68":-5,"_70":-5,"_111":112,"_24":-5,"_62":26,"_27":28,"_76":17,"_41":42,"_75":21},"emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},["D",1771690989159],"updatedAt",["D",1771692268905],"main","lockfileKey","1771692371179-esm.town_v_stevekrouse_microgpt@17-main",{"_18":158,"_20":159,"_120":160,"_122":-5,"_123":66,"_124":161,"_126":127,"_22":162,"_63":163,"_129":95,"_131":164,"_133":26,"_134":-5,"_85":165},{"_18":151,"_20":152,"_120":153,"_122":-5,"_123":66,"_124":64,"_126":127,"_22":154,"_63":155,"_129":130,"_131":156,"_133":26,"_134":-5,"_85":157},{"_18":144,"_20":145,"_120":146,"_122":-5,"_123":66,"_124":147,"_126":127,"_22":148,"_63":40,"_129":130,"_131":149,"_133":26,"_134":-5,"_85":150},{"_18":136,"_20":137,"_120":138,"_122":-5,"_123":66,"_124":139,"_126":127,"_22":140,"_63":141,"_129":130,"_131":142,"_133":26,"_134":-5,"_85":143},{"_18":118,"_20":119,"_120":121,"_122":-5,"_123":66,"_124":125,"_126":127,"_22":128,"_63":40,"_129":130,"_131":132,"_133":26,"_134":-5,"_85":135},"f1f30e5e-0f4c-11f1-89a5-42dde27851f2","value.ts","fileId","f1f30fda-0f4c-11f1-89a5-42dde27851f2","parentId","branchId","branchVersion",6,"latest",true,["D",1771695857629],"fileType","script","content","// ---- Value (Karpathy's micrograd-style scalar autograd) ----\n// Every scalar is a Value node in a dynamic computation graph.\n// This is Karpathy's approach: slow but maximally transparent.\nexport class Value {\n data: number;\n grad: number = 0;\n _backward: () => void = () => {};\n _prev: Value[];\n constructor(data: number, children: Value[] = []) {\n this.data = data;\n this._prev = children;\n }\n add(other: Value): Value {\n const out = new Value(this.data + other.data, [this, other]);\n out._backward = () => {\n this.grad += out.grad;\n other.grad += out.grad;\n };\n return out;\n }\n mul(other: Value): Value {\n const out = new Value(this.data * other.data, [this, other]);\n out._backward = () => {\n this.grad += other.data * out.grad;\n other.grad += this.data * out.grad;\n };\n return out;\n }\n pow(n: number): Value {\n const out = new Value(this.data ** n, [this]);\n out._backward = () => {\n this.grad += n * this.data ** (n - 1) * out.grad;\n };\n return out;\n }\n exp(): Value {\n const out = new Value(Math.exp(this.data), [this]);\n out._backward = () => {\n this.grad += out.data * out.grad;\n };\n return out;\n }\n log(): Value {\n const out = new Value(Math.log(this.data + 1e-10), [this]);\n out._backward = () => {\n this.grad += out.grad / (this.data + 1e-10);\n };\n return out;\n }\n relu(): Value {\n const out = new Value(this.data > 0 ? this.data : 0, [this]);\n out._backward = () => {\n this.grad += (this.data > 0 ? 1 : 0) * out.grad;\n };\n return out;\n }\n neg(): Value {\n const out = new Value(-this.data, [this]);\n out._backward = () => {\n this.grad += -out.grad;\n };\n return out;\n }\n backward() {\n const order: Value[] = [], visited = new Set();\n const topo = (t: Value) => {\n if (visited.has(t)) return;\n visited.add(t);\n for (const p of t._prev) topo(p);\n order.push(t);\n };\n topo(this);\n this.grad = 1;\n for (let i = order.length - 1; i >= 0; i--) order[i]._backward();\n }\n}\n","deleted","label",[119],"7c4545d0-0f4e-11f1-a65e-42dde27851f2","types.ts","da699e56-0f4c-11f1-982f-42dde27851f2",51,["D",1771696519176],2,"import type { Value } from \"./value.ts\";\nimport type { buildCharTokenizerFromDocs } from \"./tokenizer.ts\";\n\nexport type Params = Value[][];\n\nexport interface ModelSpec {\n vocabSize: number;\n maxLen: number;\n dModel: number;\n nHeads: number;\n nLayers: number;\n dFF: number;\n}\n\nexport interface Model {\n spec: ModelSpec;\n initParams: (seed: number) => Params;\n forward: (params: Params, tokens: Int32Array) => Value[][];\n countParams: (p: Params) => number;\n}\n\nexport type Trained = {\n tokenizer: ReturnType;\n model: Model;\n params: Params;\n};\n\nexport type TrainOpts = {\n docs: string[];\n tokenizer: ReturnType;\n model: Model;\n steps: number;\n maxLen: number;\n seed: number;\n learningRate: number;\n};\n\nexport type GenOpts = {\n trained: Trained;\n maxNewTokens: number;\n temperature: number;\n seed: number;\n};\n",[137],"f8accc26-0f4c-11f1-a8e3-42dde27851f2","tokenizer.ts","f8acccd0-0f4c-11f1-a8e3-42dde27851f2",7,["D",1771695868907],"// ---- Tokenizer ----\nexport function buildCharTokenizerFromDocs(docs: string[]) {\n const chars = [...new Set(docs.join(\"\"))].sort();\n const BOS = chars.length, vocabSize = chars.length + 1;\n const c2i = new Map(chars.map((c, i) => [c, i]));\n return {\n vocabSize,\n BOS,\n chars,\n encode(text: string): Int32Array {\n const ids = [BOS];\n for (const c of text) ids.push(c2i.get(c)!);\n ids.push(BOS);\n return new Int32Array(ids);\n },\n decode(tokens: Int32Array): string {\n let s = \"\";\n for (const t of tokens) if (t !== BOS && t < chars.length) s += chars[t];\n return s;\n },\n };\n}\n",[145],"9d9566f2-0f4e-11f1-9585-42dde27851f2","main.ts","019c8147-9f56-743d-9a85-5753000aa122",["D",1771696575074],48,"import { Value } from \"./value.ts\";\nimport { buildCharTokenizerFromDocs } from \"./tokenizer.ts\";\nimport type { GenOpts, Model, ModelSpec, Params, TrainOpts, Trained } from \"./types.ts\";\n\nfunction makeRng(seed: number) { // xorshift32\n let s = seed | 0 || 1;\n return () => {\n s ^= s << 13;\n s ^= s >> 17;\n s ^= s << 5;\n return (s >>> 0) / 4294967296;\n };\n}\n\nasync function loadDocs(): Promise {\n const text = await (await fetch(\"https://raw.githubusercontent.com/karpathy/makemore/refs/heads/master/names.txt\")).text();\n return text.trim().split(\"\\n\").map((s) => s.trim().toLowerCase()).filter((s) => s.length > 0);\n}\n\nfunction makeTransformerLanguageModel(spec: ModelSpec): Model {\n const { vocabSize, maxLen, dModel, nHeads, nLayers, dFF } = spec;\n const headDim = dModel / nHeads;\n\n function initParams(seed: number): Params {\n const rng = makeRng(seed), params: Params = [];\n const randn = (rows: number, cols: number, scale = 0.02) => {\n const arr = Array.from({ length: rows * cols }, () => {\n const u1 = rng() || 1e-10, u2 = rng();\n return new Value(Math.sqrt(-2 * Math.log(u1)) * Math.cos(6.2831853 * u2) * scale);\n });\n params.push(arr);\n return arr;\n };\n randn(vocabSize, dModel);\n randn(maxLen, dModel); // token embeddings, position embeddings\n for (let _l = 0; _l < nLayers; _l++) {\n randn(dModel, 3 * dModel);\n randn(1, 3 * dModel); // attention QKV weight + bias\n randn(dModel, dModel, 0);\n randn(1, dModel); // attention output projection (init to 0)\n randn(dModel, dFF);\n randn(1, dFF); // feedforward layer 1 weight + bias\n randn(dFF, dModel, 0);\n randn(1, dModel); // feedforward layer 2 (init to 0)\n }\n return params;\n }\n\n function getRow(w: Value[], cols: number, row: number): Value[] {\n return w.slice(row * cols, (row + 1) * cols);\n }\n\n function dot(a: Value[], b: Value[]): Value {\n let s = a[0].mul(b[0]);\n for (let i = 1; i < a.length; i++) s = s.add(a[i].mul(b[i]));\n return s;\n }\n\n // v[rows] @ w[rows, cols] -> result[cols]\n function vecmat(v: Value[], w: Value[], rows: number, cols: number): Value[] {\n const out: Value[] = [];\n for (let c = 0; c < cols; c++) {\n let s = v[0].mul(w[0 * cols + c]);\n for (let r = 1; r < rows; r++) {\n s = s.add(v[r].mul(w[r * cols + c]));\n }\n out.push(s);\n }\n return out;\n }\n\n function rmsnorm(v: Value[]): Value[] {\n let ms = v[0].mul(v[0]);\n for (let i = 1; i < v.length; i++) ms = ms.add(v[i].mul(v[i]));\n const scale = ms.mul(new Value(1 / v.length)).add(new Value(1e-5)).pow(-0.5);\n return v.map((x) => x.mul(scale));\n }\n\n function vadd(a: Value[], b: Value[]): Value[] {\n return a.map((x, i) => x.add(b[i]));\n }\n\n function forward(params: Params, tokens: Int32Array): Value[][] {\n let pi = 0;\n const tokEmb = params[pi++], posEmb = params[pi++], T = tokens.length;\n\n let x: Value[][] = [];\n for (let t = 0; t < T; t++)\n x.push(vadd(getRow(tokEmb, dModel, tokens[t]), getRow(posEmb, dModel, t % maxLen)));\n x = x.map(rmsnorm);\n\n for (let _l = 0; _l < nLayers; _l++) {\n const wQKV = params[pi++], bQKV = params[pi++];\n const wO = params[pi++], bO = params[pi++];\n\n const xNorm = x.map(rmsnorm);\n const qkv: Value[][] = [];\n for (let t = 0; t < T; t++) {\n qkv.push(vadd(vecmat(xNorm[t], wQKV, dModel, 3 * dModel), bQKV));\n }\n\n const attnOut: Value[][] = Array.from({ length: T }, () => new Array(dModel));\n\n for (let h = 0; h < nHeads; h++) {\n const off = h * headDim;\n const scaleVal = 1 / Math.sqrt(headDim);\n\n const Q: Value[][] = [], K: Value[][] = [], V: Value[][] = [];\n for (let t = 0; t < T; t++) {\n Q.push(qkv[t].slice(off, off + headDim));\n K.push(qkv[t].slice(dModel + off, dModel + off + headDim));\n V.push(qkv[t].slice(2 * dModel + off, 2 * dModel + off + headDim));\n }\n\n for (let i = 0; i < T; i++) { // causal attention\n const scores: Value[] = [];\n for (let j = 0; j <= i; j++) {\n scores.push(dot(Q[i], K[j]).mul(new Value(scaleVal)));\n }\n\n let mx = -Infinity;\n for (const s of scores) mx = Math.max(mx, s.data);\n const ex = scores.map((s) => s.add(new Value(-mx)).exp());\n const sum = ex.reduce((a, b) => a.add(b));\n const attnW = ex.map((e) => e.mul(sum.pow(-1)));\n\n for (let d = 0; d < headDim; d++)\n attnOut[i][off + d] = attnW.reduce((s, w, j) => s.add(w.mul(V[j][d])), new Value(0));\n }\n }\n\n for (let t = 0; t < T; t++) // output projection + residual\n x[t] = vadd(x[t], vadd(vecmat(attnOut[t], wO, dModel, dModel), bO));\n\n const wFF1 = params[pi++], bFF1 = params[pi++]; // MLP\n const wFF2 = params[pi++], bFF2 = params[pi++];\n const xNorm2 = x.map(rmsnorm);\n for (let t = 0; t < T; t++) {\n const h = vadd(vecmat(xNorm2[t], wFF1, dModel, dFF), bFF1).map((x) => x.relu());\n x[t] = vadd(x[t], vadd(vecmat(h, wFF2, dFF, dModel), bFF2));\n }\n }\n\n return x.map((xt) => // weight-tied logits: dot(x[t], tokEmb[v])\n Array.from({ length: vocabSize }, (_, v) => dot(xt, getRow(tokEmb, dModel, v)))\n );\n }\n\n return {\n spec,\n initParams,\n forward,\n countParams: (p: Params) => p.reduce((s, t) => s + t.length, 0),\n };\n}\n\nfunction crossEntropyLoss(logits: Value[][], targets: Int32Array): Value {\n let loss = new Value(0);\n for (let t = 0; t < logits.length; t++) {\n let mx = -Infinity;\n for (const v of logits[t]) mx = Math.max(mx, v.data);\n const ex = logits[t].map((v) => v.add(new Value(-mx)).exp());\n loss = loss.add(ex[targets[t]].mul(ex.reduce((a, b) => a.add(b)).pow(-1)).log().neg());\n }\n return loss.mul(new Value(1 / logits.length));\n}\n\nfunction trainNameModel(opts: TrainOpts): Trained {\n const { docs, tokenizer, model, steps, maxLen, seed, learningRate } = opts;\n const params = model.initParams(seed);\n const numParams = model.countParams(params);\n console.log(`num params: ${numParams}`);\n const beta1 = 0.9, beta2 = 0.95, epsAdam = 1e-8; // Adam\n const mBuf = params.map((p) => new Float64Array(p.length));\n const vBuf = params.map((p) => new Float64Array(p.length));\n const rng = makeRng(seed);\n const shuffled = [...docs];\n for (let i = shuffled.length - 1; i > 0; i--) {\n const j = Math.floor(rng() * (i + 1));\n [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];\n }\n\n const t0 = Date.now();\n for (let step = 0; step < steps; step++) {\n for (const p of params) for (const v of p) v.grad = 0; // zero grad\n const allTokens = tokenizer.encode(shuffled[step % shuffled.length]);\n const n = Math.min(maxLen, allTokens.length - 1);\n const logits = model.forward(params, allTokens.slice(0, n));\n const loss = crossEntropyLoss(logits, allTokens.slice(1, n + 1));\n loss.backward();\n const lrT = learningRate * (1 - step / steps); // Adam + LR decay\n for (let pi = 0; pi < params.length; pi++) {\n const p = params[pi], m = mBuf[pi], v = vBuf[pi];\n for (let i = 0; i < p.length; i++) {\n m[i] = beta1 * m[i] + (1 - beta1) * p[i].grad;\n v[i] = beta2 * v[i] + (1 - beta2) * p[i].grad * p[i].grad;\n const mHat = m[i] / (1 - Math.pow(beta1, step + 1));\n const vHat = v[i] / (1 - Math.pow(beta2, step + 1));\n p[i].data -= lrT * mHat / (Math.sqrt(vHat) + epsAdam);\n }\n }\n\n if (step % 100 === 0 || step === steps - 1) {\n const elapsed = ((Date.now() - t0) / 1000).toFixed(1);\n console.log(\n `step ${String(step + 1).padStart(4)} / ${steps} | loss ${\n loss.data.toFixed(4)\n } | ${elapsed}s`,\n );\n }\n }\n\n return { tokenizer, model, params };\n}\n\nfunction generateName(opts: GenOpts): string {\n const { trained: { tokenizer, model, params }, maxNewTokens, temperature, seed } = opts;\n const rng = makeRng(seed);\n let tokens = new Int32Array([tokenizer.BOS]);\n\n for (let i = 0; i < maxNewTokens; i++) {\n const logits = model.forward(params, tokens);\n const T = tokens.length, V = tokenizer.vocabSize;\n const lp = new Float64Array(V);\n for (let v = 0; v < V; v++) lp[v] = logits[T - 1][v].data / temperature;\n let mx = -Infinity;\n for (let v = 0; v < V; v++) mx = Math.max(mx, lp[v]);\n let s = 0;\n for (let v = 0; v < V; v++) { lp[v] = Math.exp(lp[v] - mx); s += lp[v]; }\n for (let v = 0; v < V; v++) lp[v] /= s;\n const r = rng();\n let cum = 0, next = 0;\n for (let v = 0; v < V; v++) { cum += lp[v]; if (r < cum) { next = v; break; } }\n if (next === tokenizer.BOS) break;\n const newTokens = new Int32Array(tokens.length + 1);\n newTokens.set(tokens);\n newTokens[tokens.length] = next;\n tokens = newTokens;\n }\n\n return tokenizer.decode(tokens);\n}\n\n// ============================================================================\n// main script\n// ============================================================================\n\nconst docs = await loadDocs();\nconsole.log(`num docs: ${docs.length}`);\n\nconst tok = buildCharTokenizerFromDocs(docs);\nconsole.log(`vocab size: ${tok.vocabSize}`);\n\nconst model = makeTransformerLanguageModel({\n vocabSize: tok.vocabSize, // 27 (a-z + BOS)\n maxLen: 8, // max sequence length (matches Karpathy's block_size)\n dModel: 16, // embedding dimension (matches Karpathy's n_embd)\n nHeads: 4, // attention heads (matches Karpathy's n_head)\n nLayers: 1, // transformer blocks (matches Karpathy's n_layer)\n dFF: 64, // feedforward hidden dim (4 * dModel, matches Karpathy)\n});\n\nconst trained = trainNameModel({\n docs,\n tokenizer: tok,\n model,\n steps: 1000, // same as Karpathy\n maxLen: 8, // crop sequences to block_size\n seed: 42, // same seed as Karpathy\n learningRate: 0.01, // same as Karpathy\n});\n\nconsole.log(\"\\n--- generation ---\");\nArray.from({ length: 20 }, (_, i) => i + 1)\n .map((i) =>\n generateName({\n trained,\n maxNewTokens: 8, // max name length = block_size\n temperature: 0.7, // balance between variety and quality\n seed: i * 7919, // spread seeds far apart for diversity (7919 is prime)\n })\n )\n .forEach((name, i) =>\n console.log(`sample ${String(i + 1).padStart(2, \" \")}: ${name}`)\n );",[152],"60b5b608-0f4c-11f1-a98e-42dde27851f2","README.md","019c8147-9f56-743d-9a85-51574f3e6a91",3,["D",1771695613949],1,"# microGPT.ts (scalar autograd edition)\n\nA direct TypeScript port of [Karpathy's microGPT](https://karpathy.github.io/2026/02/12/microgpt/) — faithful to the original's scalar `Value`-based autograd, not optimized for speed.\n\nKarpathy's microGPT is a single Python file that trains and inferences a GPT with zero dependencies. As he put it: *\"This is the full algorithmic content of what is needed. Everything else is just efficiency.\"* This port preserves that spirit **at the same level of abstraction** — every scalar is a `Value` node in a dynamically-built computation graph, just like the original.\n\n## Scalar Autograd: Same as Karpathy\n\nThe core design choice: **every number is a `Value` object**.\n\n```typescript\nclass Value {\n data: number; // one scalar\n grad: number; // one scalar\n _backward: () => void;\n _prev: Value[]; // parent nodes in the graph\n}\n```\n\nA 16-dim embedding lookup creates 16 `Value` objects. A vector-matrix multiply of `[16] × [16, 48]` creates 48 dot products, each made of 16 multiplies and 15 adds — **over 1,000 new `Value` nodes**. A single forward pass builds a computation graph with **tens of thousands of nodes**, each individually linked and walked during `.backward()`.\n\nThis is the same approach as Karpathy's Python `Value` class. It makes the chain rule visible at every single scalar operation.\n\n## Architecture\n\nSame GPT-2-style transformer as the original:\n\n```mermaid\ngraph TD\n X[Input x] --> NORM1[RMSNorm]\n NORM1 --> QKV[\"Linear → Q, K, V\"]\n QKV --> ATTN[\"Causal Multi-Head Attention
4 heads × 4-dim\"]\n ATTN --> PROJ[\"Output projection\"]\n PROJ --> ADD1[\"+ residual\"]\n X --> ADD1\n\n ADD1 --> NORM2[RMSNorm]\n NORM2 --> FF1[\"Linear → 64-dim\"]\n FF1 --> RELU2[ReLU]\n RELU2 --> FF2[\"Linear → 16-dim\"]\n FF2 --> ADD2[\"+ residual\"]\n ADD1 --> ADD2\n\n ADD2 --> LOGITS[\"Weight-tied logits
dot(x, tokEmb)\"]\n```\n\n## The Seven Components\n\nFollowing Karpathy's decomposition:\n\n1. **Dataset** — ~32k names fetched from Karpathy's `makemore` repo\n2. **Tokenizer** — Character-level: 26 letters + 1 BOS/EOS token (vocab size 27)\n3. **Autograd** — Scalar `Value` class with `add`, `mul`, `pow`, `exp`, `log`, `relu` — each operation creates a node in the graph\n4. **Architecture** — 1-layer GPT-2-style transformer (RMSNorm, causal attention, ReLU MLP, weight tying)\n5. **Loss** — Cross-entropy over next-token predictions\n6. **Optimizer** — Adam with bias correction and linear learning rate decay\n7. **Sampling** — Temperature-controlled autoregressive generation\n\n## Running\n\nThis is a [Val Town](https://val.town) script val. It trains for 1000 steps on CPU. Because every scalar is its own autograd node, it's **much slower** than a tensor-based implementation (~60x slower) — but the code is simpler and more transparent:\n\n```\nnum docs: 32033\nvocab size: 27\nnum params: 4795\nstep 1 / 1000 | loss 3.5062 | 0.0s\nstep 101 / 1000 | loss 2.7573 | ...s\n...\nstep 1000 / 1000 | loss 2.2891 | ...s\n\n--- generation ---\nsample 1: malede\nsample 2: jara\n...\n```\n\n## Hyperparameters\n\nMatched to Karpathy's defaults:\n\n| Parameter | Value | Notes |\n|---|---|---|\n| `dModel` | 16 | Embedding dimension |\n| `nHeads` | 4 | Attention heads (head dim = 4) |\n| `nLayers` | 1 | Transformer blocks |\n| `dFF` | 64 | FF hidden dim (4× `dModel`) |\n| `maxLen` | 8 | Context window |\n| `steps` | 1000 | Training iterations |\n| `learningRate` | 0.01 | With linear decay to 0 |\n| `seed` | 42 | Deterministic initialization |\n\n## Why is this slow?\n\nThis version uses **scalar autograd** — the same approach as Karpathy's original Python. Every addition, multiplication, and activation creates a new `Value` object in the computation graph. A single forward pass creates tens of thousands of nodes, and `backward()` must topologically sort and walk all of them.\n\nThe [main branch](https://www.val.town/x/stevekrouse/microgpt) uses **tensor autograd** instead — a single `matmul` operation creates one graph node with a fused backward closure that loops over all scalars in tight `for` loops. Same math, ~60x faster, but the individual scalar chain rule is hidden inside the closure.\n\n| Factor | This version (scalar Values) | Main branch (Tensors) |\n|---|---|---|\n| **Graph nodes per forward pass** | ~10,000+ `Value` objects | ~30-40 `Tensor` objects |\n| **Object allocation** | One object per scalar op | One TypedArray per tensor op |\n| **Backward traversal** | Topological sort over ~10K nodes | Topological sort over ~30 nodes |\n| **Inner loop execution** | V8 JIT over many small closures | V8 JIT over tight array loops |\n\nKarpathy's choice of scalar `Value` was deliberate: it makes the chain rule visible at the most granular level. This port preserves that choice.\n\n## Why TypeScript?\n\nTypeScript's type system makes the structure visible: `ModelSpec` describes *what* the model is, `Model` is the *behavior*, `Trained` is a frozen snapshot. But the autograd is the same scalar-level approach as the Python original — no tensor tricks, no fused kernels, just `Value.add()` and `Value.mul()` all the way down.\n",[159],"pulls","branches","forks","envVars",{"_173":40,"_175":40},"connections",{"_173":40,"_174":40},"count","unfilledCount","emptyCount",{"_18":158,"_20":159,"_120":160,"_122":-5,"_123":66,"_124":161,"_126":127,"_22":194,"_63":163,"_129":95,"_131":164,"_133":26,"_134":-5,"_182":195,"_85":196},{"_18":151,"_20":152,"_120":153,"_122":-5,"_123":66,"_124":64,"_126":127,"_22":191,"_63":155,"_129":130,"_131":156,"_133":26,"_134":-5,"_182":192,"_85":193},{"_18":144,"_20":145,"_120":146,"_122":-5,"_123":66,"_124":147,"_126":127,"_22":188,"_63":40,"_129":130,"_131":149,"_133":26,"_134":-5,"_182":189,"_85":190},{"_18":136,"_20":137,"_120":138,"_122":-5,"_123":66,"_124":139,"_126":127,"_22":185,"_63":141,"_129":130,"_131":142,"_133":26,"_134":-5,"_182":186,"_85":187},{"_18":118,"_20":119,"_120":121,"_122":-5,"_123":66,"_124":125,"_126":127,"_22":181,"_63":40,"_129":130,"_131":132,"_133":26,"_134":-5,"_182":183,"_85":184},["D",1771695857629],"children",[],[119],["D",1771696519176],[],[137],["D",1771695868907],[],[145],["D",1771696575074],[],[152],["D",1771695613949],[],[159],["D",1771690989159],{"_16":17,"_83":79},{"_16":17,"_77":-5,"_78":79,"_33":34,"_80":81,"_82":26},{"_63":64,"_20":65,"_18":66,"_67":19,"_68":69,"_70":71,"_72":28,"_73":19,"_74":42,"_75":21,"_76":17},"ctx",{"_56":-5,"_104":234,"_235":236,"_237":238,"_239":26,"_240":26,"_241":26,"_242":127,"_243":-5,"_244":-7,"_245":-7,"_246":40,"_247":26,"_248":26,"_249":-5,"_250":26,"_251":26,"_252":26,"_253":-5},"ENV",{"_205":206,"_207":208,"_209":110,"_210":211,"_212":213,"_214":215,"_216":217,"_218":219,"_220":221,"_222":223,"_224":225,"_226":227,"_228":229,"_230":231,"_232":233},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Search