[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_201":202,"_203":204},"routes/_app.x.$handle.$name",{"_101":100,"_37":49,"_109":89,"_32":57,"_13":14,"_110":-5,"_11":111,"_112":113,"_114":115,"_116":117,"_118":119,"_26":-5,"_120":-5,"_121":122,"_123":30,"_124":125,"_126":30,"_127":-5,"_128":-5,"_129":130},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":18,"_19":-5},"project",{"_47":48,"_37":49,"_42":50,"_51":-5,"_52":30,"_53":54,"_55":54,"_56":57,"_58":57,"_19":59,"_60":-5,"_61":30,"_62":-5,"_63":-5,"_64":65,"_66":67,"_68":-5,"_69":-5,"_70":-5,"_71":-5,"_72":73,"_74":73,"_75":76,"_77":76,"_78":76,"_79":73,"_80":73,"_81":82,"_83":84,"_85":86,"_87":30},"version",6,"changes",[25],"aiChatInfo",{"_20":21,"_22":-5,"_23":-5,"_24":-5},"userId","aiChatMessageId","d90a4c24-2130-45a1-8139-600d86cc501e","aiChatThreadId","aiChatThreadUserId","aiChatThreadTitle",{"_26":27,"_28":-7,"_29":30,"_31":30,"_32":33,"_34":33},"file",{"_35":36,"_37":33,"_38":39,"_40":41,"_42":43,"_44":14,"_45":46,"_29":30},"previous","deleted",false,"deletedHere","path","value.ts","latestPath","fileId","f1f30fda-0f4c-11f1-89a5-42dde27851f2","name","fileType","script","content","// ---- Value (Karpathy's micrograd-style scalar autograd) ----\n// Every scalar is a Value node in a dynamic computation graph.\n// This is Karpathy's approach: slow but maximally transparent.\nexport class Value {\n data: number;\n grad: number = 0;\n _backward: () => void = () => {};\n _prev: Value[];\n constructor(data: number, children: Value[] = []) {\n this.data = data;\n this._prev = children;\n }\n add(other: Value): Value {\n const out = new Value(this.data + other.data, [this, other]);\n out._backward = () => {\n this.grad += out.grad;\n other.grad += out.grad;\n };\n return out;\n }\n mul(other: Value): Value {\n const out = new Value(this.data * other.data, [this, other]);\n out._backward = () => {\n this.grad += other.data * out.grad;\n other.grad += this.data * out.grad;\n };\n return out;\n }\n pow(n: number): Value {\n const out = new Value(this.data ** n, [this]);\n out._backward = () => {\n this.grad += n * this.data ** (n - 1) * out.grad;\n };\n return out;\n }\n exp(): Value {\n const out = new Value(Math.exp(this.data), [this]);\n out._backward = () => {\n this.grad += out.data * out.grad;\n };\n return out;\n }\n log(): Value {\n const out = new Value(Math.log(this.data + 1e-10), [this]);\n out._backward = () => {\n this.grad += out.grad / (this.data + 1e-10);\n };\n return out;\n }\n relu(): Value {\n const out = new Value(this.data > 0 ? this.data : 0, [this]);\n out._backward = () => {\n this.grad += (this.data > 0 ? 1 : 0) * out.grad;\n };\n return out;\n }\n neg(): Value {\n const out = new Value(-this.data, [this]);\n out._backward = () => {\n this.grad += -out.grad;\n };\n return out;\n }\n backward() {\n const order: Value[] = [], visited = new Set();\n const topo = (t: Value) => {\n if (visited.has(t)) return;\n visited.add(t);\n for (const p of t._prev) topo(p);\n order.push(t);\n };\n topo(this);\n this.grad = 1;\n for (let i = order.length - 1; i >= 0; i--) order[i]._backward();\n }\n}\n","createdAt",["D",1771695857629],"branchVersion","branchId","11405b5a-0f4c-11f1-8f75-42dde27851f2","id","9c1dc470-0f41-11f1-b466-42dde27851f2","microgpt",["D",1771690989159],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","","imageUrl","a0bf3b31-15a5-4d5c-880e-4b1e22c9bc18","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","e81e7335-6d4c-4764-a902-d0962bbfa9f7","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_101":100,"_108":104},"org",{"_101":100,"_102":-5,"_103":104,"_19":59,"_105":106,"_107":30},"branch",{"_13":88,"_37":89,"_47":46,"_90":48,"_91":92,"_93":94,"_95":54,"_96":48,"_97":67,"_98":49,"_99":100},"isOwned",55,"karpathy-scalar-autograd","projectId","forkedBranchId","9c1e152e-0f41-11f1-b466-42dde27851f2","forkedBranchVersion",17,"forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","stevekrouse","handle","displayName","avatar","https://images.clerk.dev/uploaded/img_2PqHa2Gsy93xQrjh2w78Xu0cChW.jpeg","tier","pro","canRestrictVals","profileImageUrl","branchName","adoptOrgId",{"_47":48,"_37":49,"_42":197,"_51":-5,"_52":30,"_53":54,"_55":54,"_56":57,"_58":57,"_19":59,"_60":-5,"_61":30,"_62":-5,"_63":-5,"_64":65,"_66":67,"_68":-5,"_69":-5,"_70":-5,"_71":-5,"_72":73,"_74":73,"_75":76,"_77":76,"_78":76,"_79":73,"_80":73,"_81":198,"_83":199,"_85":200,"_87":30},"tree",[180,181,182,183],"files",[180,181,182,183],"counts",{"_170":65,"_171":159,"_172":65,"_173":174,"_175":176},"contents",[137,138,139,140],"upstreamProject","upstreamBranch",{"_47":92,"_42":131,"_132":133,"_37":134,"_90":48,"_13":94,"_91":-5,"_93":-5,"_135":136,"_51":-5,"_87":30,"_53":54,"_99":100,"_66":67,"_98":49},"emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{},["D",1771690989159],"updatedAt",["D",1771692268905],"main","lockfileKey","1771692371179-esm.town_v_stevekrouse_microgpt@17-main",{"_47":163,"_37":164,"_35":165,"_142":-5,"_45":46,"_44":166,"_143":144,"_42":167,"_13":153,"_38":26,"_40":168,"_29":30,"_146":-5,"_32":169},{"_47":156,"_37":157,"_35":158,"_142":-5,"_45":46,"_44":159,"_143":30,"_42":160,"_13":153,"_38":39,"_40":161,"_29":30,"_146":-5,"_32":162},{"_47":148,"_37":149,"_35":150,"_142":-5,"_45":46,"_44":151,"_143":30,"_42":152,"_13":153,"_38":39,"_40":154,"_29":30,"_146":-5,"_32":155},{"_47":141,"_37":33,"_35":36,"_142":-5,"_45":46,"_44":14,"_143":144,"_42":145,"_13":65,"_38":39,"_40":41,"_29":30,"_146":-5,"_32":147},"f1f30e5e-0f4c-11f1-89a5-42dde27851f2","parentId","latest",true,["D",1771695857629],"label",[33],"e480123a-0f4c-11f1-93b8-42dde27851f2","types.ts","da699e56-0f4c-11f1-982f-42dde27851f2",5,["D",1771695835044],1,"import type { Value } from \"./value.ts\";\nimport type { buildCharTokenizerFromDocs } from \"./tokenizer.ts\";\n\nexport type Params = Value[][];\n\nexport interface ModelSpec {\n vocabSize: number;\n maxLen: number;\n dModel: number;\n nHeads: number;\n nLayers: number;\n dFF: number;\n}\n\nexport interface Model {\n spec: ModelSpec;\n initParams: (seed: number) => Params;\n forward: (params: Params, tokens: Int32Array) => Value[][];\n countParams: (p: Params) => number;\n}\n\nexport type Trained = {\n tokenizer: ReturnType;\n model: Model;\n params: Params;\n};\n",[149],"48f7423e-0f4c-11f1-84dc-42dde27851f2","main.ts","019c8147-9f56-743d-9a85-5753000aa122",2,["D",1771695574100],"// ---- Value (Karpathy's micrograd-style scalar autograd) ----\n// Every scalar is a Value node in a dynamic computation graph.\n// This is Karpathy's approach: slow but maximally transparent.\nclass Value {\n data: number;\n grad: number = 0;\n _backward: () => void = () => {};\n _prev: Value[];\n constructor(data: number, children: Value[] = []) {\n this.data = data;\n this._prev = children;\n }\n add(other: Value): Value {\n const out = new Value(this.data + other.data, [this, other]);\n out._backward = () => {\n this.grad += out.grad;\n other.grad += out.grad;\n };\n return out;\n }\n mul(other: Value): Value {\n const out = new Value(this.data * other.data, [this, other]);\n out._backward = () => {\n this.grad += other.data * out.grad;\n other.grad += this.data * out.grad;\n };\n return out;\n }\n pow(n: number): Value {\n const out = new Value(this.data ** n, [this]);\n out._backward = () => {\n this.grad += n * this.data ** (n - 1) * out.grad;\n };\n return out;\n }\n exp(): Value {\n const out = new Value(Math.exp(this.data), [this]);\n out._backward = () => {\n this.grad += out.data * out.grad;\n };\n return out;\n }\n log(): Value {\n const out = new Value(Math.log(this.data + 1e-10), [this]);\n out._backward = () => {\n this.grad += out.grad / (this.data + 1e-10);\n };\n return out;\n }\n relu(): Value {\n const out = new Value(this.data > 0 ? this.data : 0, [this]);\n out._backward = () => {\n this.grad += (this.data > 0 ? 1 : 0) * out.grad;\n };\n return out;\n }\n neg(): Value {\n const out = new Value(-this.data, [this]);\n out._backward = () => {\n this.grad += -out.grad;\n };\n return out;\n }\n backward() {\n const order: Value[] = [], visited = new Set();\n const topo = (t: Value) => {\n if (visited.has(t)) return;\n visited.add(t);\n for (const p of t._prev) topo(p);\n order.push(t);\n };\n topo(this);\n this.grad = 1;\n for (let i = order.length - 1; i >= 0; i--) order[i]._backward();\n }\n}\n\n// ---- Seeded RNG (xorshift32) ----\nfunction makeRng(seed: number) {\n let s = seed | 0 || 1;\n return () => {\n s ^= s << 13;\n s ^= s >> 17;\n s ^= s << 5;\n return (s >>> 0) / 4294967296;\n };\n}\n\n// ---- Dataset (fetch from Karpathy's repo) ----\nasync function loadDocs(): Promise {\n const resp = await fetch(\n \"https://raw.githubusercontent.com/karpathy/makemore/refs/heads/master/names.txt\",\n );\n const text = await resp.text();\n return text.trim().split(\"\\n\").map((s) => s.trim().toLowerCase()).filter(\n (s) => s.length > 0\n );\n}\n\n// ---- Tokenizer ----\nfunction buildCharTokenizerFromDocs(docs: string[]) {\n const chars = [...new Set(docs.join(\"\"))].sort();\n const BOS = chars.length, vocabSize = chars.length + 1;\n const c2i = new Map(chars.map((c, i) => [c, i]));\n return {\n vocabSize,\n BOS,\n chars,\n encode(text: string): Int32Array {\n const ids = [BOS];\n for (const c of text) ids.push(c2i.get(c)!);\n ids.push(BOS);\n return new Int32Array(ids);\n },\n decode(tokens: Int32Array): string {\n let s = \"\";\n for (const t of tokens) if (t !== BOS && t < chars.length) s += chars[t];\n return s;\n },\n };\n}\n\n// ---- Model ----\n// Params is a flat list of Value[][] (2D weight matrices / 1D bias vectors stored as Value[])\ntype Params = Value[][];\ninterface ModelSpec {\n vocabSize: number;\n maxLen: number;\n dModel: number;\n nHeads: number;\n nLayers: number;\n dFF: number;\n}\ninterface Model {\n spec: ModelSpec;\n initParams: (seed: number) => Params;\n forward: (params: Params, tokens: Int32Array) => Value[][];\n countParams: (p: Params) => number;\n}\n\nfunction makeTransformerLanguageModel(spec: ModelSpec): Model {\n const { vocabSize, maxLen, dModel, nHeads, nLayers, dFF } = spec;\n const headDim = dModel / nHeads;\n\n function initParams(seed: number): Params {\n const rng = makeRng(seed), params: Params = [];\n const randn = (rows: number, cols: number, scale = 0.02): Value[] => {\n const arr: Value[] = [];\n for (let i = 0; i < rows * cols; i++) {\n const u1 = rng() || 1e-10, u2 = rng();\n arr.push(\n new Value(\n Math.sqrt(-2 * Math.log(u1)) * Math.cos(6.2831853 * u2) * scale,\n ),\n );\n }\n params.push(arr);\n return arr;\n };\n randn(vocabSize, dModel);\n randn(maxLen, dModel); // token embeddings, position embeddings\n for (let _l = 0; _l < nLayers; _l++) {\n randn(dModel, 3 * dModel);\n randn(1, 3 * dModel); // attention QKV weight + bias\n randn(dModel, dModel, 0);\n randn(1, dModel); // attention output projection (init to 0)\n randn(dModel, dFF);\n randn(1, dFF); // feedforward layer 1 weight + bias\n randn(dFF, dModel, 0);\n randn(1, dModel); // feedforward layer 2 (init to 0)\n }\n return params;\n }\n\n // Helper: row of a weight matrix stored as flat Value[]\n function getRow(w: Value[], cols: number, row: number): Value[] {\n return w.slice(row * cols, (row + 1) * cols);\n }\n\n // Dot product of two Value arrays\n function dot(a: Value[], b: Value[]): Value {\n let s = a[0].mul(b[0]);\n for (let i = 1; i < a.length; i++) s = s.add(a[i].mul(b[i]));\n return s;\n }\n\n // Matrix-vector multiply: w[rows, cols] @ v[cols] -> result[rows]\n function matvec(w: Value[], rows: number, cols: number, v: Value[]): Value[] {\n const out: Value[] = [];\n for (let r = 0; r < rows; r++) {\n out.push(dot(getRow(w, cols, r), v));\n }\n return out;\n }\n\n // Vector-matrix multiply: v[rows] @ w[rows, cols] -> result[cols]\n function vecmat(v: Value[], w: Value[], rows: number, cols: number): Value[] {\n const out: Value[] = [];\n for (let c = 0; c < cols; c++) {\n let s = v[0].mul(w[0 * cols + c]);\n for (let r = 1; r < rows; r++) {\n s = s.add(v[r].mul(w[r * cols + c]));\n }\n out.push(s);\n }\n return out;\n }\n\n // Add bias vector to a vector\n function addBias(v: Value[], b: Value[]): Value[] {\n return v.map((x, i) => x.add(b[i]));\n }\n\n // ReLU on a vector\n function reluVec(v: Value[]): Value[] {\n return v.map((x) => x.relu());\n }\n\n // RMSNorm on a vector\n function rmsnormVec(v: Value[]): Value[] {\n const D = v.length;\n let ms = v[0].mul(v[0]);\n for (let i = 1; i < D; i++) ms = ms.add(v[i].mul(v[i]));\n // ms = mean of squares + eps\n const eps = new Value(1e-5);\n const scale = ms.mul(new Value(1 / D)).add(eps).pow(-0.5);\n return v.map((x) => x.mul(scale));\n }\n\n // Embedding lookup: return the row of the embedding table\n function embeddingLookup(table: Value[], D: number, idx: number): Value[] {\n return getRow(table, D, idx);\n }\n\n // Add two vectors elementwise\n function addVec(a: Value[], b: Value[]): Value[] {\n return a.map((x, i) => x.add(b[i]));\n }\n\n function forward(params: Params, tokens: Int32Array): Value[][] {\n let pi = 0;\n const tokEmb = params[pi++], posEmb = params[pi++], T = tokens.length;\n\n // Build sequence of hidden states: x[t] is a Value[] of length dModel\n let x: Value[][] = [];\n for (let t = 0; t < T; t++) {\n const te = embeddingLookup(tokEmb, dModel, tokens[t]);\n const pe = embeddingLookup(posEmb, dModel, t % maxLen);\n x.push(addVec(te, pe));\n }\n // Initial RMSNorm\n x = x.map((v) => rmsnormVec(v));\n\n for (let _l = 0; _l < nLayers; _l++) {\n const wQKV = params[pi++], bQKV = params[pi++];\n const wO = params[pi++], bO = params[pi++];\n\n // RMSNorm before attention\n const xNorm: Value[][] = x.map((v) => rmsnormVec(v));\n\n // Compute QKV for each position: xNorm[t] @ wQKV + bQKV -> [3*dModel]\n const qkv: Value[][] = [];\n for (let t = 0; t < T; t++) {\n qkv.push(addBias(vecmat(xNorm[t], wQKV, dModel, 3 * dModel), bQKV));\n }\n\n // Multi-head causal attention\n const attnOut: Value[][] = Array.from({ length: T }, () =>\n new Array(dModel).fill(null)\n );\n\n for (let h = 0; h < nHeads; h++) {\n const off = h * headDim;\n const scaleVal = 1 / Math.sqrt(headDim);\n\n // Extract Q, K, V for this head: each is Value[][headDim]\n const Q: Value[][] = [], K: Value[][] = [], V: Value[][] = [];\n for (let t = 0; t < T; t++) {\n Q.push(qkv[t].slice(off, off + headDim));\n K.push(qkv[t].slice(dModel + off, dModel + off + headDim));\n V.push(qkv[t].slice(2 * dModel + off, 2 * dModel + off + headDim));\n }\n\n // Causal attention\n for (let i = 0; i < T; i++) {\n // Compute attention scores for position i\n const scores: Value[] = [];\n for (let j = 0; j <= i; j++) {\n scores.push(dot(Q[i], K[j]).mul(new Value(scaleVal)));\n }\n\n // Softmax (numerically stable)\n let mx = -Infinity;\n for (const s of scores) mx = Math.max(mx, s.data);\n const expScores = scores.map((s) => s.add(new Value(-mx)).exp());\n let sumExp = expScores[0];\n for (let j = 1; j < expScores.length; j++) {\n sumExp = sumExp.add(expScores[j]);\n }\n const attnW = expScores.map((e) => e.mul(sumExp.pow(-1)));\n\n // Weighted sum of values\n for (let d = 0; d < headDim; d++) {\n let s = attnW[0].mul(V[0][d]);\n for (let j = 1; j <= i; j++) {\n s = s.add(attnW[j].mul(V[j][d]));\n }\n attnOut[i][off + d] = s;\n }\n }\n }\n\n // Output projection + residual\n for (let t = 0; t < T; t++) {\n const proj = addBias(vecmat(attnOut[t], wO, dModel, dModel), bO);\n x[t] = addVec(x[t], proj);\n }\n\n // MLP with RMSNorm, ReLU\n const wFF1 = params[pi++], bFF1 = params[pi++];\n const wFF2 = params[pi++], bFF2 = params[pi++];\n const xNorm2: Value[][] = x.map((v) => rmsnormVec(v));\n for (let t = 0; t < T; t++) {\n const h1 = reluVec(\n addBias(vecmat(xNorm2[t], wFF1, dModel, dFF), bFF1),\n );\n const h2 = addBias(vecmat(h1, wFF2, dFF, dModel), bFF2);\n x[t] = addVec(x[t], h2);\n }\n }\n\n // Weight-tied output head: logits[t][v] = dot(x[t], tokEmb[v])\n const logits: Value[][] = [];\n for (let t = 0; t < T; t++) {\n const row: Value[] = [];\n for (let v = 0; v < vocabSize; v++) {\n const embRow = embeddingLookup(tokEmb, dModel, v);\n row.push(dot(x[t], embRow));\n }\n logits.push(row);\n }\n return logits;\n }\n\n return {\n spec,\n initParams,\n forward,\n countParams: (p: Params) => p.reduce((s, t) => s + t.length, 0),\n };\n}\n\n// ---- Cross-entropy loss (scalar Value version) ----\nfunction crossEntropyLoss(logits: Value[][], targets: Int32Array): Value {\n const T = logits.length;\n let totalLoss = new Value(0);\n for (let t = 0; t < T; t++) {\n const row = logits[t], V = row.length;\n // Numerically stable softmax + cross-entropy\n let mx = -Infinity;\n for (let v = 0; v < V; v++) mx = Math.max(mx, row[v].data);\n const expVals = row.map((r) => r.add(new Value(-mx)).exp());\n let sumExp = expVals[0];\n for (let v = 1; v < V; v++) sumExp = sumExp.add(expVals[v]);\n const logProb = expVals[targets[t]].mul(sumExp.pow(-1)).log();\n totalLoss = totalLoss.add(logProb.neg());\n }\n return totalLoss.mul(new Value(1 / T));\n}\n\n// ---- Training (Karpathy-style: 1 doc per step, Adam optimizer) ----\ntype Trained = {\n tokenizer: ReturnType;\n model: Model;\n params: Params;\n};\n\nfunction trainNameModel(opts: {\n docs: string[];\n tokenizer: ReturnType;\n model: Model;\n steps: number;\n maxLen: number;\n seed: number;\n learningRate: number;\n}): Trained {\n const { docs, tokenizer, model, steps, maxLen, seed, learningRate } = opts;\n const params = model.initParams(seed);\n const numParams = model.countParams(params);\n console.log(`num params: ${numParams}`);\n\n // Adam optimizer state\n const beta1 = 0.9, beta2 = 0.95, epsAdam = 1e-8;\n const mBuf: Float64Array[] = params.map((p) => new Float64Array(p.length));\n const vBuf: Float64Array[] = params.map((p) => new Float64Array(p.length));\n\n // Shuffle docs with our seeded RNG\n const rng = makeRng(seed);\n const shuffled = [...docs];\n for (let i = shuffled.length - 1; i > 0; i--) {\n const j = Math.floor(rng() * (i + 1));\n [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];\n }\n\n const t0 = Date.now();\n for (let step = 0; step < steps; step++) {\n // Zero gradients (every Value in every param)\n for (const p of params) {\n for (const v of p) v.grad = 0;\n }\n\n // Pick one doc, encode it: [BOS, ...chars, BOS]\n const doc = shuffled[step % shuffled.length];\n const allTokens = tokenizer.encode(doc);\n const n = Math.min(maxLen, allTokens.length - 1);\n const input = allTokens.slice(0, n);\n const target = allTokens.slice(1, n + 1);\n\n // Forward + loss + backward\n const logits = model.forward(params, input);\n const loss = crossEntropyLoss(logits, target);\n loss.backward();\n\n // Adam update with linear LR decay\n const lrT = learningRate * (1 - step / steps);\n for (let pi = 0; pi < params.length; pi++) {\n const p = params[pi], m = mBuf[pi], v = vBuf[pi];\n for (let i = 0; i < p.length; i++) {\n m[i] = beta1 * m[i] + (1 - beta1) * p[i].grad;\n v[i] = beta2 * v[i] + (1 - beta2) * p[i].grad * p[i].grad;\n const mHat = m[i] / (1 - Math.pow(beta1, step + 1));\n const vHat = v[i] / (1 - Math.pow(beta2, step + 1));\n p[i].data -= lrT * mHat / (Math.sqrt(vHat) + epsAdam);\n }\n }\n\n if (step % 100 === 0 || step === steps - 1) {\n const elapsed = ((Date.now() - t0) / 1000).toFixed(1);\n console.log(\n `step ${String(step + 1).padStart(4)} / ${steps} | loss ${\n loss.data.toFixed(4)\n } | ${elapsed}s`,\n );\n }\n }\n\n return { tokenizer, model, params };\n}\n\n// ---- Sampling ----\nfunction generateName(\n opts: {\n trained: Trained;\n maxNewTokens: number;\n temperature: number;\n seed: number;\n },\n): string {\n const {\n trained: { tokenizer, model, params },\n maxNewTokens,\n temperature,\n seed,\n } = opts;\n const rng = makeRng(seed);\n let tokens = new Int32Array([tokenizer.BOS]);\n\n for (let i = 0; i < maxNewTokens; i++) {\n const logits = model.forward(params, tokens);\n const T = tokens.length, V = tokenizer.vocabSize;\n // Take logits from the last position, apply temperature\n const lp = new Float64Array(V);\n for (let v = 0; v < V; v++) {\n lp[v] = logits[T - 1][v].data / temperature;\n }\n // Softmax\n let mx = -Infinity;\n for (let v = 0; v < V; v++) mx = Math.max(mx, lp[v]);\n let s = 0;\n for (let v = 0; v < V; v++) {\n lp[v] = Math.exp(lp[v] - mx);\n s += lp[v];\n }\n for (let v = 0; v < V; v++) lp[v] /= s;\n // Sample from distribution\n const r = rng();\n let cum = 0, next = 0;\n for (let v = 0; v < V; v++) {\n cum += lp[v];\n if (r < cum) {\n next = v;\n break;\n }\n }\n if (next === tokenizer.BOS) break;\n const newTokens = new Int32Array(tokens.length + 1);\n newTokens.set(tokens);\n newTokens[tokens.length] = next;\n tokens = newTokens;\n }\n\n return tokenizer.decode(tokens);\n}\n\n// ============================================================================\n// main script\n// ============================================================================\n\nconst docs = await loadDocs();\nconsole.log(`num docs: ${docs.length}`);\n\nconst tok = buildCharTokenizerFromDocs(docs);\nconsole.log(`vocab size: ${tok.vocabSize}`);\n\nconst model = makeTransformerLanguageModel({\n vocabSize: tok.vocabSize, // 27 (a-z + BOS)\n maxLen: 8, // max sequence length (matches Karpathy's block_size)\n dModel: 16, // embedding dimension (matches Karpathy's n_embd)\n nHeads: 4, // attention heads (matches Karpathy's n_head)\n nLayers: 1, // transformer blocks (matches Karpathy's n_layer)\n dFF: 64, // feedforward hidden dim (4 * dModel, matches Karpathy)\n});\n\nconst trained = trainNameModel({\n docs,\n tokenizer: tok,\n model,\n steps: 1000, // same as Karpathy\n maxLen: 8, // crop sequences to block_size\n seed: 42, // same seed as Karpathy\n learningRate: 0.01, // same as Karpathy\n});\n\nconsole.log(\"\\n--- generation ---\");\nArray.from({ length: 20 }, (_, i) => i + 1)\n .map((i) =>\n generateName({\n trained,\n maxNewTokens: 8, // max name length = block_size\n temperature: 0.7, // balance between variety and quality\n seed: i * 7919, // spread seeds far apart for diversity (7919 is prime)\n })\n )\n .forEach((name, i) =>\n console.log(`sample ${String(i + 1).padStart(2, \" \")}: ${name}`)\n );",[157],"60b5b608-0f4c-11f1-a98e-42dde27851f2","README.md","019c8147-9f56-743d-9a85-51574f3e6a91",3,["D",1771695613949],"# microGPT.ts (scalar autograd edition)\n\nA direct TypeScript port of [Karpathy's microGPT](https://karpathy.github.io/2026/02/12/microgpt/) — faithful to the original's scalar `Value`-based autograd, not optimized for speed.\n\nKarpathy's microGPT is a single Python file that trains and inferences a GPT with zero dependencies. As he put it: *\"This is the full algorithmic content of what is needed. Everything else is just efficiency.\"* This port preserves that spirit **at the same level of abstraction** — every scalar is a `Value` node in a dynamically-built computation graph, just like the original.\n\n## Scalar Autograd: Same as Karpathy\n\nThe core design choice: **every number is a `Value` object**.\n\n```typescript\nclass Value {\n data: number; // one scalar\n grad: number; // one scalar\n _backward: () => void;\n _prev: Value[]; // parent nodes in the graph\n}\n```\n\nA 16-dim embedding lookup creates 16 `Value` objects. A vector-matrix multiply of `[16] × [16, 48]` creates 48 dot products, each made of 16 multiplies and 15 adds — **over 1,000 new `Value` nodes**. A single forward pass builds a computation graph with **tens of thousands of nodes**, each individually linked and walked during `.backward()`.\n\nThis is the same approach as Karpathy's Python `Value` class. It makes the chain rule visible at every single scalar operation.\n\n## Architecture\n\nSame GPT-2-style transformer as the original:\n\n```mermaid\ngraph TD\n X[Input x] --> NORM1[RMSNorm]\n NORM1 --> QKV[\"Linear → Q, K, V\"]\n QKV --> ATTN[\"Causal Multi-Head Attention
4 heads × 4-dim\"]\n ATTN --> PROJ[\"Output projection\"]\n PROJ --> ADD1[\"+ residual\"]\n X --> ADD1\n\n ADD1 --> NORM2[RMSNorm]\n NORM2 --> FF1[\"Linear → 64-dim\"]\n FF1 --> RELU2[ReLU]\n RELU2 --> FF2[\"Linear → 16-dim\"]\n FF2 --> ADD2[\"+ residual\"]\n ADD1 --> ADD2\n\n ADD2 --> LOGITS[\"Weight-tied logits
dot(x, tokEmb)\"]\n```\n\n## The Seven Components\n\nFollowing Karpathy's decomposition:\n\n1. **Dataset** — ~32k names fetched from Karpathy's `makemore` repo\n2. **Tokenizer** — Character-level: 26 letters + 1 BOS/EOS token (vocab size 27)\n3. **Autograd** — Scalar `Value` class with `add`, `mul`, `pow`, `exp`, `log`, `relu` — each operation creates a node in the graph\n4. **Architecture** — 1-layer GPT-2-style transformer (RMSNorm, causal attention, ReLU MLP, weight tying)\n5. **Loss** — Cross-entropy over next-token predictions\n6. **Optimizer** — Adam with bias correction and linear learning rate decay\n7. **Sampling** — Temperature-controlled autoregressive generation\n\n## Running\n\nThis is a [Val Town](https://val.town) script val. It trains for 1000 steps on CPU. Because every scalar is its own autograd node, it's **much slower** than a tensor-based implementation (~60x slower) — but the code is simpler and more transparent:\n\n```\nnum docs: 32033\nvocab size: 27\nnum params: 4795\nstep 1 / 1000 | loss 3.5062 | 0.0s\nstep 101 / 1000 | loss 2.7573 | ...s\n...\nstep 1000 / 1000 | loss 2.2891 | ...s\n\n--- generation ---\nsample 1: malede\nsample 2: jara\n...\n```\n\n## Hyperparameters\n\nMatched to Karpathy's defaults:\n\n| Parameter | Value | Notes |\n|---|---|---|\n| `dModel` | 16 | Embedding dimension |\n| `nHeads` | 4 | Attention heads (head dim = 4) |\n| `nLayers` | 1 | Transformer blocks |\n| `dFF` | 64 | FF hidden dim (4× `dModel`) |\n| `maxLen` | 8 | Context window |\n| `steps` | 1000 | Training iterations |\n| `learningRate` | 0.01 | With linear decay to 0 |\n| `seed` | 42 | Deterministic initialization |\n\n## Why is this slow?\n\nThis version uses **scalar autograd** — the same approach as Karpathy's original Python. Every addition, multiplication, and activation creates a new `Value` object in the computation graph. A single forward pass creates tens of thousands of nodes, and `backward()` must topologically sort and walk all of them.\n\nThe [main branch](https://www.val.town/x/stevekrouse/microgpt) uses **tensor autograd** instead — a single `matmul` operation creates one graph node with a fused backward closure that loops over all scalars in tight `for` loops. Same math, ~60x faster, but the individual scalar chain rule is hidden inside the closure.\n\n| Factor | This version (scalar Values) | Main branch (Tensors) |\n|---|---|---|\n| **Graph nodes per forward pass** | ~10,000+ `Value` objects | ~30-40 `Tensor` objects |\n| **Object allocation** | One object per scalar op | One TypedArray per tensor op |\n| **Backward traversal** | Topological sort over ~10K nodes | Topological sort over ~30 nodes |\n| **Inner loop execution** | V8 JIT over many small closures | V8 JIT over tight array loops |\n\nKarpathy's choice of scalar `Value` was deliberate: it makes the chain rule visible at the most granular level. This port preserves that choice.\n\n## Why TypeScript?\n\nTypeScript's type system makes the structure visible: `ModelSpec` describes *what* the model is, `Model` is the *behavior*, `Trained` is a frozen snapshot. But the autograd is the same scalar-level approach as the Python original — no tensor tricks, no fused kernels, just `Value.add()` and `Value.mul()` all the way down.\n",[164],"pulls","branches","forks","envVars",{"_177":65,"_179":65},"connections",{"_177":65,"_178":65},"count","unfilledCount","emptyCount",{"_47":163,"_37":164,"_35":165,"_142":-5,"_45":46,"_44":166,"_143":144,"_42":194,"_13":153,"_38":26,"_40":168,"_29":30,"_146":-5,"_185":195,"_32":196},{"_47":156,"_37":157,"_35":158,"_142":-5,"_45":46,"_44":159,"_143":30,"_42":191,"_13":153,"_38":39,"_40":161,"_29":30,"_146":-5,"_185":192,"_32":193},{"_47":148,"_37":149,"_35":150,"_142":-5,"_45":46,"_44":151,"_143":30,"_42":188,"_13":153,"_38":39,"_40":154,"_29":30,"_146":-5,"_185":189,"_32":190},{"_47":141,"_37":33,"_35":36,"_142":-5,"_45":46,"_44":14,"_143":144,"_42":184,"_13":65,"_38":39,"_40":41,"_29":30,"_146":-5,"_185":186,"_32":187},["D",1771695857629],"children",[],[33],["D",1771695835044],[],[149],["D",1771695574100],[],[157],["D",1771695613949],[],[164],["D",1771690989159],{"_101":100,"_108":104},{"_101":100,"_102":-5,"_103":104,"_19":59,"_105":106,"_107":30},{"_13":88,"_37":89,"_47":46,"_90":48,"_91":92,"_93":94,"_95":54,"_96":48,"_97":67,"_98":49,"_99":100},"ctx",{"_81":-5,"_128":234,"_235":236,"_237":238,"_239":30,"_240":30,"_241":30,"_242":144,"_243":-5,"_244":-7,"_245":-7,"_246":65,"_247":30,"_248":30,"_249":-5,"_250":30,"_251":30,"_252":30,"_253":-5},"ENV",{"_205":206,"_207":208,"_209":134,"_210":211,"_212":213,"_214":215,"_216":217,"_218":219,"_220":221,"_222":223,"_224":225,"_226":227,"_228":229,"_230":231,"_232":233},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v6