[{"_1":2,"_176":-5,"_177":-5},"loaderData",{"_3":4,"_93":94},"root",{"_5":6,"_36":37,"_66":67},"ctx",{"_7":8,"_23":-5,"_24":25,"_26":27,"_28":20,"_29":20,"_30":20,"_31":-5,"_32":33,"_34":20,"_35":20},"auth",{"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20,"_21":22},"sessionClaims","sessionId","sessionStatus","userId","actor","orgId","orgRole","orgSlug","orgPermissions","factorVerificationAge","isAuthenticated",false,"tokenType","session_token","user","preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesChatPanelState","notificationCount",0,"flagDenols","isTeamOrgMember","ENV",{"_38":39,"_40":41,"_42":43,"_44":45,"_46":47,"_48":49,"_50":51,"_52":53,"_54":55,"_56":57,"_58":59,"_60":61,"_62":63,"_64":65},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","VAL_RUNTIME_NAME","prod","RENDER_GIT_COMMIT","95c2b214125f68b24ec9fda4034953728efbbf27","RENDER_GIT_BRANCH","main","CLERK_PUBLISHABLE_KEY","pk_live_Y2xlcmsudmFsLnRvd24k","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","API_URL","https://api.val.town","clerkState",{"_68":69},"__internal_clerk_state",{"_70":71,"_72":47,"_73":74,"_75":74,"_76":20,"_77":74,"_78":74,"_79":74,"_80":74,"_81":82,"_91":20,"_92":20},"__clerk_ssr_state",{"_21":22,"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20},"__publishableKey","__proxyUrl","","__domain","__isSatellite","__signInUrl","__signUpUrl","__afterSignInUrl","__afterSignUpUrl","__clerk_debug",{"_83":20,"_19":20,"_84":74,"_85":86,"_87":74,"_88":47,"_89":20,"_90":74},"isSignedIn","proxyUrl","reason","session-token-and-uat-missing","message","publishableKey","isSatellite","domain","__telemetryDisabled","__telemetryDebug","routes/embed.x.$handle.$name.$",{"_95":96,"_97":98,"_99":100,"_101":102,"_153":154,"_173":174,"_175":-5},"handle","stevekrouse","name","autollm","path","main.ts","project",{"_103":104,"_97":98,"_105":106,"_107":-5,"_108":20,"_109":110,"_111":74,"_112":74,"_12":113,"_114":-5,"_115":20,"_116":-5,"_117":-5,"_118":33,"_14":119,"_120":-5,"_121":-5,"_122":123,"_124":123,"_125":126,"_127":126,"_128":126,"_129":123,"_130":123,"_23":131,"_134":135,"_139":140,"_152":20},"id","445cde1a-11e4-11f1-97e5-42dde27851f2","createdAt",["D",1771980752229],"deletedAt","legacyEnvVars","privacy","public","description","imageUrl","a0bf3b31-15a5-4d5c-880e-4b1e22c9bc18","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount","e81e7335-6d4c-4764-a902-d0962bbfa9f7","tursoName","tursoHostname","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm",{"_95":96,"_132":133},"profileImageUrl","https://images.clerk.dev/uploaded/img_2PqHa2Gsy93xQrjh2w78Xu0cChW.jpeg","org",{"_95":96,"_136":133,"_12":113,"_137":138},"avatar","tier","pro","branch",{"_141":142,"_97":45,"_103":143,"_144":104,"_145":-5,"_146":-5,"_147":-5,"_148":-5,"_149":-5,"_150":98,"_151":96},"version",26,"445d2d34-11e4-11f1-97e5-42dde27851f2","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","isOwned","file",{"_103":155,"_97":100,"_156":157,"_158":-5,"_159":143,"_160":161,"_162":163,"_105":164,"_141":165,"_166":167,"_168":169,"_170":20,"_171":-5,"_99":172},"05e75316-11e6-11f1-8441-42dde27851f2","fileId","445f2eb8-11e4-11f1-97e5-42dde27851f2","parentId","branchId","branchVersion",24,"latest",true,["D",1771981506429],21,"fileType","script","content","// microgpt_conal.ts\n// Conal-style reverse-mode AD in TypeScript: every computation returns\n// (value, backprop-function). Backprop functions compose automatically.\n//\n// Goal: char-level next-token predictor trained on the full\n// Karpathy makemore names.txt (~32K names), then sample new names.\n// Architecture: embedding -> concat -> hidden(tanh) -> output\n//\n// Run with: npx tsx microgpt_conal.ts\n\ntype Mat = Float32Array; // row-major, shape (rows, cols)\n\nfunction nowMs() {\n return Date.now();\n}\n\n// ---------- fetch the full dataset ----------\nconst NAMES_URL =\n \"https://raw.githubusercontent.com/karpathy/makemore/refs/heads/master/names.txt\";\n\nconst resp = await fetch(NAMES_URL);\nconst text = await resp.text();\nconst allDocs = text.trim().split(\"\\n\").map((l) => l.trim()).filter(Boolean);\nconsole.log(`Loaded ${allDocs.length} names from names.txt`);\n\nfunction shuffle(a: T[], seed = 0) {\n let x = seed >>> 0;\n function rnd() {\n x = (1664525 * x + 1013904223) >>> 0;\n return x / 2 ** 32;\n }\n for (let i = a.length - 1; i > 0; i--) {\n const j = Math.floor(rnd() * (i + 1));\n [a[i], a[j]] = [a[j], a[i]];\n }\n}\n\nshuffle(allDocs, 0);\nconst splitIdx = Math.floor(allDocs.length * 0.9);\nconst trainDocs = allDocs.slice(0, splitIdx);\nconst testDocs = allDocs.slice(splitIdx);\nconsole.log(`Train: ${trainDocs.length}, Test: ${testDocs.length}`);\n\n// vocab\nconst BOS = \".\";\nconst charSet = new Set();\nfor (const d of allDocs) for (const ch of d) charSet.add(ch);\nconst chars = Array.from(charSet).sort();\nconst vocab = [BOS, ...chars];\nconst stoi = new Map(vocab.map((c, i) => [c, i]));\nconst itos = vocab;\nconst V = vocab.length;\nconsole.log(`Vocab size: ${V}`);\n\nfunction encodeName(s: string): number[] {\n const ids: number[] = [stoi.get(BOS)!];\n for (const ch of s) ids.push(stoi.get(ch)!);\n ids.push(stoi.get(BOS)!);\n return ids;\n}\n\n// dataset: context window K predicts next char\nconst K = 5;\nfunction buildDataset(docs: string[]): { xs: number[][]; ys: number[] } {\n const xs: number[][] = [];\n const ys: number[] = [];\n for (const d of docs) {\n const t = encodeName(d);\n const ctx = new Array(K).fill(stoi.get(BOS)!);\n for (let i = 1; i < t.length; i++) {\n xs.push(ctx.slice());\n ys.push(t[i]);\n ctx.shift();\n ctx.push(t[i]);\n }\n }\n return { xs, ys };\n}\n\nconst { xs, ys } = buildDataset(trainDocs);\nconst N = xs.length;\nconst { xs: testXs, ys: testYs } = buildDataset(testDocs);\nconsole.log(`Training examples: ${N}, Test examples: ${testXs.length}`);\n\n// ---------- Conal-style AD core ----------\n//\n// ELI5: A Node is \"a value\" plus \"a function that knows how to send blame backward\".\n// - v: the computed number(s)\n// - back(g): given upstream sensitivity g, propagate it to this node's inputs\n//\n// This back(g) *is* the derivative object, represented in reverse form.\nclass Node {\n v: T;\n back: (g: any) => void;\n constructor(v: T, back?: (g: any) => void) {\n this.v = v;\n this.back = back ?? (() => {});\n }\n}\n\n// A Param is a trainable knob θ.\n// ELI5: It stores its current value, and a gradient bucket.\nclass Param {\n v: Mat;\n g: Mat;\n rows: number;\n cols: number;\n\n constructor(rows: number, cols: number, scale = 0.01, seed = 0) {\n this.rows = rows;\n this.cols = cols;\n this.v = new Float32Array(rows * cols);\n this.g = new Float32Array(rows * cols);\n let x = (seed >>> 0) || 1;\n function rndn() {\n let s = 0;\n for (let i = 0; i < 6; i++) {\n x = (1664525 * x + 1013904223) >>> 0;\n s += x / 2 ** 32;\n }\n return s - 3;\n }\n for (let i = 0; i < this.v.length; i++) this.v[i] = scale * rndn();\n }\n\n zeroGrad() {\n this.g.fill(0);\n }\n\n node(): Node {\n return new Node(this.v, (g: Mat) => {\n for (let i = 0; i < this.g.length; i++) this.g[i] += g[i];\n });\n }\n}\n\n// ---------- math helpers ----------\nfunction mi(r: number, c: number, cols: number) {\n return r * cols + c;\n}\n\nfunction softmax1(logits: Float64Array): Float64Array {\n let max = -Infinity;\n for (let i = 0; i < logits.length; i++) if (logits[i] > max) max = logits[i];\n let sum = 0;\n const exps = new Float64Array(logits.length);\n for (let i = 0; i < logits.length; i++) {\n exps[i] = Math.exp(logits[i] - max);\n sum += exps[i];\n }\n for (let i = 0; i < logits.length; i++) exps[i] /= sum;\n return exps;\n}\n\n// ---------- primitives (each returns Node(value, backprop)) ----------\n\n// gather_rows: E:(V,Emb), idx:(B,K) -> out:(B, K*Emb) (concatenated!)\n// ELI5: look up secret codes for each letter, then glue them together into one long vector\nfunction gatherAndConcat(\n E: Node,\n idx: number[][],\n V: number,\n Emb: number,\n): Node {\n const B = idx.length;\n const outDim = K * Emb;\n const out = new Float32Array(B * outDim);\n\n for (let b = 0; b < B; b++) {\n for (let k = 0; k < K; k++) {\n const row = idx[b][k];\n for (let d = 0; d < Emb; d++) {\n out[b * outDim + k * Emb + d] = E.v[mi(row, d, Emb)];\n }\n }\n }\n\n const back = (g: Float32Array) => {\n const dE = new Float32Array(V * Emb);\n for (let b = 0; b < B; b++) {\n for (let k = 0; k < K; k++) {\n const row = idx[b][k];\n for (let d = 0; d < Emb; d++) {\n dE[mi(row, d, Emb)] += g[b * outDim + k * Emb + d];\n }\n }\n }\n E.back(dE);\n };\n\n return new Node(out, back);\n}\n\n// matmul: X:(B,inD) @ W^T where W:(outD,inD) + bias:(outD) => out:(B,outD)\n// ELI5: each hidden neuron looks at the whole input and forms an opinion\nfunction linearLayer(\n X: Node,\n W: Node,\n bias: Node,\n B: number,\n inD: number,\n outD: number,\n): Node {\n const out = new Float32Array(B * outD);\n\n for (let b = 0; b < B; b++) {\n for (let o = 0; o < outD; o++) {\n let s = bias.v[o];\n for (let i = 0; i < inD; i++) s += X.v[b * inD + i] * W.v[mi(o, i, inD)];\n out[b * outD + o] = s;\n }\n }\n\n const back = (g: Float32Array) => {\n // dX = g @ W\n const dX = new Float32Array(B * inD);\n for (let b = 0; b < B; b++) {\n for (let i = 0; i < inD; i++) {\n let s = 0;\n for (let o = 0; o < outD; o++) s += g[b * outD + o] * W.v[mi(o, i, inD)];\n dX[b * inD + i] = s;\n }\n }\n\n // dW = g^T @ X\n const dW = new Float32Array(outD * inD);\n for (let o = 0; o < outD; o++) {\n for (let i = 0; i < inD; i++) {\n let s = 0;\n for (let b = 0; b < B; b++) s += g[b * outD + o] * X.v[b * inD + i];\n dW[mi(o, i, inD)] = s;\n }\n }\n\n // dBias = sum over batch\n const dBias = new Float32Array(outD);\n for (let b = 0; b < B; b++) {\n for (let o = 0; o < outD; o++) {\n dBias[o] += g[b * outD + o];\n }\n }\n\n X.back(dX);\n W.back(dW);\n bias.back(dBias);\n };\n\n return new Node(out, back);\n}\n\n// tanh activation: elementwise tanh\n// ELI5: squish each number into the range (-1, 1) — keeps things from exploding\nfunction tanhAct(\n x: Node,\n len: number,\n): Node {\n const out = new Float32Array(len);\n for (let i = 0; i < len; i++) out[i] = Math.tanh(x.v[i]);\n\n const back = (g: Float32Array) => {\n // dtanh/dx = 1 - tanh(x)^2\n const dx = new Float32Array(len);\n for (let i = 0; i < len; i++) dx[i] = g[i] * (1 - out[i] * out[i]);\n x.back(dx);\n };\n\n return new Node(out, back);\n}\n\n// cross-entropy loss from logits, mean over batch\nfunction xentFromLogits(\n logits: Node,\n y: number[],\n B: number,\n V: number,\n): Node {\n const probs = new Float64Array(B * V);\n let loss = 0;\n\n for (let b = 0; b < B; b++) {\n const row = new Float64Array(V);\n for (let v = 0; v < V; v++) row[v] = logits.v[b * V + v];\n const p = softmax1(row);\n for (let v = 0; v < V; v++) probs[b * V + v] = p[v];\n loss += -Math.log(p[y[b]] + 1e-12);\n }\n loss /= B;\n\n const back = (gscalar: number) => {\n const dLogits = new Float32Array(B * V);\n for (let b = 0; b < B; b++) {\n for (let v = 0; v < V; v++) {\n let g = probs[b * V + v];\n if (v === y[b]) g -= 1.0;\n dLogits[b * V + v] = (g / B) * gscalar;\n }\n }\n logits.back(dLogits);\n };\n\n return new Node(loss, back);\n}\n\n// ---------- model ----------\n//\n// Architecture (Bengio-style MLP, like Karpathy's makemore part 2):\n// emb = E[ctx] -> (B, K*Emb) concatenated embeddings\n// h = tanh(emb @ W1^T + b1) -> (B, H) hidden layer\n// logits = h @ W2^T + b2 -> (B, V) output logits\n// loss = cross-entropy(logits, target)\n\nconst Emb = 10; // embedding dimension per character\nconst H = 128; // hidden layer size\nconst inDim = K * Emb; // 3 * 10 = 30\n\n// Kaiming-ish init for tanh: scale = gain / sqrt(fan_in)\n// gain for tanh ≈ 5/3\nconst w1Scale = (5 / 3) / Math.sqrt(inDim);\nconst w2Scale = 0.01; // small init for output layer\n\nconst pE = new Param(V, Emb, 0.1, 1); // embedding table\nconst pW1 = new Param(H, inDim, w1Scale, 2); // hidden weights\nconst pB1 = new Param(1, H, 0, 3); // hidden biases (zero init)\nconst pW2 = new Param(V, H, w2Scale, 4); // output weights\nconst pB2 = new Param(1, V, 0, 5); // output biases (zero init)\n\nconst allParams = [pE, pW1, pB1, pW2, pB2];\nconst totalParams = allParams.reduce((s, p) => s + p.v.length, 0);\nconsole.log(`Model parameters: ${totalParams}`);\n\nfunction forwardBatch(xb: number[][], yb: number[]): Node {\n const B = xb.length;\n const En = pE.node();\n const W1n = pW1.node();\n const B1n = pB1.node();\n const W2n = pW2.node();\n const B2n = pB2.node();\n\n const emb = gatherAndConcat(En, xb, V, Emb);\n const preact = linearLayer(emb, W1n, B1n, B, inDim, H);\n const h = tanhAct(preact, B * H);\n const logits = linearLayer(h, W2n, B2n, B, H, V);\n return xentFromLogits(logits, yb, B, V);\n}\n\n// evaluate loss on a dataset (no gradient needed)\nfunction evalLoss(exs: number[][], eys: number[]): number {\n const chunkSize = 1024;\n let totalLoss = 0;\n let totalCount = 0;\n for (let i = 0; i < exs.length; i += chunkSize) {\n const end = Math.min(i + chunkSize, exs.length);\n const xb = exs.slice(i, end);\n const yb = eys.slice(i, end);\n const B = xb.length;\n\n const En = new Node(pE.v);\n const W1n = new Node(pW1.v);\n const B1n = new Node(pB1.v);\n const W2n = new Node(pW2.v);\n const B2n = new Node(pB2.v);\n\n const emb = gatherAndConcat(En, xb, V, Emb);\n const preact = linearLayer(emb, W1n, B1n, B, inDim, H);\n const h = tanhAct(preact, B * H);\n const logits = linearLayer(h, W2n, B2n, B, H, V);\n const loss = xentFromLogits(logits, yb, B, V);\n totalLoss += loss.v * B;\n totalCount += B;\n }\n return totalLoss / totalCount;\n}\n\n// ---------- training ----------\nfunction randint(n: number, seedObj: { x: number }) {\n seedObj.x = (1664525 * seedObj.x + 1013904223) >>> 0;\n return seedObj.x % n;\n}\n\nconst TIMEOUT_MS = 10_000;\nconst seedObj = { x: 123 };\nconst t0 = nowMs();\nlet steps = 0;\nlet lastLoss = NaN;\n\nconsole.log(`\\nTraining for ${TIMEOUT_MS / 1000}s...`);\n\nwhile (true) {\n if (nowMs() - t0 >= TIMEOUT_MS) break;\n\n const B = 64;\n const xb: number[][] = new Array(B);\n const yb: number[] = new Array(B);\n for (let i = 0; i < B; i++) {\n const j = randint(N, seedObj);\n xb[i] = xs[j];\n yb[i] = ys[j];\n }\n\n for (const p of allParams) p.zeroGrad();\n\n const loss = forwardBatch(xb, yb);\n lastLoss = loss.v;\n\n // ELI5: kick off reverse mode — \"the loss depends on itself by 1\"\n loss.back(1.0);\n\n // learning rate with step decay\n const lr = steps < 2500 ? 0.5 : 0.05;\n for (const p of allParams) {\n for (let i = 0; i < p.v.length; i++) p.v[i] -= lr * p.g[i];\n }\n\n steps++;\n\n if (steps % 1000 === 0) {\n const elapsed = ((nowMs() - t0) / 1000).toFixed(1);\n console.log(` step ${steps} | loss ${lastLoss.toFixed(4)} | lr ${lr} | ${elapsed}s`);\n }\n}\n\nconst elapsed = (nowMs() - t0) / 1000;\nconsole.log(`\\nTraining done!`);\nconsole.log(` Elapsed: ${elapsed.toFixed(3)}s`);\nconsole.log(` Steps: ${steps}`);\nconsole.log(` Final train batch loss: ${lastLoss.toFixed(4)}`);\n\nconst trainLoss = evalLoss(xs, ys);\nconst testLoss = evalLoss(testXs, testYs);\nconsole.log(` Full train loss: ${trainLoss.toFixed(4)}`);\nconsole.log(` Full test loss: ${testLoss.toFixed(4)}`);\n\n// ---------- sampling ----------\nfunction sample(n = 20, maxlen = 16, temperature = 0.8) {\n const out: string[] = [];\n for (let s = 0; s < n; s++) {\n const ctx = new Array(K).fill(stoi.get(BOS)!);\n let name = \"\";\n for (let t = 0; t < maxlen; t++) {\n // forward pass for single example\n const emb = new Float32Array(inDim);\n for (let k = 0; k < K; k++) {\n const row = ctx[k];\n for (let d = 0; d < Emb; d++) emb[k * Emb + d] = pE.v[mi(row, d, Emb)];\n }\n\n // hidden = tanh(emb @ W1^T + b1)\n const h = new Float32Array(H);\n for (let o = 0; o < H; o++) {\n let s2 = pB1.v[o];\n for (let i = 0; i < inDim; i++) s2 += emb[i] * pW1.v[mi(o, i, inDim)];\n h[o] = Math.tanh(s2);\n }\n\n // logits = h @ W2^T + b2\n const logits = new Float64Array(V);\n for (let v = 0; v < V; v++) {\n let s2 = pB2.v[v];\n for (let i = 0; i < H; i++) s2 += h[i] * pW2.v[mi(v, i, H)];\n logits[v] = s2 / Math.max(1e-8, temperature);\n }\n const p = softmax1(logits);\n\n // categorical sample\n seedObj.x = (1664525 * seedObj.x + 1013904223) >>> 0;\n let r = seedObj.x / 2 ** 32;\n let acc = 0;\n let nxt = 0;\n for (let v = 0; v < V; v++) {\n acc += p[v];\n if (r <= acc) {\n nxt = v;\n break;\n }\n }\n\n if (nxt === stoi.get(BOS)!) break;\n name += itos[nxt];\n ctx.shift();\n ctx.push(nxt);\n }\n out.push(name);\n }\n return out;\n}\n\nconsole.log(\"\\nGenerated names:\");\nfor (const [i, s] of sample(20).entries()) {\n console.log(String(i + 1).padStart(2, \" \"), \":\", s);\n}\n","deleted","label",[100],"permalink","https://www.val.town/x/stevekrouse/autollm/code/main.ts","extension","actionData","errors"]