[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_165":166,"_167":168},"routes/_app.x.$handle.$name",{"_95":94,"_32":45,"_103":85,"_27":104,"_13":14,"_105":-5,"_11":106,"_107":108,"_109":110,"_111":112,"_113":114,"_20":-5,"_115":-5,"_116":-5,"_117":25,"_118":119,"_120":49,"_121":-5,"_122":-5,"_123":124},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_43":44,"_32":45,"_38":46,"_47":-5,"_48":49,"_50":51,"_52":51,"_53":-5,"_54":-5,"_18":55,"_56":31,"_57":49,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":78,"_79":80,"_81":82,"_83":25},"version",5,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_33":34,"_35":37,"_38":39,"_40":14,"_41":42,"_24":25},"previous",{"_30":31,"_32":28,"_33":34,"_35":36},"deleted",false,"deletedHere","path","main.tsx","latestPath","fileId","9231f658-2792-11f0-9d52-569c3dd06744","name","fileType","http","content","import { Hono } from \"https://deno.land/x/hono@v3.11.7/mod.ts\";\nimport { get_encoding } from \"npm:tiktoken\";\n\nconst app = new Hono();\n\nconst html = `\n\n\n\n \n \n Word Counter, Byte Counter, and Token Counter\n \n \n \n \n \n \n \n \n \n \n \n \n\n\n

Counter

All-in-one Word Counter, Byte Counter, and Token Counter.

Words

\n 0\n

Characters

\n 0\n

Twitter

\n 0\n /280\n

Sentences

\n 0\n

Paragraphs

\n 0\n

\n\n \n\n

Bytes

\n 0\n

\n\n

\n Tokenizer Model:\n \n

\n 0 tokens\n

Co-written with Claude.
Heavily inspired by https://mothereff.in/byte-counter, https://wordcounter.io/, and https://tiktokenizer.vercel.app/

\n A labspace.ai experiment\n

Counter

All-in-one Word Counter, Byte Counter, and Token Counter.

Words

\n 0\n

Characters

\n 0\n

Twitter

\n 0\n /280\n

Sentences

\n 0\n

Paragraphs

\n 0\n

\n\n \n\n

Bytes

\n 0\n

\n\n

\n Tokenizer Model:\n \n

\n 0 tokens\n

Co-written with Claude.
Heavily inspired by https://mothereff.in/byte-counter, https://wordcounter.io/, and https://tiktokenizer.vercel.app/

\n A labspace.ai experiment\n

\n \n\n\n`;\n\napp.get(\"/\", (c) => c.html(html));\n\napp.post(\"/tokenize\", async (c) => {\n try {\n const body = await c.req.json();\n const { text, model } = body;\n\n const encoder = get_encoding(model);\n const segments = getTiktokenSegments(encoder, text);\n\n encoder.free();\n\n return c.json({ count: segments.reduce((acc, seg) => acc + seg.tokens.length, 0), segments });\n } catch (error) {\n return c.json({ error: error.message }, 500);\n }\n});\n\nfunction getTiktokenSegments(encoder, inputText) {\n try {\n const tokens = encoder.encode(inputText);\n const segments = [];\n\n let currentSegment = '';\n let tokenAcc = [];\n\n for (let idx = 0; idx < tokens.length; idx++) {\n const token = tokens[idx];\n const tokenBytes = encoder.decode_single_token_bytes(token);\n const tokenText = new TextDecoder().decode(new Uint8Array(tokenBytes));\n \n currentSegment += tokenText;\n tokenAcc.push({ id: token, idx });\n\n // Check if we've reached a valid segment boundary\n if (inputText.startsWith(currentSegment)) {\n segments.push({ \n text: currentSegment, \n tokens: tokenAcc \n });\n\n // Remove the matched segment from the input text\n inputText = inputText.slice(currentSegment.length);\n \n // Reset for next segment\n currentSegment = '';\n tokenAcc = [];\n }\n }\n\n return segments;\n } catch (error) {\n throw new Error(`Error in getTiktokenSegments: ${error.message}`);\n }\n}\n\n// Export app.fetch for Val Town, otherwise export app — this is only for hono apps\nexport default (typeof Deno !== \"undefined\" && Deno.env.get(\"valtown\")) ? app.fetch : app;\n","createdAt",["D",1729630605759],"branchVersion","branchId","923012f2-2792-11f0-9d52-569c3dd06744","id","922fd8b4-2792-11f0-9d52-569c3dd06744","counter",["D",1746217091978],"deletedAt","legacyEnvVars",true,"privacy","public","httpPrivacy","description","imageUrl","a8270e74-6479-475a-8025-9b69544ee0e7","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","8319defa-6e1a-46d4-ba66-2114ffd3360b","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_95":94,"_102":98},"org",{"_95":94,"_96":-5,"_97":98,"_18":55,"_99":100,"_101":25},"branch",{"_13":84,"_32":85,"_43":42,"_86":44,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":45,"_93":94},"isOwned",9,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","yawnxyz","handle","displayName","avatar","https://images.clerk.dev/oauth_github/img_2NnaHhpxNuH1xWRIRjQNoo16TVc.jpeg","tier","pro","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_43":44,"_32":45,"_38":161,"_47":-5,"_48":49,"_50":51,"_52":51,"_53":-5,"_54":-5,"_18":55,"_56":31,"_57":49,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":162,"_79":163,"_81":164,"_83":25},"tree",[152,153],"files",[152,153],"counts",{"_142":61,"_143":138,"_144":61,"_145":146,"_147":148},"contents",[126,127],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{"_31":125},"counter.labspace.ai",{"_43":135,"_32":136,"_30":137,"_129":-5,"_41":42,"_40":138,"_130":49,"_38":139,"_13":61,"_33":20,"_35":140,"_24":25,"_133":-5,"_27":141},{"_43":128,"_32":28,"_30":31,"_129":-5,"_41":42,"_40":14,"_130":25,"_38":131,"_13":132,"_33":34,"_35":37,"_24":25,"_133":-5,"_27":134},"92456da0-2792-11f0-9d52-569c3dd06744","parentId","latest",["D",1729630605759],4,"label",[28],"9231f96e-2792-11f0-9d52-569c3dd06744","README.md","9231f9dc-2792-11f0-9d52-569c3dd06744",1,["D",1729373169069],"Migrated from folder: labspace/counter",[136],"pulls","branches","forks","envVars",{"_149":61,"_151":61},"connections",{"_149":61,"_150":61},"count","unfilledCount","emptyCount",{"_43":135,"_32":136,"_30":137,"_129":-5,"_41":42,"_40":138,"_130":49,"_38":158,"_13":61,"_33":20,"_35":140,"_24":25,"_133":-5,"_155":159,"_27":160},{"_43":128,"_32":28,"_30":31,"_129":-5,"_41":42,"_40":14,"_130":25,"_38":154,"_13":132,"_33":34,"_35":37,"_24":25,"_133":-5,"_155":156,"_27":157},["D",1729630605759],"children",[],[28],["D",1729373169069],[],[136],["D",1746217091978],{"_95":94,"_102":98},{"_95":94,"_96":-5,"_97":98,"_18":55,"_99":100,"_101":25},{"_13":84,"_32":85,"_43":42,"_86":44,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":45,"_93":94},"ctx",{"_77":-5,"_122":198,"_199":200,"_201":202,"_203":25,"_204":25,"_205":25,"_206":49,"_207":-5,"_208":-7,"_209":-7,"_210":61,"_211":25,"_212":25,"_213":-5,"_214":25,"_215":25,"_216":25,"_217":-5},"ENV",{"_169":170,"_171":172,"_173":85,"_174":175,"_176":177,"_178":179,"_180":181,"_182":183,"_184":185,"_186":187,"_188":189,"_190":191,"_192":193,"_194":195,"_196":197},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v5