[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_162":163,"_164":165},"routes/_app.x.$handle.$name",{"_11":12,"_13":14,"_15":-7,"_18":19,"_17":-7,"_120":-5,"_20":121,"_22":122,"_123":124,"_125":126,"_26":127,"_24":128,"_129":-5,"_130":-5,"_131":56,"_132":133,"_134":53,"_135":-5,"_136":-5,"_137":138},"routes/_app.x.$handle.$name.code.$",{"_11":12,"_13":14,"_15":16,"_17":-7,"_18":19,"_20":21,"_22":23,"_24":25,"_26":27,"_28":-5,"_29":-5,"_30":-5,"_31":-5},"handle","yawnxyz","name","counter","branchName","main","version","path","main.tsx","project",{"_32":67,"_13":14,"_44":68,"_69":-5,"_70":53,"_71":72,"_73":72,"_74":-5,"_75":-5,"_76":77,"_78":37,"_79":53,"_80":-5,"_81":-5,"_82":64,"_83":84,"_85":-5,"_86":-5,"_87":-5,"_88":-5,"_89":90,"_91":90,"_92":93,"_94":93,"_95":93,"_96":90,"_97":90,"_98":99,"_100":101,"_102":103,"_104":56},"tree",[50,51],"file",{"_32":33,"_34":35,"_36":37,"_13":19,"_38":39,"_40":41,"_17":42,"_43":-5,"_44":45,"_46":47,"_48":-5,"_18":49},"contents",[],"upstream","pull","ahead","behind","id","9259cc14-2792-11f0-9d52-569c3dd06744","branchId","923012f2-2792-11f0-9d52-569c3dd06744","fileId","9231f658-2792-11f0-9d52-569c3dd06744","content","import { Hono } from \"https://deno.land/x/hono@v3.11.7/mod.ts\";\nimport { get_encoding } from \"npm:tiktoken\";\n\nconst app = new Hono();\n\nconst html = `\n\n\n\n \n \n \n Word Counter, Byte Counter, and Token Counter\n \n \n \n \n \n \n \n \n \n \n \n \n\n\n

Counter

All-in-one Word Counter, Byte Counter, and Token Counter. Save results to browser history.

Words

\n 0\n

Characters

\n 0\n

Twitter

\n 0\n /280\n

Sentences

\n 0\n

Paragraphs

\n 0\n

\n\n \n\n

Bytes

\n 0\n

\n\n

\n Choose Tokenizer\n

The tokenizers are still a work-in-progress, and sometimes break on certain characters, emojis, and more. o200k_base is the most stable tokenizer.

\n \n

Co-written with Claude.
Heavily inspired by https://mothereff.in/byte-counter, https://wordcounter.io/, and https://tiktokenizer.vercel.app/

\n A labspace.ai experiment\n

\n \n \n\n\n`;\n\napp.get(\"/\", (c) => c.html(html));\n\napp.post(\"/tokenize\", async (c) => {\n try {\n const body = await c.req.json();\n const { text, model } = body;\n if (!text) {\n return c.json({ count: 0, segments: [] });\n }\n\n // Pre-process text to handle emojis\n const processedText = replaceEmojisWithCodePoints(text);\n\n const encoder = get_encoding(model);\n const segments = getTiktokenSegments(encoder, processedText);\n\n encoder.free();\n\n return c.json({ count: segments.reduce((acc, seg) => acc + seg.tokens.length, 0), segments });\n } catch (error) {\n return c.json({ error: error.message }, 500);\n }\n});\n\nfunction replaceEmojisWithCodePoints(text) {\n console.log(\"Replacing emojis with code points:\", text);\n\n // More comprehensive emoji regex that includes additional ranges and combined emojis\n return text.replace(\n /[\\u{1F000}-\\u{1F9FF}\\u{2600}-\\u{27BF}\\u{2B50}\\u{2B06}\\u{2934}-\\u{2935}\\u{2B05}-\\u{2B07}\\u{2B1B}-\\u{2B1C}\\u{2B50}\\u{2B55}\\u{3030}\\u{303D}\\u{3297}\\u{3299}\\u{2702}-\\u{27B0}\\u{1F170}-\\u{1F251}]|[\\u{23E9}-\\u{23EC}\\u{23F0}\\u{23F3}\\u{25FD}-\\u{25FE}\\u{2614}-\\u{2615}\\u{2648}-\\u{2653}\\u{267F}\\u{2693}\\u{26A1}\\u{26AA}-\\u{26AB}\\u{26BD}-\\u{26BE}\\u{26C4}-\\u{26C5}\\u{26CE}\\u{26D4}\\u{26EA}\\u{26F2}-\\u{26F3}\\u{26F5}\\u{26FA}\\u{26FD}\\u{2705}\\u{2728}\\u{274C}\\u{274E}\\u{2753}-\\u{2755}\\u{2757}\\u{2795}-\\u{2797}\\u{27B0}\\u{27BF}\\u{2B1B}-\\u{2B1C}\\u{2B50}\\u{2B55}]|\\u{200D}[\\u{2640}\\u{2642}\\u{2695}\\u{2696}\\u{2708}\\u{2764}]|\\u{FE0F}|[\\u{1F468}\\u{1F469}][\\u{1F3FB}-\\u{1F3FF}]?\\u{200D}[\\u{2695}\\u{2696}\\u{2708}\\u{2764}]|\\u{1F468}\\u{200D}\\u{1F469}\\u{200D}[\\u{1F466}\\u{1F467}]|[\\u{1F468}\\u{1F469}]\\u{200D}[\\u{1F466}\\u{1F467}]/gu,\n (match) => {\n console.log(\"match:\", match);\n const codePoint = match.codePointAt(0)?.toString(16);\n return `[U+${codePoint}]`;\n }\n );\n}\n\nfunction getTiktokenSegments(encoder, inputText) {\n try {\n const tokens = encoder.encode(inputText);\n const segments = [];\n\n let currentSegment = '';\n let tokenAcc = [];\n\n for (let idx = 0; idx < tokens.length; idx++) {\n const token = tokens[idx];\n const tokenBytes = encoder.decode_single_token_bytes(token);\n const tokenText = new TextDecoder().decode(new Uint8Array(tokenBytes));\n \n currentSegment += tokenText;\n tokenAcc.push({ id: token, idx });\n\n // Check if we've reached a valid segment boundary\n if (inputText.startsWith(currentSegment)) {\n segments.push({ \n text: currentSegment, \n tokens: tokenAcc \n });\n\n // Remove the matched segment from the input text\n inputText = inputText.slice(currentSegment.length);\n \n // Reset for next segment\n currentSegment = '';\n tokenAcc = [];\n }\n }\n\n return segments;\n } catch (error) {\n throw new Error(`Error in getTiktokenSegments: ${error.message}`);\n }\n}\n\n// Export app.fetch for Val Town, otherwise export app — this is only for hono apps\nexport default (typeof Deno !== \"undefined\" && Deno.env.get(\"valtown\")) ? app.fetch : app;\n","fileType","http",8,"parentId","createdAt",["D",1730409518987],"branchVersion",9,"label",[19],{"_32":59,"_13":60,"_36":61,"_43":-5,"_34":35,"_46":62,"_52":53,"_44":63,"_17":64,"_40":24,"_38":-5,"_55":56,"_48":-5,"_57":65,"_18":66},{"_32":33,"_13":19,"_36":37,"_43":-5,"_34":35,"_46":47,"_52":53,"_44":54,"_17":42,"_40":41,"_38":-5,"_55":56,"_48":-5,"_57":58,"_18":49},"latest",true,["D",1730409518987],"deleted",false,"children",[],"9231f96e-2792-11f0-9d52-569c3dd06744","README.md","9231f9dc-2792-11f0-9d52-569c3dd06744",1,["D",1729373169069],0,[],[60],"922fd8b4-2792-11f0-9d52-569c3dd06744",["D",1746217091978],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","imageUrl","userId","a8270e74-6479-475a-8025-9b69544ee0e7","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount","orgId","8319defa-6e1a-46d4-ba66-2114ffd3360b","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_11":12,"_119":115},"org",{"_11":12,"_113":-5,"_114":115,"_76":77,"_116":117,"_118":56},"branch",{"_17":47,"_13":16,"_32":35,"_105":67,"_106":-5,"_107":-5,"_108":-5,"_109":-5,"_110":-5,"_111":14,"_112":12},"isOwned","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","displayName","avatar","https://images.clerk.dev/oauth_github/img_2NnaHhpxNuH1xWRIRjQNoo16TVc.jpeg","tier","pro","canRestrictVals","profileImageUrl","adoptOrgId",{"_32":67,"_13":14,"_44":158,"_69":-5,"_70":53,"_71":72,"_73":72,"_74":-5,"_75":-5,"_76":77,"_78":37,"_79":53,"_80":-5,"_81":-5,"_82":64,"_83":84,"_85":-5,"_86":-5,"_87":-5,"_88":-5,"_89":90,"_91":90,"_92":93,"_94":93,"_95":93,"_96":90,"_97":90,"_98":159,"_100":160,"_102":161,"_104":56},[152,153],"files",[152,153],"counts",{"_142":64,"_143":64,"_144":64,"_145":146,"_147":148},[],{"_32":33,"_13":19,"_36":37,"_43":-5,"_34":35,"_46":47,"_52":53,"_44":140,"_17":42,"_40":41,"_38":39,"_55":56,"_48":-5,"_18":141},"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{"_37":139},"counter.labspace.ai",["D",1730409518987],[19],"pulls","branches","forks","envVars",{"_149":64,"_151":64},"connections",{"_149":64,"_150":64},"count","unfilledCount","emptyCount",{"_32":59,"_13":60,"_36":61,"_43":-5,"_34":35,"_46":62,"_52":53,"_44":63,"_17":64,"_40":24,"_38":-5,"_55":56,"_48":-5,"_57":156,"_18":157},{"_32":33,"_13":19,"_36":37,"_43":-5,"_34":35,"_46":47,"_52":53,"_44":54,"_17":42,"_40":41,"_38":-5,"_55":56,"_48":-5,"_57":154,"_18":155},[],[19],[],[60],["D",1746217091978],{"_11":12,"_119":115},{"_11":12,"_113":-5,"_114":115,"_76":77,"_116":117,"_118":56},{"_17":47,"_13":16,"_32":35,"_105":67,"_106":-5,"_107":-5,"_108":-5,"_109":-5,"_110":-5,"_111":14,"_112":12},"ctx",{"_98":-5,"_195":196,"_136":197,"_198":199,"_200":201,"_202":56,"_203":56,"_204":56,"_205":53,"_206":-5,"_207":-7,"_208":-7,"_209":64,"_210":56,"_211":56,"_212":-5},"ENV",{"_166":167,"_168":169,"_170":16,"_171":172,"_173":174,"_175":176,"_177":178,"_179":180,"_181":182,"_183":184,"_185":186,"_187":188,"_189":190,"_191":192,"_193":194},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","3bdf5109b605ebb2de94b34b62dae107321f9c84","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town","aiChatModels",[213,214,215,216,217],[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","navCollapsed","currentOrgId",{"_218":241,"_48":242,"_221":243,"_74":244,"_224":53,"_225":53,"_226":227,"_228":53},{"_218":237,"_48":238,"_221":239,"_74":240,"_224":53,"_225":53,"_226":227,"_228":56},{"_218":233,"_48":234,"_221":235,"_74":236,"_224":53,"_225":53,"_226":227,"_228":56},{"_218":229,"_48":230,"_221":231,"_74":232,"_224":53,"_225":53,"_226":227,"_228":56},{"_218":219,"_48":220,"_221":222,"_74":223,"_224":53,"_225":53,"_226":227,"_228":56},"value","fable","Claude Fable 5.1","shortLabel","Fable","Most capable; 2× Opus pricing","imageInput","thinking","contextWindowSize",1000000,"isDefault","opus55","Claude Opus 5.5","Opus","Highly capable for complex work","sonnet55","Claude Sonnet 5.5","Sonnet","Balanced speed and capability","haiku55","Claude Haiku 5.5","Haiku 5.5","Fast; good for straightforward edits","deepseekflash","DeepSeek V4.1 Flash","DeepSeek Flash","Smart and very cheap"]