[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_169":170,"_171":172},"routes/_app.x.$handle.$name",{"_11":12,"_13":14,"_15":16,"_18":19,"_17":-7,"_127":-5,"_20":128,"_22":129,"_130":131,"_132":133,"_25":134,"_24":-5,"_135":-5,"_136":-5,"_137":54,"_138":139,"_140":45,"_141":-5,"_142":-5,"_143":144},"routes/_app.x.$handle.$name.branch.$branch.code.$",{"_11":12,"_13":14,"_15":16,"_17":-7,"_18":19,"_20":21,"_22":23,"_24":-7,"_25":26,"_27":-5,"_28":-5,"_29":-5,"_30":-5,"_31":-5},"handle","yawnxyz","name","counter","branchName","main","version","path","","project",{"_34":74,"_13":14,"_46":75,"_76":-5,"_77":45,"_78":79,"_80":79,"_81":-5,"_82":-5,"_83":84,"_85":38,"_86":45,"_87":-5,"_88":-5,"_89":62,"_90":91,"_92":-5,"_93":-5,"_94":-5,"_95":-5,"_96":97,"_98":97,"_99":100,"_101":100,"_102":100,"_103":97,"_104":97,"_105":106,"_107":108,"_109":110,"_111":54},"tree",[65,66],"file","contents",[32,33],"upstream","pull","ahead","behind","envVars",{"_34":57,"_13":58,"_37":59,"_39":-5,"_40":41,"_42":60,"_44":45,"_46":61,"_17":62,"_49":24,"_51":63,"_53":54,"_55":-5,"_18":64},{"_34":35,"_13":36,"_37":38,"_39":-5,"_40":41,"_42":43,"_44":45,"_46":47,"_17":48,"_49":50,"_51":52,"_53":54,"_55":-5,"_18":56},"id","9259cc14-2792-11f0-9d52-569c3dd06744","main.tsx","fileId","9231f658-2792-11f0-9d52-569c3dd06744","parentId","branchId","923012f2-2792-11f0-9d52-569c3dd06744","branchVersion",9,"latest",true,"createdAt",["D",1730409518987],8,"fileType","http","content","import { Hono } from \"https://deno.land/x/hono@v3.11.7/mod.ts\";\nimport { get_encoding } from \"npm:tiktoken\";\n\nconst app = new Hono();\n\nconst html = `\n\n\n\n \n \n \n Word Counter, Byte Counter, and Token Counter\n \n \n \n \n \n \n \n \n \n \n \n \n\n\n

Counter

All-in-one Word Counter, Byte Counter, and Token Counter. Save results to browser history.

Words

\n 0\n

Characters

\n 0\n

Twitter

\n 0\n /280\n

Sentences

\n 0\n

Paragraphs

\n 0\n

\n\n \n\n

Bytes

\n 0\n

\n\n

\n Choose Tokenizer\n

The tokenizers are still a work-in-progress, and sometimes break on certain characters, emojis, and more. o200k_base is the most stable tokenizer.

\n \n

Co-written with Claude.
Heavily inspired by https://mothereff.in/byte-counter, https://wordcounter.io/, and https://tiktokenizer.vercel.app/

\n A labspace.ai experiment\n

\n \n \n\n\n`;\n\napp.get(\"/\", (c) => c.html(html));\n\napp.post(\"/tokenize\", async (c) => {\n try {\n const body = await c.req.json();\n const { text, model } = body;\n if (!text) {\n return c.json({ count: 0, segments: [] });\n }\n\n // Pre-process text to handle emojis\n const processedText = replaceEmojisWithCodePoints(text);\n\n const encoder = get_encoding(model);\n const segments = getTiktokenSegments(encoder, processedText);\n\n encoder.free();\n\n return c.json({ count: segments.reduce((acc, seg) => acc + seg.tokens.length, 0), segments });\n } catch (error) {\n return c.json({ error: error.message }, 500);\n }\n});\n\nfunction replaceEmojisWithCodePoints(text) {\n console.log(\"Replacing emojis with code points:\", text);\n\n // More comprehensive emoji regex that includes additional ranges and combined emojis\n return text.replace(\n /[\\u{1F000}-\\u{1F9FF}\\u{2600}-\\u{27BF}\\u{2B50}\\u{2B06}\\u{2934}-\\u{2935}\\u{2B05}-\\u{2B07}\\u{2B1B}-\\u{2B1C}\\u{2B50}\\u{2B55}\\u{3030}\\u{303D}\\u{3297}\\u{3299}\\u{2702}-\\u{27B0}\\u{1F170}-\\u{1F251}]|[\\u{23E9}-\\u{23EC}\\u{23F0}\\u{23F3}\\u{25FD}-\\u{25FE}\\u{2614}-\\u{2615}\\u{2648}-\\u{2653}\\u{267F}\\u{2693}\\u{26A1}\\u{26AA}-\\u{26AB}\\u{26BD}-\\u{26BE}\\u{26C4}-\\u{26C5}\\u{26CE}\\u{26D4}\\u{26EA}\\u{26F2}-\\u{26F3}\\u{26F5}\\u{26FA}\\u{26FD}\\u{2705}\\u{2728}\\u{274C}\\u{274E}\\u{2753}-\\u{2755}\\u{2757}\\u{2795}-\\u{2797}\\u{27B0}\\u{27BF}\\u{2B1B}-\\u{2B1C}\\u{2B50}\\u{2B55}]|\\u{200D}[\\u{2640}\\u{2642}\\u{2695}\\u{2696}\\u{2708}\\u{2764}]|\\u{FE0F}|[\\u{1F468}\\u{1F469}][\\u{1F3FB}-\\u{1F3FF}]?\\u{200D}[\\u{2695}\\u{2696}\\u{2708}\\u{2764}]|\\u{1F468}\\u{200D}\\u{1F469}\\u{200D}[\\u{1F466}\\u{1F467}]|[\\u{1F468}\\u{1F469}]\\u{200D}[\\u{1F466}\\u{1F467}]/gu,\n (match) => {\n console.log(\"match:\", match);\n const codePoint = match.codePointAt(0)?.toString(16);\n return `[U+${codePoint}]`;\n }\n );\n}\n\nfunction getTiktokenSegments(encoder, inputText) {\n try {\n const tokens = encoder.encode(inputText);\n const segments = [];\n\n let currentSegment = '';\n let tokenAcc = [];\n\n for (let idx = 0; idx < tokens.length; idx++) {\n const token = tokens[idx];\n const tokenBytes = encoder.decode_single_token_bytes(token);\n const tokenText = new TextDecoder().decode(new Uint8Array(tokenBytes));\n \n currentSegment += tokenText;\n tokenAcc.push({ id: token, idx });\n\n // Check if we've reached a valid segment boundary\n if (inputText.startsWith(currentSegment)) {\n segments.push({ \n text: currentSegment, \n tokens: tokenAcc \n });\n\n // Remove the matched segment from the input text\n inputText = inputText.slice(currentSegment.length);\n \n // Reset for next segment\n currentSegment = '';\n tokenAcc = [];\n }\n }\n\n return segments;\n } catch (error) {\n throw new Error(`Error in getTiktokenSegments: ${error.message}`);\n }\n}\n\n// Export app.fetch for Val Town, otherwise export app — this is only for hono apps\nexport default (typeof Deno !== \"undefined\" && Deno.env.get(\"valtown\")) ? app.fetch : app;\n","deleted",false,"label",[36],"9231f96e-2792-11f0-9d52-569c3dd06744","README.md","9231f9dc-2792-11f0-9d52-569c3dd06744",1,["D",1729373169069],0,"Migrated from folder: labspace/counter",[58],{"_34":57,"_13":58,"_37":59,"_39":-5,"_40":41,"_42":60,"_44":45,"_46":71,"_17":62,"_49":24,"_51":63,"_53":54,"_55":-5,"_68":72,"_18":73},{"_34":35,"_13":36,"_37":38,"_39":-5,"_40":41,"_42":43,"_44":45,"_46":67,"_17":48,"_49":50,"_51":52,"_53":54,"_55":-5,"_68":69,"_18":70},["D",1730409518987],"children",[],[36],["D",1729373169069],[],[58],"922fd8b4-2792-11f0-9d52-569c3dd06744",["D",1746217091978],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","imageUrl","userId","a8270e74-6479-475a-8025-9b69544ee0e7","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount","orgId","8319defa-6e1a-46d4-ba66-2114ffd3360b","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_11":12,"_126":122},"org",{"_11":12,"_120":-5,"_121":122,"_83":84,"_123":124,"_125":54},"branch",{"_17":43,"_13":16,"_34":41,"_112":74,"_113":-5,"_114":-5,"_115":-5,"_116":-5,"_117":-5,"_118":14,"_119":12},"isOwned","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","displayName","avatar","https://images.clerk.dev/oauth_github/img_2NnaHhpxNuH1xWRIRjQNoo16TVc.jpeg","tier","pro","canRestrictVals","profileImageUrl","adoptOrgId",{"_34":74,"_13":14,"_46":165,"_76":-5,"_77":45,"_78":79,"_80":79,"_81":-5,"_82":-5,"_83":84,"_85":38,"_86":45,"_87":-5,"_88":-5,"_89":62,"_90":91,"_92":-5,"_93":-5,"_94":-5,"_95":-5,"_96":97,"_98":97,"_99":100,"_101":100,"_102":100,"_103":97,"_104":97,"_105":166,"_107":167,"_109":168,"_111":54},[159,160],"files",[159,160],"counts",{"_150":62,"_151":60,"_152":62,"_31":153,"_154":155},[146,147],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{"_38":145},"counter.labspace.ai",{"_34":57,"_13":58,"_37":59,"_39":-5,"_40":41,"_42":60,"_44":45,"_46":61,"_17":62,"_49":24,"_51":63,"_53":54,"_55":-5,"_18":149},{"_34":35,"_13":36,"_37":38,"_39":-5,"_40":41,"_42":43,"_44":45,"_46":47,"_17":48,"_49":50,"_51":52,"_53":54,"_55":-5,"_18":148},[36],[58],"pulls","branches","forks",{"_156":62,"_158":62},"connections",{"_156":62,"_157":62},"count","unfilledCount","emptyCount",{"_34":57,"_13":58,"_37":59,"_39":-5,"_40":41,"_42":60,"_44":45,"_46":71,"_17":62,"_49":24,"_51":63,"_53":54,"_55":-5,"_68":163,"_18":164},{"_34":35,"_13":36,"_37":38,"_39":-5,"_40":41,"_42":43,"_44":45,"_46":67,"_17":48,"_49":50,"_51":52,"_53":54,"_55":-5,"_68":161,"_18":162},[],[36],[],[58],["D",1746217091978],{"_11":12,"_126":122},{"_11":12,"_120":-5,"_121":122,"_83":84,"_123":124,"_125":54},{"_17":43,"_13":16,"_34":41,"_112":74,"_113":-5,"_114":-5,"_115":-5,"_116":-5,"_117":-5,"_118":14,"_119":12},"ctx",{"_105":-5,"_142":202,"_203":204,"_205":206,"_207":54,"_208":54,"_209":54,"_210":45,"_211":-5,"_212":-7,"_213":-7,"_214":62,"_215":54,"_216":54,"_217":-5,"_218":54,"_219":54,"_220":54,"_221":-5},"ENV",{"_173":174,"_175":176,"_177":16,"_178":179,"_180":181,"_182":183,"_184":185,"_186":187,"_188":189,"_190":191,"_192":193,"_194":195,"_196":197,"_198":199,"_200":201},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]