[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_160":161,"_162":163},"routes/_app.x.$handle.$name",{"_11":12,"_13":14,"_88":-7,"_89":90,"_29":-7,"_91":-5,"_15":92,"_93":94,"_95":96,"_97":98,"_99":100,"_101":-5,"_102":-5,"_103":-5,"_104":75,"_105":106,"_107":38,"_108":-5,"_109":-5,"_110":111},"routes/_app.x.$handle.$name.branches",{"_11":12,"_13":14,"_15":16,"_17":18},"handle","yawnxyz","name","counter","project",{"_20":28,"_13":14,"_22":36,"_35":-5,"_37":38,"_39":40,"_41":40,"_42":-5,"_43":-5,"_44":45,"_46":47,"_48":38,"_49":-5,"_50":-5,"_51":52,"_53":54,"_55":-5,"_56":-5,"_57":-5,"_58":-5,"_59":60,"_61":60,"_62":63,"_64":63,"_65":63,"_66":60,"_67":60,"_68":69,"_70":71,"_72":73,"_74":75},"branches",[19],{"_20":21,"_22":23,"_24":25,"_13":26,"_27":28,"_29":30,"_31":-5,"_32":-5,"_33":34,"_35":-5},"id","923012f2-2792-11f0-9d52-569c3dd06744","createdAt",["D",1746217091978],"updatedAt",["D",1730409518987],"main","projectId","922fd8b4-2792-11f0-9d52-569c3dd06744","version",9,"forkedBranchId","forkedBranchVersion","lockfileKey","1746243533430-esm.town_v_yawnxyz_counter@9-main","deletedAt",["D",1746217091978],"legacyEnvVars",true,"privacy","public","httpPrivacy","description","imageUrl","userId","a8270e74-6479-475a-8025-9b69544ee0e7","pinnedFileId","9231f658-2792-11f0-9d52-569c3dd06744","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","8319defa-6e1a-46d4-ba66-2114ffd3360b","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_11":12,"_87":83},"org",{"_11":12,"_81":-5,"_82":83,"_44":45,"_84":85,"_86":75},"branch",{"_29":30,"_13":26,"_20":21,"_27":28,"_31":-5,"_32":-5,"_76":-5,"_77":-5,"_78":-5,"_79":14,"_80":12},"isOwned",false,"forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","displayName","avatar","https://images.clerk.dev/oauth_github/img_2NnaHhpxNuH1xWRIRjQNoo16TVc.jpeg","tier","pro","canRestrictVals","profileImageUrl","branchName","path","","adoptOrgId",{"_20":28,"_13":14,"_22":156,"_35":-5,"_37":38,"_39":40,"_41":40,"_42":-5,"_43":-5,"_44":45,"_46":47,"_48":38,"_49":-5,"_50":-5,"_51":52,"_53":54,"_55":-5,"_56":-5,"_57":-5,"_58":-5,"_59":60,"_61":60,"_62":63,"_64":63,"_65":63,"_66":60,"_67":60,"_68":157,"_70":158,"_72":159,"_74":75},"tree",[147,148],"files",[147,148],"counts",{"_138":52,"_17":134,"_139":52,"_140":141,"_142":143},"contents",[113,114],"file","upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{"_47":112},"counter.labspace.ai",{"_20":131,"_13":132,"_117":133,"_118":-5,"_119":21,"_120":134,"_121":38,"_22":135,"_29":52,"_124":101,"_126":136,"_128":75,"_129":-5,"_89":137},{"_20":115,"_13":116,"_117":47,"_118":-5,"_119":21,"_120":30,"_121":38,"_22":122,"_29":123,"_124":125,"_126":127,"_128":75,"_129":-5,"_89":130},"9259cc14-2792-11f0-9d52-569c3dd06744","main.tsx","fileId","parentId","branchId","branchVersion","latest",["D",1730409518987],8,"fileType","http","content","import { Hono } from \"https://deno.land/x/hono@v3.11.7/mod.ts\";\nimport { get_encoding } from \"npm:tiktoken\";\n\nconst app = new Hono();\n\nconst html = `\n\n\n\n \n \n \n Word Counter, Byte Counter, and Token Counter\n \n \n \n \n \n \n \n \n \n \n \n \n\n\n

Counter

All-in-one Word Counter, Byte Counter, and Token Counter. Save results to browser history.

Words

\n 0\n

Characters

\n 0\n

Twitter

\n 0\n /280\n

Sentences

\n 0\n

Paragraphs

\n 0\n

\n\n \n\n

Bytes

\n 0\n

\n\n

\n Choose Tokenizer\n

The tokenizers are still a work-in-progress, and sometimes break on certain characters, emojis, and more. o200k_base is the most stable tokenizer.

\n \n

Co-written with Claude.
Heavily inspired by https://mothereff.in/byte-counter, https://wordcounter.io/, and https://tiktokenizer.vercel.app/

\n A labspace.ai experiment\n

\n \n \n\n\n`;\n\napp.get(\"/\", (c) => c.html(html));\n\napp.post(\"/tokenize\", async (c) => {\n try {\n const body = await c.req.json();\n const { text, model } = body;\n if (!text) {\n return c.json({ count: 0, segments: [] });\n }\n\n // Pre-process text to handle emojis\n const processedText = replaceEmojisWithCodePoints(text);\n\n const encoder = get_encoding(model);\n const segments = getTiktokenSegments(encoder, processedText);\n\n encoder.free();\n\n return c.json({ count: segments.reduce((acc, seg) => acc + seg.tokens.length, 0), segments });\n } catch (error) {\n return c.json({ error: error.message }, 500);\n }\n});\n\nfunction replaceEmojisWithCodePoints(text) {\n console.log(\"Replacing emojis with code points:\", text);\n\n // More comprehensive emoji regex that includes additional ranges and combined emojis\n return text.replace(\n /[\\u{1F000}-\\u{1F9FF}\\u{2600}-\\u{27BF}\\u{2B50}\\u{2B06}\\u{2934}-\\u{2935}\\u{2B05}-\\u{2B07}\\u{2B1B}-\\u{2B1C}\\u{2B50}\\u{2B55}\\u{3030}\\u{303D}\\u{3297}\\u{3299}\\u{2702}-\\u{27B0}\\u{1F170}-\\u{1F251}]|[\\u{23E9}-\\u{23EC}\\u{23F0}\\u{23F3}\\u{25FD}-\\u{25FE}\\u{2614}-\\u{2615}\\u{2648}-\\u{2653}\\u{267F}\\u{2693}\\u{26A1}\\u{26AA}-\\u{26AB}\\u{26BD}-\\u{26BE}\\u{26C4}-\\u{26C5}\\u{26CE}\\u{26D4}\\u{26EA}\\u{26F2}-\\u{26F3}\\u{26F5}\\u{26FA}\\u{26FD}\\u{2705}\\u{2728}\\u{274C}\\u{274E}\\u{2753}-\\u{2755}\\u{2757}\\u{2795}-\\u{2797}\\u{27B0}\\u{27BF}\\u{2B1B}-\\u{2B1C}\\u{2B50}\\u{2B55}]|\\u{200D}[\\u{2640}\\u{2642}\\u{2695}\\u{2696}\\u{2708}\\u{2764}]|\\u{FE0F}|[\\u{1F468}\\u{1F469}][\\u{1F3FB}-\\u{1F3FF}]?\\u{200D}[\\u{2695}\\u{2696}\\u{2708}\\u{2764}]|\\u{1F468}\\u{200D}\\u{1F469}\\u{200D}[\\u{1F466}\\u{1F467}]|[\\u{1F468}\\u{1F469}]\\u{200D}[\\u{1F466}\\u{1F467}]/gu,\n (match) => {\n console.log(\"match:\", match);\n const codePoint = match.codePointAt(0)?.toString(16);\n return `[U+${codePoint}]`;\n }\n );\n}\n\nfunction getTiktokenSegments(encoder, inputText) {\n try {\n const tokens = encoder.encode(inputText);\n const segments = [];\n\n let currentSegment = '';\n let tokenAcc = [];\n\n for (let idx = 0; idx < tokens.length; idx++) {\n const token = tokens[idx];\n const tokenBytes = encoder.decode_single_token_bytes(token);\n const tokenText = new TextDecoder().decode(new Uint8Array(tokenBytes));\n \n currentSegment += tokenText;\n tokenAcc.push({ id: token, idx });\n\n // Check if we've reached a valid segment boundary\n if (inputText.startsWith(currentSegment)) {\n segments.push({ \n text: currentSegment, \n tokens: tokenAcc \n });\n\n // Remove the matched segment from the input text\n inputText = inputText.slice(currentSegment.length);\n \n // Reset for next segment\n currentSegment = '';\n tokenAcc = [];\n }\n }\n\n return segments;\n } catch (error) {\n throw new Error(`Error in getTiktokenSegments: ${error.message}`);\n }\n}\n\n// Export app.fetch for Val Town, otherwise export app — this is only for hono apps\nexport default (typeof Deno !== \"undefined\" && Deno.env.get(\"valtown\")) ? app.fetch : app;\n","deleted","label",[116],"9231f96e-2792-11f0-9d52-569c3dd06744","README.md","9231f9dc-2792-11f0-9d52-569c3dd06744",1,["D",1729373169069],"Migrated from folder: labspace/counter",[132],"pulls","forks","envVars",{"_144":52,"_146":52},"connections",{"_144":52,"_145":52},"count","unfilledCount","emptyCount",{"_20":131,"_13":132,"_117":133,"_118":-5,"_119":21,"_120":134,"_121":38,"_22":153,"_29":52,"_124":101,"_126":136,"_128":75,"_129":-5,"_150":154,"_89":155},{"_20":115,"_13":116,"_117":47,"_118":-5,"_119":21,"_120":30,"_121":38,"_22":149,"_29":123,"_124":125,"_126":127,"_128":75,"_129":-5,"_150":151,"_89":152},["D",1730409518987],"children",[],[116],["D",1729373169069],[],[132],["D",1746217091978],{"_11":12,"_87":83},{"_11":12,"_81":-5,"_82":83,"_44":45,"_84":85,"_86":75},{"_29":30,"_13":26,"_20":21,"_27":28,"_31":-5,"_32":-5,"_76":-5,"_77":-5,"_78":-5,"_79":14,"_80":12},"ctx",{"_68":-5,"_109":193,"_194":195,"_196":197,"_198":75,"_199":75,"_200":75,"_201":38,"_202":-5,"_203":-7,"_204":-7,"_205":52,"_206":75,"_207":75,"_208":-5,"_209":75,"_210":75,"_211":75,"_212":-5},"ENV",{"_164":165,"_166":167,"_168":26,"_169":170,"_171":172,"_173":174,"_175":176,"_177":178,"_179":180,"_181":182,"_183":184,"_185":186,"_187":188,"_189":190,"_191":192},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Branches1

Branches
1