Counter

All-in-one Word Counter, Byte Counter, and Token Counter. Save results to browser history.

Words

\n 0\n

Characters

\n 0\n

Twitter

\n 0\n /280\n

Sentences

\n 0\n

Paragraphs

\n 0\n

\n\n \n\n

Bytes

\n 0\n

\n\n

\n Choose Tokenizer\n

The tokenizers are still a work-in-progress, and sometimes break on certain characters, emojis, and more. o200k_base is the most stable tokenizer.

\n \n

Co-written with Claude.
Heavily inspired by https://mothereff.in/byte-counter, https://wordcounter.io/, and https://tiktokenizer.vercel.app/

\n A labspace.ai experiment\n

\n \n \n\n\n`;\n\napp.get(\"/\", (c) => c.html(html));\n\napp.post(\"/tokenize\", async (c) => {\n try {\n const body = await c.req.json();\n const { text, model } = body;\n if (!text) {\n return c.json({ count: 0, segments: [] });\n }\n\n // Pre-process text to handle emojis\n const processedText = replaceEmojisWithCodePoints(text);\n\n const encoder = get_encoding(model);\n const segments = getTiktokenSegments(encoder, processedText);\n\n encoder.free();\n\n return c.json({ count: segments.reduce((acc, seg) => acc + seg.tokens.length, 0), segments });\n } catch (error) {\n return c.json({ error: error.message }, 500);\n }\n});\n\nfunction replaceEmojisWithCodePoints(text) {\n console.log(\"Replacing emojis with code points:\", text);\n\n // More comprehensive emoji regex that includes additional ranges and combined emojis\n return text.replace(\n /[\\u{1F000}-\\u{1F9FF}\\u{2600}-\\u{27BF}\\u{2B50}\\u{2B06}\\u{2934}-\\u{2935}\\u{2B05}-\\u{2B07}\\u{2B1B}-\\u{2B1C}\\u{2B50}\\u{2B55}\\u{3030}\\u{303D}\\u{3297}\\u{3299}\\u{2702}-\\u{27B0}\\u{1F170}-\\u{1F251}]|[\\u{23E9}-\\u{23EC}\\u{23F0}\\u{23F3}\\u{25FD}-\\u{25FE}\\u{2614}-\\u{2615}\\u{2648}-\\u{2653}\\u{267F}\\u{2693}\\u{26A1}\\u{26AA}-\\u{26AB}\\u{26BD}-\\u{26BE}\\u{26C4}-\\u{26C5}\\u{26CE}\\u{26D4}\\u{26EA}\\u{26F2}-\\u{26F3}\\u{26F5}\\u{26FA}\\u{26FD}\\u{2705}\\u{2728}\\u{274C}\\u{274E}\\u{2753}-\\u{2755}\\u{2757}\\u{2795}-\\u{2797}\\u{27B0}\\u{27BF}\\u{2B1B}-\\u{2B1C}\\u{2B50}\\u{2B55}]|\\u{200D}[\\u{2640}\\u{2642}\\u{2695}\\u{2696}\\u{2708}\\u{2764}]|\\u{FE0F}|[\\u{1F468}\\u{1F469}][\\u{1F3FB}-\\u{1F3FF}]?\\u{200D}[\\u{2695}\\u{2696}\\u{2708}\\u{2764}]|\\u{1F468}\\u{200D}\\u{1F469}\\u{200D}[\\u{1F466}\\u{1F467}]|[\\u{1F468}\\u{1F469}]\\u{200D}[\\u{1F466}\\u{1F467}]/gu,\n (match) => {\n console.log(\"match:\", match);\n const codePoint = match.codePointAt(0)?.toString(16);\n return `[U+${codePoint}]`;\n }\n );\n}\n\nfunction getTiktokenSegments(encoder, inputText) {\n try {\n const tokens = encoder.encode(inputText);\n const segments = [];\n\n let currentSegment = '';\n let tokenAcc = [];\n\n for (let idx = 0; idx < tokens.length; idx++) {\n const token = tokens[idx];\n const tokenBytes = encoder.decode_single_token_bytes(token);\n const tokenText = new TextDecoder().decode(new Uint8Array(tokenBytes));\n \n currentSegment += tokenText;\n tokenAcc.push({ id: token, idx });\n\n // Check if we've reached a valid segment boundary\n if (inputText.startsWith(currentSegment)) {\n segments.push({ \n text: currentSegment, \n tokens: tokenAcc \n });\n\n // Remove the matched segment from the input text\n inputText = inputText.slice(currentSegment.length);\n \n // Reset for next segment\n currentSegment = '';\n tokenAcc = [];\n }\n }\n\n return segments;\n } catch (error) {\n throw new Error(`Error in getTiktokenSegments: ${error.message}`);\n }\n}\n\n// Export app.fetch for Val Town, otherwise export app — this is only for hono apps\nexport default (typeof Deno !== \"undefined\" && Deno.env.get(\"valtown\")) ? app.fetch : app;\n","deleted","label",[99],"permalink","https://www.val.town/x/yawnxyz/counter/code/main.tsx","extension","actionData","errors"]