1
2
3
4
5
6
7
8
9
10
11
12
import { Tokenizer } from "https://esm.town/v/zzz/Tokenizer";
// Demo of tokenizer to mimic behavior of https://platform.openai.com/tokenizer
// Tokenizer uses "gpt-3.5-turbo" model by default but this demo uses davinci to match the playground
export const TokenizerDemo = (async () => {
const tokens = await Tokenizer(
"Hello Val Town",
"text-davinci-003",
);
console.log(tokens);
return `Tokens: ${tokens.length}`;
})();