TokenizerDemo
Script
v6
Fork
1
2
3
4
5
6
7
8
9
10
11
12
import
{
Tokenizer
}
from
"https://esm.town/v/zzz/Tokenizer"
;
// Demo of tokenizer to mimic behavior of https://platform.openai.com/tokenizer
// Tokenizer uses "gpt-3.5-turbo" model by default but this demo uses davinci to match the playground
export
const
TokenizerDemo
=
(
async
(
)
=>
{
const
tokens
=
await
Tokenizer
(
"Hello Val Town"
,
"text-davinci-003"
,
)
;
console
.
log
(
tokens
)
;
return
`Tokens:
${
tokens
.
length
}
`
;
}
)
(
)
;