1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
export async function magnitTokenize(value: string, specification: Array<{ regex: RegExp, key?: string, captureGroups?: string[], groupJoinValue?: string, resultCallback?: (value: string) => Promise<string> }>|null = null)
{
class MagnitTokenizerSpecification extends Array<MagnitTokenizerSpecificationItem> { }
// key is optional because you can leave it as null in order
// to skip any tokens that match for that regex.
class MagnitTokenizerSpecificationItem
{
regex: RegExp;
key?: string;
captureGroups?: string[];
groupJoinValue?: string;
resultCallback?: (value: string) => Promise<string>
constructor(regex: RegExp, options: { key?:string, captureGroups?: string[], groupJoinValue?:string, resultCallback?:(value:string) =>Promise<string> } | null = null)
{
this.regex = regex;
this.key = options?.key,
this.captureGroups = options?.captureGroups;
this.groupJoinValue = options?.groupJoinValue;
this.resultCallback = options?.resultCallback;
}
}
class MagnitToken
{
startIndex: number = 0;
type?: string;
value?: any;
}
class MagnitTokenizer
{
specification: MagnitTokenizerSpecification;
private input?: string;
private cursor: number = 0;
private previousCursorPosition: number;
private lookahead?: MagnitToken | null;
constructor(specification: MagnitTokenizerSpecification)
{
this.specification = specification;
this.input = "";
this.previousCursorPosition = -1;
}
async parse(input: string, specification: MagnitTokenizerSpecification | null = null) : Promise<MagnitToken[]>
{
if(specification != null)
{
this.specification = specification;
}
return await this.getTokens(input);
}
private async getTokens(input: string) : Promise<MagnitToken[]>
{
this.input = input;
this.cursor = 0;
this.lookahead = await this.getNextToken();
const tokens: MagnitToken[] = [];
while(this.hasMoreTokens())
{
this.previousCursorPosition = this.getCursorPosition();
tokens.push(await this.consume(this.lookahead?.type!));
}
if(this.lookahead != null)
{
tokens.push(await this.consume(this.lookahead?.type!))
}
return tokens;
}
private async consume(tokenType: string) : Promise<MagnitToken>
{
const token = this.lookahead;
if(token == null)
{
throw new Error(`Unexpected end of input. Expected: "${tokenType}".`);
}
if(token.type != tokenType)
{
throw new Error(`Unexpected token type: "${token.type}". Expected: "${tokenType}".`)
}
this.lookahead = await this.getNextToken();
return token;
}
isEndOfFile(): boolean
{
if(this.input == null) return true;
return this.cursor == this.input?.length;
}
hasMoreTokens(): boolean
👆 This is a val. Vals are TypeScript snippets of code, written in the browser and run on our servers. Create scheduled functions, email yourself, and persist small pieces of data — all from the browser.