export async function magnitTokenize(value: string, specification: Array<{ regex: RegExp, key?: string, captureGroups?: string[], groupJoinValue?: string, resultCallback?: (value: string) => Promise<string> }>|null = null)
{
class MagnitTokenizerSpecification extends Array<MagnitTokenizerSpecificationItem> { }
class MagnitTokenizerSpecificationItem
{
regex: RegExp;
key?: string;
captureGroups?: string[];
groupJoinValue?: string;
resultCallback?: (value: string) => Promise<string>
constructor(regex: RegExp, options: { key?:string, captureGroups?: string[], groupJoinValue?:string, resultCallback?:(value:string) =>Promise<string> } | null = null)
{
this.regex = regex;
this.key = options?.key,
this.captureGroups = options?.captureGroups;
this.groupJoinValue = options?.groupJoinValue;
this.resultCallback = options?.resultCallback;
}
}
class MagnitToken
{
startIndex: number = 0;
type?: string;
value?: any;
}
class MagnitTokenizer
{
specification: MagnitTokenizerSpecification;
private input?: string;
private cursor: number = 0;
private previousCursorPosition: number;
private lookahead?: MagnitToken | null;
constructor(specification: MagnitTokenizerSpecification)
{
this.specification = specification;
this.input = "";
this.previousCursorPosition = -1;
}
async parse(input: string, specification: MagnitTokenizerSpecification | null = null) : Promise<MagnitToken[]>
{
if(specification != null)
{
this.specification = specification;
}
return await this.getTokens(input);
}
private async getTokens(input: string) : Promise<MagnitToken[]>
{
this.input = input;
this.cursor = 0;
this.lookahead = await this.getNextToken();
const tokens: MagnitToken[] = [];
while(this.hasMoreTokens())
{
this.previousCursorPosition = this.getCursorPosition();
tokens.push(await this.consume(this.lookahead?.type!));
}
if(this.lookahead != null)
{
tokens.push(await this.consume(this.lookahead?.type!))
}
return tokens;
}
private async consume(tokenType: string) : Promise<MagnitToken>
{
const token = this.lookahead;
if(token == null)
{
throw new Error(`Unexpected end of input. Expected: "${tokenType}".`);
}
if(token.type != tokenType)
{
throw new Error(`Unexpected token type: "${token.type}". Expected: "${tokenType}".`)
}
this.lookahead = await this.getNextToken();
return token;
}
isEndOfFile(): boolean
{
if(this.input == null) return true;
return this.cursor == this.input?.length;
}
hasMoreTokens(): boolean