[{"_1":2,"_168":-5,"_169":-5},"loaderData",{"_3":4,"_91":92},"root",{"_5":6,"_34":35,"_64":65},"ctx",{"_7":8,"_23":-5,"_24":25,"_26":27,"_28":20,"_29":20,"_30":20,"_31":32,"_33":20},"auth",{"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20,"_21":22},"sessionClaims","sessionId","sessionStatus","userId","actor","orgId","orgRole","orgSlug","orgPermissions","factorVerificationAge","isAuthenticated",false,"tokenType","session_token","user","preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","notificationCount",0,"flagDenols","ENV",{"_36":37,"_38":39,"_40":41,"_42":43,"_44":45,"_46":47,"_48":49,"_50":51,"_52":53,"_54":55,"_56":57,"_58":59,"_60":61,"_62":63},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","VAL_RUNTIME_NAME","prod","RENDER_GIT_COMMIT","230fbbe1911b69d46be82b6db4e617b331d282a6","RENDER_GIT_BRANCH","main","CLERK_PUBLISHABLE_KEY","pk_live_Y2xlcmsudmFsLnRvd24k","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","API_URL","https://api.val.town","clerkState",{"_66":67},"__internal_clerk_state",{"_68":69,"_70":45,"_71":72,"_73":72,"_74":20,"_75":72,"_76":72,"_77":72,"_78":72,"_79":80,"_89":20,"_90":20},"__clerk_ssr_state",{"_21":22,"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5,"_18":-5,"_19":20},"__publishableKey","__proxyUrl","","__domain","__isSatellite","__signInUrl","__signUpUrl","__afterSignInUrl","__afterSignUpUrl","__clerk_debug",{"_81":20,"_19":20,"_82":72,"_83":84,"_85":72,"_86":45,"_87":20,"_88":72},"isSignedIn","proxyUrl","reason","session-token-and-uat-missing","message","publishableKey","isSatellite","domain","__telemetryDisabled","__telemetryDebug","routes/embed.x.$handle.$name.$",{"_93":94,"_95":96,"_97":98,"_99":100,"_147":148,"_165":166,"_167":-5},"handle","catapart","name","magnitTokenize","path","main.tsx","project",{"_101":102,"_95":96,"_103":104,"_105":-5,"_106":20,"_107":108,"_109":-5,"_110":-5,"_12":111,"_112":-5,"_113":114,"_115":-5,"_116":32,"_14":117,"_118":119,"_120":119,"_121":122,"_123":122,"_124":122,"_125":119,"_126":119,"_23":127,"_130":131,"_133":134,"_146":20},"id","2e4a9142-26d2-11f0-a428-569c3dd06744","createdAt",["D",1746134461009],"deletedAt","legacyEnvVars","privacy","public","description","imageUrl","1faed022-fe27-46f0-b3c6-964a960dc3a7","pinnedFileId","pinPreview",true,"screenshotVersion","likeCount","2942ce29-335d-4452-ad9e-7de780992d60","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm",{"_93":94,"_128":129},"profileImageUrl","https://secure.gravatar.com/avatar/9a63b236fbb8ba96b0fc6cb28bfc0ab5&d=identicon","org",{"_93":94,"_132":129},"avatar","branch",{"_135":136,"_95":43,"_101":137,"_138":102,"_139":-5,"_140":-5,"_141":-5,"_142":-5,"_143":-5,"_144":96,"_145":94},"version",4,"2e4ae05c-26d2-11f0-a428-569c3dd06744","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","isOwned","file",{"_101":149,"_95":98,"_150":151,"_152":-5,"_153":137,"_154":136,"_155":114,"_103":156,"_135":157,"_158":159,"_160":161,"_162":20,"_163":-5,"_97":164},"2e608c72-26d2-11f0-a428-569c3dd06744","fileId","2e4e1e16-26d2-11f0-a428-569c3dd06744","parentId","branchId","branchVersion","latest",["D",1698082250000],3,"fileType","script","content","export async function magnitTokenize(value: string, specification: Array<{ regex: RegExp, key?: string, captureGroups?: string[], groupJoinValue?: string, resultCallback?: (value: string) => Promise }>|null = null)\n{\n class MagnitTokenizerSpecification extends Array { }\n \n // key is optional because you can leave it as null in order\n // to skip any tokens that match for that regex.\n class MagnitTokenizerSpecificationItem\n {\n regex: RegExp;\n key?: string;\n captureGroups?: string[];\n groupJoinValue?: string;\n resultCallback?: (value: string) => Promise\n \n constructor(regex: RegExp, options: { key?:string, captureGroups?: string[], groupJoinValue?:string, resultCallback?:(value:string) =>Promise } | null = null)\n {\n this.regex = regex;\n this.key = options?.key,\n this.captureGroups = options?.captureGroups;\n this.groupJoinValue = options?.groupJoinValue;\n this.resultCallback = options?.resultCallback;\n }\n }\n \n class MagnitToken\n {\n startIndex: number = 0;\n type?: string;\n value?: any;\n }\n \n class MagnitTokenizer\n {\n specification: MagnitTokenizerSpecification;\n \n private input?: string;\n private cursor: number = 0;\n private previousCursorPosition: number;\n private lookahead?: MagnitToken | null;\n \n constructor(specification: MagnitTokenizerSpecification)\n {\n this.specification = specification;\n this.input = \"\";\n this.previousCursorPosition = -1;\n }\n \n async parse(input: string, specification: MagnitTokenizerSpecification | null = null) : Promise\n {\n if(specification != null)\n {\n this.specification = specification;\n }\n \n return await this.getTokens(input);\n }\n private async getTokens(input: string) : Promise\n {\n this.input = input;\n this.cursor = 0;\n \n this.lookahead = await this.getNextToken();\n \n const tokens: MagnitToken[] = [];\n while(this.hasMoreTokens())\n {\n this.previousCursorPosition = this.getCursorPosition();\n tokens.push(await this.consume(this.lookahead?.type!));\n }\n if(this.lookahead != null)\n {\n tokens.push(await this.consume(this.lookahead?.type!))\n }\n \n return tokens;\n }\n \n private async consume(tokenType: string) : Promise\n {\n const token = this.lookahead;\n \n if(token == null)\n {\n throw new Error(`Unexpected end of input. Expected: \"${tokenType}\".`);\n }\n if(token.type != tokenType)\n {\n throw new Error(`Unexpected token type: \"${token.type}\". Expected: \"${tokenType}\".`)\n }\n \n this.lookahead = await this.getNextToken();\n return token;\n }\n \n isEndOfFile(): boolean\n {\n if(this.input == null) return true;\n return this.cursor == this.input?.length;\n }\n hasMoreTokens(): boolean\n {\n if(this.input == null) return false;\n return this.cursor < this.input?.length;\n }\n \n getCursorPosition() : number\n {\n return this.cursor;\n }\n \n async getNextToken(): Promise\n {\n if(!this.hasMoreTokens() || this.specification == null || this.input == null)\n {\n return null;\n }\n \n let input: string = this.input.substring(this.cursor);\n let startIndex : number = this.cursor;\n for(const item of this.specification)\n {\n const tokenType: string | undefined = item.key;\n const tokenValue: string | null = this.match(item, input);\n if(tokenValue == null)\n {\n continue;\n }\n if(tokenType == null)\n {\n return await this.getNextToken();\n }\n \n const token = new MagnitToken();\n token.startIndex = startIndex;\n token.type = tokenType;\n token.value = (item.resultCallback == null) ? tokenValue : await item.resultCallback(tokenValue);\n \n return token;\n }\n \n throw new Error(`Unexpected token: \"${input[0]}\".`)\n }\n private match(item: MagnitTokenizerSpecificationItem, input : string) : string | null\n {\n let match = input.match(item.regex);\n if(match == null || match.length == 0)\n {\n return null;\n }\n \n let value: string = match[0];\n this.cursor += value.length;\n \n if(item.captureGroups != null && match.groups != null)\n {\n let combinedCaptureValue: string = \"\";\n for(let i = 0; i < item.captureGroups.length; i++)\n {\n const captureGroupName: string = item.captureGroups[i];\n const capturedValue = match.groups[captureGroupName];\n if(capturedValue != null)\n {\n combinedCaptureValue += capturedValue;\n if(item.groupJoinValue !+ null && item.captureGroups.length > (i + 1))\n {\n combinedCaptureValue += item.groupJoinValue;\n }\n }\n }\n value = combinedCaptureValue;\n }\n \n return value;\n }\n }\n \n // regex is checked against string in the order that it is defined in the specification\n // whitespace first, then comments, then good matches, then a utility match to catch any \n // unknown characters.\n const defaultSpecification = [\n // Whitespace\n new MagnitTokenizerSpecificationItem(new RegExp(/^\\s+/)), // undefined/null as value; matches will be skipped.\n \n // Comments\n new MagnitTokenizerSpecificationItem(new RegExp(/^\\/\\/.*/)), // undefined/null as value; matches will be skipped.\n new MagnitTokenizerSpecificationItem(new RegExp(/^\\/\\*[\\s\\S]*?\\*\\//)), // undefined/null as value; matches will be skipped.\n \n // Variables\n new MagnitTokenizerSpecificationItem(\n new RegExp(/^\\$[^(\\/\\/)(\\/\\*)]*/), \n { \n key:\"VARIABLE\", // a unique name to use as a label for all matching items.\n resultCallback: async (result: string) => \n {\n // mutate the result to fit your needs\n let value = result;\n if(value.startsWith('$'))\n {\n value = value.substring(1);\n }\n return value;\n } \n }\n ),\n \n // Utility\n new MagnitTokenizerSpecificationItem(new RegExp(/^[\\s\\S]*/), { key:\"UNKNOWN\" }),\n ];\n\n if(specification == null)\n {\n specification = defaultSpecification;\n }\n const tokenizer = new MagnitTokenizer(specification);\n return await tokenizer.parse(value);\n}","deleted","label",[98],"permalink","https://www.val.town/x/catapart/magnitTokenize/code/main.tsx","extension","actionData","errors"]