[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_163":164,"_165":166},"routes/_app.x.$handle.$name",{"_95":94,"_32":45,"_103":85,"_27":36,"_13":14,"_104":-5,"_11":105,"_106":107,"_108":109,"_110":111,"_112":113,"_20":-5,"_114":-5,"_115":-5,"_116":25,"_117":118,"_119":49,"_120":-5,"_121":-5,"_122":123},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_43":44,"_32":45,"_38":46,"_47":-5,"_48":49,"_50":51,"_52":51,"_53":-5,"_54":-5,"_18":55,"_56":31,"_57":49,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":78,"_79":80,"_81":82,"_83":25},"version",2,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_33":34,"_35":37,"_38":39,"_40":14,"_41":42,"_24":25},"previous",{"_30":31,"_32":28,"_33":34,"_35":36},"deleted",false,"deletedHere","path","main.tsx","latestPath","fileId","96799b76-2792-11f0-9d52-569c3dd06744","name","fileType","http","content","","import { Hono } from \"https://deno.land/x/hono@v3.11.7/mod.ts\";\nimport { get_encoding } from \"npm:tiktoken\";\nimport \"jsr:@std/dotenv/load\"; // needed for deno run; not req for smallweb or valtown\n\nconst app = new Hono();\n\nconst html = `\n\n\n\n \n \n GitHub Repository Parser\n \n \n \n\n\n

GitHub Repository Parser

Parse and analyze GitHub repositories.

\n \n \n

Bytes

\n 0\n

0 tokens (GPT-3.5/4 - cl100k_base)

0 tokens (GPT-3 - p50k_base)

0 tokens (GPT-2 - r50k_base)

0 tokens (4o - o200k_base)

\n A labspace.ai experiment\n

\n \n\n\n`;\n\napp.get(\"/\", (c) => c.html(html));\n\napp.post(\"/parse\", async (c) => {\n try {\n const body = await c.req.json();\n const repoUrl = body.repoUrl;\n \n if (!repoUrl) {\n throw new Error(\"Repository URL is missing\");\n }\n\n const [owner, repo] = repoUrl.split('/').slice(-2);\n \n if (!owner || !repo) {\n throw new Error(\"Invalid repository URL format\");\n }\n\n const content = await fetchRepositoryContent(owner, repo);\n const tokenCounts = await countTokens(content);\n const tokenizedText = await getTokenizedText(content);\n return c.json({ content, tokenCounts, tokenizedText });\n } catch (error) {\n return c.json({ error: error.message }, 400);\n }\n});\n\napp.post(\"/tokenize\", async (c) => {\n try {\n const body = await c.req.json();\n const { text, model } = body;\n\n const encoder = get_encoding(model);\n const segments = getTiktokenSegments(encoder, text);\n\n encoder.free();\n\n return c.json({ count: segments.reduce((acc, seg) => acc + seg.tokens.length, 0), segments });\n } catch (error) {\n return c.json({ error: error.message }, 500);\n }\n});\n\nfunction getTiktokenSegments(encoder, inputText) {\n try {\n const tokens = encoder.encode(inputText);\n const segments = [];\n\n let currentSegment = '';\n let tokenAcc = [];\n\n for (let idx = 0; idx < tokens.length; idx++) {\n const token = tokens[idx];\n const tokenBytes = encoder.decode_single_token_bytes(token);\n const tokenText = new TextDecoder().decode(new Uint8Array(tokenBytes));\n \n currentSegment += tokenText;\n tokenAcc.push({ id: token, idx });\n\n // Check if we've reached a valid segment boundary\n if (inputText.startsWith(currentSegment)) {\n segments.push({ \n text: currentSegment, \n tokens: tokenAcc \n });\n\n // Remove the matched segment from the input text\n inputText = inputText.slice(currentSegment.length);\n \n // Reset for next segment\n currentSegment = '';\n tokenAcc = [];\n }\n }\n\n return segments;\n } catch (error) {\n throw new Error(`Error in getTiktokenSegments: ${error.message}`);\n }\n}\n\nasync function fetchRepositoryContent(owner, repo) {\n const apiUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/main?recursive=1`;\n const response = await fetch(apiUrl, {\n headers: {\n 'Authorization': `token ${Deno.env.get(\"GITHUB_API_KEY\")}`,\n 'Accept': 'application/vnd.github.v3+json'\n }\n });\n\n if (!response.ok) {\n throw new Error(`GitHub API request failed: ${response.statusText}`);\n }\n\n const data = await response.json();\n const files = data.tree.filter(item => item.type === 'blob' && !shouldIgnoreFile(item.path));\n \n let content = '';\n for (const file of files) {\n const fileContent = await fetchFileContent(owner, repo, file.path);\n content += `File: ${file.path}\\n\\n${fileContent}\\n\\n`;\n }\n\n return content;\n}\n\nasync function fetchFileContent(owner, repo, path) {\n const apiUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${path}`;\n const response = await fetch(apiUrl, {\n headers: {\n 'Authorization': `token ${Deno.env.get(\"GITHUB_API_KEY\")}`,\n 'Accept': 'application/vnd.github.v3+json'\n }\n });\n\n if (!response.ok) {\n throw new Error(`Failed to fetch file content: ${response.statusText}`);\n }\n\n const data = await response.json();\n return atob(data.content);\n}\n\nfunction shouldIgnoreFile(path) {\n const ignoredPatterns = [\n /package-lock\\.json$/,\n /\\.(pdf|png|jpg|jpeg|gif|ico|svg|woff|woff2|eot|ttf|otf)$/,\n /\\.(mp4|avi|webm|mov|mp3|wav|flac|ogg|webp)$/,\n /(npm-debug|yarn-debug|yarn-error)\\.log$/,\n /(tsconfig|jest\\.config)\\./,\n /^\\.git\\//\n ];\n\n return ignoredPatterns.some(pattern => pattern.test(path));\n}\n\nasync function countTokens(text) {\n const encoders = {\n cl100k_base: get_encoding(\"cl100k_base\"),\n p50k_base: get_encoding(\"p50k_base\"),\n r50k_base: get_encoding(\"r50k_base\"),\n o200k_base: get_encoding(\"o200k_base\"),\n };\n\n const counts = {};\n for (const [name, encoder] of Object.entries(encoders)) {\n counts[name] = encoder.encode(text).length;\n encoder.free();\n }\n\n return counts;\n}\n\nasync function getTokenizedText(text) {\n const encoder = get_encoding(\"cl100k_base\");\n const tokens = encoder.encode(text);\n const tokenizedHtml = tokens.map(token => {\n const tokenBytes = encoder.decode_single_token_bytes(token);\n const tokenText = new TextDecoder().decode(new Uint8Array(tokenBytes));\n return `${tokenText === '\\n' ? '↵' : tokenText}`;\n }).join('');\n encoder.free();\n return tokenizedHtml;\n}\n\n// Export app.fetch for Val Town, otherwise export app — this is only for hono apps\nexport default (typeof Deno !== \"undefined\" && Deno.env.get(\"valtown\")) ? app.fetch : app;\n","createdAt",["D",1729379862823],"branchVersion","branchId","96777da0-2792-11f0-9d52-569c3dd06744","id","96773d86-2792-11f0-9d52-569c3dd06744","githubParser",["D",1746217099157],"deletedAt","legacyEnvVars",true,"privacy","public","httpPrivacy","description","imageUrl","a8270e74-6479-475a-8025-9b69544ee0e7","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","8319defa-6e1a-46d4-ba66-2114ffd3360b","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_95":94,"_102":98},"org",{"_95":94,"_96":-5,"_97":98,"_18":55,"_99":100,"_101":25},"branch",{"_13":84,"_32":85,"_43":42,"_86":44,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":45,"_93":94},"isOwned",15,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","yawnxyz","handle","displayName","avatar","https://images.clerk.dev/oauth_github/img_2NnaHhpxNuH1xWRIRjQNoo16TVc.jpeg","tier","pro","canRestrictVals","profileImageUrl","branchName","adoptOrgId",{"_43":44,"_32":45,"_38":159,"_47":-5,"_48":49,"_50":51,"_52":51,"_53":-5,"_54":-5,"_18":55,"_56":31,"_57":49,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":160,"_79":161,"_81":162,"_83":25},"tree",[150,151],"files",[150,151],"counts",{"_140":61,"_141":131,"_142":61,"_143":144,"_145":146},"contents",[125,126],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{"_31":124},"gh.labspace.ai",{"_43":134,"_32":135,"_30":136,"_128":-5,"_41":42,"_40":131,"_129":49,"_38":137,"_13":61,"_33":20,"_35":138,"_24":25,"_132":-5,"_27":139},{"_43":127,"_32":28,"_30":31,"_128":-5,"_41":42,"_40":14,"_129":25,"_38":130,"_13":131,"_33":34,"_35":37,"_24":25,"_132":-5,"_27":133},"967ef27e-2792-11f0-9d52-569c3dd06744","parentId","latest",["D",1729379862823],1,"label",[28],"9679a0f8-2792-11f0-9d52-569c3dd06744","README.md","9679a170-2792-11f0-9d52-569c3dd06744",["D",1729379842953],"Migrated from folder: labspace/githubParser",[135],"pulls","branches","forks","envVars",{"_147":61,"_149":61},"connections",{"_147":61,"_148":61},"count","unfilledCount","emptyCount",{"_43":134,"_32":135,"_30":136,"_128":-5,"_41":42,"_40":131,"_129":49,"_38":156,"_13":61,"_33":20,"_35":138,"_24":25,"_132":-5,"_153":157,"_27":158},{"_43":127,"_32":28,"_30":31,"_128":-5,"_41":42,"_40":14,"_129":25,"_38":152,"_13":131,"_33":34,"_35":37,"_24":25,"_132":-5,"_153":154,"_27":155},["D",1729379862823],"children",[],[28],["D",1729379842953],[],[135],["D",1746217099157],{"_95":94,"_102":98},{"_95":94,"_96":-5,"_97":98,"_18":55,"_99":100,"_101":25},{"_13":84,"_32":85,"_43":42,"_86":44,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":45,"_93":94},"ctx",{"_77":-5,"_121":196,"_197":198,"_199":200,"_201":25,"_202":25,"_203":25,"_204":49,"_205":-5,"_206":-7,"_207":-7,"_208":61,"_209":25,"_210":25,"_211":-5,"_212":25,"_213":25,"_214":25,"_215":-5},"ENV",{"_167":168,"_169":170,"_171":85,"_172":173,"_174":175,"_176":177,"_178":179,"_180":181,"_182":183,"_184":185,"_186":187,"_188":189,"_190":191,"_192":193,"_194":195},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v2