[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_165":166,"_167":168},"routes/_app.x.$handle.$name",{"_95":94,"_32":45,"_103":85,"_27":104,"_13":14,"_105":-5,"_11":106,"_107":108,"_109":110,"_111":112,"_113":114,"_20":-5,"_115":-5,"_116":-5,"_117":25,"_118":119,"_120":49,"_121":-5,"_122":-5,"_123":124},"routes/_app.x.$handle.$name.branch.$branch.version.$version.diff",{"_11":12,"_13":14,"_15":16,"_17":-5,"_18":-5},"project",{"_43":44,"_32":45,"_38":46,"_47":-5,"_48":49,"_50":51,"_52":51,"_53":-5,"_54":-5,"_18":55,"_56":31,"_57":49,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":78,"_79":80,"_81":82,"_83":25},"version",8,"changes",[19],"aiChatInfo","userId",{"_20":21,"_22":23,"_24":25,"_26":25,"_27":28,"_29":28},"file",{"_30":31,"_32":28,"_33":34,"_35":37,"_38":39,"_40":14,"_41":42,"_24":25},"previous",{"_30":31,"_32":28,"_33":34,"_35":36},"deleted",false,"deletedHere","path","main.tsx","latestPath","fileId","96799b76-2792-11f0-9d52-569c3dd06744","name","fileType","http","content","import { Hono } from \"https://deno.land/x/hono@v3.11.7/mod.ts\";\nimport { get_encoding } from \"npm:tiktoken\";\nimport \"jsr:@std/dotenv/load\"; // needed for deno run; not req for smallweb or valtown\n\nconst app = new Hono();\n\nconst html = `\n\n\n\n \n \n GitHub Repository Parser\n \n \n \n\n\n

GitHub Repository Parser

Parse and analyze GitHub repositories.

\n \n

\n \n \n

Bytes

\n 0\n

0 tokens (GPT-3.5/4 - cl100k_base)

0 tokens (GPT-3 - p50k_base)

0 tokens (GPT-2 - r50k_base)

0 tokens (4o - o200k_base)

\n Source code\n

\n A labspace.ai experiment\n

\n \n\n\n`;\n\napp.get(\"/\", (c) => c.html(html));\n\napp.post(\"/parse\", async (c) => {\n try {\n const body = await c.req.json();\n const repoUrl = body.repoUrl;\n\n if (!repoUrl) {\n throw new Error(\"Repository URL is missing\");\n }\n\n const [owner, repo] = repoUrl.split('/').slice(-2);\n\n if (!owner || !repo) {\n throw new Error(\"Invalid repository URL format\");\n }\n\n const content = await fetchRepositoryContent(owner, repo);\n const tokenCounts = await countTokens(content);\n const tokenizedText = await getTokenizedText(content);\n return c.json({ content, tokenCounts, tokenizedText });\n } catch (error) {\n return c.json({ error: error.message }, 400);\n }\n});\n\napp.post(\"/tokenize\", async (c) => {\n try {\n const body = await c.req.json();\n const { text, model } = body;\n\n const encoder = get_encoding(model);\n const segments = getTiktokenSegments(encoder, text);\n\n encoder.free();\n\n return c.json({ count: segments.reduce((acc, seg) => acc + seg.tokens.length, 0), segments });\n } catch (error) {\n return c.json({ error: error.message }, 500);\n }\n});\n\nfunction getTiktokenSegments(encoder, inputText) {\n try {\n const tokens = encoder.encode(inputText);\n const segments = [];\n\n let currentSegment = '';\n let tokenAcc = [];\n\n for (let idx = 0; idx < tokens.length; idx++) {\n const token = tokens[idx];\n const tokenBytes = encoder.decode_single_token_bytes(token);\n const tokenText = new TextDecoder().decode(new Uint8Array(tokenBytes));\n\n currentSegment += tokenText;\n tokenAcc.push({ id: token, idx });\n\n // Check if we've reached a valid segment boundary\n if (inputText.startsWith(currentSegment)) {\n segments.push({\n text: currentSegment,\n tokens: tokenAcc\n });\n\n // Remove the matched segment from the input text\n inputText = inputText.slice(currentSegment.length);\n\n // Reset for next segment\n currentSegment = '';\n tokenAcc = [];\n }\n }\n\n return segments;\n } catch (error) {\n throw new Error(`Error in getTiktokenSegments: ${error.message}`);\n }\n}\n\nasync function fetchRepositoryContent(owner, repo) {\n const apiUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/main?recursive=1`;\n const response = await fetch(apiUrl, {\n headers: {\n 'Authorization': `token ${Deno.env.get(\"GITHUB_API_KEY\")}`,\n 'Accept': 'application/vnd.github.v3+json'\n }\n });\n\n if (!response.ok) {\n throw new Error(`GitHub API request failed: ${response.statusText}`);\n }\n\n const data = await response.json();\n const files = data.tree.filter(item => item.type === 'blob' && !shouldIgnoreFile(item.path));\n\n let content = '';\n for (const file of files) {\n const fileContent = await fetchFileContent(owner, repo, file.path);\n content += `File: ${file.path}\\n\\n${fileContent}\\n\\n`;\n }\n\n return content;\n}\n\nasync function fetchFileContent(owner, repo, path) {\n const apiUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${path}`;\n const response = await fetch(apiUrl, {\n headers: {\n 'Authorization': `token ${Deno.env.get(\"GITHUB_API_KEY\")}`,\n 'Accept': 'application/vnd.github.v3+json'\n }\n });\n\n if (!response.ok) {\n throw new Error(`Failed to fetch file content: ${response.statusText}`);\n }\n\n const data = await response.json();\n return atob(data.content);\n}\n\nfunction shouldIgnoreFile(path) {\n const ignoredPatterns = [\n /package-lock\\.json$/,\n /\\.(pdf|png|jpg|jpeg|gif|ico|svg|woff|woff2|eot|ttf|otf)$/,\n /\\.(mp4|avi|webm|mov|mp3|wav|flac|ogg|webp)$/,\n /(npm-debug|yarn-debug|yarn-error)\\.log$/,\n /(tsconfig|jest\\.config)\\./,\n /^\\.git\\//\n ];\n\n return ignoredPatterns.some(pattern => pattern.test(path));\n}\n\nasync function countTokens(text) {\n const encoders = {\n cl100k_base: get_encoding(\"cl100k_base\"),\n p50k_base: get_encoding(\"p50k_base\"),\n r50k_base: get_encoding(\"r50k_base\"),\n o200k_base: get_encoding(\"o200k_base\"),\n };\n\n const counts = {};\n for (const [name, encoder] of Object.entries(encoders)) {\n counts[name] = encoder.encode(text).length;\n encoder.free();\n }\n\n return counts;\n}\n\nasync function getTokenizedText(text) {\n const encoder = get_encoding(\"cl100k_base\");\n const tokens = encoder.encode(text);\n const tokenizedHtml = tokens.map(token => {\n const tokenBytes = encoder.decode_single_token_bytes(token);\n const tokenText = new TextDecoder().decode(new Uint8Array(tokenBytes));\n return `${tokenText === '\\n' ? '↵' : tokenText}`;\n }).join('');\n encoder.free();\n return tokenizedHtml;\n}\n\n// Export app.fetch for Val Town, otherwise export app — this is only for hono apps\nexport default (typeof Deno !== \"undefined\" && Deno.env.get(\"valtown\")) ? app.fetch : app;\n","import { Hono } from \"https://deno.land/x/hono@v3.11.7/mod.ts\";\nimport { get_encoding } from \"npm:tiktoken\";\nimport \"jsr:@std/dotenv/load\"; // needed for deno run; not req for smallweb or valtown\n\nconst app = new Hono();\n\nconst ignoredPatternsString = getIgnoredPatterns()\n .map(pattern => pattern.toString().slice(1, -1))\n .join('\\n');\n\n\nconst html = `\n\n\n\n \n \n GitHub Repository Parser\n \n \n \n \n\n\n

GitHub Repository Parser

Parse and analyze GitHub repositories.

Sometimes GitHub will block your request (e.g. respond with \"forbidden\"), in which case you'll just have to try again later. Or clone this project and use your own API key ;)

\n \n

\n \n \n

Bytes

\n 0\n

0 tokens (GPT-3.5/4 - cl100k_base)

0 tokens (GPT-3 - p50k_base)

0 tokens (GPT-2 - r50k_base)

0 tokens (4o - o200k_base)

\n Source code\n

\n A labspace.ai experiment\n

\n \n\n\n`;\n\napp.get(\"/\", (c) => c.html(html));\n\napp.post(\"/parse\", async (c) => {\n try {\n const body = await c.req.json();\n let { repoUrl, ignoredPatterns } = body;\n\n if (!repoUrl) {\n throw new Error(\"Repository URL is missing\");\n }\n\n // Convert ignored patterns back to RegExp objects\n ignoredPatterns = ignoredPatterns.map(pattern => new RegExp(pattern));\n\n // Extract branch if specified\n let branch = null;\n if (repoUrl.includes('#')) {\n [repoUrl, branch] = repoUrl.split('#');\n }\n\n const [owner, repo] = repoUrl.split('/').slice(-2);\n\n if (!owner || !repo) {\n throw new Error(\"Invalid repository URL format\");\n }\n\n const content = await fetchRepositoryContent(owner, repo, branch, ignoredPatterns);\n const tokenCounts = await countTokens(content);\n const tokenizedText = await getTokenizedText(content);\n return c.json({ content, tokenCounts, tokenizedText });\n } catch (error) {\n console.error(`Error parsing repository: ${error.message}`);\n return c.json({ error: `Failed to parse repository: ${error.message}` }, 400);\n }\n});\n\napp.post(\"/tokenize\", async (c) => {\n try {\n const body = await c.req.json();\n const { text, model } = body;\n\n const encoder = get_encoding(model);\n const segments = getTiktokenSegments(encoder, text);\n\n encoder.free();\n\n return c.json({ count: segments.reduce((acc, seg) => acc + seg.tokens.length, 0), segments });\n } catch (error) {\n return c.json({ error: error.message }, 500);\n }\n});\n\nfunction getTiktokenSegments(encoder, inputText) {\n try {\n const tokens = encoder.encode(inputText);\n const segments = [];\n\n let currentSegment = '';\n let tokenAcc = [];\n\n for (let idx = 0; idx < tokens.length; idx++) {\n const token = tokens[idx];\n const tokenBytes = encoder.decode_single_token_bytes(token);\n const tokenText = new TextDecoder().decode(new Uint8Array(tokenBytes));\n\n currentSegment += tokenText;\n tokenAcc.push({ id: token, idx });\n\n // Check if we've reached a valid segment boundary\n if (inputText.startsWith(currentSegment)) {\n segments.push({\n text: currentSegment,\n tokens: tokenAcc\n });\n\n // Remove the matched segment from the input text\n inputText = inputText.slice(currentSegment.length);\n\n // Reset for next segment\n currentSegment = '';\n tokenAcc = [];\n }\n }\n\n return segments;\n } catch (error) {\n throw new Error(`Error in getTiktokenSegments: ${error.message}`);\n }\n}\n\nasync function fetchRepositoryContent(owner, repo, specifiedBranch = null, ignoredPatterns) {\n // First, try to get the default branch or use the specified branch\n const repoInfoUrl = `https://api.github.com/repos/${owner}/${repo}`;\n const repoInfoResponse = await fetch(repoInfoUrl, {\n headers: {\n 'Authorization': `token ${Deno.env.get(\"GITHUB_API_KEY\")}`,\n 'Accept': 'application/vnd.github.v3+json'\n }\n });\n\n if (!repoInfoResponse.ok) {\n if (repoInfoResponse.status === 404) {\n throw new Error(`Repository not found or not accessible: ${owner}/${repo}`);\n }\n throw new Error(`Failed to fetch repository info: ${repoInfoResponse.statusText}`);\n }\n\n const repoInfo = await repoInfoResponse.json();\n const defaultBranch = specifiedBranch || repoInfo.default_branch;\n\n const apiUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${defaultBranch}?recursive=1`;\n const response = await fetch(apiUrl, {\n headers: {\n 'Authorization': `token ${Deno.env.get(\"GITHUB_API_KEY\")}`,\n 'Accept': 'application/vnd.github.v3+json'\n }\n });\n\n if (!response.ok) {\n if (response.status === 404) {\n throw new Error(`Failed to fetch repository content. The ${defaultBranch} branch might not exist.`);\n }\n throw new Error(`GitHub API request failed: ${response.statusText}`);\n }\n\n const data = await response.json();\n const files = data.tree.filter(item => item.type === 'blob' && !shouldIgnoreFile(item.path, ignoredPatterns));\n\n let content = '';\n for (const file of files) {\n try {\n const fileContent = await fetchFileContent(owner, repo, file.path, defaultBranch);\n content += `File: ${file.path}\\n\\n${fileContent}\\n\\n`;\n } catch (error) {\n console.error(`Failed to fetch content for ${file.path}: ${error.message}`);\n content += `File: ${file.path}\\n\\nError: Failed to fetch content\\n\\n`;\n }\n }\n\n return content;\n}\n\nasync function fetchFileContent(owner, repo, path, branch) {\n const apiUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${path}`;\n const response = await fetch(apiUrl, {\n headers: {\n 'Authorization': `token ${Deno.env.get(\"GITHUB_API_KEY\")}`,\n 'Accept': 'application/vnd.github.v3+json'\n }\n });\n\n if (!response.ok) {\n throw new Error(`Failed to fetch file content: ${response.statusText}`);\n }\n\n const data = await response.json();\n return atob(data.content);\n}\n\nfunction getIgnoredPatterns() {\n return [\n // Folders\n /^\\.github\\//,\n /^dist\\//,\n /^public\\//,\n /^static\\//,\n /^node_modules\\//,\n /^vendor\\//,\n /^test\\//,\n /^\\.git\\//,\n\n // Files\n /\\.npmrc$/,\n /\\.gitignore$/,\n /\\.prettierrc$/,\n /package-lock\\.json$/,\n /yarn\\.lock$/,\n /tsconfig\\.json$/,\n /\\.yaml$/,\n /\\.yml$/,\n /^LICENSE$/,\n\n // Lint configuration files\n /\\.eslintrc(\\.json|\\.js|\\.yml|\\.yaml)?$/,\n /\\.prettierrc(\\.json|\\.js|\\.yml|\\.yaml)?$/,\n /\\.stylelintrc(\\.json|\\.js|\\.yml|\\.yaml)?$/,\n /\\.jshintrc$/,\n /\\.jscsrc$/,\n /\\.editorconfig$/,\n /\\.babelrc$/,\n /\\.flowconfig$/,\n /\\.postcssrc$/,\n /\\.htmlhintrc$/,\n /\\.csslintrc$/,\n /\\.sasslintrc$/,\n /\\.lesshintrc$/,\n /\\.markdownlintrc$/,\n /\\.coffeelintrc$/,\n /\\.eslintignore$/,\n /\\.prettierignore$/,\n /\\.stylelintignore$/,\n\n // Additional configuration files\n /\\.env(\\..+)?$/,\n /\\.dockerignore$/,\n /Dockerfile$/,\n /docker-compose\\.yml$/,\n /\\.travis\\.yml$/,\n /\\.circleci\\/config\\.yml$/,\n /\\.gitlab-ci\\.yml$/,\n /\\.github\\/workflows\\/.*\\.yml$/,\n /\\.vscode\\/.*$/,\n /\\.idea\\/.*$/,\n\n // Additional ignored patterns from .npmignore\n /^samples\\//,\n /^addons\\//,\n /^libs\\//,\n /\\.njsproj$/,\n /^\\..*$/, // This covers all dot files/folders\n /\\.log$/,\n /^tools\\//,\n /^bower\\.json$/,\n /^\\.lambdatest$/,\n /^tunnel\\.pid$/,\n /^pnpm-workspace\\.yaml$/,\n\n // File types\n /\\.md$/, // Ignore all .md files except README.md\n /\\.(pdf|png|jpg|jpeg|gif|ico|svg|woff|woff2|eot|ttf|otf)$/,\n /\\.(mp4|avi|webm|mov|mp3|wav|flac|ogg|webp)$/,\n /(npm-debug|yarn-debug|yarn-error)\\.log$/,\n ];\n}\n\nfunction shouldIgnoreFile(path, ignoredPatterns) {\n // Special case for README.md\n if (path.toLowerCase() === 'readme.md') {\n return false;\n }\n\n return ignoredPatterns.some(pattern => pattern.test(path));\n}\n\nasync function countTokens(text) {\n const encoders = {\n cl100k_base: get_encoding(\"cl100k_base\"),\n p50k_base: get_encoding(\"p50k_base\"),\n r50k_base: get_encoding(\"r50k_base\"),\n o200k_base: get_encoding(\"o200k_base\"),\n };\n\n const counts = {};\n for (const [name, encoder] of Object.entries(encoders)) {\n counts[name] = encoder.encode(text).length;\n encoder.free();\n }\n\n return counts;\n}\n\nasync function getTokenizedText(text) {\n const encoder = get_encoding(\"cl100k_base\");\n const tokens = encoder.encode(text);\n const tokenizedHtml = tokens.map(token => {\n const tokenBytes = encoder.decode_single_token_bytes(token);\n const tokenText = new TextDecoder().decode(new Uint8Array(tokenBytes));\n return `${tokenText === '\\n' ? '↵' : tokenText}`;\n }).join('');\n encoder.free();\n return tokenizedHtml;\n}\n\n// Export app.fetch for Val Town, otherwise export app — this is only for hono apps\nexport default (typeof Deno !== \"undefined\" && Deno.env.get(\"valtown\")) ? app.fetch : app;\n","createdAt",["D",1729553485558],"branchVersion","branchId","96777da0-2792-11f0-9d52-569c3dd06744","id","96773d86-2792-11f0-9d52-569c3dd06744","githubParser",["D",1746217099157],"deletedAt","legacyEnvVars",true,"privacy","public","httpPrivacy","description","imageUrl","a8270e74-6479-475a-8025-9b69544ee0e7","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount",0,"orgId","8319defa-6e1a-46d4-ba66-2114ffd3360b","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_95":94,"_102":98},"org",{"_95":94,"_96":-5,"_97":98,"_18":55,"_99":100,"_101":25},"branch",{"_13":84,"_32":85,"_43":42,"_86":44,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":45,"_93":94},"isOwned",15,"main","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","yawnxyz","handle","displayName","avatar","https://images.clerk.dev/oauth_github/img_2NnaHhpxNuH1xWRIRjQNoo16TVc.jpeg","tier","pro","canRestrictVals","profileImageUrl","branchName","","adoptOrgId",{"_43":44,"_32":45,"_38":161,"_47":-5,"_48":49,"_50":51,"_52":51,"_53":-5,"_54":-5,"_18":55,"_56":31,"_57":49,"_58":-5,"_59":-5,"_60":61,"_62":63,"_64":-5,"_65":-5,"_66":-5,"_67":-5,"_68":69,"_70":69,"_71":72,"_73":72,"_74":72,"_75":69,"_76":69,"_77":162,"_79":163,"_81":164,"_83":25},"tree",[152,153],"files",[152,153],"counts",{"_142":61,"_143":138,"_144":61,"_145":146,"_147":148},"contents",[126,127],"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{"_31":125},"gh.labspace.ai",{"_43":135,"_32":136,"_30":137,"_129":-5,"_41":42,"_40":138,"_130":49,"_38":139,"_13":61,"_33":20,"_35":140,"_24":25,"_133":-5,"_27":141},{"_43":128,"_32":28,"_30":31,"_129":-5,"_41":42,"_40":14,"_130":25,"_38":131,"_13":132,"_33":34,"_35":37,"_24":25,"_133":-5,"_27":134},"96bf7dc6-2792-11f0-9d52-569c3dd06744","parentId","latest",["D",1729553485558],7,"label",[28],"9679a0f8-2792-11f0-9d52-569c3dd06744","README.md","9679a170-2792-11f0-9d52-569c3dd06744",1,["D",1729379842953],"Migrated from folder: labspace/githubParser",[136],"pulls","branches","forks","envVars",{"_149":61,"_151":61},"connections",{"_149":61,"_150":61},"count","unfilledCount","emptyCount",{"_43":135,"_32":136,"_30":137,"_129":-5,"_41":42,"_40":138,"_130":49,"_38":158,"_13":61,"_33":20,"_35":140,"_24":25,"_133":-5,"_155":159,"_27":160},{"_43":128,"_32":28,"_30":31,"_129":-5,"_41":42,"_40":14,"_130":25,"_38":154,"_13":132,"_33":34,"_35":37,"_24":25,"_133":-5,"_155":156,"_27":157},["D",1729553485558],"children",[],[28],["D",1729379842953],[],[136],["D",1746217099157],{"_95":94,"_102":98},{"_95":94,"_96":-5,"_97":98,"_18":55,"_99":100,"_101":25},{"_13":84,"_32":85,"_43":42,"_86":44,"_87":-5,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":45,"_93":94},"ctx",{"_77":-5,"_122":198,"_199":200,"_201":202,"_203":25,"_204":25,"_205":25,"_206":49,"_207":-5,"_208":-7,"_209":-7,"_210":61,"_211":25,"_212":25,"_213":-5,"_214":25,"_215":25,"_216":25,"_217":-5},"ENV",{"_169":170,"_171":172,"_173":85,"_174":175,"_176":177,"_178":179,"_180":181,"_182":183,"_184":185,"_186":187,"_188":189,"_190":191,"_192":193,"_194":195,"_196":197},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]

Diff · v8