[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_163":164,"_165":166},"routes/_app.x.$handle.$name",{"_11":12,"_13":14,"_15":16,"_18":19,"_17":-7,"_123":-5,"_20":124,"_22":125,"_126":127,"_128":129,"_26":130,"_24":131,"_132":-5,"_133":-5,"_134":52,"_135":136,"_137":43,"_138":-5,"_139":-5,"_140":141},"routes/_app.x.$handle.$name.branch.$branch.code.$",{"_11":12,"_13":14,"_15":16,"_17":-7,"_18":19,"_20":21,"_22":23,"_24":25,"_26":27,"_28":-5,"_29":-5,"_30":-5,"_31":-5,"_32":-5},"handle","yawnxyz","name","githubParser","branchName","main","version","path","main.tsx","project",{"_33":70,"_13":14,"_44":71,"_72":-5,"_73":43,"_74":75,"_76":75,"_77":-5,"_78":-5,"_79":80,"_81":36,"_82":43,"_83":-5,"_84":-5,"_85":66,"_86":87,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":93,"_94":93,"_95":96,"_97":96,"_98":96,"_99":93,"_100":93,"_101":102,"_103":104,"_105":106,"_107":52},"tree",[55,56],"file",{"_33":34,"_13":19,"_35":36,"_37":-5,"_38":39,"_40":41,"_42":43,"_44":45,"_17":46,"_47":48,"_49":50,"_51":52,"_53":-5,"_18":54},"contents",[],"upstream","pull","ahead","behind","envVars","id","96f9e4de-2792-11f0-9d52-569c3dd06744","fileId","96799b76-2792-11f0-9d52-569c3dd06744","parentId","branchId","96777da0-2792-11f0-9d52-569c3dd06744","branchVersion",15,"latest",true,"createdAt",["D",1738446498357],14,"fileType","http","content","import { Hono } from \"https://deno.land/x/hono@v3.11.7/mod.ts\";\nimport { get_encoding } from \"npm:tiktoken\";\nimport \"jsr:@std/dotenv/load\"; // needed for deno run; not req for smallweb or valtown\n\nconst app = new Hono();\n\nconst ignoredPatternsString = getIgnoredPatterns()\n .map(pattern => pattern.toString().slice(1, -1))\n .join('\\n');\n\n\nconst html = `\n\n\n\n \n \n \n GitHub Repository Parser\n \n \n \n \n \n \n \n \n \n \n \n \n \n\n\n

GitHub Repository Parser

Parse and analyze GitHub repositories.

Sometimes GitHub will block your request (e.g. respond with \"forbidden\"), in which case you'll just have to try again later. Or clone this project and use your own API key ;)

\n \n

Select File Types to Include:

\n \n

\n \n \n

\n\n

Bytes

\n 0\n

0 tokens (GPT-3.5/4 - cl100k_base)

0 tokens (GPT-3 - p50k_base)

0 tokens (GPT-2 - r50k_base)

0 tokens (4o - o200k_base)

\n Source code\n

\n A labspace.ai experiment\n

\n \n\n\n`;\n\napp.get(\"/\", (c) => c.html(html));\n\napp.post(\"/parse\", async (c) => {\n try {\n console.log('Received parse request');\n const body = await c.req.json();\n console.log('Request body:', body);\n \n let { repoUrl, ignoredPatterns, selectedTypes } = body;\n\n if (!repoUrl) {\n throw new Error(\"Repository URL is missing\");\n }\n\n // Convert ignored patterns back to RegExp objects\n ignoredPatterns = ignoredPatterns.map(pattern => new RegExp(pattern));\n console.log('Processing with selected types:', selectedTypes);\n\n // Extract branch if specified\n let branch = null;\n if (repoUrl.includes('#')) {\n [repoUrl, branch] = repoUrl.split('#');\n }\n\n const [owner, repo] = repoUrl.split('/').slice(-2);\n\n if (!owner || !repo) {\n throw new Error(\"Invalid repository URL format\");\n }\n\n console.log(`Fetching content for ${owner}/${repo}`);\n const content = await fetchRepositoryContent(owner, repo, branch, ignoredPatterns, selectedTypes);\n const tokenCounts = await countTokens(content);\n \n console.log('Successfully processed repository');\n return c.json({ content, tokenCounts });\n } catch (error) {\n console.error(`Error parsing repository: ${error.message}`);\n return c.json({ error: `Failed to parse repository: ${error.message}` }, 400);\n }\n});\n\napp.post(\"/tokenize\", async (c) => {\n try {\n const body = await c.req.json();\n const { text, model } = body;\n\n const encoder = get_encoding(model);\n const segments = getTiktokenSegments(encoder, text);\n\n encoder.free();\n\n return c.json({ count: segments.reduce((acc, seg) => acc + seg.tokens.length, 0), segments });\n } catch (error) {\n return c.json({ error: error.message }, 500);\n }\n});\n\nfunction getTiktokenSegments(encoder, inputText) {\n try {\n const tokens = encoder.encode(inputText);\n const segments = [];\n\n let currentSegment = '';\n let tokenAcc = [];\n\n for (let idx = 0; idx < tokens.length; idx++) {\n const token = tokens[idx];\n const tokenBytes = encoder.decode_single_token_bytes(token);\n const tokenText = new TextDecoder().decode(new Uint8Array(tokenBytes));\n\n currentSegment += tokenText;\n tokenAcc.push({ id: token, idx });\n\n // Check if we've reached a valid segment boundary\n if (inputText.startsWith(currentSegment)) {\n segments.push({\n text: currentSegment,\n tokens: tokenAcc\n });\n\n // Remove the matched segment from the input text\n inputText = inputText.slice(currentSegment.length);\n\n // Reset for next segment\n currentSegment = '';\n tokenAcc = [];\n }\n }\n\n return segments;\n } catch (error) {\n throw new Error(`Error in getTiktokenSegments: ${error.message}`);\n }\n}\n\nfunction getFileExtension(filename) {\n const parts = filename.split('.');\n return parts.length > 1 ? '.' + parts.pop().toLowerCase() : 'no-extension';\n}\n\n\nfunction getFileType(path) {\n // Handle directories\n if (path.endsWith('/')) {\n return path;\n }\n\n // Handle files\n const ext = getFileExtension(path);\n if (ext === 'no-extension') {\n // Return the filename for files without extensions\n return path.split('/').pop();\n }\n return ext;\n}\n\nasync function fetchRepositoryContent(owner, repo, specifiedBranch = null, ignoredPatterns, selectedTypes = null) {\n // First, try to get the default branch or use the specified branch\n const repoInfoUrl = `https://api.github.com/repos/${owner}/${repo}`;\n const repoInfoResponse = await fetch(repoInfoUrl, {\n headers: {\n 'Authorization': `token ${Deno.env.get(\"GITHUB_API_KEY\")}`,\n 'Accept': 'application/vnd.github.v3+json'\n }\n });\n\n if (!repoInfoResponse.ok) {\n if (repoInfoResponse.status === 404) {\n throw new Error(`Repository not found or not accessible: ${owner}/${repo}`);\n }\n throw new Error(`Failed to fetch repository info: ${repoInfoResponse.statusText}`);\n }\n\n const repoInfo = await repoInfoResponse.json();\n const defaultBranch = specifiedBranch || repoInfo.default_branch;\n\n const apiUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${defaultBranch}?recursive=1`;\n const response = await fetch(apiUrl, {\n headers: {\n 'Authorization': `token ${Deno.env.get(\"GITHUB_API_KEY\")}`,\n 'Accept': 'application/vnd.github.v3+json'\n }\n });\n\n if (!response.ok) {\n if (response.status === 404) {\n throw new Error(`Failed to fetch repository content. The ${defaultBranch} branch might not exist.`);\n }\n throw new Error(`GitHub API request failed: ${response.statusText}`);\n }\n\n const data = await response.json();\n console.log('Selected types for filtering:', selectedTypes);\n \n const files = data.tree.filter(item => {\n if (item.type !== 'blob') return false;\n \n const fileType = getFileType(item.path);\n console.log(`Checking file ${item.path} with type ${fileType}`);\n \n // If no types are selected, include all non-ignored files\n if (!selectedTypes || selectedTypes.length === 0) {\n return !shouldIgnoreFile(item.path, ignoredPatterns);\n }\n \n // Otherwise, check if the file type is selected and not ignored\n return selectedTypes.includes(fileType) && !shouldIgnoreFile(item.path, ignoredPatterns);\n });\n\n console.log('Filtered files:', files.map(f => f.path));\n\n let content = '';\n for (const file of files) {\n try {\n const fileContent = await fetchFileContent(owner, repo, file.path, defaultBranch);\n content += `File: ${file.path}\\n\\n${fileContent}\\n\\n`;\n } catch (error) {\n console.error(`Failed to fetch content for ${file.path}: ${error.message}`);\n content += `File: ${file.path}\\n\\nError: Failed to fetch content\\n\\n`;\n }\n }\n\n if (!content) {\n console.log('No content was gathered from any files');\n throw new Error('No files were found matching the selected criteria');\n }\n\n return content;\n}\n\nasync function fetchFileContent(owner, repo, path, branch) {\n const apiUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${path}`;\n const response = await fetch(apiUrl, {\n headers: {\n 'Authorization': `token ${Deno.env.get(\"GITHUB_API_KEY\")}`,\n 'Accept': 'application/vnd.github.v3+json'\n }\n });\n\n if (!response.ok) {\n throw new Error(`Failed to fetch file content: ${response.statusText}`);\n }\n\n const data = await response.json();\n return atob(data.content);\n}\n\nfunction getIgnoredPatterns() {\n return [\n // Folders\n /^\\.github\\//,\n /^dist\\//,\n /^public\\//,\n /^static\\//,\n /^node_modules\\//,\n /^vendor\\//,\n /^test\\//,\n /^\\.git\\//,\n\n // Files\n /\\.npmrc$/,\n /\\.gitignore$/,\n /\\.prettierrc$/,\n /package-lock\\.json$/,\n /yarn\\.lock$/,\n /tsconfig\\.json$/,\n /\\.yaml$/,\n /\\.yml$/,\n /^LICENSE$/,\n\n // Lint configuration files\n /\\.eslintrc(\\.json|\\.js|\\.yml|\\.yaml)?$/,\n /\\.prettierrc(\\.json|\\.js|\\.yml|\\.yaml)?$/,\n /\\.stylelintrc(\\.json|\\.js|\\.yml|\\.yaml)?$/,\n /\\.jshintrc$/,\n /\\.jscsrc$/,\n /\\.editorconfig$/,\n /\\.babelrc$/,\n /\\.flowconfig$/,\n /\\.postcssrc$/,\n /\\.htmlhintrc$/,\n /\\.csslintrc$/,\n /\\.sasslintrc$/,\n /\\.lesshintrc$/,\n /\\.markdownlintrc$/,\n /\\.coffeelintrc$/,\n /\\.eslintignore$/,\n /\\.prettierignore$/,\n /\\.stylelintignore$/,\n\n // Additional configuration files\n /\\.env(\\..+)?$/,\n /\\.dockerignore$/,\n /Dockerfile$/,\n /docker-compose\\.yml$/,\n /\\.travis\\.yml$/,\n /\\.circleci\\/config\\.yml$/,\n /\\.gitlab-ci\\.yml$/,\n /\\.github\\/workflows\\/.*\\.yml$/,\n /\\.vscode\\/.*$/,\n /\\.idea\\/.*$/,\n\n // Additional ignored patterns from .npmignore\n /^samples\\//,\n /^addons\\//,\n /^libs\\//,\n /\\.njsproj$/,\n /^\\..*$/, // This covers all dot files/folders\n /\\.log$/,\n /^tools\\//,\n /^bower\\.json$/,\n /^\\.lambdatest$/,\n /^tunnel\\.pid$/,\n /^pnpm-workspace\\.yaml$/,\n\n // File types\n /\\.md$/, // Ignore all .md files except README.md\n /\\.(pdf|png|jpg|jpeg|gif|ico|svg|woff|woff2|eot|ttf|otf)$/,\n /\\.(mp4|avi|webm|mov|mp3|wav|flac|ogg|webp)$/,\n /(npm-debug|yarn-debug|yarn-error)\\.log$/,\n ];\n}\n\nfunction shouldIgnoreFile(path, ignoredPatterns) {\n // Special case for README.md\n if (path.toLowerCase() === 'readme.md') {\n return false;\n }\n\n return ignoredPatterns.some(pattern => pattern.test(path));\n}\n\nasync function countTokens(text) {\n const encoders = {\n cl100k_base: get_encoding(\"cl100k_base\"),\n p50k_base: get_encoding(\"p50k_base\"),\n r50k_base: get_encoding(\"r50k_base\"),\n o200k_base: get_encoding(\"o200k_base\"),\n };\n\n const counts = {};\n for (const [name, encoder] of Object.entries(encoders)) {\n counts[name] = encoder.encode(text).length;\n encoder.free();\n }\n\n return counts;\n}\n\nasync function getTokenizedText(text) {\n const encoder = get_encoding(\"cl100k_base\");\n const tokens = encoder.encode(text);\n const tokenizedHtml = tokens.map(token => {\n const tokenBytes = encoder.decode_single_token_bytes(token);\n const tokenText = new TextDecoder().decode(new Uint8Array(tokenBytes));\n return `${tokenText === '\\n' ? '↵' : tokenText}`;\n }).join('');\n encoder.free();\n return tokenizedHtml;\n}\n\n// Add new endpoint to handle file type fetching\napp.post(\"/file-types\", async (c) => {\n try {\n const { repoUrl } = await c.req.json();\n \n if (!repoUrl) {\n throw new Error(\"Repository URL is missing\");\n }\n \n let branch = null;\n if (repoUrl.includes('#')) {\n [repoUrl, branch] = repoUrl.split('#');\n }\n \n const [owner, repo] = repoUrl.split('/').slice(-2);\n \n if (!owner || !repo) {\n throw new Error(\"Invalid repository URL format\");\n }\n \n const fileTypes = await getRepositoryFileTypes(owner, repo, branch);\n return c.json(fileTypes);\n } catch (error) {\n return c.json({ error: error.message }, 400);\n }\n});\n\n// Add new function to get repository file types\nasync function getRepositoryFileTypes(owner, repo, specifiedBranch = null) {\n const repoInfoUrl = `https://api.github.com/repos/${owner}/${repo}`;\n const repoInfoResponse = await fetch(repoInfoUrl, {\n headers: {\n 'Authorization': `token ${Deno.env.get(\"GITHUB_API_KEY\")}`,\n 'Accept': 'application/vnd.github.v3+json'\n }\n });\n \n if (!repoInfoResponse.ok) {\n throw new Error(`Failed to fetch repository info: ${repoInfoResponse.statusText}`);\n }\n \n const repoInfo = await repoInfoResponse.json();\n const defaultBranch = specifiedBranch || repoInfo.default_branch;\n \n const apiUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${defaultBranch}?recursive=1`;\n const response = await fetch(apiUrl, {\n headers: {\n 'Authorization': `token ${Deno.env.get(\"GITHUB_API_KEY\")}`,\n 'Accept': 'application/vnd.github.v3+json'\n }\n });\n \n if (!response.ok) {\n throw new Error(`GitHub API request failed: ${response.statusText}`);\n }\n \n const data = await response.json();\n \n // Count occurrences of each file type\n const typeCount = new Map();\n data.tree.forEach(item => {\n const type = getFileType(item.path);\n typeCount.set(type, (typeCount.get(type) || 0) + 1);\n });\n \n // Convert to array and sort by count\n return Array.from(typeCount.entries())\n .map(([type, count]) => ({ type, count }))\n .sort((a, b) => b.count - a.count);\n}\n\n// Export app.fetch for Val Town, otherwise export app — this is only for hono apps\nexport default (typeof Deno !== \"undefined\" && Deno.env.get(\"valtown\")) ? app.fetch : app;\n","deleted",false,"label",[19],{"_33":61,"_13":62,"_35":63,"_37":-5,"_38":39,"_40":64,"_42":43,"_44":65,"_17":66,"_47":24,"_49":67,"_51":52,"_53":-5,"_58":68,"_18":69},{"_33":34,"_13":19,"_35":36,"_37":-5,"_38":39,"_40":41,"_42":43,"_44":57,"_17":46,"_47":48,"_49":50,"_51":52,"_53":-5,"_58":59,"_18":60},["D",1738446498357],"children",[],[19],"9679a0f8-2792-11f0-9d52-569c3dd06744","README.md","9679a170-2792-11f0-9d52-569c3dd06744",1,["D",1729379842953],0,"Migrated from folder: labspace/githubParser",[],[62],"96773d86-2792-11f0-9d52-569c3dd06744",["D",1746217099157],"deletedAt","legacyEnvVars","privacy","public","httpPrivacy","description","imageUrl","userId","a8270e74-6479-475a-8025-9b69544ee0e7","pinnedFileId","pinPreview","pinMigrated","screenshotVersion","likeCount","orgId","8319defa-6e1a-46d4-ba66-2114ffd3360b","tursoName","tursoHostname","tursoMeasuredAt","pinned","userPerm","r","valPerm","blobPerm","rw","emailPerm","sqlitePerm","projectPerm","telemetryPerm","user",{"_11":12,"_122":118},"org",{"_11":12,"_116":-5,"_117":118,"_79":80,"_119":120,"_121":52},"branch",{"_17":41,"_13":16,"_33":39,"_108":70,"_109":-5,"_110":-5,"_111":-5,"_112":-5,"_113":-5,"_114":14,"_115":12},"isOwned","projectId","forkedBranchId","forkedBranchVersion","forkedProjectPrivacy","forkedProjectId","forkedProjectOrgId","projectName","userHandle","displayName","avatar","https://images.clerk.dev/oauth_github/img_2NnaHhpxNuH1xWRIRjQNoo16TVc.jpeg","tier","pro","canRestrictVals","profileImageUrl","adoptOrgId",{"_33":70,"_13":14,"_44":159,"_72":-5,"_73":43,"_74":75,"_76":75,"_77":-5,"_78":-5,"_79":80,"_81":36,"_82":43,"_83":-5,"_84":-5,"_85":66,"_86":87,"_88":-5,"_89":-5,"_90":-5,"_91":-5,"_92":93,"_94":93,"_95":96,"_97":96,"_98":96,"_99":93,"_100":93,"_101":160,"_103":161,"_105":162,"_107":52},[153,154],"files",[153,154],"counts",{"_144":66,"_145":64,"_146":66,"_32":147,"_148":149},[],{"_33":34,"_13":19,"_35":36,"_37":-5,"_38":39,"_40":41,"_42":43,"_44":45,"_17":46,"_47":48,"_49":50,"_51":52,"_53":-5,"_18":143},"upstreamProject","upstreamBranch","emptyEnvVars","flags",[],"hasHttpFiles","grants","orgs","customDomains",{"_36":142},"gh.labspace.ai",[19],"pulls","branches","forks",{"_150":66,"_152":66},"connections",{"_150":66,"_151":66},"count","unfilledCount","emptyCount",{"_33":61,"_13":62,"_35":63,"_37":-5,"_38":39,"_40":64,"_42":43,"_44":65,"_17":66,"_47":24,"_49":67,"_51":52,"_53":-5,"_58":157,"_18":158},{"_33":34,"_13":19,"_35":36,"_37":-5,"_38":39,"_40":41,"_42":43,"_44":57,"_17":46,"_47":48,"_49":50,"_51":52,"_53":-5,"_58":155,"_18":156},[],[19],[],[62],["D",1746217099157],{"_11":12,"_122":118},{"_11":12,"_116":-5,"_117":118,"_79":80,"_119":120,"_121":52},{"_17":41,"_13":16,"_33":39,"_108":70,"_109":-5,"_110":-5,"_111":-5,"_112":-5,"_113":-5,"_114":14,"_115":12},"ctx",{"_101":-5,"_139":196,"_197":198,"_199":200,"_201":52,"_202":52,"_203":52,"_204":43,"_205":-5,"_206":-7,"_207":-7,"_208":66,"_209":52,"_210":52,"_211":-5,"_212":52,"_213":52,"_214":52,"_215":-5},"ENV",{"_167":168,"_169":170,"_171":16,"_172":173,"_174":175,"_176":177,"_178":179,"_180":181,"_182":183,"_184":185,"_186":187,"_188":189,"_190":191,"_192":193,"_194":195},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","RENDER_GIT_COMMIT","e87eb04e707cf64e7e532c8e8ec395f209e17f15","RENDER_GIT_BRANCH","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","VANITY_WEB_API_TEMPLATE","https://{label}.val.run/","DENOLS_URL","https://lsp.val.town","NODE_ENV","production","CLOUDFLARE_TURNSTILE_SITE_KEY","0x4AAAAAACtaIZJH7s9rBMZ6","GOOGLE_CLIENT_ID","875989696899-jtsqkhlbdv5jrlsk8631p7ipjg3c6o62.apps.googleusercontent.com","STRIPE_PUBLISHABLE_KEY","pk_live_51N2zvWEpZTTLipIkVEwXrUG9AT0yeOOEYxszarZT1jpsZV58Ly6wlg3SltZpHN26CUalOBmUSY7adQRQiSkCZz2500qSXNkf8Y","API_URL","https://api.val.town",[],"preferencesKeybindings","default","preferencesTheme","light","preferencesFixers","preferencesCopilot","preferencesSelectionAi","preferencesCloseBrackets","preferencesChatPanelState","preferencesDprint","preferencesDenofmt","notificationCount","flagDenols","isBusinessOrgMember","businessOrgHandle","isBusinessOrgAdmin","isHigherTownieTier","navCollapsed","currentOrgId"]